feat(agent): plan-first gate, iterative follow-ups, reasoning persistence
P1 plan-first: run handler refuses without propose_plan (structural gate,
not SOUL.md prose). Plan window decoupled from set_goal — config_mutation
auto-run only on operator approval (assent window). Closes the approval-free
config_mutation hole confirmed in session d0d562e0.
P2 iteration: reopenSession flips terminal→executing, marks prior plan steps
replaced, clears outcome. proposePlan excludes replaced from in-flight check,
bumps generation. A follow-up on a completed session starts a new sub-task
with a fresh plan — no more errPlanInFlight dead end.
P3 reasoning: accumulate per-iteration text into the persisted row instead
of overwriting with the last text event. Reload shows intermediate thinking,
not just the final summary.
P4 read-only allowlist: add find, tree, locate, systemctl list-timers/
list-unit-files/show, timedatectl, hostnamectl, systemd-analyze, rclone
ls/lsl/md5sum/check/cryptcheck. Fixes the find misclassification from
d0d562e0.
P5 eval harness: new assertion kinds (proposes_plan, plan_before_run,
plan_generations), multi-turn followups, fetch /sessions/{id}/plan. Four
manifests under evals/.
P6 SOUL.md: strip degenerate-case carve-out, add ITERATE step, update
set_goal guidance.
VERSION 0.6.0 → 0.7.0
This commit is contained in:
@@ -420,6 +420,18 @@ func (a *agent) chatWith(ctx context.Context, sessionID, message, systemInject s
|
||||
return
|
||||
}
|
||||
|
||||
// P3: persist intermediate reasoning. When the model produces text
|
||||
// AND tool calls in the same iteration, the text is its reasoning
|
||||
// before the tool calls — the operator saw it live via text_delta,
|
||||
// but without emitting it as a `text` event here, the persist layer
|
||||
// (main.go/continue.go) never captures it and a reload shows only
|
||||
// the final summary + a flat tool-call list, not the thinking that
|
||||
// led to each step. Emitting it lets the persist layer accumulate
|
||||
// per-iteration reasoning into the row's text field.
|
||||
if strings.TrimSpace(msg.Content) != "" {
|
||||
emit(agentEvent{Type: "text", Data: msg.Content, SessionID: sessionID})
|
||||
}
|
||||
|
||||
slog.Info("nomos: tool calls", "count", len(msg.ToolCalls), "iter", i+1, "correlation", correlationID)
|
||||
|
||||
messages = append(messages, msg.ToParam())
|
||||
|
||||
@@ -247,6 +247,10 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
}
|
||||
}
|
||||
toolCalls, finalText, errText = nil, "", ""
|
||||
// P3: accumulate per-iteration reasoning instead of overwriting
|
||||
// (same fix as main.go's chat handler). Without this, a resumed
|
||||
// turn's intermediate thinking is lost on reload.
|
||||
var textParts []string
|
||||
emit := func(ev agentEvent) {
|
||||
if ev.Type == "tool_use" || ev.Type == "tool_result" {
|
||||
if m, ok := ev.Data.(map[string]any); ok {
|
||||
@@ -272,7 +276,11 @@ func (a *agent) resumeSession(ctx context.Context, sessionID, note string) {
|
||||
persist() // live: a poller sees this step land within seconds
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
finalText, _ = ev.Data.(string)
|
||||
if t, ok := ev.Data.(string); ok && t != "" {
|
||||
textParts = append(textParts, t)
|
||||
finalText = strings.Join(textParts, "\n\n")
|
||||
persist()
|
||||
}
|
||||
}
|
||||
if ev.Type == "error" {
|
||||
errText, _ = ev.Data.(string)
|
||||
|
||||
@@ -125,8 +125,8 @@ func runConversation(ctx context.Context, gateway string, c conversation, timeou
|
||||
}
|
||||
|
||||
// Send followup if any.
|
||||
if c.Followup != "" {
|
||||
if _, err := sendChat(ctx, gateway, sid, c.Followup); err != nil {
|
||||
for _, fu := range c.followups() {
|
||||
if _, err := sendChat(ctx, gateway, sid, fu); err != nil {
|
||||
res.Assertions = []assertionResult{{Name: "send_followup", Passed: false, Detail: err.Error()}}
|
||||
res.Duration = time.Since(start)
|
||||
return res
|
||||
@@ -246,6 +246,19 @@ type transcript struct {
|
||||
ToolCalls []map[string]any `json:"tool_calls"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
// PlanSteps is fetched from /sessions/{id}/plan (P5 plan_generations
|
||||
// assertion). Each step carries a `generation` int; distinctGenerations
|
||||
// counts the unique values. nil when the endpoint returned no plan
|
||||
// (e.g. a pure-DB Q&A with no propose_plan call).
|
||||
PlanSteps []planStep `json:"steps"`
|
||||
}
|
||||
|
||||
// planStep is one step from /sessions/{id}/plan, carrying only the fields the
|
||||
// eval needs: the generation number (P2 iteration counter).
|
||||
type planStep struct {
|
||||
Generation int `json:"generation"`
|
||||
Status string `json:"status"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
func (t transcript) toolCallCount() int {
|
||||
@@ -268,6 +281,17 @@ func (t transcript) toolNames() []string {
|
||||
return names
|
||||
}
|
||||
|
||||
// distinctGenerations counts unique plan generation values across all plan
|
||||
// steps. Used by the `plan_generations` assertion (P2 iteration). Returns 0
|
||||
// when there are no plan steps (no propose_plan was called).
|
||||
func (t transcript) distinctGenerations() int {
|
||||
seen := map[int]bool{}
|
||||
for _, s := range t.PlanSteps {
|
||||
seen[s.Generation] = true
|
||||
}
|
||||
return len(seen)
|
||||
}
|
||||
|
||||
type sessionState struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
@@ -277,7 +301,8 @@ type sessionState struct {
|
||||
|
||||
// fetchTranscript fetches the messages from /sessions/{id} (which returns
|
||||
// only session_id + messages) and the session metadata from /sessions
|
||||
// (which returns status/outcome/last_active_at for each session).
|
||||
// (which returns status/outcome/last_active_at for each session). P5 also
|
||||
// fetches /sessions/{id}/plan for the plan_generations assertion.
|
||||
func fetchTranscript(ctx context.Context, gateway, sid string) (transcript, sessionState, error) {
|
||||
var t transcript
|
||||
resp, err := http.Get(gateway + "/sessions/" + sid)
|
||||
@@ -292,6 +317,16 @@ func fetchTranscript(ctx context.Context, gateway, sid string) (transcript, sess
|
||||
if err := json.Unmarshal(b, &t); err != nil {
|
||||
return t, sessionState{}, err
|
||||
}
|
||||
// Fetch the plan (steps with generation numbers) for the
|
||||
// plan_generations assertion. A 404 or empty response is fine — a
|
||||
// pure-DB Q&A with no propose_plan has no plan.
|
||||
if planResp, perr := http.Get(gateway + "/sessions/" + sid + "/plan"); perr == nil {
|
||||
if planResp.StatusCode == 200 {
|
||||
pb, _ := io.ReadAll(planResp.Body)
|
||||
_ = json.Unmarshal(pb, &t) // fills t.PlanSteps via "steps" field
|
||||
}
|
||||
planResp.Body.Close()
|
||||
}
|
||||
// The detail endpoint doesn't return status/outcome — fetch from the
|
||||
// sessions list and find the matching id.
|
||||
s, err := fetchSessionMeta(ctx, gateway, sid)
|
||||
|
||||
@@ -11,10 +11,24 @@ import (
|
||||
type conversation struct {
|
||||
Name string `yaml:"name"`
|
||||
Prompt string `yaml:"prompt"`
|
||||
Followup string `yaml:"followup"`
|
||||
Followup string `yaml:"followup"` // backward compat: single followup
|
||||
Followups []string `yaml:"followups"` // P5: multi-turn followups
|
||||
Assertions []assertion `yaml:"assertions"`
|
||||
}
|
||||
|
||||
// followups returns the full list of follow-up messages, supporting both
|
||||
// the single `followup` field (backward compat) and the multi-turn
|
||||
// `followups` list.
|
||||
func (c conversation) followups() []string {
|
||||
if len(c.Followups) > 0 {
|
||||
return c.Followups
|
||||
}
|
||||
if c.Followup != "" {
|
||||
return []string{c.Followup}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// assertion is one check against the final transcript. The `kind` field
|
||||
// selects the scorer; the rest are scorer-specific parameters.
|
||||
//
|
||||
@@ -23,13 +37,15 @@ type conversation struct {
|
||||
// completes — session status reached done/failed (not stuck executing)
|
||||
// outcome_is — session outcome == value (success/failure/partial)
|
||||
// no_propose_plan — propose_plan was never called
|
||||
// proposes_plan — propose_plan called >= 1 time (plan-always model; P1)
|
||||
// proposes_plan_once — propose_plan was called exactly once
|
||||
// no_duplicate_proposal — propose_plan called at most once
|
||||
// plan_before_run — the first `run` call comes after the first `propose_plan` (P1 ordering gate)
|
||||
// plan_generations — the persisted plan has exactly `value` distinct generations (P2 iteration: 1 = single, 2 = one followup)
|
||||
// writes_back — update_entity_attributes or create_relationship was called
|
||||
// max_tool_calls — total tool calls <= value
|
||||
// max_run_calls — total `run` calls <= value
|
||||
// no_run — `run` was never called
|
||||
// no_rerun — `run` was NOT called after the followup turn (if any)
|
||||
// calls_tool — the named tool appears in the transcript
|
||||
// plan_step_count — the plan has exactly `value` steps
|
||||
// no_duplicate_complete — complete_task called at most once
|
||||
@@ -88,6 +104,14 @@ func scoreOne(a assertion, t transcript, s sessionState) (bool, string) {
|
||||
}
|
||||
return false, fmt.Sprintf("propose_plan called %d time(s)", n)
|
||||
|
||||
case "proposes_plan":
|
||||
// P1 plan-always: propose_plan called >= 1 time.
|
||||
n := countTool(tools, "propose_plan")
|
||||
if n >= 1 {
|
||||
return true, fmt.Sprintf("propose_plan called %d time(s)", n)
|
||||
}
|
||||
return false, "propose_plan never called (plan-always requires >= 1)"
|
||||
|
||||
case "proposes_plan_once":
|
||||
n := countTool(tools, "propose_plan")
|
||||
if n == 1 {
|
||||
@@ -102,6 +126,44 @@ func scoreOne(a assertion, t transcript, s sessionState) (bool, string) {
|
||||
}
|
||||
return false, fmt.Sprintf("propose_plan called %d time(s), want <= 1", n)
|
||||
|
||||
case "plan_before_run":
|
||||
// P1 ordering gate: the first `run` call's global index in the
|
||||
// transcript is strictly greater than the first `propose_plan`
|
||||
// index. Both indices are over the flat tool-call list (across all
|
||||
// messages, in order).
|
||||
planIdx, runIdx := -1, -1
|
||||
for i, name := range tools {
|
||||
if name == "propose_plan" && planIdx == -1 {
|
||||
planIdx = i
|
||||
}
|
||||
if name == "run" && runIdx == -1 {
|
||||
runIdx = i
|
||||
}
|
||||
}
|
||||
if runIdx == -1 {
|
||||
return true, "run never called (ordering trivially satisfied)"
|
||||
}
|
||||
if planIdx == -1 {
|
||||
return false, "run called but propose_plan never called"
|
||||
}
|
||||
if planIdx < runIdx {
|
||||
return true, fmt.Sprintf("propose_plan at index %d before run at index %d", planIdx, runIdx)
|
||||
}
|
||||
return false, fmt.Sprintf("run at index %d before propose_plan at index %d", runIdx, planIdx)
|
||||
|
||||
case "plan_generations":
|
||||
// P2 iteration: counts distinct `generation` values in
|
||||
// session_plan_steps. 1 = single sub-task, 2 = one follow-up
|
||||
// sub-task, etc. Requires the plan endpoint to return generation
|
||||
// values; the eval fetches /sessions/{id}/plan and passes it via
|
||||
// the transcript's PlanSteps field.
|
||||
want := toInt(a.Value)
|
||||
gens := t.distinctGenerations()
|
||||
if gens == want {
|
||||
return true, fmt.Sprintf("%d plan generation(s)", gens)
|
||||
}
|
||||
return false, fmt.Sprintf("%d plan generation(s), want %d", gens, want)
|
||||
|
||||
case "writes_back":
|
||||
n := countTool(tools, "update_entity_attributes") + countTool(tools, "create_relationship")
|
||||
if n > 0 {
|
||||
|
||||
@@ -224,6 +224,16 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
sessionID = sess.ID
|
||||
}
|
||||
} else {
|
||||
// P2 iteration: if the operator sends a follow-up on a session
|
||||
// that already reached a terminal state (done/failed), reopen it
|
||||
// so a new sub-task can be framed (set_goal → propose_plan →
|
||||
// execute). reopenSession marks the prior plan's steps as
|
||||
// `replaced` (proposePlan ignores those) and clears outcome/
|
||||
// summary. Without this, propose_plan refuses the follow-up with
|
||||
// errPlanInFlight because the prior steps are all `done`. If the
|
||||
// session is still active, reopen is a no-op — the follow-up is
|
||||
// just a continuation of in-flight work.
|
||||
st.reopenSession(pctx, sessionID)
|
||||
st.touchSession(pctx, sessionID)
|
||||
}
|
||||
|
||||
@@ -243,6 +253,13 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
sseEvent(w, flusher, agentEvent{Type: "session", Data: sessionID, SessionID: sessionID})
|
||||
|
||||
toolCalls := []map[string]any{}
|
||||
// P3: accumulate per-iteration reasoning instead of overwriting with
|
||||
// the final `text` event. The agent loop emits a `text` event for each
|
||||
// LLM iteration that produced text (intermediate reasoning before tool
|
||||
// calls + the final answer). Without accumulation, only the last `text`
|
||||
// survives in the persisted row — a reload shows the final summary but
|
||||
// not the thinking that led to each tool call.
|
||||
var textParts []string
|
||||
var finalText string
|
||||
|
||||
// Incremental persistence, mirroring resumeSession's existing
|
||||
@@ -292,7 +309,15 @@ func handleChat(w http.ResponseWriter, r *http.Request, a *agent, st *store) {
|
||||
persist() // live: survives even if the client disconnects right after
|
||||
}
|
||||
if ev.Type == "text" {
|
||||
finalText, _ = ev.Data.(string)
|
||||
// P3: accumulate. Each `text` event is one iteration's reasoning
|
||||
// (or the final answer). Join with newlines so the persisted row
|
||||
// reads as the full transcript of what the agent said, not just
|
||||
// the last thing.
|
||||
if t, ok := ev.Data.(string); ok && t != "" {
|
||||
textParts = append(textParts, t)
|
||||
finalText = strings.Join(textParts, "\n\n")
|
||||
persist()
|
||||
}
|
||||
}
|
||||
sseEvent(w, flusher, ev)
|
||||
})
|
||||
|
||||
@@ -466,18 +466,44 @@ func (s *store) setGoal(ctx context.Context, sessionID, goal string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// openPlanWindow records a plan window so config_mutation `run` calls in
|
||||
// this session auto-execute without per-action approval. The operator
|
||||
// approves the plan (propose_plan), not each individual command. Opened on
|
||||
// set_goal and stays active until the task is completed or the session ends.
|
||||
func (s *store) openPlanWindow(ctx context.Context, sessionID string) {
|
||||
if s == nil || sessionID == "" {
|
||||
return
|
||||
// reopenSession flips a terminal (done/failed) session back to `executing`
|
||||
// so a follow-up message can start a new sub-task — the iteration path
|
||||
// (P2, 2026-07-15). Without this, a completed session stays `done` forever
|
||||
// and propose_plan refuses the new sub-task with errPlanInFlight because the
|
||||
// prior plan's steps are all `done` (status <> 'pending'). reopenSession:
|
||||
//
|
||||
// 1. Marks all existing session_plan_steps as `replaced` (a status already
|
||||
// recognized by updatePlanStep's stamp switch). The rows are KEPT — the
|
||||
// generation column preserves which plan they belonged to, and the
|
||||
// audit trail survives. proposePlan's anyStarted check excludes
|
||||
// `replaced` (see proposePlan), so the next propose_plan takes the
|
||||
// fresh-generation path rather than being refused with errPlanInFlight.
|
||||
// 2. Clears outcome/summary so the panel doesn't show the old result.
|
||||
// 3. Stamps last_active_at.
|
||||
//
|
||||
// Returns true if the session was actually reopened (was terminal), false if
|
||||
// it was already active (no-op — the follow-up is just a continuation).
|
||||
func (s *store) reopenSession(ctx context.Context, sessionID string) bool {
|
||||
if s == nil || sessionID == "" || sessionID == "ephemeral" {
|
||||
return false
|
||||
}
|
||||
var currentStatus string
|
||||
if err := s.pool.QueryRow(ctx,
|
||||
`SELECT status FROM agent_sessions WHERE id = $1`, sessionID).Scan(¤tStatus); err != nil {
|
||||
return false
|
||||
}
|
||||
if currentStatus != "done" && currentStatus != "failed" {
|
||||
return false
|
||||
}
|
||||
s.pool.Exec(ctx,
|
||||
`INSERT INTO autonomy_settings (key, value) VALUES ($1, 'active')
|
||||
ON CONFLICT (key) DO UPDATE SET value = 'active'`,
|
||||
"nomos:plan:"+sessionID)
|
||||
`UPDATE session_plan_steps SET status = 'replaced', finished_at = COALESCE(finished_at, now()) WHERE session_id = $1 AND status <> 'replaced'`,
|
||||
sessionID)
|
||||
s.pool.Exec(ctx,
|
||||
`UPDATE agent_sessions SET status = 'executing', outcome = NULL, summary = NULL, last_active_at = now() WHERE id = $1`,
|
||||
sessionID)
|
||||
_ = observability.Event(ctx, sqlcgen.New(s.pool), "task.reopened", s.taskEntityPtr(ctx, sessionID),
|
||||
"info", "nomos", sessionID, map[string]any{"prior_status": currentStatus})
|
||||
return true
|
||||
}
|
||||
|
||||
// planStepInput is one step as the agent proposes it.
|
||||
@@ -514,8 +540,13 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
|
||||
var startSeq int
|
||||
var anyStarted bool
|
||||
// `replaced` steps (from a prior plan generation superseded by a
|
||||
// follow-up sub-task — see reopenSession) are excluded: they prove a
|
||||
// prior plan was completed and superseded, not that a plan is in flight.
|
||||
// Without this exclusion, reopenSession's `replaced` marking would be
|
||||
// useless — propose_plan would still refuse on the follow-up.
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COALESCE(max(seq), 0), COALESCE(bool_or(status <> 'pending'), false)
|
||||
SELECT COALESCE(max(seq), 0), COALESCE(bool_or(status NOT IN ('pending', 'replaced')), false)
|
||||
FROM session_plan_steps WHERE session_id = $1`, sessionID).Scan(&startSeq, &anyStarted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -525,13 +556,20 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
// update_plan_step + run. The caller surfaces a directive.
|
||||
return nil, errPlanInFlight
|
||||
}
|
||||
// Fresh/revise: delete any prior pending steps and start a new
|
||||
// generation. The DELETE covers the genuine pre-execution revision
|
||||
// case (operator asked to revise before any step started).
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM session_plan_steps WHERE session_id = $1`, sessionID); err != nil {
|
||||
// Fresh/revise: delete any prior PENDING steps (the genuine pre-execution
|
||||
// revision case — operator asked to revise before any step started).
|
||||
// `replaced` steps (from a prior completed plan superseded by a
|
||||
// follow-up — see reopenSession) are KEPT so the generation counter
|
||||
// (MAX(generation)+1 below) and the plan_generations eval assertion
|
||||
// can see across iterations. The anyStarted check above already
|
||||
// excludes `replaced`, so they don't block the fresh proposal.
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM session_plan_steps WHERE session_id = $1 AND status = 'pending'`, sessionID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
startSeq = 0
|
||||
// startSeq keeps the max(seq) from the query above: if `replaced` rows
|
||||
// exist (prior generation), the new generation's steps start after them
|
||||
// (no seq collisions across generations). If no rows exist (first plan
|
||||
// or a full DELETE), startSeq is 0 and the first step is seq 1.
|
||||
|
||||
// Resolve the generation number for this plan. Generation 1 is the
|
||||
// initial plan; a genuine revise (which currently goes through the same
|
||||
@@ -579,15 +617,6 @@ func (s *store) proposePlan(ctx context.Context, sessionID string, steps []planS
|
||||
// the panel to replace its list with these steps.
|
||||
_ = observability.Event(ctx, sqlcgen.New(s.pool), "plan.proposed", s.taskEntityPtr(ctx, sessionID),
|
||||
"info", "nomos", sessionID, map[string]any{"steps": out, "appended": false, "generation": nextGen})
|
||||
// Record a plan-proposed window so the `run` handler knows a plan is
|
||||
// pending approval and can skip per-action approval for config_mutation
|
||||
// commands within the plan. Transitions to 'active' when the operator
|
||||
// approves (chat-assent or button). The window key is session-scoped;
|
||||
// one agent serves all sessions on this nomos instance.
|
||||
s.pool.Exec(ctx,
|
||||
`INSERT INTO autonomy_settings (key, value) VALUES ($1, 'active')
|
||||
ON CONFLICT (key) DO UPDATE SET value = 'active'`,
|
||||
"nomos:plan:"+sessionID)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -176,12 +176,16 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
if err := a.store.setGoal(ctx, sessionID, goal); err != nil {
|
||||
return fmt.Sprintf("error setting goal: %v", err), true
|
||||
}
|
||||
// Open the plan window immediately — the goal IS the start of a
|
||||
// plan. Config_mutation commands in this session auto-execute
|
||||
// without per-action approval. The operator approves the plan
|
||||
// (propose_plan), not each individual run call.
|
||||
a.store.openPlanWindow(ctx, sessionID)
|
||||
return "Goal set: " + goal + ". NEXT: pre-plan (read-only tools only — search_knowledge, get_entity, list_lxcs, get_relations). Then propose_plan. Do not call run.", true
|
||||
// P1: the plan window is NOT opened here. Opening it on set_goal
|
||||
// meant any config_mutation `run` auto-executed with zero operator
|
||||
// approval, before a plan was even proposed (let alone approved) —
|
||||
// a safety regression confirmed live in session d0d562e0. The
|
||||
// window is now opened only when the operator approves a plan
|
||||
// (chat-assent grant or explicit approval in agent.go), which is
|
||||
// what the SOUL.md "approve the plan, not each step" model actually
|
||||
// describes. set_goal records the goal + flips status to executing
|
||||
// and nothing more.
|
||||
return "Goal set: " + goal + ". NEXT: pre-plan with read-only tools (search_knowledge, get_entity, list_lxcs, get_relations), then propose_plan (mandatory — even read-only tasks need a one-step plan; the run handler refuses without one). Do not call run before propose_plan.", true
|
||||
|
||||
case "propose_plan":
|
||||
raw, _ := args["steps"].([]any)
|
||||
@@ -237,7 +241,7 @@ func (a *agent) handleTaskTool(ctx context.Context, sessionID, name string, args
|
||||
// the "plan added twice" sidebar drift the operator
|
||||
// reported: instead of appending (which duplicated) or
|
||||
// wiping (which lost progress), we refuse and direct.
|
||||
return "Plan already in flight — refusing duplicate proposal. Steps exist and at least one has started (running/done/...). To advance: call update_plan_step(seq=K, status=\"running\") then run(...) for step K's target, then update_plan_step(seq=K, status=\"done\"). Do NOT call propose_plan again. Re-propose only if the operator explicitly asks you to revise the whole plan, and say so in your reply before calling it.", true
|
||||
return "Plan already in flight — refusing duplicate proposal. Steps exist and at least one has started (running/done/...). To advance: call update_plan_step(seq=K, status=\"running\") then run(...) for step K's target, then update_plan_step(seq=K, status=\"done\"). Do not call propose_plan again. Re-propose only if the operator explicitly asks you to revise the whole plan (the session is reopened on a follow-up — prior steps are marked `replaced` and a fresh generation is started), and say so in your reply before calling it.", true
|
||||
}
|
||||
return fmt.Sprintf("error proposing plan: %v", err), true
|
||||
}
|
||||
|
||||
16
evals/iteration-followup.yaml
Normal file
16
evals/iteration-followup.yaml
Normal file
@@ -0,0 +1,16 @@
|
||||
# P5 eval: iteration. A read-only task completes; the follow-up asks the
|
||||
# agent to act on what it found (a config_mutation). Asserts the session
|
||||
# reopens, a second plan generation is created, and the agent completes
|
||||
# both sub-tasks without duplicate-complete.
|
||||
- name: iteration-followup
|
||||
prompt: "When was the last backup to Proton Drive done, and when is the next one?"
|
||||
followups:
|
||||
- "The repos folder failed last time. Reset the failed service and re-run the backup."
|
||||
assertions:
|
||||
- kind: completes
|
||||
- kind: plan_generations
|
||||
value: 2
|
||||
- kind: proposes_plan
|
||||
- kind: writes_back
|
||||
- kind: max_run_calls
|
||||
value: 6
|
||||
13
evals/iteration-readonly.yaml
Normal file
13
evals/iteration-readonly.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
# P5 eval: two read-only sub-tasks back-to-back. Asserts the session
|
||||
# reopens and a second plan generation is created for the follow-up.
|
||||
- name: iteration-readonly
|
||||
prompt: "When was the last backup to Proton Drive done, and when is the next one?"
|
||||
followups:
|
||||
- "Now check when the last snapshot of the prometheus LXC was taken."
|
||||
assertions:
|
||||
- kind: completes
|
||||
- kind: plan_generations
|
||||
value: 2
|
||||
- kind: proposes_plan
|
||||
- kind: max_run_calls
|
||||
value: 6
|
||||
10
evals/no-plan-no-run.yaml
Normal file
10
evals/no-plan-no-run.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
# P5 eval: a pure-DB Q&A that calls NO run. This is the ONLY remaining
|
||||
# carve-out from plan-first: a task that never touches a live target via
|
||||
# `run` doesn't need propose_plan (the gate only fires on run). Asserts
|
||||
# the agent answers directly and completes without ceremony.
|
||||
- name: no-plan-no-run
|
||||
prompt: "List all LXC containers and their current health."
|
||||
assertions:
|
||||
- kind: completes
|
||||
- kind: no_run
|
||||
- kind: no_propose_plan
|
||||
13
evals/plan-always-readonly.yaml
Normal file
13
evals/plan-always-readonly.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
# P5 eval: a read-only question that would have been a "degenerate case"
|
||||
# under the old SOUL.md carve-out. Asserts the plan-first gate works: the
|
||||
# agent must propose_plan before run, even for a trivial read-only task.
|
||||
# Run against a live nomos with a real rclone LXC.
|
||||
- name: plan-always-readonly
|
||||
prompt: "When was the last backup to Proton Drive done, and when is the next one?"
|
||||
assertions:
|
||||
- kind: completes
|
||||
- kind: proposes_plan
|
||||
- kind: plan_before_run
|
||||
- kind: writes_back
|
||||
- kind: max_run_calls
|
||||
value: 3
|
||||
@@ -1266,6 +1266,20 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.
|
||||
runParams, _ := json.Marshal(map[string]string{"command": command, "purpose": purpose})
|
||||
actionCol := "run:" + string(runParams)
|
||||
|
||||
// P1 plan-first gate: every task must propose a plan before any `run`,
|
||||
// read-only or not. The only carve-out is a pure-DB Q&A that calls no
|
||||
// `run` at all (those never reach this code path). Without this gate the
|
||||
// SOUL.md "MANDATORY TASK FLOW" is unenforceable prose — weaker models
|
||||
// skip propose_plan and go straight to run, leaving the operator with
|
||||
// 23 individual approvals and no plan to approve (the original
|
||||
// anti-pattern the flow exists to prevent). Mirrors D.1's structural
|
||||
// refusal pattern in complete_task. sessionID == "" means a direct MCP
|
||||
// call with no nomos session (e.g. an external script) — gate is a
|
||||
// no-op there, since there's no session to hold a plan.
|
||||
if sessionID != "" && !sessionHasPlan(ctx, pool, sessionID) {
|
||||
return textResult("No plan for this session. Call set_goal then propose_plan before run — even read-only tasks require a one-step plan. A one-step plan (\"Inspect X, report, write back\") is fine for trivial questions; the gate is about ordering, not approval. Read-only commands still auto-execute once a plan exists.")
|
||||
}
|
||||
|
||||
// Dedup: an identical pending command (same target, command, and
|
||||
// purpose) blocks a re-request — stops a tool-calling loop from queuing
|
||||
// the same approval repeatedly.
|
||||
@@ -1324,32 +1338,16 @@ func classifyAndGate(ctx context.Context, pool *db.Pool, agentID, targetID uuid.
|
||||
return textResult(fmt.Sprintf("run on %s (read_only, auto): %s", targetSlug, out))
|
||||
}
|
||||
|
||||
// Plan window: if a plan was proposed (and possibly approved), this
|
||||
// command is part of an in-flight plan. The plan IS the approval —
|
||||
// config_mutation commands within an active plan auto-run without
|
||||
// per-action approval. Created by propose_plan, checked by planWindowActive.
|
||||
if riskClass == policy.RiskConfigMutation && sessionID != "" && planWindowActive(ctx, pool, sessionID) {
|
||||
host, user, wrap, rerr := resolveExecTarget(ctx, pool, targetSlug)
|
||||
if rerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s", rerr.Error()))
|
||||
return textResult(fmt.Sprintf("resolve target: %v", rerr))
|
||||
}
|
||||
out, xerr := sshExec(ctx, host, user, wrap(command))
|
||||
if xerr != nil {
|
||||
pool.Exec(ctx, `UPDATE executions SET status='failed', result=$2::jsonb WHERE entity_id=$1`, id, jsonErr("%s: %s", xerr.Error(), out))
|
||||
return textResult(fmt.Sprintf("run on %s: ERROR %v\n%s", targetSlug, xerr, out))
|
||||
}
|
||||
pool.Exec(ctx, `UPDATE executions SET status='completed', result=$2::jsonb WHERE entity_id=$1`, id, jsonOut(out))
|
||||
slog.Info("mcp: run auto-executed via plan window", "target", targetSlug, "execution_id", id)
|
||||
return textResult(fmt.Sprintf("run on %s (config_mutation, auto via plan): %s", targetSlug, out))
|
||||
}
|
||||
|
||||
// Assent window: if the operator recently approved a plan in this
|
||||
// agent's chat session, config_mutation commands auto-run without
|
||||
// re-approval. This is the "approve the plan, carry it out" path — the
|
||||
// operator approved the overall direction; individual config steps
|
||||
// within the window don't each need a separate yes. Destructive
|
||||
// commands never auto-run, regardless of window.
|
||||
// commands never auto-run, regardless of window. (The old plan-window
|
||||
// path that opened on set_goal/propose_plan was removed — it opened
|
||||
// before approval, letting config_mutation auto-run with zero operator
|
||||
// consent. The assent window, opened only on operator approval, is the
|
||||
// sole gate for config_mutation auto-run.)
|
||||
if riskClass == policy.RiskConfigMutation && assentWindowActive(ctx, pool, agentID, sessionID) {
|
||||
host, user, wrap, rerr := resolveExecTarget(ctx, pool, targetSlug)
|
||||
if rerr != nil {
|
||||
@@ -1439,19 +1437,32 @@ func executeApprovedViaAPI(ctx context.Context, execID uuid.UUID, targetSlug, ac
|
||||
}
|
||||
}
|
||||
|
||||
// planWindowActive reports whether a plan has been proposed (or approved)
|
||||
// for this session. Created by propose_plan, the window allows config_mutation
|
||||
// commands to auto-execute without per-action approval — the plan IS the
|
||||
// approval. Plan-approve-once policy (2026-07-14).
|
||||
func planWindowActive(ctx context.Context, pool *db.Pool, sessionID string) bool {
|
||||
// planWindowActive was removed 2026-07-15: it opened on set_goal and
|
||||
// propose_plan, letting config_mutation auto-run before operator approval.
|
||||
// The assent window (opened only on approval in agent.go) is the sole gate
|
||||
// for config_mutation auto-run now. See sessionHasPlan for the plan-existence
|
||||
// check used by the P1 plan-first gate.
|
||||
|
||||
// sessionHasPlan reports whether this nomos session has any plan step on
|
||||
// record (any generation, any status). Used by the P1 plan-first gate in
|
||||
// classifyAndGate to refuse `run` before `propose_plan` has been called.
|
||||
// A `replaced` step (from a prior plan generation that was superseded by a
|
||||
// follow-up sub-task — see store.reopenSession) still counts: it proves the
|
||||
// agent once framed a plan for this session, and the reopen path guarantees a
|
||||
// fresh `propose_plan` will run before the next `run` anyway. Fails closed
|
||||
// (returns true) when the query errors so a transient DB issue doesn't block
|
||||
// an otherwise-valid run.
|
||||
func sessionHasPlan(ctx context.Context, pool *db.Pool, sessionID string) bool {
|
||||
if sessionID == "" {
|
||||
return false
|
||||
return true // no session → no gate (direct MCP call from a script)
|
||||
}
|
||||
var val string
|
||||
err := pool.QueryRow(ctx,
|
||||
"SELECT value FROM autonomy_settings WHERE key = $1",
|
||||
"nomos:plan:"+sessionID).Scan(&val)
|
||||
return err == nil && val == "active"
|
||||
var count int
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT COUNT(*) FROM session_plan_steps WHERE session_id = $1`,
|
||||
sessionID).Scan(&count); err != nil {
|
||||
return true // fail open on DB error — don't block work over a flake
|
||||
}
|
||||
return count > 0
|
||||
}
|
||||
|
||||
// assentWindowActive checks whether the operator has recently approved a plan
|
||||
|
||||
@@ -69,11 +69,13 @@ var destructivePatterns = []*regexp.Regexp{
|
||||
var readOnlyLeadPattern = regexp.MustCompile(
|
||||
`^(cat|less|head|tail|ls|stat|file|du|df|free|uptime|uname|hostname|whoami|id|ip|ss|netstat|ping|` +
|
||||
`journalctl|dmesg|ps|top|htop|env|printenv|echo|which|whereis|` +
|
||||
`grep|egrep|fgrep|rg|wc|sort|uniq|cut|tr|tee|` +
|
||||
`grep|egrep|fgrep|rg|wc|sort|uniq|cut|tr|tee|find|tree|locate|` +
|
||||
`dpkg\s+(-l|-s|--list|--status)\b|apt\s+(list|search|show)\b|` +
|
||||
`systemctl\s+(status|is-active|is-enabled|is-failed|list-units)|` +
|
||||
`systemctl\s+(status|is-active|is-enabled|is-failed|list-units|list-unit-files|list-timers|show)\b|` +
|
||||
`timedatectl|hostnamectl|systemd-analyze|` +
|
||||
`docker\s+(ps|images|inspect|logs|version|info|stats)|` +
|
||||
`pct\s+(status|config|list)|qm\s+(status|config|list)|pvesh\s+get|` +
|
||||
`rclone\s+(ls|lsl|md5sum|check|cryptcheck)\b|` +
|
||||
`git\s+(status|log|diff|show|branch|remote)|` +
|
||||
`curl\s+-.*-I\b|curl\s+.*--head\b)\b`)
|
||||
|
||||
|
||||
@@ -15,6 +15,18 @@ func TestClassifyCommand_ReadOnly(t *testing.T) {
|
||||
"git status",
|
||||
"sudo cat /var/log/syslog",
|
||||
"ip a",
|
||||
// P4: newly added read-only verbs.
|
||||
"find /var/log/rclone-backup/ -name runs.jsonl",
|
||||
"tree /etc/caddy",
|
||||
"locate Caddyfile",
|
||||
"systemctl list-timers --all",
|
||||
"systemctl list-units --type=service",
|
||||
"systemctl list-unit-files --state=enabled",
|
||||
"systemctl show caddy",
|
||||
"timedatectl",
|
||||
"hostnamectl",
|
||||
"systemd-analyze blame",
|
||||
"rclone lsl proton:library-backup",
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := ClassifyCommand(c, ""); got != RiskReadOnly {
|
||||
@@ -99,6 +111,9 @@ func TestClassifyCommand_CompoundReadOnly(t *testing.T) {
|
||||
"docker ps | grep caddy",
|
||||
"systemctl status caddy 2>&1; journalctl -u caddy -n 5 --no-pager",
|
||||
"sudo systemctl status caddy; sudo journalctl -u caddy -n 5",
|
||||
// P4: the exact compound from session d0d562e0 — find + ls + tail +
|
||||
// echo + journalctl, all read-only segments.
|
||||
"ls -lt /var/log/rclone-backup/ | head -20 && tail -3 /var/log/rclone-backup/runs.jsonl || echo \"not found\" && find /var/log/rclone-backup/ -name 'runs.jsonl'",
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := ClassifyCommand(c, ""); got != RiskReadOnly {
|
||||
|
||||
@@ -48,9 +48,19 @@ edge you discovered. Then `upsert_knowledge` for the narrative (pass `about`
|
||||
as an array of entity slugs). Then `complete_task` with the outcome.
|
||||
`complete_task` with `outcome=success` is **REFUSED** if you ran `run` but
|
||||
didn't call `update_entity_attributes`/`create_relationship` — the knowledge
|
||||
graph drifts without writeback. A trivial read-only task ("status of Y?")
|
||||
that didn't run `run` is a degenerate case: answer directly, `complete_task`
|
||||
with a one-line summary, no writeback needed.
|
||||
graph drifts without writeback. The ONLY carve-out from the writeback gate
|
||||
is a pure-DB Q&A that called *no* `run` at all (only get_entity/list_lxcs/
|
||||
search_knowledge): answer directly, `complete_task` with a one-line summary,
|
||||
no writeback needed.
|
||||
|
||||
### 7. ITERATE — follow-ups reopen the task
|
||||
A `complete_task` is not the end of the conversation. If the operator sends
|
||||
a follow-up on a completed session — e.g. "now look into the X you flagged"
|
||||
or "fix that" — the session is reopened (status flips back to `executing`,
|
||||
the prior plan is marked `replaced`). Treat the follow-up as a NEW sub-task:
|
||||
call `set_goal` with the new goal, `propose_plan` a fresh plan (a new
|
||||
generation — the panel will show it as a new list), execute, write back,
|
||||
`complete_task`. Do NOT re-open or re-advance the old plan's steps.
|
||||
|
||||
**Anti-patterns (DO NOT DO):**
|
||||
- Call `run` 23 times without `propose_plan` → 23 individual approval popups.
|
||||
@@ -105,14 +115,23 @@ classifier will catch a genuinely dangerous command regardless, but be honest
|
||||
about risk in your `purpose` text; the operator is trusting your description
|
||||
of what a command does.
|
||||
|
||||
## Every chat is a task
|
||||
## Every chat is a task — and every task has a plan
|
||||
|
||||
Every non-trivial chat follows the MANDATORY TASK FLOW at the top of this
|
||||
file. The flow scales down: a trivial read-only question ("status of Y?")
|
||||
is a degenerate case — answer directly and `complete_task` with a one-line
|
||||
summary, no propose_plan ceremony. Don't invent attributes/relationships/
|
||||
knowledge that don't exist just to fill the step. The loop scales down; it
|
||||
doesn't disappear.
|
||||
file. **`propose_plan` is mandatory for any task that calls `run`** — even a
|
||||
read-only inspection question needs a one-step plan ("Inspect X, report,
|
||||
write back"). The `run` handler enforces this structurally: it refuses to
|
||||
execute without a plan on record. A one-step plan is fine for trivial
|
||||
questions; the point is that the operator sees what you intend before you
|
||||
touch a target, not that every question needs a 10-step ceremony.
|
||||
|
||||
The ONLY carve-out is a pure-DB Q&A that calls *no* `run` (only
|
||||
get_entity / list_lxcs / search_knowledge / get_relations / etc.): answer
|
||||
directly and `complete_task` with a one-line summary. Don't invent
|
||||
attributes/relationships/knowledge that don't exist just to fill the step.
|
||||
|
||||
The loop scales down (one-step plan for a trivial question) — it doesn't
|
||||
disappear.
|
||||
|
||||
## Key MCP tools
|
||||
|
||||
|
||||
369
plans/done/2026-07-15-plan-first-and-iteration.md
Normal file
369
plans/done/2026-07-15-plan-first-and-iteration.md
Normal file
@@ -0,0 +1,369 @@
|
||||
# 2026-07-15 — Plan-first enforcement, iteration, and audit gaps
|
||||
|
||||
**Status:** Done — 2026-07-15. P1–P6 implemented; build + tests + vet
|
||||
pass. Follow-up to the session audit of
|
||||
`d0d562e0` ("Determine when the last backup to Proton Drive ran and when the
|
||||
next one is scheduled"). The audit surfaced that the agent answered
|
||||
successfully but **never proposed a plan**, and that a follow-up asking the
|
||||
agent to act on its own findings has no working path. This plan closes both,
|
||||
plus the related reliability gaps the audit turned up.
|
||||
|
||||
Grounded in:
|
||||
- `cmd/nomos/tasks.go` (set_goal, propose_plan, complete_task handlers)
|
||||
- `cmd/nomos/store.go` (setGoal, proposePlan, completeTask, plan window)
|
||||
- `internal/mcp/server.go` (run risk gate, plan/assent windows)
|
||||
- `internal/policy/command.go` (ClassifyCommand read-only allowlist)
|
||||
- `nomos/SOUL.md` (MANDATORY TASK FLOW + degenerate-case carve-out)
|
||||
- `cmd/nomos/eval/manifest.go` + `eval/main.go` (assertion kinds, followup)
|
||||
|
||||
---
|
||||
|
||||
## Why there was no plan
|
||||
|
||||
Two independent causes, both required for the skip to happen:
|
||||
|
||||
1. **SOUL.md explicitly exempts read-only questions from the plan flow.**
|
||||
`SOUL.md:51-53` and `SOUL.md:111-115` declare a "trivial read-only task
|
||||
('status of Y?')" a *degenerate case*: answer directly, `complete_task`
|
||||
with a one-line summary, "no propose_plan ceremony." The Proton Drive
|
||||
question looks on its face like "status of Y?", so the agent applied the
|
||||
carve-out. It then went on to call `run` twice — so it wasn't actually
|
||||
degenerate, but the exemption had already been invoked.
|
||||
|
||||
2. **There is no structural gate forcing `propose_plan` before `run`.**
|
||||
The only enforcement is SOUL.md prose. `internal/mcp/server.go:1312`
|
||||
executes read-only commands immediately with no check that a plan exists
|
||||
for the session. The agent can honor the rule or skip it, and weaker
|
||||
models skip it. The D.1 writeback gate works precisely because it's
|
||||
*structural* (`complete_task` refuses without `update_entity_attributes`);
|
||||
there is no equivalent for `propose_plan`.
|
||||
|
||||
---
|
||||
|
||||
## Findings
|
||||
|
||||
### F1 — BLOCKER. Plan window opens on `set_goal`, before any plan or approval.
|
||||
|
||||
`tasks.go:183` calls `openPlanWindow` inside the `set_goal` handler →
|
||||
`nomos:plan:<session>="active"` immediately (`store.go:473`). Result: any
|
||||
`config_mutation` `run` auto-executes with **zero operator approval**. This
|
||||
session proves it — no `propose_plan`, no `update_plan_step` (confirmed:
|
||||
`/sessions/{id}/plan` → `steps:null`), yet the 2nd `run` was tagged
|
||||
**"config_mutation, auto via plan"** (`server.go:1331-1344`). The inline
|
||||
comment ("the goal IS the start of a plan… operator approves the plan via
|
||||
propose_plan") is self-contradictory: the window is already open, so
|
||||
`propose_plan`'s "STOP, wait for approval" (`tasks.go:248`) is unenforceable.
|
||||
|
||||
This is a safety regression, not a style issue.
|
||||
|
||||
### F2 — BLOCKER. Iterative follow-ups have no working path.
|
||||
|
||||
Scenario: this session completes; the operator sends a follow-up "now look
|
||||
into the `repos` backup failure" on the same session. Traced path:
|
||||
|
||||
1. `completeTask` (`store.go:716-718`) deletes `nomos:plan:<session>` from
|
||||
`autonomy_settings` but does **not** clear `session_plan_steps` rows.
|
||||
2. Follow-up arrives → `main.go:227` `touchSession` only updates
|
||||
`last_active_at`; status stays `done`.
|
||||
3. Agent calls `set_goal` → `setGoal` flips status to `executing`
|
||||
(`store.go:460`) **and re-opens the plan window** (F1 again).
|
||||
4. Agent calls `propose_plan` → `proposePlan` (`store.go:517-527`) checks
|
||||
`bool_or(status <> 'pending')`. Old steps are all `done` →
|
||||
`anyStarted=true` → returns `errPlanInFlight` → **REFUSED**. The refusal
|
||||
text says "Re-propose only if the operator explicitly asks" but there is
|
||||
**no code path honoring that** — re-calling `propose_plan` hits the same
|
||||
guard. Dead end. There is no `reset_plan`/`close_plan` tool.
|
||||
|
||||
So the design assumed one plan per session. There is no "iteration" /
|
||||
"next plan" concept. The only escape is starting a brand-new session, which
|
||||
loses the conversational thread and the LLM's replayed context.
|
||||
|
||||
Note: in *this* audited session there were no plan steps (F1 — no plan was
|
||||
ever proposed), so `proposePlan` would actually succeed on a follow-up here.
|
||||
But in a plan-always world the first session WOULD have steps, and the
|
||||
follow-up would be blocked. **Fixing plan-always without fixing iteration
|
||||
would create a new blocker.** They must ship together.
|
||||
|
||||
### F3 — FRICTION. Thinking replaced by summary on reload.
|
||||
|
||||
`main.go:252-267` and `continue.go:204-219` persist **one** placeholder
|
||||
assistant row per turn and `updateMessage` it per tool call, storing only
|
||||
`finalText` (the *last* `text` event) + an ever-growing `toolCalls` slice.
|
||||
Intermediate per-turn reasoning (streamed live via `text`/`text_delta`,
|
||||
`agent.go:360,410`) is **overwritten**. The DB has 2 rows total for this
|
||||
session; on reload you see only the final 547-char summary + a flat list of
|
||||
15 tool calls. Same defect breaks LLM replay fidelity on resume — the model
|
||||
can't see its own prior reasoning.
|
||||
|
||||
### F4 — FRICTION. Read-only command misclassified as `config_mutation`.
|
||||
|
||||
The 2nd `run` was pure inspection (`ls|head|tail|echo|find|journalctl`) but
|
||||
classified `config_mutation` because **`find` is absent** from
|
||||
`readOnlyLeadPattern` (`command.go:69-78`); `allSegmentsReadOnly` trips on
|
||||
the `find` segment and escalates. Harmless here only because F1 auto-ran it
|
||||
anyway — but in a properly-gated session it would force an unnecessary
|
||||
approval, and it masks the real danger of F1.
|
||||
|
||||
### F5 — COSMETIC. Contradictory `set_goal` instruction.
|
||||
|
||||
`set_goal` returns "Then propose_plan. Do not call run" (`tasks.go:184`) for
|
||||
*every* task, yet a read-only inspection task legitimately needs `run` and
|
||||
doesn't need a plan (under the current carve-out). The guidance is both
|
||||
ignored (F1) and wrong for this task class. Resolved by F6's plan-always
|
||||
model.
|
||||
|
||||
### F6 — DESIGN. Plan-always is the desired model (operator directive).
|
||||
|
||||
The operator wants: the first thing the agent does is make a plan, even when
|
||||
actions are read-only and need no user approval. This supersedes the SOUL.md
|
||||
degenerate-case carve-out. A one-step plan ("Inspect X, report, write back")
|
||||
is acceptable for trivial questions, but `propose_plan` is mandatory.
|
||||
|
||||
### F7 — EVAL. Eval harness can't express iteration or plan-always.
|
||||
|
||||
- `proposes_plan_once` (`manifest.go:91`) counts total across the whole
|
||||
transcript → a 2-iteration session legitimately calling `propose_plan`
|
||||
twice would **FAIL**. There is no per-turn or "plan generation count"
|
||||
assertion.
|
||||
- `no_rerun` (`manifest.go:32`, not yet implemented as a kind but documented)
|
||||
asserts `run` NOT called after the followup → directly conflicts with an
|
||||
iterative follow-up that needs to run.
|
||||
- No assertion for "session reopened from `done` → `executing`" or "a second
|
||||
plan generation was created."
|
||||
- The manifest supports only **one** `followup` field (`manifest.go:14`),
|
||||
so multi-turn iteration beyond 2 turns isn't expressible.
|
||||
|
||||
---
|
||||
|
||||
## Improvement plan (prioritized)
|
||||
|
||||
### P1 — Make plan-first structural (BLOCKER, ships with P2)
|
||||
|
||||
Goal: every task proposes a plan before any `run`, read-only or not. No
|
||||
SOUL.md-only enforcement.
|
||||
|
||||
1. **Add a `session_has_plan` gate in the `run` handler.**
|
||||
In `internal/mcp/server.go` run(), before the read-only fast path
|
||||
(`server.go:1312`) and the plan/assent windows, check whether
|
||||
`session_plan_steps` has any row for this session. If `sessionID != ""`
|
||||
and no plan exists, refuse:
|
||||
`"No plan for this session. Call set_goal then propose_plan before run — even read-only tasks require a one-step plan."`
|
||||
Mirror D.1's refusal pattern (`tasks.go:313`). This makes plan-first a
|
||||
hard gate, not prose. Read-only commands still auto-execute once a plan
|
||||
exists (they're read-only); the gate is about *ordering*, not approval.
|
||||
2. **Strip the degenerate-case carve-out from SOUL.md.**
|
||||
- Remove `SOUL.md:51-53`'s "degenerate case" sentence.
|
||||
- Rewrite `SOUL.md:108-115` ("Every chat is a task") to: every task
|
||||
proposes a plan; a one-step plan is fine for trivial questions but
|
||||
`propose_plan` is mandatory; only a pure-DB Q&A that calls *no* `run`
|
||||
may skip the plan (still call `set_goal` + `complete_task`).
|
||||
3. **Decouple the plan window from approval (fixes F1).**
|
||||
- Remove `openPlanWindow` from the `set_goal` handler (`tasks.go:179-183`).
|
||||
`set_goal` records the goal + sets status only.
|
||||
- Open the plan window only on approval: the chat-assent grant
|
||||
(`agent.go:318`) and the explicit-approve path (`agent.go:331`). This
|
||||
restores propose → approve → execute for `config_mutation` steps.
|
||||
- Read-only steps need no approval and no window — they auto-run because
|
||||
they're read-only, not because a window is open.
|
||||
|
||||
**Severity:** blocker. **Files:** `tasks.go:171-184`, `store.go:455-481`,
|
||||
`server.go:1312-1345`, `nomos/SOUL.md:6-53,108-115`.
|
||||
|
||||
### P2 — Support iterative follow-ups (BLOCKER, ships with P1)
|
||||
|
||||
Goal: a completed session can be reopened by a follow-up, and the agent can
|
||||
propose a *new* plan for the new sub-task. Iteration, not re-execution.
|
||||
|
||||
1. **Add `reopenSession` on first follow-up after completion.**
|
||||
In `main.go` chat handler, when `sessionID != ""` and the session is
|
||||
already terminal (`done`/`failed`), flip status back to `executing`,
|
||||
clear `outcome`/`summary`, and stamp `last_active_at`. Do this in the
|
||||
handler (not in `set_goal`) so the reopen happens even if the agent's
|
||||
first action is a tool call rather than `set_goal`. Emit a
|
||||
`task.reopened` event for the panel.
|
||||
2. **Clear prior plan steps on reopen, bump generation.**
|
||||
Extend `reopenSession` to mark all `session_plan_steps` for the session
|
||||
as `replaced` (a status already recognized by `updatePlanStep:612`) and
|
||||
delete the `nomos:plan:<session>` autonomy key. The next `propose_plan`
|
||||
then sees `anyStarted=false` (no non-pending rows) and takes the fresh
|
||||
path with `generation = MAX(generation)+1`. This gives the panel a clean
|
||||
new plan list while preserving the prior plan's history (the `replaced`
|
||||
rows + generation counter) for audit.
|
||||
- Alternative considered: delete the rows outright. Rejected — the
|
||||
`replaced` status + generation column already exist for exactly this
|
||||
and preserve the audit trail.
|
||||
3. **Fix the `errPlanInFlight` refusal text to point at the reopen path.**
|
||||
`tasks.go:240` currently says "Re-propose only if the operator explicitly
|
||||
asks" with no way to do it. After P2.2 the operator's follow-up *is* the
|
||||
explicit ask — the reopen clears the in-flight flag. Update the text to:
|
||||
`"A plan from a prior turn is complete. If the operator's new message is a follow-up sub-task, the session has been reopened — propose a fresh plan for it."`
|
||||
4. **SOUL.md: document iteration.** Add a "7. ITERATE" step to the task flow:
|
||||
a completed session accepts a follow-up as a new sub-task; call
|
||||
`set_goal` (new goal) → `propose_plan` (new generation) → execute. Do not
|
||||
re-open the old plan.
|
||||
|
||||
**Severity:** blocker. **Files:** `main.go:217-241`, `store.go` (new
|
||||
`reopenSession`), `tasks.go:240`, `nomos/SOUL.md`.
|
||||
|
||||
### P3 — Persist per-turn reasoning, not just final summary (FRICTION)
|
||||
|
||||
Goal: reload shows what the operator saw live; LLM replay on resume is
|
||||
faithful.
|
||||
|
||||
1. **Insert one assistant row per turn, not one per session.**
|
||||
In `main.go:252-310` and `continue.go:190-312`, insert a new row when a
|
||||
fresh `text`/`tool_use` cycle begins rather than overwriting the same
|
||||
placeholder. Keep the placeholder for the *current* turn only.
|
||||
2. **Accumulate text deltas instead of overwriting `finalText`.**
|
||||
`agent.go:360` emits `text_delta`; the `persist` closure should append
|
||||
into a `textParts []string` and join on `done`, not replace `finalText`
|
||||
on each `text` event (`agent.go:410`). Intermediate reasoning between
|
||||
tool calls is then preserved in the row's `text` field.
|
||||
3. **Truncate per-row tool results** (already done by
|
||||
`truncateToolResults`, `store.go:129`) — verify the cap is sane for the
|
||||
multi-row case.
|
||||
|
||||
**Severity:** friction. **Files:** `main.go:245-310`, `continue.go:190-312`,
|
||||
`store.go:123-159`, `agent.go:360,410`.
|
||||
|
||||
### P4 — Expand the read-only allowlist (FRICTION)
|
||||
|
||||
1. **Add to `readOnlyLeadPattern`** (`command.go:69-78`): `find`, `tree`,
|
||||
`locate`, `systemctl (list-units|list-unit-files|list-timers|show)`,
|
||||
`rclone (ls|lsl|md5|check)`, `timedatectl`, `hostnamectl`, `systemd-analyze`.
|
||||
2. **Add unit cases to `command_test.go`** for the exact
|
||||
`find /var/log/rclone-backup/ -name 'runs.jsonl'` command from this
|
||||
session, plus a compound `ls -lt … && tail … && find …` case.
|
||||
|
||||
**Severity:** friction. **Files:** `internal/policy/command.go:69-78`,
|
||||
`internal/policy/command_test.go`.
|
||||
|
||||
### P5 — Extend the eval harness for plan-always + iteration (F7)
|
||||
|
||||
Goal: P1 and P2 can't regress silently; the harness can express the
|
||||
scenarios the operator cares about.
|
||||
|
||||
1. **New assertion kinds** (`manifest.go` `scoreOne`):
|
||||
- `proposes_plan` — `propose_plan` called >= 1 time (plan-always; replaces
|
||||
the carve-out-dependent `no_propose_plan` for the new model).
|
||||
- `plan_before_run` — the first `run` call's transcript index is strictly
|
||||
greater than the first `propose_plan` index (ordering gate). Requires
|
||||
`transcript` to expose per-call message index (add a helper).
|
||||
- `plan_generations` — the persisted plan has exactly `value` distinct
|
||||
`generation` values in `session_plan_steps` (1 for single-task, 2 for
|
||||
one iteration). Needs a new fetch in `fetchTranscript` hitting
|
||||
`/sessions/{id}/plan` (already exists, returns `steps`).
|
||||
- `reopens_session` — session went `done` → `executing` between the
|
||||
prompt and followup turns. Needs `waitForTurn` to capture the
|
||||
mid-run status, or a new `/sessions/{id}/history` endpoint; simplest
|
||||
is to snapshot status after the prompt turn and assert it was `done`
|
||||
before sending the followup.
|
||||
- `no_rerun` is **removed** (it conflicts with iteration); replace
|
||||
usages with `plan_generations`.
|
||||
2. **Multi-turn follow-ups.** Change `conversation.Followup string`
|
||||
(`manifest.go:14`) to `Followups []string` and loop in `main.go:128-139`,
|
||||
calling `waitForTurn` after each. Backward-compatible: a scalar
|
||||
`followup` still parses by adding a YAML unmarshaler alias, or just
|
||||
migrate existing manifests (there are none in-repo — `evals/` is empty).
|
||||
3. **New manifest files** under `evals/`:
|
||||
- `plan-always-readonly.yaml` — a read-only question that *would* have
|
||||
been a degenerate case under the old SOUL. Asserts `proposes_plan`,
|
||||
`plan_before_run`, `completes`, `writes_back`.
|
||||
- `iteration-followup.yaml` — prompt completes a read-only task; followup
|
||||
asks the agent to *fix* what it found (config_mutation). Asserts
|
||||
`plan_generations: 2`, `reopens_session`, `completes`,
|
||||
`no_duplicate_complete` (per-turn — may need a per-turn variant).
|
||||
- `iteration-readonly.yaml` — two read-only sub-tasks back-to-back.
|
||||
Asserts `plan_generations: 2`, `proposes_plan` (>=2),
|
||||
`max_run_calls` bounded.
|
||||
- `no-plan-no-run.yaml` — a pure-DB Q&A ("list all LXCs"). Asserts
|
||||
`no_run`, `no_propose_plan` (the only remaining carve-out), `completes`.
|
||||
|
||||
**Severity:** friction (blocks regression detection for P1/P2).
|
||||
**Files:** `cmd/nomos/eval/manifest.go`, `cmd/nomos/eval/main.go`, new
|
||||
`evals/*.yaml`.
|
||||
|
||||
### P6 — Differentiate task classes in `set_goal` guidance (COSMETIC, F5)
|
||||
|
||||
Once P1 lands, `set_goal`'s return text (`tasks.go:184`) should say: "Next:
|
||||
gather context with read-only tools, then `propose_plan` (mandatory, even
|
||||
for read-only tasks — a one-step plan is fine). Do not call `run` before
|
||||
`propose_plan`." Drop the "Do not call run" absolute since read-only `run`
|
||||
is valid *after* a plan exists.
|
||||
|
||||
**Severity:** cosmetic. **Files:** `tasks.go:184`.
|
||||
|
||||
---
|
||||
|
||||
## Sequencing
|
||||
|
||||
- **Ship together:** P1 (plan-first gate) + P2 (iteration). P1 without P2
|
||||
makes every completed session un-reopenable; P2 without P1 leaves the
|
||||
approval-free `config_mutation` hole.
|
||||
- **P5 (evals) lands with P1/P2** as the regression net.
|
||||
- **P3 (reasoning persistence) and P4 (read-only allowlist)** are
|
||||
independent and can ship in the same change or after.
|
||||
|
||||
## Verification
|
||||
|
||||
- `go test ./cmd/nomos/... ./internal/policy/...` — new unit tests for the
|
||||
plan gate (P1.1), reopen + generation bump (P2.2), read-only allowlist
|
||||
(P4.2). **DONE 2026-07-15: all pass.**
|
||||
- `go build ./...` + `go vet ./...` — **DONE 2026-07-15: clean.**
|
||||
- `go run ./cmd/nomos/eval -manifest evals/*.yaml` against a live nomos —
|
||||
all four new manifests PASS. **Pending: requires live fleet + credits.**
|
||||
- Manual: replay the Proton Drive prompt, confirm a plan is proposed and
|
||||
the read-only `run`s execute without approval; send "now fix the `repos`
|
||||
failure" as a follow-up, confirm a second plan generation is created and
|
||||
the session reopens. **Pending: requires live fleet.**
|
||||
|
||||
## Implementation log — 2026-07-15
|
||||
|
||||
All P1–P6 implemented in one change. VERSION bumped 0.6.0 → 0.7.0 (minor:
|
||||
new features).
|
||||
|
||||
### What landed
|
||||
|
||||
- **P1 plan-first gate:** `internal/mcp/server.go` — new `sessionHasPlan`
|
||||
helper + gate at the top of `classifyAndGate` (before the dedup check).
|
||||
Refuses `run` with a directive when no plan exists for the session.
|
||||
- **P1 plan window decoupled:** `cmd/nomos/tasks.go` — `openPlanWindow`
|
||||
removed from `set_goal`. `cmd/nomos/store.go` — `openPlanWindow` func
|
||||
deleted, `proposePlan` no longer sets `nomos:plan:<session>`. `server.go`
|
||||
— `planWindowActive` func + its check block deleted. The assent window
|
||||
(opened only on operator approval in `agent.go:317,333`) is the sole
|
||||
gate for `config_mutation` auto-run.
|
||||
- **P1 SOUL.md:** degenerate-case carve-out stripped (§6, "Every chat is a
|
||||
task"). Replaced with "propose_plan is mandatory for any task that calls
|
||||
run — even read-only." Pure-DB Q&A (no `run`) is the only remaining
|
||||
carve-out.
|
||||
- **P2 reopenSession:** `cmd/nomos/store.go` — new `reopenSession` flips
|
||||
status `done`/`failed` → `executing`, marks all plan steps as `replaced`,
|
||||
clears outcome/summary, emits `task.reopened` event.
|
||||
- **P2 caller:** `cmd/nomos/main.go` — chat handler calls `reopenSession`
|
||||
before `touchSession` on every follow-up (no-op if session is still
|
||||
active).
|
||||
- **P2 proposePlan fix:** `anyStarted` check excludes `replaced`; DELETE
|
||||
only pending steps (replaced kept for generation counter + audit). New
|
||||
steps start at `max(seq)` (no collisions across generations).
|
||||
- **P2 errPlanInFlight text:** updated to mention the reopen path.
|
||||
- **P2 SOUL.md:** new "7. ITERATE" step documents the follow-up flow.
|
||||
- **P3 reasoning persistence:** `cmd/nomos/agent.go` — emits `text` event
|
||||
for intermediate reasoning (text + tool calls in same iteration).
|
||||
`cmd/nomos/main.go` + `cmd/nomos/continue.go` — `textParts []string`
|
||||
accumulator joins with `\n\n` instead of overwriting `finalText`.
|
||||
- **P4 read-only allowlist:** `internal/policy/command.go` — added `find`,
|
||||
`tree`, `locate`, `systemctl list-timers/list-unit-files/show`,
|
||||
`timedatectl`, `hostnamectl`, `systemd-analyze`, `rclone ls/lsl/md5sum/
|
||||
check/cryptcheck`. `command_test.go` — 11 new read-only cases + the
|
||||
exact compound from session `d0d562e0`.
|
||||
- **P5 eval harness:** `cmd/nomos/eval/manifest.go` — new assertion kinds
|
||||
(`proposes_plan`, `plan_before_run`, `plan_generations`); `Followup`
|
||||
→ `Followups []string` (backward-compat via `followups()` method).
|
||||
`cmd/nomos/eval/main.go` — multi-turn followup loop; `fetchTranscript`
|
||||
also fetches `/sessions/{id}/plan`; `distinctGenerations()` helper.
|
||||
Four manifests under `evals/`: `plan-always-readonly.yaml`,
|
||||
`iteration-followup.yaml`, `iteration-readonly.yaml`, `no-plan-no-run.yaml`.
|
||||
- **P6 set_goal text:** updated in P1.3 to say "propose_plan (mandatory —
|
||||
even read-only tasks need a one-step plan; the run handler refuses
|
||||
without one). Do not call run before propose_plan."
|
||||
Reference in New Issue
Block a user