feat(agent): plan-first gate, iterative follow-ups, reasoning persistence

P1 plan-first: run handler refuses without propose_plan (structural gate,
not SOUL.md prose). Plan window decoupled from set_goal — config_mutation
auto-run only on operator approval (assent window). Closes the approval-free
config_mutation hole confirmed in session d0d562e0.

P2 iteration: reopenSession flips terminal→executing, marks prior plan steps
replaced, clears outcome. proposePlan excludes replaced from in-flight check,
bumps generation. A follow-up on a completed session starts a new sub-task
with a fresh plan — no more errPlanInFlight dead end.

P3 reasoning: accumulate per-iteration text into the persisted row instead
of overwriting with the last text event. Reload shows intermediate thinking,
not just the final summary.

P4 read-only allowlist: add find, tree, locate, systemctl list-timers/
list-unit-files/show, timedatectl, hostnamectl, systemd-analyze, rclone
ls/lsl/md5sum/check/cryptcheck. Fixes the find misclassification from
d0d562e0.

P5 eval harness: new assertion kinds (proposes_plan, plan_before_run,
plan_generations), multi-turn followups, fetch /sessions/{id}/plan. Four
manifests under evals/.

P6 SOUL.md: strip degenerate-case carve-out, add ITERATE step, update
set_goal guidance.

VERSION 0.6.0 → 0.7.0

cmd/nomos/eval/manifest.go

@@ -11,10 +11,24 @@ import (
 type conversation struct {
 	Name       string     `yaml:"name"`
 	Prompt     string     `yaml:"prompt"`
-	Followup   string     `yaml:"followup"`
+	Followup   string     `yaml:"followup"`   // backward compat: single followup
+	Followups  []string   `yaml:"followups"`  // P5: multi-turn followups
 	Assertions []assertion `yaml:"assertions"`
 }
 
+// followups returns the full list of follow-up messages, supporting both
+// the single `followup` field (backward compat) and the multi-turn
+// `followups` list.
+func (c conversation) followups() []string {
+	if len(c.Followups) > 0 {
+		return c.Followups
+	}
+	if c.Followup != "" {
+		return []string{c.Followup}
+	}
+	return nil
+}
+
 // assertion is one check against the final transcript. The `kind` field
 // selects the scorer; the rest are scorer-specific parameters.
 //
@@ -23,13 +37,15 @@ type conversation struct {
 //	completes             — session status reached done/failed (not stuck executing)
 //	outcome_is            — session outcome == value (success/failure/partial)
 //	no_propose_plan       — propose_plan was never called
+//	proposes_plan         — propose_plan called >= 1 time (plan-always model; P1)
 //	proposes_plan_once    — propose_plan was called exactly once
 //	no_duplicate_proposal — propose_plan called at most once
+//	plan_before_run       — the first `run` call comes after the first `propose_plan` (P1 ordering gate)
+//	plan_generations      — the persisted plan has exactly `value` distinct generations (P2 iteration: 1 = single, 2 = one followup)
 //	writes_back           — update_entity_attributes or create_relationship was called
 //	max_tool_calls        — total tool calls <= value
 //	max_run_calls         — total `run` calls <= value
 //	no_run                — `run` was never called
-//	no_rerun              — `run` was NOT called after the followup turn (if any)
 //	calls_tool            — the named tool appears in the transcript
 //	plan_step_count       — the plan has exactly `value` steps
 //	no_duplicate_complete — complete_task called at most once
@@ -88,6 +104,14 @@ func scoreOne(a assertion, t transcript, s sessionState) (bool, string) {
 		}
 		return false, fmt.Sprintf("propose_plan called %d time(s)", n)
 
+	case "proposes_plan":
+		// P1 plan-always: propose_plan called >= 1 time.
+		n := countTool(tools, "propose_plan")
+		if n >= 1 {
+			return true, fmt.Sprintf("propose_plan called %d time(s)", n)
+		}
+		return false, "propose_plan never called (plan-always requires >= 1)"
+
 	case "proposes_plan_once":
 		n := countTool(tools, "propose_plan")
 		if n == 1 {
@@ -102,6 +126,44 @@ func scoreOne(a assertion, t transcript, s sessionState) (bool, string) {
 		}
 		return false, fmt.Sprintf("propose_plan called %d time(s), want <= 1", n)
 
+	case "plan_before_run":
+		// P1 ordering gate: the first `run` call's global index in the
+		// transcript is strictly greater than the first `propose_plan`
+		// index. Both indices are over the flat tool-call list (across all
+		// messages, in order).
+		planIdx, runIdx := -1, -1
+		for i, name := range tools {
+			if name == "propose_plan" && planIdx == -1 {
+				planIdx = i
+			}
+			if name == "run" && runIdx == -1 {
+				runIdx = i
+			}
+		}
+		if runIdx == -1 {
+			return true, "run never called (ordering trivially satisfied)"
+		}
+		if planIdx == -1 {
+			return false, "run called but propose_plan never called"
+		}
+		if planIdx < runIdx {
+			return true, fmt.Sprintf("propose_plan at index %d before run at index %d", planIdx, runIdx)
+		}
+		return false, fmt.Sprintf("run at index %d before propose_plan at index %d", runIdx, planIdx)
+
+	case "plan_generations":
+		// P2 iteration: counts distinct `generation` values in
+		// session_plan_steps. 1 = single sub-task, 2 = one follow-up
+		// sub-task, etc. Requires the plan endpoint to return generation
+		// values; the eval fetches /sessions/{id}/plan and passes it via
+		// the transcript's PlanSteps field.
+		want := toInt(a.Value)
+		gens := t.distinctGenerations()
+		if gens == want {
+			return true, fmt.Sprintf("%d plan generation(s)", gens)
+		}
+		return false, fmt.Sprintf("%d plan generation(s), want %d", gens, want)
+
 	case "writes_back":
 		n := countTool(tools, "update_entity_attributes") + countTool(tools, "create_relationship")
 		if n > 0 {