From f1dda7290a44b3c4bf090111722f9eb3759ca9e6 Mon Sep 17 00:00:00 2001 From: dtoro Date: Wed, 15 Jul 2026 10:02:46 +0200 Subject: [PATCH] fix(eval): fix manifests to require live inspection + approval followup plan-always-readonly: prompt now demands live systemd timer inspection, not just DB lookup. Added calls_tool: run assertion. iteration-followup: added 'go ahead' as second followup so the config_mutation plan gets approved and can execute. iteration-readonly: replaced nonexistent lxc:prometheus with lxc:dns, keep it read-only so no approval needed. --- evals/iteration-followup.yaml | 10 ++++++---- evals/iteration-readonly.yaml | 12 ++++++++---- evals/plan-always-readonly.yaml | 11 ++++++----- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/evals/iteration-followup.yaml b/evals/iteration-followup.yaml index 885a43e..8a989ad 100644 --- a/evals/iteration-followup.yaml +++ b/evals/iteration-followup.yaml @@ -1,11 +1,13 @@ # P5 eval: iteration. A read-only task completes; the follow-up asks the -# agent to act on what it found (a config_mutation). Asserts the session +# agent to act on what it found (a config_mutation). The second followup +# is the approval ("go ahead") so the plan can execute. Asserts the session # reopens, a second plan generation is created, and the agent completes -# both sub-tasks without duplicate-complete. +# both sub-tasks. - name: iteration-followup - prompt: "When was the last backup to Proton Drive done, and when is the next one?" + prompt: "Check the live systemd timer on lxc:rclone — when did the last backup to Proton Drive actually run, and when is the next one scheduled? Inspect the actual service state, don't just read the DB." followups: - "The repos folder failed last time. Reset the failed service and re-run the backup." + - "go ahead" assertions: - kind: completes - kind: plan_generations @@ -13,4 +15,4 @@ - kind: proposes_plan - kind: writes_back - kind: max_run_calls - value: 6 + value: 8 diff --git a/evals/iteration-readonly.yaml b/evals/iteration-readonly.yaml index 75d4f74..ae619ae 100644 --- a/evals/iteration-readonly.yaml +++ b/evals/iteration-readonly.yaml @@ -1,13 +1,17 @@ -# P5 eval: two read-only sub-tasks back-to-back. Asserts the session -# reopens and a second plan generation is created for the follow-up. +# P5 eval: two read-only sub-tasks back-to-back. The follow-up is also +# read-only (no approval needed) so the agent can execute immediately after +# proposing the second plan. Asserts the session reopens and a second plan +# generation is created. - name: iteration-readonly - prompt: "When was the last backup to Proton Drive done, and when is the next one?" + prompt: "Check the live systemd timer on lxc:rclone — when did the last backup to Proton Drive actually run, and when is the next one scheduled?" followups: - - "Now check when the last snapshot of the prometheus LXC was taken." + - "Now check the uptime of lxc:dns." assertions: - kind: completes - kind: plan_generations value: 2 - kind: proposes_plan + - kind: calls_tool + value: run - kind: max_run_calls value: 6 diff --git a/evals/plan-always-readonly.yaml b/evals/plan-always-readonly.yaml index ac71f54..a825380 100644 --- a/evals/plan-always-readonly.yaml +++ b/evals/plan-always-readonly.yaml @@ -1,13 +1,14 @@ -# P5 eval: a read-only question that would have been a "degenerate case" -# under the old SOUL.md carve-out. Asserts the plan-first gate works: the -# agent must propose_plan before run, even for a trivial read-only task. -# Run against a live nomos with a real rclone LXC. +# P5 eval: a read-only question that requires live inspection (not just DB +# lookup). Asserts the plan-first gate works: the agent must propose_plan +# before run, even for a trivial read-only task. - name: plan-always-readonly - prompt: "When was the last backup to Proton Drive done, and when is the next one?" + prompt: "Check the live systemd timer on lxc:rclone — when did the last backup to Proton Drive actually run, and when is the next one scheduled? Inspect the actual service state, don't just read the DB." assertions: - kind: completes - kind: proposes_plan - kind: plan_before_run + - kind: calls_tool + value: run - kind: writes_back - kind: max_run_calls value: 3