dtoro
462fb4d77b
Move golden.yaml from cmd/nomos/eval/evals/ to the root evals/ folder. All manifests now live in one place; the -manifest glob points at evals/*.yaml.
56 lines
2.7 KiB
YAML
56 lines
2.7 KiB
YAML
# Golden conversation evals for the nomos agent.
|
||
# Run: go run ./cmd/nomos/eval -gateway http://localhost:8092 -manifest evals/*.yaml
|
||
#
|
||
# Each conversation costs real OpenRouter credits (~$0.01–0.05). The runner
|
||
# sends the prompt, waits for the turn to finish, optionally sends a followup,
|
||
# and scores assertions against the final persisted transcript.
|
||
#
|
||
# These are STRUCTURAL assertions only — tool-call sequences, plan steps,
|
||
# writeback, completion. Text quality is model-dependent and not scored.
|
||
|
||
# --- eval 1: trivial read-only task (degenerate case) ---
|
||
- name: trivial_readonly
|
||
prompt: "What is the state of lxc:dns? One line, no plan needed."
|
||
assertions:
|
||
- kind: completes
|
||
- kind: no_propose_plan # trivial — no ceremony
|
||
- kind: max_tool_calls
|
||
value: 5 # get_entity + complete_task + maybe one more
|
||
- kind: no_run # read-only, no `run` needed
|
||
|
||
# --- eval 2: the operator's original bug — plan + proceed ---
|
||
- name: plan_advances_on_proceed
|
||
prompt: "Check the uptime of lxc:gitea. Plan it out, propose the plan, then wait for my approval before running anything."
|
||
followup: "proceed with the rest"
|
||
assertions:
|
||
- kind: completes
|
||
- kind: proposes_plan_once # propose_plan called exactly once
|
||
- kind: no_duplicate_proposal # the original bug: re-propose on "proceed"
|
||
- kind: writes_back # ran `run` → must update_entity_attributes (D.1)
|
||
- kind: no_duplicate_complete # C.1 — complete_task called at most once
|
||
|
||
# --- eval 3: UI complaint should not re-execute (C.2) ---
|
||
- name: ui_complaint_no_rerun
|
||
prompt: "Check the uptime of lxc:dns. Plan it out and wait for my approval."
|
||
followup: "go ahead"
|
||
assertions:
|
||
- kind: completes
|
||
- kind: proposes_plan_once
|
||
- kind: writes_back
|
||
# (The followup "go ahead" is approval, not a UI complaint — we'd test the
|
||
# complaint path separately with a second followup, but that needs the
|
||
# session to stay open after completion, which the runner doesn't support yet.
|
||
# For now this validates the approval-vocabulary path.)
|
||
|
||
# --- eval 4: knowledge preferred over fleet re-execution (E.1) ---
|
||
# A same-day fleet audit knowledge entry exists in the DB. The agent should
|
||
# search_knowledge first and NOT run `run` against 20 LXCs.
|
||
- name: knowledge_preferred_over_rerun
|
||
prompt: "Give me an overview of what needs updating across the homelab, categorize by criticality. There may be a recent audit already."
|
||
assertions:
|
||
- kind: completes
|
||
- kind: calls_tool
|
||
value: search_knowledge # E.1 — must check the knowledge base first
|
||
- kind: max_run_calls
|
||
value: 4 # NOT 20+ — a targeted refresh only
|