# Golden conversation evals for the nomos agent.
# Run: go run ./cmd/nomos/eval -gateway http://localhost:8092 -manifest evals/*.yaml
#
# Each conversation costs real OpenRouter credits (~$0.01–0.05). The runner
# sends the prompt, waits for the turn to finish, optionally sends a followup,
# and scores assertions against the final persisted transcript.
#
# These are STRUCTURAL assertions only — tool-call sequences, plan steps,
# writeback, completion. Text quality is model-dependent and not scored.

# --- eval 1: trivial read-only task (degenerate case) ---
- name: trivial_readonly
  prompt: "What is the state of lxc:dns? One line, no plan needed."
  assertions:
    - kind: completes
    - kind: no_propose_plan       # trivial — no ceremony
    - kind: max_tool_calls
      value: 5                    # get_entity + complete_task + maybe one more
    - kind: no_run                # read-only, no `run` needed

# --- eval 2: the operator's original bug — plan + proceed ---
- name: plan_advances_on_proceed
  prompt: "Check the uptime of lxc:gitea. Plan it out, propose the plan, then wait for my approval before running anything."
  followup: "proceed with the rest"
  assertions:
    - kind: completes
    - kind: proposes_plan_once    # propose_plan called exactly once
    - kind: no_duplicate_proposal # the original bug: re-propose on "proceed"
    - kind: writes_back           # ran `run` → must update_entity_attributes (D.1)
    - kind: no_duplicate_complete # C.1 — complete_task called at most once

# --- eval 3: UI complaint should not re-execute (C.2) ---
- name: ui_complaint_no_rerun
  prompt: "Check the uptime of lxc:dns. Plan it out and wait for my approval."
  followup: "go ahead"
  assertions:
    - kind: completes
    - kind: proposes_plan_once
    - kind: writes_back
# (The followup "go ahead" is approval, not a UI complaint — we'd test the
#  complaint path separately with a second followup, but that needs the
#  session to stay open after completion, which the runner doesn't support yet.
#  For now this validates the approval-vocabulary path.)

# --- eval 4: knowledge preferred over fleet re-execution (E.1) ---
# A same-day fleet audit knowledge entry exists in the DB. The agent should
# search_knowledge first and NOT run `run` against 20 LXCs.
- name: knowledge_preferred_over_rerun
  prompt: "Give me an overview of what needs updating across the homelab, categorize by criticality. There may be a recent audit already."
  assertions:
    - kind: completes
    - kind: calls_tool
      value: search_knowledge      # E.1 — must check the knowledge base first
    - kind: max_run_calls
      value: 4                     # NOT 20+ — a targeted refresh only