Implement Sophia MVP stages 4-7 (introspection, queries, persistence, polish)

Brings the system from "engrams cluster + form synapses" to a complete
end-to-end demo: ingest text, watch it cluster, ask questions, restart
with state intact.

- Stage 4: birth introspection (taxonomy/goals/open_questions via LLM,
  bounded by the global parallel-op semaphore), per-engram memory log,
  click-to-inspect side panel.
- Stage 5: queries as conversations. POST /api/galaxy/:id/query embeds
  the question, materializes a pinned Query-Engram at the donut center,
  runs broadcast retrieval (global cosine scan + 1-hop synaptic
  expansion with attenuation) and fans out responder LLM calls. The
  integrator runs every 2s on accumulated snippets and streams the
  refining answer back over SSE; responders briefly transition to
  Conversing on the WS bus so the right dots light up.
- Stage 6: snapshot persistence. sled-backed store keyed by galaxy id,
  JSON-encoded values (bincode chokes on internally-tagged enums like
  Manifest/MemoryKind), 60s periodic snapshot task, hydrate-on-boot,
  DELETE /api/galaxy/:id wired through. State survives kill -9.
- Stage 7: HUD additions (sim ticks/sec, LLM queue depth, FPS) via a
  new GET /api/stats polled at 1Hz. `sophia demo` subcommand boots the
  server then auto-ingests a 50-paragraph corpus baked into the binary
  with include_str!. README quickstart added.

Token caps for query_responder/integrator bumped (gemma-4-e4b is a
thinking model — output budget must cover hidden reasoning + visible
answer, otherwise content comes back empty). Pinned engrams skip
physics; their tick scheduling is also skipped at materialization so
they stay perfectly still at the donut center.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/sophia-core/src/dto.rs

@@ -10,6 +10,8 @@ use serde::{Deserialize, Serialize};
 use crate::engram::EngramState;
 use crate::ids::{EngramId, GalaxyId, SynapseId};
 use crate::manifest::Manifest;
+use crate::memory::{Introspection, Memory};
+use crate::query::{QueryId, QueryStatus};
 use crate::synapse::Synapse;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -62,6 +64,10 @@ pub struct EngramDetail {
     pub manifest: Option<Manifest>,
     pub slate_dim: Option<usize>,
     pub slate_norm: Option<f32>,
+    pub introspection: Introspection,
+    pub memories: Vec<Memory>,
+    /// True for Query-Engrams (Stage 5).
+    pub pinned: bool,
 }
 
 /// Position frame: a downsampled bundle of all engram positions at a moment in
@@ -97,6 +103,37 @@ pub enum SimEvent {
         major_radius: f32,
         minor_radius: f32,
     },
+    /// An engram's lifecycle state changed (e.g., responder lit up while
+    /// answering a query, then returned to idle). Frontend uses this to
+    /// re-paint the engram's color in real time.
+    EngramStateChanged {
+        id: EngramId,
+        state: EngramState,
+    },
+    /// A query was started — a Query-Engram materialized at `position` with
+    /// the question text. Frontend uses this to register the query for the
+    /// answer panel + visual highlight.
+    QueryStarted {
+        query: QueryId,
+        engram: EngramId,
+        position: [f32; 3],
+        text: String,
+    },
+    /// The integrator produced a new running answer. `version` increments
+    /// monotonically per query; clients can ignore stale frames.
+    QueryAnswerUpdated {
+        query: QueryId,
+        version: u32,
+        status: QueryStatus,
+        answer: String,
+    },
+    /// Terminal event for a query. The Query-Engram has been moved to the
+    /// `Memorize` state and the SSE stream is closed.
+    QueryFinished {
+        query: QueryId,
+        status: QueryStatus,
+        responder_count: u32,
+    },
     /// Position frame is sent over the wire as a binary frame; this variant
     /// carries the in-process payload.
     #[serde(skip)]

crates/sophia-core/src/engram.rs

@@ -1,8 +1,11 @@
+use std::collections::VecDeque;
+
 use glam::Vec3;
 use serde::{Deserialize, Serialize};
 
 use crate::ids::EngramId;
 use crate::manifest::Manifest;
+use crate::memory::{Introspection, Memory};
 use crate::slate::Slate;
 
 /// Engram state machine — see §5 of `docs/system-analysis.md`.
@@ -31,7 +34,9 @@ impl EngramState {
 ///
 /// Stage 2 adds `manifest` (the source content) and `slate` (an embedding for
 /// fast similarity). Stage 4 adds taxonomy/goals/memories on top of that.
-#[derive(Debug, Clone)]
+/// Stage 6 adds Serialize/Deserialize so engrams can round-trip through the
+/// `sophia-store` snapshot file.
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Engram {
     pub id: EngramId,
     /// Slot index for the visualization's `InstancedMesh`. Assigned at birth,
@@ -51,4 +56,14 @@ pub struct Engram {
     /// Embedding vector — System-1 layer of the Universal Slate. `None` for
     /// synthetic seed engrams.
     pub slate: Option<Slate>,
+    /// LLM-generated self-description (Stage 4). Filled async after birth;
+    /// empty until the introspection task completes.
+    pub introspection: Introspection,
+    /// Per-engram memory log (Stage 4). Bounded — older entries evicted from
+    /// the front when full.
+    pub memories: VecDeque<Memory>,
+    /// Pinned engrams skip physics integration — they sit at their birth
+    /// position permanently. Used for Query-Engrams (Stage 5), which
+    /// materialize at the galaxy center and shouldn't drift into the tube.
+    pub pinned: bool,
 }

crates/sophia-core/src/galaxy.rs

@@ -59,8 +59,9 @@ pub struct Galaxy {
     pub id: GalaxyId,
     pub name: String,
     /// Galaxy origin in world coords. Always `Vec3::ZERO` in v1; reserved for
-    /// future multi-galaxy layouts.
-    #[serde(skip)]
+    /// future multi-galaxy layouts. Serialized so persistence round-trips
+    /// the full state (Stage 6) — defaults to ZERO if missing from disk.
+    #[serde(default)]
     pub center: Vec3,
     pub shape: GalaxyShape,
 }

crates/sophia-core/src/lib.rs

@@ -8,6 +8,9 @@ pub mod engram;
 pub mod galaxy;
 pub mod ids;
 pub mod manifest;
+pub mod memory;
+pub mod persistence;
+pub mod query;
 pub mod slate;
 pub mod synapse;
 
@@ -16,6 +19,9 @@ pub use engram::{Engram, EngramState};
 pub use galaxy::{Galaxy, GalaxyShape};
 pub use ids::{EngramId, GalaxyId, SynapseId};
 pub use manifest::Manifest;
+pub use memory::{Introspection, Memory, MemoryKind, MAX_MEMORIES};
+pub use persistence::GalaxySnapshot;
+pub use query::{QueryId, QueryStatus};
 pub use slate::Slate;
 pub use synapse::{canonical_pair, Synapse};
 

crates/sophia-core/src/memory.rs

@@ -0,0 +1,52 @@
+//! Per-Engram memory entries — a running log of meaningful events from the
+//! engram's own perspective. Stage 4 only records synapse formation; Stage 5
+//! will add conversation memorialization.
+
+use serde::{Deserialize, Serialize};
+
+use crate::ids::EngramId;
+use crate::query::QueryId;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum MemoryKind {
+    Born,
+    Introspected,
+    /// A synapse formed with another engram.
+    SynapseFormed { with: EngramId, weight: f32 },
+    /// This engram contributed a snippet to a query's running answer.
+    /// `snippet` is the engram's own LLM-generated POV on the question.
+    QueryParticipated { query: QueryId, snippet: String },
+    /// (Query-engrams only.) The query completed and produced a final answer.
+    /// Stored on the Query-Engram itself so the inspector can show the
+    /// resolution after the fact.
+    QueryAnswered { answer: String, responder_count: u32 },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Memory {
+    /// Time relative to the start of the simulation (ms).
+    pub at_ms: u32,
+    pub kind: MemoryKind,
+}
+
+/// Hard cap on a single engram's memory list. Older entries are evicted from
+/// the front of the deque when full. Stage 4 picks 64 — generous enough for
+/// the demo, small enough not to bloat the Hello/inspector payloads.
+pub const MAX_MEMORIES: usize = 64;
+
+/// Output of LLM-driven self-introspection at birth (Stage 4). Stored on the
+/// `Engram` itself, not the `Manifest`, since it's the engram's *interpretation*
+/// of its source content rather than the source itself.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct Introspection {
+    pub taxonomy: Vec<String>,
+    pub goals: Vec<String>,
+    pub open_questions: Vec<String>,
+}
+
+impl Introspection {
+    pub fn is_empty(&self) -> bool {
+        self.taxonomy.is_empty() && self.goals.is_empty() && self.open_questions.is_empty()
+    }
+}

crates/sophia-core/src/persistence.rs

@@ -0,0 +1,31 @@
+//! Persistence types (Stage 6).
+//!
+//! `GalaxySnapshot` is the on-disk shape of a galaxy: its metadata plus the
+//! full set of Engrams and Synapses it contains. This is what the periodic
+//! snapshot task in `sophia-bin` writes to the sled store, and what the
+//! `SimHandle::hydrate_galaxy` command reads back at boot.
+//!
+//! Pinned engrams (Stage 5 Query-Engrams) are intentionally excluded from
+//! snapshots. Queries are ephemeral — their orchestrator task is gone after
+//! a server restart, and a half-rendered query-engram with no SSE channel
+//! would be confusing in the UI. Letting them die with the process is the
+//! cleaner default.
+
+use serde::{Deserialize, Serialize};
+
+use crate::engram::Engram;
+use crate::galaxy::Galaxy;
+use crate::synapse::Synapse;
+
+/// One galaxy's full persistent state. Encoded with `bincode` and stored
+/// under the galaxy's id in the sled tree.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GalaxySnapshot {
+    pub galaxy: Galaxy,
+    /// All non-pinned engrams. The order is preserved so `instance_idx`
+    /// values remain meaningful after a round-trip — the sim re-uses each
+    /// engram's stored `instance_idx` rather than re-numbering, so the
+    /// position-frame slot mapping stays stable across restarts.
+    pub engrams: Vec<Engram>,
+    pub synapses: Vec<Synapse>,
+}

crates/sophia-core/src/query.rs

@@ -0,0 +1,45 @@
+//! Query domain types (Stage 5).
+//!
+//! A "query" is a question that materializes as a special pinned Engram at
+//! the galaxy center. The Query-Engram broadcasts to nearby resonating
+//! Engrams; their snippets are integrated into a streaming answer.
+//!
+//! `QueryId` is distinct from `EngramId` even though every query has a
+//! backing query-engram — keeping them separate lets the orchestrator track
+//! query state (responders, snippets, answer versions) independently of the
+//! engram's lifecycle in the world.
+
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct QueryId(pub Uuid);
+
+impl QueryId {
+    pub fn new() -> Self {
+        Self(Uuid::now_v7())
+    }
+}
+
+impl Default for QueryId {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Lifecycle phase of a query, as exposed to SSE consumers.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum QueryStatus {
+    /// Query-Engram materialized; broadcast not yet started.
+    Pending,
+    /// Phase A/B retrieval in flight; responders being polled.
+    Responding,
+    /// All responses gathered; integrator producing final answer.
+    Integrating,
+    /// Final answer published; query-engram transitioned to Memorize.
+    Done,
+    /// Aborted (e.g. embedding failure or sim shutdown).
+    Failed,
+}