Implement Sophia MVP stages 4-7 (introspection, queries, persistence, polish)
Brings the system from "engrams cluster + form synapses" to a complete end-to-end demo: ingest text, watch it cluster, ask questions, restart with state intact. - Stage 4: birth introspection (taxonomy/goals/open_questions via LLM, bounded by the global parallel-op semaphore), per-engram memory log, click-to-inspect side panel. - Stage 5: queries as conversations. POST /api/galaxy/:id/query embeds the question, materializes a pinned Query-Engram at the donut center, runs broadcast retrieval (global cosine scan + 1-hop synaptic expansion with attenuation) and fans out responder LLM calls. The integrator runs every 2s on accumulated snippets and streams the refining answer back over SSE; responders briefly transition to Conversing on the WS bus so the right dots light up. - Stage 6: snapshot persistence. sled-backed store keyed by galaxy id, JSON-encoded values (bincode chokes on internally-tagged enums like Manifest/MemoryKind), 60s periodic snapshot task, hydrate-on-boot, DELETE /api/galaxy/:id wired through. State survives kill -9. - Stage 7: HUD additions (sim ticks/sec, LLM queue depth, FPS) via a new GET /api/stats polled at 1Hz. `sophia demo` subcommand boots the server then auto-ingests a 50-paragraph corpus baked into the binary with include_str!. README quickstart added. Token caps for query_responder/integrator bumped (gemma-4-e4b is a thinking model — output budget must cover hidden reasoning + visible answer, otherwise content comes back empty). Pinned engrams skip physics; their tick scheduling is also skipped at materialization so they stay perfectly still at the donut center. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -15,10 +15,13 @@ sophia-core = { workspace = true }
|
||||
sophia-server = { workspace = true }
|
||||
sophia-sim = { workspace = true }
|
||||
sophia-llm = { workspace = true }
|
||||
sophia-store = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
|
||||
@@ -1,19 +1,31 @@
|
||||
//! Sophia binary. Reads `config.toml`, initializes tracing, starts the server.
|
||||
//! Sophia binary. Reads `config.toml`, initializes tracing, opens the
|
||||
//! persistence store, hydrates any saved galaxies into the sim, and starts
|
||||
//! the server. A background task snapshots the sim to disk on a fixed
|
||||
//! cadence (Stage 6).
|
||||
//!
|
||||
//! `sophia demo` (Stage 7) — same boot path, plus a background task that
|
||||
//! waits for the server to be reachable then ingests a curated corpus
|
||||
//! (compiled into the binary). Useful for screen recordings and first-run
|
||||
//! demos: a single command takes you from cold boot to a populated galaxy.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
use sophia_core::GalaxyShape;
|
||||
use sophia_llm::{LmStudioClient, LmStudioConfig, TokenCapsAll};
|
||||
use sophia_server::{serve, ServerConfig};
|
||||
use sophia_sim::SimHandle;
|
||||
use sophia_store::Store;
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Config {
|
||||
server: ServerSection,
|
||||
#[serde(default)]
|
||||
storage: StorageSection,
|
||||
lm_studio: LmStudioConfig,
|
||||
#[allow(dead_code)] // wired in to LLM call sites in Stages 4+
|
||||
token_caps: TokenCapsAll,
|
||||
#[serde(default)]
|
||||
galaxy_defaults: GalaxyDefaultsSection,
|
||||
@@ -26,6 +38,21 @@ struct ServerSection {
|
||||
static_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct StorageSection {
|
||||
data_dir: String,
|
||||
snapshot_interval_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for StorageSection {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
data_dir: "data".to_string(),
|
||||
snapshot_interval_secs: 60,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GalaxyDefaultsSection {
|
||||
major_radius: f32,
|
||||
@@ -92,11 +119,175 @@ fn load_config() -> Result<Config> {
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// Hydrate the sim from disk. Returns the count of galaxies restored.
|
||||
/// If the store is empty, materializes a fresh "default" galaxy and writes
|
||||
/// the first snapshot so a subsequent kill+restart already finds it.
|
||||
async fn hydrate_or_seed(sim: &SimHandle, store: &Store) -> Result<usize> {
|
||||
let saved = store.list_galaxies().context("listing stored galaxies")?;
|
||||
if saved.is_empty() {
|
||||
let info = sim.create_galaxy("default".to_string()).await?;
|
||||
tracing::info!("no saved galaxies — created default {:?}", info.id);
|
||||
// Persist the empty galaxy now so the very next restart rehydrates
|
||||
// (with the same id) instead of creating a fresh one — keeps the
|
||||
// browser's bookmarked galaxy id stable across restarts.
|
||||
let snaps = sim.snapshot_all().await?;
|
||||
for snap in &snaps {
|
||||
store.save_galaxy(snap)?;
|
||||
}
|
||||
return Ok(0);
|
||||
}
|
||||
let n = saved.len();
|
||||
for snap in saved {
|
||||
let id = snap.galaxy.id;
|
||||
let engrams = snap.engrams.len();
|
||||
let synapses = snap.synapses.len();
|
||||
match sim.hydrate_galaxy(snap).await {
|
||||
Ok(info) => {
|
||||
tracing::info!(
|
||||
"hydrated galaxy {:?} ({}, {} engrams, {} synapses)",
|
||||
info.id,
|
||||
info.name,
|
||||
engrams,
|
||||
synapses
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("hydration failed for {:?}: {e}", id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Stage 7: corpus to ingest when invoked as `sophia demo`. Compiled into
|
||||
/// the binary so the command works from any cwd.
|
||||
const DEMO_CORPUS: &str = include_str!("../../../assets/demo_corpus.txt");
|
||||
|
||||
/// Stage 7 demo task. Polls `/healthz` until the server is reachable, then
|
||||
/// posts the curated corpus into the first galaxy. Logs progress; failures
|
||||
/// are non-fatal — the server stays up either way.
|
||||
fn spawn_demo_ingest(host: String, port: u16) {
|
||||
tokio::spawn(async move {
|
||||
let base = format!("http://{host}:{port}");
|
||||
let client = match reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(60))
|
||||
.build()
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: reqwest client failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Wait for server reachable. Cap at 30 s so a misconfigured boot
|
||||
// doesn't loop forever.
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(30);
|
||||
let mut reachable = false;
|
||||
while std::time::Instant::now() < deadline {
|
||||
if client.get(format!("{base}/healthz")).send().await.is_ok() {
|
||||
reachable = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
if !reachable {
|
||||
tracing::warn!("demo: server never became reachable, aborting ingest");
|
||||
return;
|
||||
}
|
||||
|
||||
// Pick the first galaxy (the auto-created or just-hydrated default).
|
||||
let galaxies: Vec<serde_json::Value> = match client
|
||||
.get(format!("{base}/api/galaxy"))
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
{
|
||||
Ok(r) => match r.json().await {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: parsing galaxies failed: {e}");
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: listing galaxies failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(first) = galaxies.first() else {
|
||||
tracing::warn!("demo: no galaxies to ingest into");
|
||||
return;
|
||||
};
|
||||
let Some(gid) = first.get("id").and_then(|v| v.as_str()) else {
|
||||
tracing::warn!("demo: galaxy entry missing id");
|
||||
return;
|
||||
};
|
||||
|
||||
let texts: Vec<&str> = DEMO_CORPUS
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.collect();
|
||||
tracing::info!("demo: ingesting {} paragraphs into galaxy {}", texts.len(), gid);
|
||||
|
||||
let body = serde_json::json!({ "texts": texts });
|
||||
match client
|
||||
.post(format!("{base}/api/galaxy/{gid}/ingest"))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
{
|
||||
Ok(_) => {
|
||||
tracing::info!(
|
||||
"demo: ingest succeeded — open {base} to watch the cluster form"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: ingest failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Background task: every `interval`, snapshot every galaxy and write to
|
||||
/// disk. Errors are logged and ignored — we don't want a transient disk
|
||||
/// hiccup to take down the simulation.
|
||||
fn spawn_snapshot_task(sim: SimHandle, store: Store, interval: Duration) {
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(interval);
|
||||
// First tick fires immediately by default; we want the first save
|
||||
// *after* an interval has passed so boot doesn't double-write.
|
||||
ticker.tick().await;
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
match sim.snapshot_all().await {
|
||||
Ok(snaps) => {
|
||||
for snap in &snaps {
|
||||
if let Err(e) = store.save_galaxy(snap) {
|
||||
tracing::warn!("snapshot save failed: {e}");
|
||||
}
|
||||
}
|
||||
tracing::debug!("snapshot tick: persisted {} galaxies", snaps.len());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("snapshot_all failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let cfg = load_config()?;
|
||||
|
||||
// Stage 7 CLI: only one subcommand right now (`demo`). Anything else
|
||||
// boots the server normally.
|
||||
let demo_mode = std::env::args().nth(1).as_deref() == Some("demo");
|
||||
|
||||
let server_cfg = ServerConfig {
|
||||
host: cfg.server.host,
|
||||
port: cfg.server.port,
|
||||
@@ -125,9 +316,28 @@ async fn main() -> Result<()> {
|
||||
default_shape.minor_radius
|
||||
);
|
||||
|
||||
let sim = sophia_sim::spawn_sim(default_shape);
|
||||
let info = sim.create_galaxy("default".to_string()).await?;
|
||||
tracing::info!("default galaxy ready: id={:?} name={}", info.id, info.name);
|
||||
let store = Store::open(&cfg.storage.data_dir)
|
||||
.with_context(|| format!("opening sled store at {}", cfg.storage.data_dir))?;
|
||||
tracing::info!("persistence store open at {}", cfg.storage.data_dir);
|
||||
|
||||
serve(server_cfg, sim, llm).await
|
||||
let sim = sophia_sim::spawn_sim(default_shape);
|
||||
let restored = hydrate_or_seed(&sim, &store).await?;
|
||||
tracing::info!("hydrated {restored} galaxies from disk");
|
||||
|
||||
spawn_snapshot_task(
|
||||
sim.clone(),
|
||||
store.clone(),
|
||||
Duration::from_secs(cfg.storage.snapshot_interval_secs.max(1)),
|
||||
);
|
||||
tracing::info!(
|
||||
"snapshot task scheduled every {}s",
|
||||
cfg.storage.snapshot_interval_secs
|
||||
);
|
||||
|
||||
if demo_mode {
|
||||
tracing::info!("demo mode: will auto-ingest curated corpus once server is reachable");
|
||||
spawn_demo_ingest(server_cfg.host.clone(), server_cfg.port);
|
||||
}
|
||||
|
||||
serve(server_cfg, sim, llm, cfg.token_caps, store).await
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user