Compare commits
2 Commits
43c4d270e6
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| bae084cd76 | |||
| 8688f632bf |
10
.gitignore
vendored
Normal file
10
.gitignore
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
/target
|
||||
**/node_modules
|
||||
web/dist
|
||||
**/.DS_Store
|
||||
*.log
|
||||
sled_data/
|
||||
/data/
|
||||
.claude/
|
||||
config.local.toml
|
||||
.env
|
||||
2668
Cargo.lock
generated
Normal file
2668
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
75
Cargo.toml
Normal file
75
Cargo.toml
Normal file
@@ -0,0 +1,75 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/sophia-core",
|
||||
"crates/sophia-sim",
|
||||
"crates/sophia-llm",
|
||||
"crates/sophia-store",
|
||||
"crates/sophia-server",
|
||||
"crates/sophia-bin",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.0.1"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
rust-version = "1.80"
|
||||
authors = ["Sophia"]
|
||||
|
||||
[workspace.dependencies]
|
||||
# Internal
|
||||
sophia-core = { path = "crates/sophia-core" }
|
||||
sophia-sim = { path = "crates/sophia-sim" }
|
||||
sophia-llm = { path = "crates/sophia-llm" }
|
||||
sophia-store = { path = "crates/sophia-store" }
|
||||
sophia-server = { path = "crates/sophia-server" }
|
||||
|
||||
# Errors / runtime
|
||||
anyhow = "1"
|
||||
thiserror = "1"
|
||||
tokio = { version = "1.40", features = ["full"] }
|
||||
async-trait = "0.1"
|
||||
|
||||
# Serde
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
|
||||
# Math, ids
|
||||
glam = { version = "0.28", features = ["serde"] }
|
||||
uuid = { version = "1", features = ["v7", "serde"] }
|
||||
|
||||
# HTTP/WS server (used by sophia-server only)
|
||||
axum = { version = "0.7", features = ["ws", "macros"] }
|
||||
tower = "0.5"
|
||||
tower-http = { version = "0.6", features = ["fs", "trace", "cors"] }
|
||||
|
||||
# HTTP client (LM Studio)
|
||||
reqwest = { version = "0.12", features = ["json", "stream"] }
|
||||
|
||||
# Sim
|
||||
kiddo = "4"
|
||||
rand = { version = "0.8", features = ["small_rng"] }
|
||||
|
||||
# Async helpers
|
||||
futures-util = { version = "0.3", default-features = false, features = ["std", "sink"] }
|
||||
tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
bytes = "1"
|
||||
|
||||
# Config
|
||||
toml = "0.8"
|
||||
|
||||
# Persistence (Stage 6) — embedded KV with serde_json-encoded values. We
|
||||
# pick JSON over bincode because `Manifest` and `MemoryKind` are
|
||||
# internally-tagged enums (`#[serde(tag = "kind")]`), which require a
|
||||
# self-describing format on the wire. Snapshots are 60-second cadence;
|
||||
# JSON's verbosity isn't a real cost at MVP scale.
|
||||
sled = "0.34"
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
opt-level = 3
|
||||
251
README.md
251
README.md
@@ -1,125 +1,156 @@
|
||||
# Sophia
|
||||
|
||||
Sophia is a system that provides an organic, self organizing, self actualizing, agent based database.
|
||||
A knowledge engine where each idea is an autonomous agent — an *Engram* — that
|
||||
moves, resonates with similar ideas, forms synapses with them, and answers
|
||||
questions by gathering its own perspective into a single integrated reply.
|
||||
|
||||
## Main components
|
||||
The whole system is a 3D simulation you can watch: ingested text becomes glowing
|
||||
dots that fly out from the centre of a torus, drift toward similar peers, and
|
||||
light up when a query's broadcast wave touches them.
|
||||
|
||||
- Space of Recollection: A 3-diementional space where all enitites live in.
|
||||
- The Space fo Recollection is an ever increasing Space, it grows with the number of Engrams, and contains them all.
|
||||
- Cycle: A fourth dimension that allows the simulation of the world to progress and evolve. Moves forward by default.
|
||||
- Engrams: Autonous entities that inhabit the Space during cycles.
|
||||
- The Great Reflection: It's source of all Engrams, a portal in or out of the Space of Recollection, opens or collapses to allow engrams to materiaze.
|
||||
```
|
||||
ingest text → embed → spawn engram at center
|
||||
↓
|
||||
spiral into the tube
|
||||
↓
|
||||
gravity pulls toward similar engrams
|
||||
↓
|
||||
co-resonating pairs form synapses
|
||||
↓
|
||||
query broadcast → cosine + 1-hop synaptic →
|
||||
N responders contribute snippets →
|
||||
streamed integrated answer
|
||||
```
|
||||
|
||||
## Layers of the System
|
||||
## Stack
|
||||
|
||||
- On the highest level of the sytems there is a user, a user which created Engrams
|
||||
- Engrams can hold any type of data, text, files, images, etc. This source data contibute to their self diefinition.
|
||||
- The user can query the sytem to retrive information, which should be retrived based on the knowledge available in the system.
|
||||
- On the middle level there are Egnrams of Engrams (though syntesis), which evolve their own sef-definition
|
||||
- On the lowest level there are Engrams interacting with each other.
|
||||
- **Rust** workspace — domain types, event-driven sim, axum HTTP/WS server,
|
||||
sled persistence, LM Studio client
|
||||
- **Three.js + WebGL** — additive-blended particle cloud, post-processing
|
||||
bloom, custom GLSL for per-engram pulse and hue variance
|
||||
- **LM Studio** — local LLM (`gemma-4-e4b`) for introspection + query
|
||||
responders + integrator; embeddings via `nomic-embed-text-v1.5`
|
||||
|
||||
## Engrams
|
||||
The full design lives in [`docs/system-analysis.md`](docs/system-analysis.md).
|
||||
|
||||
- Engrams are autonomous agents, each with its own information and goals.
|
||||
- Engrams can develop/destroy relationships with other Engrams, this connecttions are called "Synapsis" and hold information too.
|
||||
- Engrams are composed of:
|
||||
- A Manifest: A description of their attributes.
|
||||
- Taxonomy: Who am I?
|
||||
- Goals: What is my will ?
|
||||
- Memories: What have I experienced?
|
||||
- State: Idle, syntetizing, searching, etc.
|
||||
- Engrams have the following Goals:
|
||||
- I wanna understand and define myself.
|
||||
- I wanna be close to other Engrams that are similar to me.
|
||||
- I wanna be unique or unquely part of another Engram (though syntesis).
|
||||
- I wanna record and remember my history.
|
||||
- Engrams have at least the following function:
|
||||
- move()
|
||||
- introspect()
|
||||
- compare(engramB)
|
||||
- memorize()
|
||||
- recall()
|
||||
- syntetize(engramB)
|
||||
- Engrams:
|
||||
- Their size is a function of how many other Engrams I have syntetized
|
||||
- Their visibiliy (how far can I see other engrams) is a function of how big they are
|
||||
- Memory is a function of interactions with other Engrams.
|
||||
## Quickstart
|
||||
|
||||
## Questions to clarify mechanics
|
||||
You need:
|
||||
|
||||
Space & Movement:
|
||||
1. Is the 3D space continuous or discrete (grid)? Does it have boundaries or is it truly
|
||||
unbounded?
|
||||
- Its a continuous and unbounded space.
|
||||
1. What determines an Engram's position when it first materializes? Random? Embedding-based?
|
||||
- The Great reflection inhabits the center and ends of space like a dounut, new Engrams start at its centerm decaing ones move to the edge.
|
||||
1. How does move() work — does each Engram compute forces (attraction/repulsion) from
|
||||
neighbors, or is it goal-directed pathfinding?
|
||||
- Each engram and Engram of Engram has a self proplusion, relative to their size, but there are external factors that affect it, like curiosity (new engrams have mroe) or gravity (theres a tendency to explore areas with more Engrams)
|
||||
Engram Lifecycle:
|
||||
1. When two Engrams synthesize(), do the originals disappear, or does a new parent Engram
|
||||
form while children persist inside it? (Absorption vs. federation)
|
||||
- There is a federation, only true duplicates after a threshold get abosorved.
|
||||
1. Can Engrams die/decay? If nothing references or interacts with an Engram for many cycles,
|
||||
does it fade?
|
||||
- Yes
|
||||
1. Is there an upper bound on synthesis depth? (Engram of Engrams of Engrams...)
|
||||
- No upper bound, but as Engrams of Engrams grow, they start acting as a collective.
|
||||
Synapses:
|
||||
1. Are Synapses directional (A knows B, but B doesn't know A) or always bidirectional?
|
||||
- The are bidirectional, allways
|
||||
1. What information does a Synapse hold — just weight/strength, or richer metadata (e.g.,
|
||||
"related because X")?
|
||||
- Synapsis hold richer metdata, specfiically what does the relationship betweent the two is, also can hold memories.
|
||||
1. Do Synapses decay over time without reinforcement?
|
||||
- Synampsis do not decay, but are affected by absoprtion processes.
|
||||
- Rust 1.80+ (via `rustup`)
|
||||
- Node 20+ + `npm`
|
||||
- LM Studio running locally with both `google/gemma-4-e4b` and
|
||||
`text-embedding-nomic-embed-text-v1.5` loaded; copy your developer API
|
||||
token from LM Studio into `config.local.toml`:
|
||||
```toml
|
||||
[lm_studio]
|
||||
api_token = "lms-..."
|
||||
```
|
||||
|
||||
The Great Reflection (I/O portal):
|
||||
10. When a user queries the system, does the query itself become a temporary Engram that
|
||||
"searches" the space? Or is retrieval handled externally?
|
||||
- Yes, are engrams too.
|
||||
1. Does ingestion happen in bulk or one-at-a-time? Can the system handle streaming input?
|
||||
- Both. Bulk ingeestion is called seeding.
|
||||
Build the frontend once:
|
||||
|
||||
Cycles & Simulation:
|
||||
12. What happens in a single Cycle? Does every Engram get one action, or is it
|
||||
continuous/event-driven?
|
||||
- Continuous and event driven, events can come from inside and outside.
|
||||
1. Is the simulation always running in the background, or only advances when triggered
|
||||
- Alleawy running and evolving
|
||||
1. "Moves forward by default" — can Cycles move backward? Is there a concept of rewinding
|
||||
state?
|
||||
- Yes, there is a timeline, which can be reconstructed with the indiviudal memories.
|
||||
```bash
|
||||
(cd web && npm install && npm run build)
|
||||
```
|
||||
|
||||
Intelligence layer:
|
||||
15. What drives introspect() and compare() — LLM calls, embedding similarity, rule-based
|
||||
heuristics, or a mix?
|
||||
- A mix, but whe probably need a sort of universal slate to which any manifest can be computed to, and that helps with the comparison in spite of the difference in formats.
|
||||
1. How much autonomy do Engrams have? Is each one making LLM calls independently, or is
|
||||
there a shared "consciousness" scheduler?
|
||||
- They have their own consciousness on the level of following their set of rules, ideally eeasily computable, and the can also call LLMs on their own.
|
||||
Then run the auto-ingesting demo:
|
||||
|
||||
---
|
||||
Possible improvements & extensions
|
||||
```bash
|
||||
cargo run --release -- demo
|
||||
# → http://127.0.0.1:7777
|
||||
```
|
||||
|
||||
- Gravity model: Engrams could exert gravitational pull proportional to their size,
|
||||
naturally clustering related knowledge without explicit pathfinding.
|
||||
- Yes.
|
||||
- Dreams / Defragmentation cycles: Periodic "offline" phases where the system reorganizes
|
||||
- Not for now.
|
||||
more aggressively (like sleep consolidation in neuroscience).
|
||||
- Attention mechanism: Queries could emit a "signal wave" through the space — Engrams that
|
||||
resonate propagate it further, creating activation patterns for retrieval.
|
||||
- Thats a cool idea, lets explore
|
||||
- Forgetting curve: Engrams/Synapses that are never accessed could gradually lose fidelity
|
||||
or sink to the edges of space, implementing Ebbinghaus-style decay.
|
||||
- Lets explore-
|
||||
- Conflict resolution: When two Engrams hold contradictory information, synthesis could
|
||||
produce a "tension" Engram that flags the contradiction.
|
||||
- Explore
|
||||
- Visualization: The 3D space is naturally suited for a real-time WebGL/Three.js
|
||||
visualization where users can watch their knowledge self-organize.
|
||||
- I do want a way to see it. lets explore it.
|
||||
- Multi-user spaces: Multiple users contributing Engrams to a shared Space, with
|
||||
ownership/provenance tracked.
|
||||
- not for now
|
||||
The bin boots the server, hydrates any saved galaxies (`data/`), then spawns a
|
||||
background task that ingests `assets/demo_corpus.txt` (~50 paragraphs across
|
||||
cooking, ML, and history). Open the URL and you should see three jewel-tone
|
||||
clusters form within ~30 s.
|
||||
|
||||
Without `demo`, the server boots empty and you ingest manually via the bottom-
|
||||
right text area, or:
|
||||
|
||||
```bash
|
||||
curl -X POST -H 'content-type: application/json' \
|
||||
-d '{"texts":["Pasta carbonara uses ...","..."]}' \
|
||||
http://127.0.0.1:7777/api/galaxy/<gid>/ingest
|
||||
```
|
||||
|
||||
## Asking a question
|
||||
|
||||
Use the bottom-left **ask** box. The query materialises as a pinned engram at
|
||||
the donut center; relevant engrams light up briefly and contribute a snippet;
|
||||
the integrated answer streams back over SSE. Try:
|
||||
|
||||
- `What did we ingest about pasta?`
|
||||
- `How does gradient descent work?`
|
||||
- `When did the Berlin Wall fall?`
|
||||
|
||||
## API
|
||||
|
||||
| route | what |
|
||||
| --- | --- |
|
||||
| `GET /healthz` | process status + LM Studio reachability |
|
||||
| `GET /api/stats` | live HUD numbers (`ticks_per_sec`, `llm_queue_depth`, …) |
|
||||
| `GET /api/galaxy` | list galaxies |
|
||||
| `POST /api/galaxy` | `{name}` → new galaxy |
|
||||
| `DELETE /api/galaxy/:id` | remove from sim + disk |
|
||||
| `POST /api/galaxy/:id/seed?n=200` | spawn N synthetic engrams |
|
||||
| `POST /api/galaxy/:id/ingest` | `{texts: [...]}` |
|
||||
| `POST /api/galaxy/:id/resize` | `{major_radius, minor_radius}` |
|
||||
| `POST /api/galaxy/:id/query` | `{text}` → `{query_id}` |
|
||||
| `GET /api/query/:id` | latest snapshot |
|
||||
| `GET /api/query/:id/stream` | SSE updates of running answer |
|
||||
| `GET /api/galaxy/:gid/engrams/:eid` | inspector payload (manifest, memories…) |
|
||||
| `WS /ws/galaxy/:id/events` | event stream (positions, synapses, query lifecycle) |
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
crates/
|
||||
sophia-core/ # domain types, no I/O
|
||||
sophia-sim/ # event scheduler, physics, kiddo spatial index, broadcast
|
||||
sophia-llm/ # LM Studio client + caveman prompts (introspection, query)
|
||||
sophia-store/ # sled+JSON snapshot persistence
|
||||
sophia-server/ # axum HTTP + WS, query orchestrator, SSE answer stream
|
||||
sophia-bin/ # main; loads config, hydrates store, runs demo subcommand
|
||||
web/ # Vite + TypeScript + Three.js
|
||||
assets/ # demo corpus
|
||||
data/ # sled DB (gitignored)
|
||||
docs/ # design + plan
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
[`config.toml`](config.toml) is versioned and contains defaults; copy what you
|
||||
want to override into `config.local.toml` (gitignored — for the LM Studio
|
||||
token, or for tweaking the snapshot interval, etc.). Keys:
|
||||
|
||||
- `[server]` — host, port, static dir
|
||||
- `[storage]` — `data_dir`, `snapshot_interval_secs`
|
||||
- `[lm_studio]` — base URL, model names, API token, parallel-op cap
|
||||
- `[galaxy_defaults]` — `major_radius`, `minor_radius`
|
||||
- `[token_caps]` — per-call input/output caps for each LLM use site
|
||||
(note: `gemma-4-e4b` is a thinking model; output caps must budget for
|
||||
hidden reasoning *plus* visible content)
|
||||
|
||||
## Persistence
|
||||
|
||||
Each galaxy snapshots to one row in a sled DB at `data/`, JSON-encoded, every
|
||||
60 s. Kill the server with `Ctrl-C` or `kill -9`; on restart it loads the
|
||||
latest snapshot and resumes — same engram positions, slates, manifests,
|
||||
memories, and synapses.
|
||||
|
||||
## Stage map
|
||||
|
||||
The implementation grew incrementally; each stage produced a runnable demo:
|
||||
|
||||
0. Scaffolding — workspace + black canvas
|
||||
1. Engrams in space — physics, scheduler, position WS stream
|
||||
2. Slate + LM Studio — embed, ingest, manifest
|
||||
3. Gravity + clustering + synapses
|
||||
4. Birth introspection (taxonomy/goals) + memories + click-to-inspect
|
||||
5. Queries as conversations — broadcast, responders, integrator, SSE
|
||||
6. Persistence + replay-on-boot
|
||||
7. Polish — HUD (FPS, ticks/sec, LLM queue), `demo` subcommand
|
||||
|
||||
Each is verifiable end-to-end and the full plan is in
|
||||
[`docs/system-analysis.md`](docs/system-analysis.md).
|
||||
|
||||
50
assets/demo_corpus.txt
Normal file
50
assets/demo_corpus.txt
Normal file
@@ -0,0 +1,50 @@
|
||||
Pasta carbonara uses guanciale, eggs, pecorino romano, and freshly cracked black pepper.
|
||||
Spaghetti aglio e olio combines garlic, olive oil, chili flakes, and parsley over al dente pasta.
|
||||
Risotto requires arborio rice, slowly added warm broth, and constant stirring to release starch.
|
||||
Pesto alla genovese blends basil, pine nuts, garlic, parmesan, pecorino, and olive oil.
|
||||
Cacio e pepe is just pasta, pecorino, and pepper — emulsion comes from starchy pasta water.
|
||||
Lasagne bolognese layers ragu, bechamel, and pasta sheets between melted parmesan.
|
||||
Amatriciana sauces guanciale and tomato together with a touch of pecorino.
|
||||
Gnocchi alla romana use semolina baked with butter and cheese instead of potato.
|
||||
Tiramisu layers savoiardi soaked in espresso with mascarpone whipped with eggs and sugar.
|
||||
Sourdough bread relies on a wild yeast starter and slow bulk fermentation for flavor.
|
||||
Bechamel sauce is built from a roux of butter and flour cooked into milk.
|
||||
Caramelization happens to sugars above 160 C; the Maillard reaction needs proteins and amines.
|
||||
Sushi rice is seasoned with vinegar, sugar, and salt while still warm so seasoning soaks evenly.
|
||||
Korean kimchi ferments napa cabbage with chili paste, garlic, ginger, and salted shrimp.
|
||||
French croissants get their flake from laminated layers of cold butter folded into yeasted dough.
|
||||
Backpropagation computes gradients of a loss with respect to neural network weights via the chain rule.
|
||||
Gradient descent updates model parameters in the direction that reduces the loss the most.
|
||||
Stochastic gradient descent samples mini-batches so each step costs less and adds useful noise.
|
||||
Adam adapts a per-parameter learning rate using running moments of the gradients.
|
||||
Convolutional neural networks share weights across spatial locations to exploit translation invariance.
|
||||
Recurrent networks process sequences by feeding state from one step into the next.
|
||||
Transformers attend over a whole sequence at once instead of recurring through it.
|
||||
Self-attention computes a weighted sum of values where weights are softmax of query-key dot products.
|
||||
Layer normalization stabilizes training by normalizing activations along the feature dimension.
|
||||
Dropout randomly zeroes activations during training as a regularizer.
|
||||
Residual connections let very deep networks train by giving gradients an additive path back.
|
||||
Word embeddings map discrete tokens to dense vectors learned end-to-end with the task.
|
||||
A loss function turns predictions and targets into a single scalar to minimize.
|
||||
Cross-entropy loss compares a predicted distribution to a one-hot target.
|
||||
Reinforcement learning learns a policy by trial-and-error rewards in an environment.
|
||||
Markov decision processes describe RL with states, actions, transitions, and rewards.
|
||||
Q-learning estimates the value of taking each action in each state.
|
||||
Policy gradient methods directly adjust the policy to maximize expected reward.
|
||||
The Roman Empire fell in 476 CE when Romulus Augustulus, the last western emperor, was deposed.
|
||||
Constantinople fell to the Ottomans under Mehmed II in 1453, ending the Byzantine Empire.
|
||||
The Magna Carta of 1215 limited the power of the English monarchy and protected baronial rights.
|
||||
The printing press, perfected by Gutenberg around 1440, accelerated the spread of literacy in Europe.
|
||||
The Treaty of Westphalia in 1648 ended the Thirty Years War and established sovereign-state diplomacy.
|
||||
The French Revolution began in 1789 with the storming of the Bastille on July 14.
|
||||
The Industrial Revolution started in late-eighteenth-century Britain with mechanized textile production.
|
||||
The American Civil War ran from 1861 to 1865 over slavery, states' rights, and union.
|
||||
World War I lasted from 1914 to 1918 and was triggered by the assassination of Archduke Franz Ferdinand.
|
||||
The Russian Revolution of 1917 replaced the tsarist autocracy with Bolshevik rule.
|
||||
The Great Depression began with the 1929 Wall Street Crash and lasted through the 1930s.
|
||||
World War II ended in 1945 after Germany surrendered in May and Japan in September.
|
||||
The United Nations was founded in 1945 in San Francisco to prevent another global war.
|
||||
The Cold War divided the world between US and Soviet spheres from roughly 1947 to 1991.
|
||||
The Berlin Wall fell on November 9, 1989, symbolizing the end of the Cold War in Europe.
|
||||
The Apollo 11 mission landed humans on the Moon for the first time on July 20, 1969.
|
||||
The European Union was established by the Maastricht Treaty in 1993, succeeding the EEC.
|
||||
63
config.toml
Normal file
63
config.toml
Normal file
@@ -0,0 +1,63 @@
|
||||
# Sophia runtime configuration.
|
||||
# Copy to config.local.toml to override locally (gitignored).
|
||||
|
||||
[server]
|
||||
host = "127.0.0.1"
|
||||
port = 7777
|
||||
static_dir = "web/dist"
|
||||
|
||||
# Persistence (Stage 6). `data_dir` is a directory; sled creates and manages
|
||||
# files inside it. Snapshots are written every `snapshot_interval_secs`
|
||||
# while the server runs; on boot any existing galaxy snapshots are loaded
|
||||
# and re-instantiated in the sim.
|
||||
[storage]
|
||||
data_dir = "data"
|
||||
snapshot_interval_secs = 60
|
||||
|
||||
[lm_studio]
|
||||
# LM Studio's OpenAI-compatible REST endpoint.
|
||||
base_url = "http://127.0.0.1:1234/v1"
|
||||
chat_model = "google/gemma-4-e4b"
|
||||
# Embedding model loaded in LM Studio. Often a separate small model
|
||||
# (e.g. "nomic-embed-text-v1.5"). Set this to whatever you load.
|
||||
embedding_model = "text-embedding-nomic-embed-text-v1.5"
|
||||
# Fixed parallel-op ceiling per §13.5 of docs/system-analysis.md.
|
||||
parallel_ops = 8
|
||||
# LM Studio 0.3.x+ enables auth by default. Create a token in the LM Studio
|
||||
# app under Developer → API Tokens, then put it in `config.local.toml`
|
||||
# (which is gitignored) like:
|
||||
#
|
||||
# [lm_studio]
|
||||
# api_token = "lms-..."
|
||||
#
|
||||
# config.local.toml is merged on top of this file at startup, so it only
|
||||
# needs to contain the fields you want to override. Leave api_token unset
|
||||
# if your LM Studio install has auth disabled.
|
||||
|
||||
# Default torus shape for newly-created galaxies. The world is fixed-size
|
||||
# (per the Topology Pivot — we replaced density-driven recentering with
|
||||
# manually-resizable torus volumes). Live resize available at
|
||||
# `POST /api/galaxy/:id/resize`. major_radius must exceed minor_radius.
|
||||
[galaxy_defaults]
|
||||
major_radius = 100.0
|
||||
minor_radius = 30.0
|
||||
|
||||
# Caveman token caps per §13.5.
|
||||
# Note: gemma-4-e4b is a *thinking* model — it emits a hidden reasoning
|
||||
# stream before its visible content. Every `_out` cap therefore needs to
|
||||
# budget for the hidden reasoning *plus* the visible answer. Numbers
|
||||
# verified against logs: a 150-cap responder hit "response had no content"
|
||||
# every time (all budget eaten by reasoning).
|
||||
[token_caps]
|
||||
introspection_in = 600
|
||||
introspection_out = 900
|
||||
peer_msg_in = 100
|
||||
peer_msg_out = 100
|
||||
synthesis_in = 300
|
||||
synthesis_out = 80
|
||||
query_responder_in = 400
|
||||
query_responder_out = 700
|
||||
query_integrator_in = 1200
|
||||
query_integrator_out = 1200
|
||||
hard_ceiling_in = 1500
|
||||
hard_ceiling_out = 1500
|
||||
27
crates/sophia-bin/Cargo.toml
Normal file
27
crates/sophia-bin/Cargo.toml
Normal file
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "sophia-bin"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "sophia"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
sophia-core = { workspace = true }
|
||||
sophia-server = { workspace = true }
|
||||
sophia-sim = { workspace = true }
|
||||
sophia-llm = { workspace = true }
|
||||
sophia-store = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
343
crates/sophia-bin/src/main.rs
Normal file
343
crates/sophia-bin/src/main.rs
Normal file
@@ -0,0 +1,343 @@
|
||||
//! Sophia binary. Reads `config.toml`, initializes tracing, opens the
|
||||
//! persistence store, hydrates any saved galaxies into the sim, and starts
|
||||
//! the server. A background task snapshots the sim to disk on a fixed
|
||||
//! cadence (Stage 6).
|
||||
//!
|
||||
//! `sophia demo` (Stage 7) — same boot path, plus a background task that
|
||||
//! waits for the server to be reachable then ingests a curated corpus
|
||||
//! (compiled into the binary). Useful for screen recordings and first-run
|
||||
//! demos: a single command takes you from cold boot to a populated galaxy.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
use sophia_core::GalaxyShape;
|
||||
use sophia_llm::{LmStudioClient, LmStudioConfig, TokenCapsAll};
|
||||
use sophia_server::{serve, ServerConfig};
|
||||
use sophia_sim::SimHandle;
|
||||
use sophia_store::Store;
|
||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Config {
|
||||
server: ServerSection,
|
||||
#[serde(default)]
|
||||
storage: StorageSection,
|
||||
lm_studio: LmStudioConfig,
|
||||
token_caps: TokenCapsAll,
|
||||
#[serde(default)]
|
||||
galaxy_defaults: GalaxyDefaultsSection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ServerSection {
|
||||
host: String,
|
||||
port: u16,
|
||||
static_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct StorageSection {
|
||||
data_dir: String,
|
||||
snapshot_interval_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for StorageSection {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
data_dir: "data".to_string(),
|
||||
snapshot_interval_secs: 60,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GalaxyDefaultsSection {
|
||||
major_radius: f32,
|
||||
minor_radius: f32,
|
||||
}
|
||||
|
||||
impl Default for GalaxyDefaultsSection {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
major_radius: GalaxyShape::DEFAULT_MAJOR,
|
||||
minor_radius: GalaxyShape::DEFAULT_MINOR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
tracing_subscriber::registry()
|
||||
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,sophia=debug")))
|
||||
.with(fmt::layer())
|
||||
.init();
|
||||
}
|
||||
|
||||
/// Recursively merge `local` into `base`. Tables are deep-merged; scalars
|
||||
/// and arrays are overwritten by `local`. Used to layer `config.local.toml`
|
||||
/// (gitignored, secrets) on top of `config.toml` (versioned defaults).
|
||||
fn merge_toml(base: &mut toml::Value, local: toml::Value) {
|
||||
match (base, local) {
|
||||
(toml::Value::Table(b), toml::Value::Table(l)) => {
|
||||
for (k, v) in l {
|
||||
match b.get_mut(&k) {
|
||||
Some(existing) => merge_toml(existing, v),
|
||||
None => {
|
||||
b.insert(k, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(slot, other) => *slot = other,
|
||||
}
|
||||
}
|
||||
|
||||
fn load_config() -> Result<Config> {
|
||||
let base_path = "config.toml";
|
||||
if !Path::new(base_path).exists() {
|
||||
anyhow::bail!("no config.toml found in cwd");
|
||||
}
|
||||
let base_text = std::fs::read_to_string(base_path)
|
||||
.with_context(|| format!("reading {base_path}"))?;
|
||||
let mut merged: toml::Value = toml::from_str(&base_text)
|
||||
.with_context(|| format!("parsing {base_path}"))?;
|
||||
tracing::info!("loaded config from {base_path}");
|
||||
|
||||
let local_path = "config.local.toml";
|
||||
if Path::new(local_path).exists() {
|
||||
let local_text = std::fs::read_to_string(local_path)
|
||||
.with_context(|| format!("reading {local_path}"))?;
|
||||
let local_value: toml::Value = toml::from_str(&local_text)
|
||||
.with_context(|| format!("parsing {local_path}"))?;
|
||||
merge_toml(&mut merged, local_value);
|
||||
tracing::info!("merged overrides from {local_path}");
|
||||
}
|
||||
|
||||
let cfg: Config = merged.try_into().context("deserializing merged config")?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// Hydrate the sim from disk. Returns the count of galaxies restored.
|
||||
/// If the store is empty, materializes a fresh "default" galaxy and writes
|
||||
/// the first snapshot so a subsequent kill+restart already finds it.
|
||||
async fn hydrate_or_seed(sim: &SimHandle, store: &Store) -> Result<usize> {
|
||||
let saved = store.list_galaxies().context("listing stored galaxies")?;
|
||||
if saved.is_empty() {
|
||||
let info = sim.create_galaxy("default".to_string()).await?;
|
||||
tracing::info!("no saved galaxies — created default {:?}", info.id);
|
||||
// Persist the empty galaxy now so the very next restart rehydrates
|
||||
// (with the same id) instead of creating a fresh one — keeps the
|
||||
// browser's bookmarked galaxy id stable across restarts.
|
||||
let snaps = sim.snapshot_all().await?;
|
||||
for snap in &snaps {
|
||||
store.save_galaxy(snap)?;
|
||||
}
|
||||
return Ok(0);
|
||||
}
|
||||
let n = saved.len();
|
||||
for snap in saved {
|
||||
let id = snap.galaxy.id;
|
||||
let engrams = snap.engrams.len();
|
||||
let synapses = snap.synapses.len();
|
||||
match sim.hydrate_galaxy(snap).await {
|
||||
Ok(info) => {
|
||||
tracing::info!(
|
||||
"hydrated galaxy {:?} ({}, {} engrams, {} synapses)",
|
||||
info.id,
|
||||
info.name,
|
||||
engrams,
|
||||
synapses
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("hydration failed for {:?}: {e}", id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Stage 7: corpus to ingest when invoked as `sophia demo`. Compiled into
|
||||
/// the binary so the command works from any cwd.
|
||||
const DEMO_CORPUS: &str = include_str!("../../../assets/demo_corpus.txt");
|
||||
|
||||
/// Stage 7 demo task. Polls `/healthz` until the server is reachable, then
|
||||
/// posts the curated corpus into the first galaxy. Logs progress; failures
|
||||
/// are non-fatal — the server stays up either way.
|
||||
fn spawn_demo_ingest(host: String, port: u16) {
|
||||
tokio::spawn(async move {
|
||||
let base = format!("http://{host}:{port}");
|
||||
let client = match reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(60))
|
||||
.build()
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: reqwest client failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Wait for server reachable. Cap at 30 s so a misconfigured boot
|
||||
// doesn't loop forever.
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(30);
|
||||
let mut reachable = false;
|
||||
while std::time::Instant::now() < deadline {
|
||||
if client.get(format!("{base}/healthz")).send().await.is_ok() {
|
||||
reachable = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
if !reachable {
|
||||
tracing::warn!("demo: server never became reachable, aborting ingest");
|
||||
return;
|
||||
}
|
||||
|
||||
// Pick the first galaxy (the auto-created or just-hydrated default).
|
||||
let galaxies: Vec<serde_json::Value> = match client
|
||||
.get(format!("{base}/api/galaxy"))
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
{
|
||||
Ok(r) => match r.json().await {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: parsing galaxies failed: {e}");
|
||||
return;
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: listing galaxies failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(first) = galaxies.first() else {
|
||||
tracing::warn!("demo: no galaxies to ingest into");
|
||||
return;
|
||||
};
|
||||
let Some(gid) = first.get("id").and_then(|v| v.as_str()) else {
|
||||
tracing::warn!("demo: galaxy entry missing id");
|
||||
return;
|
||||
};
|
||||
|
||||
let texts: Vec<&str> = DEMO_CORPUS
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.collect();
|
||||
tracing::info!("demo: ingesting {} paragraphs into galaxy {}", texts.len(), gid);
|
||||
|
||||
let body = serde_json::json!({ "texts": texts });
|
||||
match client
|
||||
.post(format!("{base}/api/galaxy/{gid}/ingest"))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
{
|
||||
Ok(_) => {
|
||||
tracing::info!(
|
||||
"demo: ingest succeeded — open {base} to watch the cluster form"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("demo: ingest failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Background task: every `interval`, snapshot every galaxy and write to
|
||||
/// disk. Errors are logged and ignored — we don't want a transient disk
|
||||
/// hiccup to take down the simulation.
|
||||
fn spawn_snapshot_task(sim: SimHandle, store: Store, interval: Duration) {
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(interval);
|
||||
// First tick fires immediately by default; we want the first save
|
||||
// *after* an interval has passed so boot doesn't double-write.
|
||||
ticker.tick().await;
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
match sim.snapshot_all().await {
|
||||
Ok(snaps) => {
|
||||
for snap in &snaps {
|
||||
if let Err(e) = store.save_galaxy(snap) {
|
||||
tracing::warn!("snapshot save failed: {e}");
|
||||
}
|
||||
}
|
||||
tracing::debug!("snapshot tick: persisted {} galaxies", snaps.len());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("snapshot_all failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let cfg = load_config()?;
|
||||
|
||||
// Stage 7 CLI: only one subcommand right now (`demo`). Anything else
|
||||
// boots the server normally.
|
||||
let demo_mode = std::env::args().nth(1).as_deref() == Some("demo");
|
||||
|
||||
let server_cfg = ServerConfig {
|
||||
host: cfg.server.host,
|
||||
port: cfg.server.port,
|
||||
static_dir: PathBuf::from(cfg.server.static_dir),
|
||||
};
|
||||
|
||||
let llm = LmStudioClient::new(cfg.lm_studio);
|
||||
// Best-effort startup ping — failure logs but doesn't block boot, so the
|
||||
// server starts even if LM Studio isn't running yet (the user can launch
|
||||
// it after, and `/healthz` will report current status).
|
||||
match llm.list_models().await {
|
||||
Ok(models) => tracing::info!("LM Studio reachable; {} model(s) loaded", models.len()),
|
||||
Err(e) => tracing::warn!("LM Studio unreachable at startup: {e}"),
|
||||
}
|
||||
|
||||
let default_shape = GalaxyShape {
|
||||
major_radius: cfg.galaxy_defaults.major_radius,
|
||||
minor_radius: cfg.galaxy_defaults.minor_radius,
|
||||
};
|
||||
if let Err(msg) = default_shape.validate() {
|
||||
anyhow::bail!("invalid [galaxy_defaults] in config: {msg}");
|
||||
}
|
||||
tracing::info!(
|
||||
"galaxy defaults: major_radius={} minor_radius={}",
|
||||
default_shape.major_radius,
|
||||
default_shape.minor_radius
|
||||
);
|
||||
|
||||
let store = Store::open(&cfg.storage.data_dir)
|
||||
.with_context(|| format!("opening sled store at {}", cfg.storage.data_dir))?;
|
||||
tracing::info!("persistence store open at {}", cfg.storage.data_dir);
|
||||
|
||||
let sim = sophia_sim::spawn_sim(default_shape);
|
||||
let restored = hydrate_or_seed(&sim, &store).await?;
|
||||
tracing::info!("hydrated {restored} galaxies from disk");
|
||||
|
||||
spawn_snapshot_task(
|
||||
sim.clone(),
|
||||
store.clone(),
|
||||
Duration::from_secs(cfg.storage.snapshot_interval_secs.max(1)),
|
||||
);
|
||||
tracing::info!(
|
||||
"snapshot task scheduled every {}s",
|
||||
cfg.storage.snapshot_interval_secs
|
||||
);
|
||||
|
||||
if demo_mode {
|
||||
tracing::info!("demo mode: will auto-ingest curated corpus once server is reachable");
|
||||
spawn_demo_ingest(server_cfg.host.clone(), server_cfg.port);
|
||||
}
|
||||
|
||||
serve(server_cfg, sim, llm, cfg.token_caps, store).await
|
||||
}
|
||||
13
crates/sophia-core/Cargo.toml
Normal file
13
crates/sophia-core/Cargo.toml
Normal file
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "sophia-core"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
glam = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
141
crates/sophia-core/src/dto.rs
Normal file
141
crates/sophia-core/src/dto.rs
Normal file
@@ -0,0 +1,141 @@
|
||||
//! Data-transfer objects shared between the simulation and the server.
|
||||
//!
|
||||
//! These types cross the channel between `sophia-sim` and `sophia-server`,
|
||||
//! and several are also serialized to JSON over WebSocket text frames.
|
||||
//! `PositionFrame` is broadcast as a binary frame instead — see
|
||||
//! `sophia-server::ws` for the wire encoding.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::engram::EngramState;
|
||||
use crate::ids::{EngramId, GalaxyId, SynapseId};
|
||||
use crate::manifest::Manifest;
|
||||
use crate::memory::{Introspection, Memory};
|
||||
use crate::query::{QueryId, QueryStatus};
|
||||
use crate::synapse::Synapse;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GalaxyInfo {
|
||||
pub id: GalaxyId,
|
||||
pub name: String,
|
||||
pub engram_count: usize,
|
||||
pub center: [f32; 3],
|
||||
pub major_radius: f32,
|
||||
pub minor_radius: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EngramSnapshot {
|
||||
pub id: EngramId,
|
||||
pub instance_idx: u32,
|
||||
pub position: [f32; 3],
|
||||
pub size: f32,
|
||||
pub state: EngramState,
|
||||
}
|
||||
|
||||
/// Wire-friendly synapse — same as `Synapse` but always serialized in the
|
||||
/// canonical (a, b) order. Sent over WS for both initial Hello and live
|
||||
/// `SynapseCreated` events.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SynapseDto {
|
||||
pub id: SynapseId,
|
||||
pub a: EngramId,
|
||||
pub b: EngramId,
|
||||
pub weight: f32,
|
||||
}
|
||||
|
||||
impl From<&Synapse> for SynapseDto {
|
||||
fn from(s: &Synapse) -> Self {
|
||||
Self { id: s.id, a: s.a, b: s.b, weight: s.weight }
|
||||
}
|
||||
}
|
||||
|
||||
/// Detailed view returned by `GET /api/galaxy/:gid/engrams/:eid`. Includes the
|
||||
/// manifest (potentially long) but only the slate's norm + dim — the full
|
||||
/// embedding vector would be wasteful to send on every inspector click.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct EngramDetail {
|
||||
pub id: EngramId,
|
||||
pub instance_idx: u32,
|
||||
pub position: [f32; 3],
|
||||
pub size: f32,
|
||||
pub state: EngramState,
|
||||
pub age: u32,
|
||||
pub manifest: Option<Manifest>,
|
||||
pub slate_dim: Option<usize>,
|
||||
pub slate_norm: Option<f32>,
|
||||
pub introspection: Introspection,
|
||||
pub memories: Vec<Memory>,
|
||||
/// True for Query-Engrams (Stage 5).
|
||||
pub pinned: bool,
|
||||
}
|
||||
|
||||
/// Position frame: a downsampled bundle of all engram positions at a moment in
|
||||
/// time. Encoded to a binary WS frame at the wire layer.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PositionFrame {
|
||||
pub t_ms: u32,
|
||||
/// Indexed by `instance_idx` (dense 0..n).
|
||||
pub positions: Vec<[f32; 3]>,
|
||||
}
|
||||
|
||||
/// Events emitted by the simulation to subscribed WS clients.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum SimEvent {
|
||||
Hello {
|
||||
galaxy: GalaxyInfo,
|
||||
engrams: Vec<EngramSnapshot>,
|
||||
synapses: Vec<SynapseDto>,
|
||||
},
|
||||
EngramCreated {
|
||||
snapshot: EngramSnapshot,
|
||||
},
|
||||
/// A new synapse formed between two engrams. (Stage 3 only emits creates;
|
||||
/// updates and removes arrive in later stages.)
|
||||
SynapseCreated {
|
||||
synapse: SynapseDto,
|
||||
},
|
||||
/// Torus shape changed (initial publish at subscribe + on every resize).
|
||||
/// Replaces the legacy `BBoxUpdated` event from the spherical topology.
|
||||
TorusUpdated {
|
||||
center: [f32; 3],
|
||||
major_radius: f32,
|
||||
minor_radius: f32,
|
||||
},
|
||||
/// An engram's lifecycle state changed (e.g., responder lit up while
|
||||
/// answering a query, then returned to idle). Frontend uses this to
|
||||
/// re-paint the engram's color in real time.
|
||||
EngramStateChanged {
|
||||
id: EngramId,
|
||||
state: EngramState,
|
||||
},
|
||||
/// A query was started — a Query-Engram materialized at `position` with
|
||||
/// the question text. Frontend uses this to register the query for the
|
||||
/// answer panel + visual highlight.
|
||||
QueryStarted {
|
||||
query: QueryId,
|
||||
engram: EngramId,
|
||||
position: [f32; 3],
|
||||
text: String,
|
||||
},
|
||||
/// The integrator produced a new running answer. `version` increments
|
||||
/// monotonically per query; clients can ignore stale frames.
|
||||
QueryAnswerUpdated {
|
||||
query: QueryId,
|
||||
version: u32,
|
||||
status: QueryStatus,
|
||||
answer: String,
|
||||
},
|
||||
/// Terminal event for a query. The Query-Engram has been moved to the
|
||||
/// `Memorize` state and the SSE stream is closed.
|
||||
QueryFinished {
|
||||
query: QueryId,
|
||||
status: QueryStatus,
|
||||
responder_count: u32,
|
||||
},
|
||||
/// Position frame is sent over the wire as a binary frame; this variant
|
||||
/// carries the in-process payload.
|
||||
#[serde(skip)]
|
||||
PositionFrame(PositionFrame),
|
||||
}
|
||||
69
crates/sophia-core/src/engram.rs
Normal file
69
crates/sophia-core/src/engram.rs
Normal file
@@ -0,0 +1,69 @@
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use glam::Vec3;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::ids::EngramId;
|
||||
use crate::manifest::Manifest;
|
||||
use crate::memory::{Introspection, Memory};
|
||||
use crate::slate::Slate;
|
||||
|
||||
/// Engram state machine — see §5 of `docs/system-analysis.md`.
|
||||
///
|
||||
/// Stage 1 only uses `Idle`. Later stages add `Searching`, `Conversing`,
|
||||
/// `Synthesizing`, `Memorize`, `Decaying`, `Deprecated`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum EngramState {
|
||||
Idle = 0,
|
||||
Searching = 1,
|
||||
Conversing = 2,
|
||||
Synthesizing = 3,
|
||||
Memorize = 4,
|
||||
Decaying = 5,
|
||||
Deprecated = 6,
|
||||
}
|
||||
|
||||
impl EngramState {
|
||||
pub fn as_u8(self) -> u8 {
|
||||
self as u8
|
||||
}
|
||||
}
|
||||
|
||||
/// One unit of knowledge — the fundamental agent.
|
||||
///
|
||||
/// Stage 2 adds `manifest` (the source content) and `slate` (an embedding for
|
||||
/// fast similarity). Stage 4 adds taxonomy/goals/memories on top of that.
|
||||
/// Stage 6 adds Serialize/Deserialize so engrams can round-trip through the
|
||||
/// `sophia-store` snapshot file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Engram {
|
||||
pub id: EngramId,
|
||||
/// Slot index for the visualization's `InstancedMesh`. Assigned at birth,
|
||||
/// stable for the engram's lifetime, dense within a galaxy.
|
||||
pub instance_idx: u32,
|
||||
pub position: Vec3,
|
||||
/// Velocity carried over between ticks — gives the simulation momentum so
|
||||
/// motion glides instead of jittering. Integrated with friction in
|
||||
/// `sophia_sim::physics::tick`.
|
||||
pub velocity: Vec3,
|
||||
pub size: f32,
|
||||
pub state: EngramState,
|
||||
/// Ticks since birth. Drives curiosity decay (see `physics::tick`).
|
||||
pub age: u32,
|
||||
/// What this engram represents. `None` for synthetic seed engrams.
|
||||
pub manifest: Option<Manifest>,
|
||||
/// Embedding vector — System-1 layer of the Universal Slate. `None` for
|
||||
/// synthetic seed engrams.
|
||||
pub slate: Option<Slate>,
|
||||
/// LLM-generated self-description (Stage 4). Filled async after birth;
|
||||
/// empty until the introspection task completes.
|
||||
pub introspection: Introspection,
|
||||
/// Per-engram memory log (Stage 4). Bounded — older entries evicted from
|
||||
/// the front when full.
|
||||
pub memories: VecDeque<Memory>,
|
||||
/// Pinned engrams skip physics integration — they sit at their birth
|
||||
/// position permanently. Used for Query-Engrams (Stage 5), which
|
||||
/// materialize at the galaxy center and shouldn't drift into the tube.
|
||||
pub pinned: bool,
|
||||
}
|
||||
78
crates/sophia-core/src/galaxy.rs
Normal file
78
crates/sophia-core/src/galaxy.rs
Normal file
@@ -0,0 +1,78 @@
|
||||
use glam::Vec3;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::ids::GalaxyId;
|
||||
|
||||
/// Geometry of a galaxy's Space of Recollection — a solid 3D torus volume.
|
||||
///
|
||||
/// `major_radius` is the distance from the donut's center to the centerline
|
||||
/// of the tube; `minor_radius` is the tube's own radius. Engrams live inside
|
||||
/// the tube. The torus is fixed-size (manually resized via API), not
|
||||
/// density-driven — see the Topology Pivot in the implementation plan.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct GalaxyShape {
|
||||
pub major_radius: f32,
|
||||
pub minor_radius: f32,
|
||||
}
|
||||
|
||||
impl GalaxyShape {
|
||||
pub const DEFAULT_MAJOR: f32 = 100.0;
|
||||
pub const DEFAULT_MINOR: f32 = 30.0;
|
||||
|
||||
/// "Great Reflection" — the void at the very center of the donut hole.
|
||||
/// New engrams materialize here and fly outward into the tube. From the
|
||||
/// center, every direction in the xy-plane heads toward the tube; physics
|
||||
/// + a small initial outward velocity does the rest.
|
||||
pub fn birth_point(&self, center: Vec3) -> Vec3 {
|
||||
center
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if !(self.major_radius.is_finite() && self.minor_radius.is_finite()) {
|
||||
return Err("radii must be finite");
|
||||
}
|
||||
if self.minor_radius <= 0.1 {
|
||||
return Err("minor_radius must be > 0.1");
|
||||
}
|
||||
if self.major_radius <= self.minor_radius {
|
||||
return Err("major_radius must exceed minor_radius");
|
||||
}
|
||||
if self.major_radius > 10_000.0 {
|
||||
return Err("major_radius must be <= 10_000");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GalaxyShape {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
major_radius: Self::DEFAULT_MAJOR,
|
||||
minor_radius: Self::DEFAULT_MINOR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Galaxy metadata — the set of Engrams it contains lives inside the simulation.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Galaxy {
|
||||
pub id: GalaxyId,
|
||||
pub name: String,
|
||||
/// Galaxy origin in world coords. Always `Vec3::ZERO` in v1; reserved for
|
||||
/// future multi-galaxy layouts. Serialized so persistence round-trips
|
||||
/// the full state (Stage 6) — defaults to ZERO if missing from disk.
|
||||
#[serde(default)]
|
||||
pub center: Vec3,
|
||||
pub shape: GalaxyShape,
|
||||
}
|
||||
|
||||
impl Galaxy {
|
||||
pub fn new(name: impl Into<String>, shape: GalaxyShape) -> Self {
|
||||
Self {
|
||||
id: GalaxyId::new(),
|
||||
name: name.into(),
|
||||
center: Vec3::ZERO,
|
||||
shape,
|
||||
}
|
||||
}
|
||||
}
|
||||
50
crates/sophia-core/src/ids.rs
Normal file
50
crates/sophia-core/src/ids.rs
Normal file
@@ -0,0 +1,50 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct EngramId(pub Uuid);
|
||||
|
||||
impl EngramId {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EngramId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct GalaxyId(pub Uuid);
|
||||
|
||||
impl GalaxyId {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GalaxyId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct SynapseId(pub Uuid);
|
||||
|
||||
impl SynapseId {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SynapseId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
29
crates/sophia-core/src/lib.rs
Normal file
29
crates/sophia-core/src/lib.rs
Normal file
@@ -0,0 +1,29 @@
|
||||
//! Sophia domain types. Pure data — no I/O, no async.
|
||||
//!
|
||||
//! See `docs/system-analysis.md` §1 (System Boundary) and §5 (Engram Dynamics)
|
||||
//! for the conceptual model.
|
||||
|
||||
pub mod dto;
|
||||
pub mod engram;
|
||||
pub mod galaxy;
|
||||
pub mod ids;
|
||||
pub mod manifest;
|
||||
pub mod memory;
|
||||
pub mod persistence;
|
||||
pub mod query;
|
||||
pub mod slate;
|
||||
pub mod synapse;
|
||||
|
||||
pub use dto::{EngramDetail, EngramSnapshot, GalaxyInfo, PositionFrame, SimEvent, SynapseDto};
|
||||
pub use engram::{Engram, EngramState};
|
||||
pub use galaxy::{Galaxy, GalaxyShape};
|
||||
pub use ids::{EngramId, GalaxyId, SynapseId};
|
||||
pub use manifest::Manifest;
|
||||
pub use memory::{Introspection, Memory, MemoryKind, MAX_MEMORIES};
|
||||
pub use persistence::GalaxySnapshot;
|
||||
pub use query::{QueryId, QueryStatus};
|
||||
pub use slate::Slate;
|
||||
pub use synapse::{canonical_pair, Synapse};
|
||||
|
||||
// Re-export glam::Vec3 so consumers don't all need to depend on glam directly.
|
||||
pub use glam::Vec3;
|
||||
26
crates/sophia-core/src/manifest.rs
Normal file
26
crates/sophia-core/src/manifest.rs
Normal file
@@ -0,0 +1,26 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What an Engram *is* — its source content and self-description.
|
||||
///
|
||||
/// Stage 2 only carries `Text` (paragraphs from ingest). Later stages add
|
||||
/// taxonomy/goals (Stage 4 introspection) and richer modalities.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum Manifest {
|
||||
Text { content: String },
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
pub fn short_label(&self, max_chars: usize) -> String {
|
||||
match self {
|
||||
Manifest::Text { content } => {
|
||||
if content.chars().count() <= max_chars {
|
||||
content.clone()
|
||||
} else {
|
||||
let truncated: String = content.chars().take(max_chars).collect();
|
||||
format!("{truncated}…")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
52
crates/sophia-core/src/memory.rs
Normal file
52
crates/sophia-core/src/memory.rs
Normal file
@@ -0,0 +1,52 @@
|
||||
//! Per-Engram memory entries — a running log of meaningful events from the
|
||||
//! engram's own perspective. Stage 4 only records synapse formation; Stage 5
|
||||
//! will add conversation memorialization.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::ids::EngramId;
|
||||
use crate::query::QueryId;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum MemoryKind {
|
||||
Born,
|
||||
Introspected,
|
||||
/// A synapse formed with another engram.
|
||||
SynapseFormed { with: EngramId, weight: f32 },
|
||||
/// This engram contributed a snippet to a query's running answer.
|
||||
/// `snippet` is the engram's own LLM-generated POV on the question.
|
||||
QueryParticipated { query: QueryId, snippet: String },
|
||||
/// (Query-engrams only.) The query completed and produced a final answer.
|
||||
/// Stored on the Query-Engram itself so the inspector can show the
|
||||
/// resolution after the fact.
|
||||
QueryAnswered { answer: String, responder_count: u32 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Memory {
|
||||
/// Time relative to the start of the simulation (ms).
|
||||
pub at_ms: u32,
|
||||
pub kind: MemoryKind,
|
||||
}
|
||||
|
||||
/// Hard cap on a single engram's memory list. Older entries are evicted from
|
||||
/// the front of the deque when full. Stage 4 picks 64 — generous enough for
|
||||
/// the demo, small enough not to bloat the Hello/inspector payloads.
|
||||
pub const MAX_MEMORIES: usize = 64;
|
||||
|
||||
/// Output of LLM-driven self-introspection at birth (Stage 4). Stored on the
|
||||
/// `Engram` itself, not the `Manifest`, since it's the engram's *interpretation*
|
||||
/// of its source content rather than the source itself.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct Introspection {
|
||||
pub taxonomy: Vec<String>,
|
||||
pub goals: Vec<String>,
|
||||
pub open_questions: Vec<String>,
|
||||
}
|
||||
|
||||
impl Introspection {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.taxonomy.is_empty() && self.goals.is_empty() && self.open_questions.is_empty()
|
||||
}
|
||||
}
|
||||
31
crates/sophia-core/src/persistence.rs
Normal file
31
crates/sophia-core/src/persistence.rs
Normal file
@@ -0,0 +1,31 @@
|
||||
//! Persistence types (Stage 6).
|
||||
//!
|
||||
//! `GalaxySnapshot` is the on-disk shape of a galaxy: its metadata plus the
|
||||
//! full set of Engrams and Synapses it contains. This is what the periodic
|
||||
//! snapshot task in `sophia-bin` writes to the sled store, and what the
|
||||
//! `SimHandle::hydrate_galaxy` command reads back at boot.
|
||||
//!
|
||||
//! Pinned engrams (Stage 5 Query-Engrams) are intentionally excluded from
|
||||
//! snapshots. Queries are ephemeral — their orchestrator task is gone after
|
||||
//! a server restart, and a half-rendered query-engram with no SSE channel
|
||||
//! would be confusing in the UI. Letting them die with the process is the
|
||||
//! cleaner default.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::engram::Engram;
|
||||
use crate::galaxy::Galaxy;
|
||||
use crate::synapse::Synapse;
|
||||
|
||||
/// One galaxy's full persistent state. Encoded with `bincode` and stored
|
||||
/// under the galaxy's id in the sled tree.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GalaxySnapshot {
|
||||
pub galaxy: Galaxy,
|
||||
/// All non-pinned engrams. The order is preserved so `instance_idx`
|
||||
/// values remain meaningful after a round-trip — the sim re-uses each
|
||||
/// engram's stored `instance_idx` rather than re-numbering, so the
|
||||
/// position-frame slot mapping stays stable across restarts.
|
||||
pub engrams: Vec<Engram>,
|
||||
pub synapses: Vec<Synapse>,
|
||||
}
|
||||
45
crates/sophia-core/src/query.rs
Normal file
45
crates/sophia-core/src/query.rs
Normal file
@@ -0,0 +1,45 @@
|
||||
//! Query domain types (Stage 5).
|
||||
//!
|
||||
//! A "query" is a question that materializes as a special pinned Engram at
|
||||
//! the galaxy center. The Query-Engram broadcasts to nearby resonating
|
||||
//! Engrams; their snippets are integrated into a streaming answer.
|
||||
//!
|
||||
//! `QueryId` is distinct from `EngramId` even though every query has a
|
||||
//! backing query-engram — keeping them separate lets the orchestrator track
|
||||
//! query state (responders, snippets, answer versions) independently of the
|
||||
//! engram's lifecycle in the world.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct QueryId(pub Uuid);
|
||||
|
||||
impl QueryId {
|
||||
pub fn new() -> Self {
|
||||
Self(Uuid::now_v7())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for QueryId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle phase of a query, as exposed to SSE consumers.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum QueryStatus {
|
||||
/// Query-Engram materialized; broadcast not yet started.
|
||||
Pending,
|
||||
/// Phase A/B retrieval in flight; responders being polled.
|
||||
Responding,
|
||||
/// All responses gathered; integrator producing final answer.
|
||||
Integrating,
|
||||
/// Final answer published; query-engram transitioned to Memorize.
|
||||
Done,
|
||||
/// Aborted (e.g. embedding failure or sim shutdown).
|
||||
Failed,
|
||||
}
|
||||
35
crates/sophia-core/src/slate.rs
Normal file
35
crates/sophia-core/src/slate.rs
Normal file
@@ -0,0 +1,35 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Universal Slate (per §13.1) — Stage 2 layer 1 only: a dense embedding
|
||||
/// vector. The deeper LLM-comparison layer arrives in Stage 4.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Slate(pub Vec<f32>);
|
||||
|
||||
impl Slate {
|
||||
pub fn dim(&self) -> usize {
|
||||
self.0.len()
|
||||
}
|
||||
|
||||
pub fn norm(&self) -> f32 {
|
||||
self.0.iter().map(|x| x * x).sum::<f32>().sqrt()
|
||||
}
|
||||
|
||||
/// Cosine similarity in `[-1, 1]`. Returns 0 if either vector is zero or
|
||||
/// dimensions disagree (the latter can happen across embedding model
|
||||
/// changes — see §13 risk #3).
|
||||
pub fn cosine(&self, other: &Slate) -> f32 {
|
||||
if self.0.len() != other.0.len() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut dot = 0.0_f32;
|
||||
let mut a2 = 0.0_f32;
|
||||
let mut b2 = 0.0_f32;
|
||||
for (a, b) in self.0.iter().zip(other.0.iter()) {
|
||||
dot += a * b;
|
||||
a2 += a * a;
|
||||
b2 += b * b;
|
||||
}
|
||||
let denom = (a2.sqrt() * b2.sqrt()).max(1e-9);
|
||||
dot / denom
|
||||
}
|
||||
}
|
||||
25
crates/sophia-core/src/synapse.rs
Normal file
25
crates/sophia-core/src/synapse.rs
Normal file
@@ -0,0 +1,25 @@
|
||||
//! Synapse — a bidirectional connection between two Engrams formed when
|
||||
//! they spend time close together with a high cosine similarity on their
|
||||
//! Slates. See `docs/system-analysis.md` §1 + §3 (Stage 3 of the impl plan).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::ids::{EngramId, SynapseId};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Synapse {
|
||||
pub id: SynapseId,
|
||||
/// Canonical pair endpoints — `a` is always the lexicographically smaller
|
||||
/// uuid so `(a, b)` is order-independent.
|
||||
pub a: EngramId,
|
||||
pub b: EngramId,
|
||||
/// Strength in `[-1, 1]` — currently set to the cosine similarity at
|
||||
/// formation time. Stage 3 doesn't update it; future stages may.
|
||||
pub weight: f32,
|
||||
}
|
||||
|
||||
/// Lay two ids out canonically (lower uuid first) so a `(a, b)` pair lookup
|
||||
/// is order-independent.
|
||||
pub fn canonical_pair(x: EngramId, y: EngramId) -> (EngramId, EngramId) {
|
||||
if x.0 <= y.0 { (x, y) } else { (y, x) }
|
||||
}
|
||||
19
crates/sophia-llm/Cargo.toml
Normal file
19
crates/sophia-llm/Cargo.toml
Normal file
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "sophia-llm"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
sophia-core = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
49
crates/sophia-llm/src/budget.rs
Normal file
49
crates/sophia-llm/src/budget.rs
Normal file
@@ -0,0 +1,49 @@
|
||||
//! Token caps from §13.5. Loaded from `config.toml`'s `[token_caps]` section.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Per-call cap (input/output tokens). Output is enforced via
|
||||
/// `max_tokens` in the chat request; input is best-effort and currently
|
||||
/// only documented (no tokenizer in MVP).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TokenCaps {
|
||||
pub input: u32,
|
||||
pub output: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct TokenCapsAll {
|
||||
pub introspection_in: u32,
|
||||
pub introspection_out: u32,
|
||||
pub peer_msg_in: u32,
|
||||
pub peer_msg_out: u32,
|
||||
pub synthesis_in: u32,
|
||||
pub synthesis_out: u32,
|
||||
pub query_responder_in: u32,
|
||||
pub query_responder_out: u32,
|
||||
pub query_integrator_in: u32,
|
||||
pub query_integrator_out: u32,
|
||||
pub hard_ceiling_in: u32,
|
||||
pub hard_ceiling_out: u32,
|
||||
}
|
||||
|
||||
impl TokenCapsAll {
|
||||
pub fn introspection(&self) -> TokenCaps {
|
||||
TokenCaps { input: self.introspection_in, output: self.introspection_out }
|
||||
}
|
||||
pub fn peer_msg(&self) -> TokenCaps {
|
||||
TokenCaps { input: self.peer_msg_in, output: self.peer_msg_out }
|
||||
}
|
||||
pub fn synthesis(&self) -> TokenCaps {
|
||||
TokenCaps { input: self.synthesis_in, output: self.synthesis_out }
|
||||
}
|
||||
pub fn query_responder(&self) -> TokenCaps {
|
||||
TokenCaps { input: self.query_responder_in, output: self.query_responder_out }
|
||||
}
|
||||
pub fn query_integrator(&self) -> TokenCaps {
|
||||
TokenCaps { input: self.query_integrator_in, output: self.query_integrator_out }
|
||||
}
|
||||
pub fn hard_ceiling(&self) -> TokenCaps {
|
||||
TokenCaps { input: self.hard_ceiling_in, output: self.hard_ceiling_out }
|
||||
}
|
||||
}
|
||||
243
crates/sophia-llm/src/client.rs
Normal file
243
crates/sophia-llm/src/client.rs
Normal file
@@ -0,0 +1,243 @@
|
||||
//! Minimal LM Studio HTTP client. Talks to LM Studio's OpenAI-compatible
|
||||
//! endpoints (`/v1/embeddings`, `/v1/chat/completions`, `/v1/models`).
|
||||
//!
|
||||
//! Concurrent inferences are bounded by a semaphore (per §13.5 — fixed
|
||||
//! parallel-op ceiling). The semaphore is shared across all callers so the
|
||||
//! total system-wide concurrency is N.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::budget::TokenCaps;
|
||||
|
||||
/// Max texts per single embed_batch call. Stage 2 chunks larger ingests in the
|
||||
/// server. Tuned conservatively for `nomic-embed-text-v1.5` running on a single
|
||||
/// GPU — a higher number is fine for stronger hardware.
|
||||
pub const EMBED_BATCH_LIMIT: usize = 32;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct LmStudioConfig {
|
||||
pub base_url: String,
|
||||
pub chat_model: String,
|
||||
pub embedding_model: String,
|
||||
pub parallel_ops: usize,
|
||||
/// Optional Bearer token. LM Studio 0.3.x+ enables this by default; create
|
||||
/// one in the LM Studio app under Developer → API Tokens. Either set it
|
||||
/// here in config.local.toml or expose it as `LM_STUDIO_API_TOKEN` and
|
||||
/// reference via env (Stage 2 keeps it simple — config-only).
|
||||
#[serde(default)]
|
||||
pub api_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LmError {
|
||||
#[error("HTTP error: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("LM Studio returned status {0}: {1}")]
|
||||
Status(u16, String),
|
||||
#[error("LM Studio returned malformed JSON: {0}")]
|
||||
Json(String),
|
||||
#[error("response had no content")]
|
||||
EmptyResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ChatRole {
|
||||
System,
|
||||
User,
|
||||
Assistant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ChatMessage {
|
||||
pub role: ChatRole,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
impl ChatMessage {
|
||||
pub fn system(s: impl Into<String>) -> Self {
|
||||
Self { role: ChatRole::System, content: s.into() }
|
||||
}
|
||||
pub fn user(s: impl Into<String>) -> Self {
|
||||
Self { role: ChatRole::User, content: s.into() }
|
||||
}
|
||||
}
|
||||
|
||||
/// Cheaply cloneable handle. Wrapped in `Arc` internally; `Clone` shares the
|
||||
/// semaphore so the parallel-op ceiling is global.
|
||||
#[derive(Clone)]
|
||||
pub struct LmStudioClient {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
http: reqwest::Client,
|
||||
cfg: LmStudioConfig,
|
||||
parallel: Semaphore,
|
||||
}
|
||||
|
||||
impl LmStudioClient {
|
||||
pub fn new(cfg: LmStudioConfig) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()
|
||||
.expect("reqwest client builds");
|
||||
let parallel = Semaphore::new(cfg.parallel_ops.max(1));
|
||||
Self { inner: Arc::new(Inner { http, cfg, parallel }) }
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &LmStudioConfig {
|
||||
&self.inner.cfg
|
||||
}
|
||||
|
||||
/// How many parallel-op permits are currently free. Equal to `parallel_ops`
|
||||
/// when idle, drops as concurrent chat/embedding calls run. Used by the
|
||||
/// HUD's "LLM queue" indicator — a number that consistently sits at 0
|
||||
/// means the LLM is the bottleneck.
|
||||
pub fn available_permits(&self) -> usize {
|
||||
self.inner.parallel.available_permits()
|
||||
}
|
||||
|
||||
/// The configured ceiling. Same as `config().parallel_ops`, exposed for
|
||||
/// callers that already have the client handle but not the config.
|
||||
pub fn parallel_ops(&self) -> usize {
|
||||
self.inner.cfg.parallel_ops.max(1)
|
||||
}
|
||||
|
||||
/// Apply Bearer auth header if a token is configured. LM Studio's REST
|
||||
/// server returns 401 if auth is enabled (default in 0.3.x+) and no token
|
||||
/// is sent.
|
||||
fn auth(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match &self.inner.cfg.api_token {
|
||||
Some(t) if !t.is_empty() => req.bearer_auth(t),
|
||||
_ => req,
|
||||
}
|
||||
}
|
||||
|
||||
/// Lightweight check: list models from LM Studio. Returns the available
|
||||
/// model ids on success. Used by `/healthz`.
|
||||
pub async fn list_models(&self) -> Result<Vec<String>, LmError> {
|
||||
let url = format!("{}/models", self.inner.cfg.base_url.trim_end_matches('/'));
|
||||
let resp = self.auth(self.inner.http.get(&url)).send().await?;
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(LmError::Status(status.as_u16(), body));
|
||||
}
|
||||
let v: serde_json::Value = resp.json().await.map_err(|e| LmError::Json(e.to_string()))?;
|
||||
let arr = v
|
||||
.get("data")
|
||||
.and_then(|d| d.as_array())
|
||||
.ok_or_else(|| LmError::Json("missing `data` array".into()))?;
|
||||
let ids = arr
|
||||
.iter()
|
||||
.filter_map(|m| m.get("id").and_then(|i| i.as_str()).map(String::from))
|
||||
.collect();
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// Embed a single text. Bounded by the global parallel-op semaphore.
|
||||
pub async fn embed(&self, text: &str) -> Result<Vec<f32>, LmError> {
|
||||
let mut out = self.embed_batch(std::slice::from_ref(&text.to_string())).await?;
|
||||
out.pop().ok_or(LmError::EmptyResponse)
|
||||
}
|
||||
|
||||
/// Embed many texts in a single LM Studio call. Uses OpenAI's batch-input
|
||||
/// embeddings form so we don't hammer the embedding endpoint with
|
||||
/// concurrent requests (LM Studio's embedding server isn't reliably
|
||||
/// reentrant — concurrent calls can return 500). One semaphore permit
|
||||
/// per call regardless of batch size.
|
||||
///
|
||||
/// Returns embeddings in input order. The caller is responsible for
|
||||
/// chunking very large inputs — see `EMBED_BATCH_LIMIT`.
|
||||
pub async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, LmError> {
|
||||
if texts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let _permit = self.inner.parallel.acquire().await.expect("semaphore not closed");
|
||||
let url = format!("{}/embeddings", self.inner.cfg.base_url.trim_end_matches('/'));
|
||||
let body = serde_json::json!({
|
||||
"model": self.inner.cfg.embedding_model,
|
||||
"input": texts,
|
||||
});
|
||||
let resp = self.auth(self.inner.http.post(&url).json(&body)).send().await?;
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(LmError::Status(status.as_u16(), body));
|
||||
}
|
||||
let v: serde_json::Value = resp.json().await.map_err(|e| LmError::Json(e.to_string()))?;
|
||||
let arr = v
|
||||
.get("data")
|
||||
.and_then(|d| d.as_array())
|
||||
.ok_or_else(|| LmError::Json("missing `data` array".into()))?;
|
||||
// Sort by index so we honour input order even if the server returns
|
||||
// out of order (the OpenAI spec guarantees input-order, but be safe).
|
||||
let mut indexed: Vec<(u64, Vec<f32>)> = arr
|
||||
.iter()
|
||||
.map(|m| {
|
||||
let idx = m.get("index").and_then(|i| i.as_u64()).unwrap_or(u64::MAX);
|
||||
let vec: Vec<f32> = m
|
||||
.get("embedding")
|
||||
.and_then(|e| e.as_array())
|
||||
.map(|a| a.iter().filter_map(|n| n.as_f64().map(|x| x as f32)).collect())
|
||||
.unwrap_or_default();
|
||||
(idx, vec)
|
||||
})
|
||||
.collect();
|
||||
indexed.sort_by_key(|(i, _)| *i);
|
||||
if indexed.len() != texts.len() {
|
||||
return Err(LmError::Json(format!(
|
||||
"embed_batch: requested {} but got {}",
|
||||
texts.len(),
|
||||
indexed.len()
|
||||
)));
|
||||
}
|
||||
if indexed.iter().any(|(_, v)| v.is_empty()) {
|
||||
return Err(LmError::EmptyResponse);
|
||||
}
|
||||
Ok(indexed.into_iter().map(|(_, v)| v).collect())
|
||||
}
|
||||
|
||||
/// One-shot chat completion (non-streaming). Bounded by the parallel-op
|
||||
/// semaphore. `caps.output` becomes the request's `max_tokens`.
|
||||
pub async fn chat(&self, messages: &[ChatMessage], caps: TokenCaps) -> Result<String, LmError> {
|
||||
let _permit = self.inner.parallel.acquire().await.expect("semaphore not closed");
|
||||
let url = format!("{}/chat/completions", self.inner.cfg.base_url.trim_end_matches('/'));
|
||||
let body = serde_json::json!({
|
||||
"model": self.inner.cfg.chat_model,
|
||||
"messages": messages,
|
||||
"max_tokens": caps.output,
|
||||
"temperature": 0.6,
|
||||
"stream": false,
|
||||
});
|
||||
debug!("chat: {} messages, max_tokens={}", messages.len(), caps.output);
|
||||
let resp = self.auth(self.inner.http.post(&url).json(&body)).send().await?;
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
warn!("chat status {}: {}", status, body);
|
||||
return Err(LmError::Status(status.as_u16(), body));
|
||||
}
|
||||
let v: serde_json::Value = resp.json().await.map_err(|e| LmError::Json(e.to_string()))?;
|
||||
let content = v
|
||||
.get("choices")
|
||||
.and_then(|c| c.as_array())
|
||||
.and_then(|arr| arr.first())
|
||||
.and_then(|first| first.get("message"))
|
||||
.and_then(|m| m.get("content"))
|
||||
.and_then(|c| c.as_str())
|
||||
.ok_or_else(|| LmError::Json("missing choices[0].message.content".into()))?
|
||||
.to_string();
|
||||
if content.is_empty() {
|
||||
return Err(LmError::EmptyResponse);
|
||||
}
|
||||
Ok(content)
|
||||
}
|
||||
}
|
||||
90
crates/sophia-llm/src/introspect.rs
Normal file
90
crates/sophia-llm/src/introspect.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
//! LLM-driven introspection: ask the chat model to summarise one piece of
|
||||
//! text into a small structured `Introspection` record (taxonomy, goals,
|
||||
//! open questions). Caveman budget per §13.5.
|
||||
|
||||
use serde::Deserialize;
|
||||
use sophia_core::Introspection;
|
||||
|
||||
use crate::budget::TokenCaps;
|
||||
use crate::client::{ChatMessage, LmError, LmStudioClient};
|
||||
|
||||
const SYSTEM_PROMPT: &str = "\
|
||||
You summarize one short piece of knowledge for a knowledge graph.\n\
|
||||
Output ONLY valid JSON matching this schema:\n\
|
||||
{\n \"taxonomy\": [string, ...],\n \"goals\": [string, ...],\n \"open_questions\": [string, ...]\n}\n\
|
||||
- taxonomy: 3 to 5 short topical labels, from general to specific.\n\
|
||||
- goals: 1 to 3 short statements of what this knowledge enables or answers.\n\
|
||||
- open_questions: 1 to 3 short questions this knowledge invites but doesn't answer.\n\
|
||||
Each item must be at most 12 words. No prose outside the JSON.";
|
||||
|
||||
/// Truncate excessively long input so we stay inside the input cap. Caveman
|
||||
/// budget tolerates ~400 input tokens total — content beyond ~1200 chars
|
||||
/// (≈300 tokens) is unlikely to add useful introspection, so we cut.
|
||||
const MAX_INPUT_CHARS: usize = 1200;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct IntrospectionPayload {
|
||||
#[serde(default)]
|
||||
taxonomy: Vec<String>,
|
||||
#[serde(default)]
|
||||
goals: Vec<String>,
|
||||
#[serde(default)]
|
||||
open_questions: Vec<String>,
|
||||
}
|
||||
|
||||
/// Introspect a single piece of text. Returns an empty `Introspection` if
|
||||
/// the LLM responds with non-JSON or empty content; the caller can decide
|
||||
/// whether to retry. Caps input length defensively.
|
||||
pub async fn introspect_text(
|
||||
client: &LmStudioClient,
|
||||
text: &str,
|
||||
caps: TokenCaps,
|
||||
) -> Result<Introspection, LmError> {
|
||||
let trimmed = if text.chars().count() > MAX_INPUT_CHARS {
|
||||
let head: String = text.chars().take(MAX_INPUT_CHARS).collect();
|
||||
format!("{head}…")
|
||||
} else {
|
||||
text.to_string()
|
||||
};
|
||||
let messages = [
|
||||
ChatMessage::system(SYSTEM_PROMPT),
|
||||
ChatMessage::user(format!("Content:\n\"\"\"\n{trimmed}\n\"\"\"")),
|
||||
];
|
||||
let raw = client.chat(&messages, caps).await?;
|
||||
Ok(parse_introspection(&raw))
|
||||
}
|
||||
|
||||
/// Best-effort JSON extraction. Models occasionally wrap output in ```json
|
||||
/// fences; we strip those before parsing. On failure returns empty.
|
||||
fn parse_introspection(raw: &str) -> Introspection {
|
||||
let json_text = strip_code_fences(raw.trim());
|
||||
let parsed: Result<IntrospectionPayload, _> = serde_json::from_str(json_text);
|
||||
match parsed {
|
||||
Ok(p) => Introspection {
|
||||
taxonomy: cleanup(p.taxonomy),
|
||||
goals: cleanup(p.goals),
|
||||
open_questions: cleanup(p.open_questions),
|
||||
},
|
||||
Err(_) => Introspection::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_code_fences(s: &str) -> &str {
|
||||
let s = s.trim();
|
||||
let stripped = s.strip_prefix("```json").or_else(|| s.strip_prefix("```"));
|
||||
if let Some(rest) = stripped {
|
||||
let rest = rest.trim_start_matches('\n');
|
||||
rest.strip_suffix("```").unwrap_or(rest).trim()
|
||||
} else {
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup(items: Vec<String>) -> Vec<String> {
|
||||
items
|
||||
.into_iter()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.take(8) // hard upper bound, just in case
|
||||
.collect()
|
||||
}
|
||||
13
crates/sophia-llm/src/lib.rs
Normal file
13
crates/sophia-llm/src/lib.rs
Normal file
@@ -0,0 +1,13 @@
|
||||
//! LM Studio client + caveman prompts + parallel-op budget.
|
||||
//!
|
||||
//! See `docs/system-analysis.md` §13.5 (compute budget, token caps).
|
||||
|
||||
mod budget;
|
||||
mod client;
|
||||
mod introspect;
|
||||
mod query;
|
||||
|
||||
pub use budget::{TokenCaps, TokenCapsAll};
|
||||
pub use client::{ChatMessage, ChatRole, LmError, LmStudioClient, LmStudioConfig, EMBED_BATCH_LIMIT};
|
||||
pub use introspect::introspect_text;
|
||||
pub use query::{integrate_responses, respond_to_query};
|
||||
92
crates/sophia-llm/src/query.rs
Normal file
92
crates/sophia-llm/src/query.rs
Normal file
@@ -0,0 +1,92 @@
|
||||
//! Query-time LLM prompts (Stage 5).
|
||||
//!
|
||||
//! Two distinct calls per query:
|
||||
//!
|
||||
//! - `respond_to_query`: each resonating engram (the "responder") is asked to
|
||||
//! say what *its own* knowledge contributes to the question. Caveman cap
|
||||
//! 200 in / 150 out — these are short, single-perspective snippets.
|
||||
//! - `integrate_responses`: the Query-Engram folds the snippets it has
|
||||
//! received so far into a single running answer. Cap 800 in / 300 out.
|
||||
//! Re-runs every couple of seconds while new snippets arrive, so the
|
||||
//! browser sees the answer refine.
|
||||
|
||||
use crate::budget::TokenCaps;
|
||||
use crate::client::{ChatMessage, LmError, LmStudioClient};
|
||||
|
||||
const RESPONDER_SYSTEM: &str = "\
|
||||
You are one fragment of knowledge inside a larger memory. \
|
||||
A query has been broadcast through the network. \
|
||||
Speak ONLY for what your own content contributes to the question. \
|
||||
Do not summarize the question. Do not speculate beyond your content. \
|
||||
If your content is irrelevant, reply with the single word: PASS. \
|
||||
Otherwise: 1–3 short sentences, plain prose, no headings, no bullets.";
|
||||
|
||||
const INTEGRATOR_SYSTEM: &str = "\
|
||||
You are the answering process for a question. You receive snippets from \
|
||||
fragments of knowledge that resonated with the question. Synthesize them \
|
||||
into ONE coherent answer to the user's question. Cite no sources. \
|
||||
Do not list the snippets. Do not say things like \"based on the snippets\". \
|
||||
Plain prose, 2–6 sentences. If the snippets are insufficient, say so briefly.";
|
||||
|
||||
/// Truncate one responder's source content to keep the per-call payload small.
|
||||
const MAX_RESPONDER_INPUT_CHARS: usize = 600;
|
||||
|
||||
/// Ask one resonating engram for its take on the question. Returns `None` if
|
||||
/// the responder explicitly opts out (`PASS`) or returns empty content.
|
||||
/// Keeping `Option<String>` rather than `Result<...>` lets the orchestrator
|
||||
/// distinguish "this engram had nothing useful to say" from "the LLM call
|
||||
/// failed" — both are fine, but only the latter should be logged.
|
||||
pub async fn respond_to_query(
|
||||
client: &LmStudioClient,
|
||||
question: &str,
|
||||
responder_text: &str,
|
||||
caps: TokenCaps,
|
||||
) -> Result<Option<String>, LmError> {
|
||||
let trimmed = if responder_text.chars().count() > MAX_RESPONDER_INPUT_CHARS {
|
||||
let head: String = responder_text.chars().take(MAX_RESPONDER_INPUT_CHARS).collect();
|
||||
format!("{head}…")
|
||||
} else {
|
||||
responder_text.to_string()
|
||||
};
|
||||
let user = format!(
|
||||
"Question:\n{question}\n\nYour content:\n\"\"\"\n{trimmed}\n\"\"\""
|
||||
);
|
||||
let messages = [
|
||||
ChatMessage::system(RESPONDER_SYSTEM),
|
||||
ChatMessage::user(user),
|
||||
];
|
||||
let raw = client.chat(&messages, caps).await?;
|
||||
let cleaned = raw.trim();
|
||||
if cleaned.is_empty() || cleaned.eq_ignore_ascii_case("PASS") {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(cleaned.to_string()))
|
||||
}
|
||||
|
||||
/// Fold the current set of responder snippets into one running answer. The
|
||||
/// orchestrator calls this repeatedly as more snippets arrive.
|
||||
pub async fn integrate_responses(
|
||||
client: &LmStudioClient,
|
||||
question: &str,
|
||||
snippets: &[String],
|
||||
caps: TokenCaps,
|
||||
) -> Result<String, LmError> {
|
||||
if snippets.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
// Number the snippets so the integrator can reason about distinct
|
||||
// perspectives without us having to add ids.
|
||||
let mut buf = String::new();
|
||||
for (i, s) in snippets.iter().enumerate() {
|
||||
buf.push_str(&format!("[{}] {}\n", i + 1, s.trim()));
|
||||
}
|
||||
let user = format!(
|
||||
"Question:\n{question}\n\nSnippets:\n{buf}\nAnswer the question now."
|
||||
);
|
||||
let messages = [
|
||||
ChatMessage::system(INTEGRATOR_SYSTEM),
|
||||
ChatMessage::user(user),
|
||||
];
|
||||
let raw = client.chat(&messages, caps).await?;
|
||||
Ok(raw.trim().to_string())
|
||||
}
|
||||
27
crates/sophia-server/Cargo.toml
Normal file
27
crates/sophia-server/Cargo.toml
Normal file
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "sophia-server"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
sophia-core = { workspace = true }
|
||||
sophia-sim = { workspace = true }
|
||||
sophia-llm = { workspace = true }
|
||||
sophia-store = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
axum = { workspace = true }
|
||||
tower = { workspace = true }
|
||||
tower-http = { workspace = true }
|
||||
|
||||
futures-util = { workspace = true }
|
||||
tokio-stream = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
452
crates/sophia-server/src/lib.rs
Normal file
452
crates/sophia-server/src/lib.rs
Normal file
@@ -0,0 +1,452 @@
|
||||
//! HTTP / WebSocket server.
|
||||
|
||||
use std::convert::Infallible;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use axum::extract::{Path, Query, State, WebSocketUpgrade};
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::sse::{Event as SseEvent, KeepAlive, Sse};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{delete, get, post};
|
||||
use axum::{Json, Router};
|
||||
use futures_util::stream::{self, Stream, StreamExt};
|
||||
use serde::Deserialize;
|
||||
use sophia_core::{EngramId, GalaxyId, GalaxyShape, Manifest, QueryId, QueryStatus, Slate};
|
||||
use sophia_llm::{introspect_text, LmStudioClient, TokenCapsAll, EMBED_BATCH_LIMIT};
|
||||
use sophia_sim::{IngestItem, SimHandle};
|
||||
use sophia_store::Store;
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
use tower_http::services::{ServeDir, ServeFile};
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
mod query;
|
||||
mod ws;
|
||||
|
||||
use query::QueryRegistry;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServerConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub static_dir: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AppState {
|
||||
sim: SimHandle,
|
||||
llm: LmStudioClient,
|
||||
token_caps: TokenCapsAll,
|
||||
queries: Arc<QueryRegistry>,
|
||||
store: Store,
|
||||
}
|
||||
|
||||
pub fn build_router(
|
||||
cfg: &ServerConfig,
|
||||
sim: SimHandle,
|
||||
llm: LmStudioClient,
|
||||
token_caps: TokenCapsAll,
|
||||
store: Store,
|
||||
) -> Router {
|
||||
let index = cfg.static_dir.join("index.html");
|
||||
let static_service = ServeDir::new(&cfg.static_dir).fallback(ServeFile::new(&index));
|
||||
let state = Arc::new(AppState {
|
||||
sim,
|
||||
llm,
|
||||
token_caps,
|
||||
queries: Arc::new(QueryRegistry::new()),
|
||||
store,
|
||||
});
|
||||
|
||||
Router::new()
|
||||
.route("/healthz", get(healthz))
|
||||
.route("/api/galaxy", post(create_galaxy).get(list_galaxies))
|
||||
.route("/api/galaxy/:id", delete(delete_galaxy))
|
||||
.route("/api/galaxy/:id/seed", post(seed_galaxy))
|
||||
.route("/api/galaxy/:id/ingest", post(ingest_galaxy))
|
||||
.route("/api/galaxy/:id/resize", post(resize_galaxy))
|
||||
.route("/api/galaxy/:id/query", post(start_query))
|
||||
.route("/api/galaxy/:gid/engrams/:eid", get(get_engram))
|
||||
.route("/api/query/:id", get(get_query))
|
||||
.route("/api/query/:id/stream", get(query_stream))
|
||||
.route("/api/stats", get(stats))
|
||||
.route("/ws/galaxy/:id/events", get(ws_events))
|
||||
.with_state(state)
|
||||
.fallback_service(static_service)
|
||||
.layer(TraceLayer::new_for_http())
|
||||
}
|
||||
|
||||
pub async fn serve(
|
||||
cfg: ServerConfig,
|
||||
sim: SimHandle,
|
||||
llm: LmStudioClient,
|
||||
token_caps: TokenCapsAll,
|
||||
store: Store,
|
||||
) -> Result<()> {
|
||||
let app = build_router(&cfg, sim, llm, token_caps, store);
|
||||
let addr = format!("{}:{}", cfg.host, cfg.port);
|
||||
let listener = tokio::net::TcpListener::bind(&addr).await?;
|
||||
tracing::info!("sophia-server listening on http://{}", addr);
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------- routes ----------
|
||||
|
||||
async fn healthz(State(s): State<Arc<AppState>>) -> Response {
|
||||
// Stage 2: ping LM Studio's /models endpoint and report what's loaded.
|
||||
let cfg = s.llm.config();
|
||||
let (lm_ok, lm_models, lm_err) = match s.llm.list_models().await {
|
||||
Ok(models) => (true, models, None),
|
||||
Err(e) => (false, Vec::new(), Some(e.to_string())),
|
||||
};
|
||||
let chat_loaded = lm_models.iter().any(|m| m == &cfg.chat_model);
|
||||
let embed_loaded = lm_models.iter().any(|m| m == &cfg.embedding_model);
|
||||
Json(serde_json::json!({
|
||||
"status": "ok",
|
||||
"stage": 2,
|
||||
"lm_studio": {
|
||||
"reachable": lm_ok,
|
||||
"configured_chat_model": cfg.chat_model,
|
||||
"configured_embedding_model": cfg.embedding_model,
|
||||
"chat_model_loaded": chat_loaded,
|
||||
"embedding_model_loaded": embed_loaded,
|
||||
"models_available": lm_models,
|
||||
"error": lm_err,
|
||||
}
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `GET /api/stats` — cheap snapshot for the HUD. Polled at ~1 Hz by the
|
||||
/// browser; every field is integer so the panel stays readable. The
|
||||
/// `llm_queue_depth` is `parallel_ops - available_permits` (i.e. how many
|
||||
/// LLM calls are currently in flight).
|
||||
async fn stats(State(s): State<Arc<AppState>>) -> Response {
|
||||
let sim_stats = s.sim.stats().await;
|
||||
let permits_free = s.llm.available_permits();
|
||||
let parallel_ops = s.llm.parallel_ops();
|
||||
let in_flight = parallel_ops.saturating_sub(permits_free);
|
||||
Json(serde_json::json!({
|
||||
"ticks_per_sec": sim_stats.ticks_per_sec,
|
||||
"engrams_total": sim_stats.engrams_total,
|
||||
"synapses_total": sim_stats.synapses_total,
|
||||
"galaxies": sim_stats.galaxies,
|
||||
"llm_queue_depth": in_flight,
|
||||
"llm_parallel_cap": parallel_ops,
|
||||
}))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreateGalaxyBody {
|
||||
name: String,
|
||||
}
|
||||
|
||||
async fn create_galaxy(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Json(body): Json<CreateGalaxyBody>,
|
||||
) -> Response {
|
||||
match s.sim.create_galaxy(body.name).await {
|
||||
Ok(info) => {
|
||||
// Persist immediately so a kill+restart in the first 60 s
|
||||
// doesn't lose the new galaxy. Failure here is non-fatal —
|
||||
// the next periodic snapshot tick will pick it up.
|
||||
if let Ok(snaps) = s.sim.snapshot_all().await {
|
||||
for snap in &snaps {
|
||||
if snap.galaxy.id == info.id {
|
||||
if let Err(e) = s.store.save_galaxy(snap) {
|
||||
tracing::warn!("initial save of new galaxy failed: {e}");
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Json(info).into_response()
|
||||
}
|
||||
Err(e) => sim_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_galaxies(State(s): State<Arc<AppState>>) -> Response {
|
||||
match s.sim.list_galaxies().await {
|
||||
Ok(list) => Json(list).into_response(),
|
||||
Err(e) => sim_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_galaxy(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<GalaxyId>,
|
||||
) -> Response {
|
||||
if let Err(e) = s.sim.delete_galaxy(id).await {
|
||||
return sim_error(e);
|
||||
}
|
||||
if let Err(e) = s.store.delete_galaxy(id) {
|
||||
// Sim succeeded but disk didn't — log and report; the galaxy is
|
||||
// gone from memory so this is a 500-ish situation, but the next
|
||||
// periodic snapshot will repair (it just won't write the deleted
|
||||
// galaxy back since it's not in `snapshot_all`).
|
||||
tracing::warn!("sim deleted galaxy {:?} but store delete failed: {e}", id);
|
||||
return (StatusCode::INTERNAL_SERVER_ERROR, format!("store: {e}")).into_response();
|
||||
}
|
||||
(StatusCode::NO_CONTENT, "").into_response()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SeedQuery {
|
||||
n: usize,
|
||||
}
|
||||
|
||||
async fn seed_galaxy(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<GalaxyId>,
|
||||
Query(q): Query<SeedQuery>,
|
||||
) -> Response {
|
||||
if q.n == 0 || q.n > 5_000 {
|
||||
return (StatusCode::BAD_REQUEST, "n must be in 1..=5000").into_response();
|
||||
}
|
||||
match s.sim.seed(id, q.n).await {
|
||||
Ok(ids) => Json(serde_json::json!({ "engram_ids": ids })).into_response(),
|
||||
Err(e) => sim_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct IngestBody {
|
||||
texts: Vec<String>,
|
||||
}
|
||||
|
||||
/// Embed all texts via LM Studio's batch-input embeddings endpoint, then
|
||||
/// ship the embedded items to the sim. Chunked at `EMBED_BATCH_LIMIT` so
|
||||
/// large ingests don't time out a single LM Studio call. Chunks are issued
|
||||
/// sequentially: LM Studio's embedding endpoint isn't reliably reentrant
|
||||
/// (concurrent calls return 500), and a batch of ~32 already saturates the
|
||||
/// embedding model on most setups.
|
||||
async fn ingest_galaxy(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<GalaxyId>,
|
||||
Json(body): Json<IngestBody>,
|
||||
) -> Response {
|
||||
if body.texts.is_empty() {
|
||||
return (StatusCode::BAD_REQUEST, "texts must not be empty").into_response();
|
||||
}
|
||||
if body.texts.len() > 200 {
|
||||
return (StatusCode::BAD_REQUEST, "max 200 texts per call").into_response();
|
||||
}
|
||||
|
||||
// Filter empty/whitespace-only entries up front so chunk indices match.
|
||||
let texts: Vec<String> = body.texts.into_iter().map(|t| t.trim().to_string()).collect();
|
||||
if texts.iter().any(|t| t.is_empty()) {
|
||||
return (StatusCode::BAD_REQUEST, "no empty/whitespace-only texts").into_response();
|
||||
}
|
||||
|
||||
let mut all_vecs: Vec<Vec<f32>> = Vec::with_capacity(texts.len());
|
||||
for chunk in texts.chunks(EMBED_BATCH_LIMIT) {
|
||||
match s.llm.embed_batch(chunk).await {
|
||||
Ok(mut v) => all_vecs.append(&mut v),
|
||||
Err(e) => {
|
||||
return (StatusCode::BAD_GATEWAY, format!("embed_batch: {e}")).into_response();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pair (text, slate) → IngestItem, but keep texts so we can launch a
|
||||
// post-ingest introspection task per engram below.
|
||||
let texts_for_introspection: Vec<String> = texts.clone();
|
||||
let items: Vec<IngestItem> = texts
|
||||
.into_iter()
|
||||
.zip(all_vecs)
|
||||
.map(|(content, vec)| IngestItem {
|
||||
manifest: Manifest::Text { content },
|
||||
slate: Slate(vec),
|
||||
})
|
||||
.collect();
|
||||
|
||||
tracing::info!("ingest: {} items into galaxy {:?}", items.len(), id);
|
||||
let result = s.sim.ingest(id, items).await;
|
||||
|
||||
// On success, fire-and-forget one LLM introspection task per engram.
|
||||
// The LM Studio client's semaphore (parallel_ops) bounds total
|
||||
// concurrency, so even 200 spawned futures only run N at a time.
|
||||
// Updates trickle in over the next several seconds via
|
||||
// `SimHandle::update_introspection`.
|
||||
if let Ok(ids) = &result {
|
||||
for (engram_id, text) in ids.iter().zip(texts_for_introspection) {
|
||||
let llm = s.llm.clone();
|
||||
let sim = s.sim.clone();
|
||||
let caps = s.token_caps.introspection();
|
||||
let galaxy_id = id;
|
||||
let engram_id = *engram_id;
|
||||
tokio::spawn(async move {
|
||||
match introspect_text(&llm, &text, caps).await {
|
||||
Ok(intro) => {
|
||||
if intro.is_empty() {
|
||||
tracing::debug!(?engram_id, "introspection returned empty");
|
||||
} else if let Err(e) =
|
||||
sim.update_introspection(galaxy_id, engram_id, intro).await
|
||||
{
|
||||
tracing::warn!(?engram_id, "update_introspection failed: {e}");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(?engram_id, "introspect_text failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(ids) => Json(serde_json::json!({ "engram_ids": ids })).into_response(),
|
||||
Err(e) => sim_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ResizeBody {
|
||||
major_radius: f32,
|
||||
minor_radius: f32,
|
||||
}
|
||||
|
||||
async fn resize_galaxy(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<GalaxyId>,
|
||||
Json(body): Json<ResizeBody>,
|
||||
) -> Response {
|
||||
let shape = GalaxyShape {
|
||||
major_radius: body.major_radius,
|
||||
minor_radius: body.minor_radius,
|
||||
};
|
||||
if let Err(msg) = shape.validate() {
|
||||
return (StatusCode::BAD_REQUEST, msg).into_response();
|
||||
}
|
||||
match s.sim.resize(id, shape).await {
|
||||
Ok(info) => Json(info).into_response(),
|
||||
Err(e) => sim_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_engram(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path((gid, eid)): Path<(GalaxyId, EngramId)>,
|
||||
) -> Response {
|
||||
match s.sim.get_engram(gid, eid).await {
|
||||
Ok(Some(detail)) => Json(detail).into_response(),
|
||||
Ok(None) => (StatusCode::NOT_FOUND, "engram not found in this galaxy").into_response(),
|
||||
Err(e) => sim_error(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ws_events(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<GalaxyId>,
|
||||
upgrade: WebSocketUpgrade,
|
||||
) -> Response {
|
||||
upgrade.on_upgrade(move |socket| ws::run_galaxy_socket(socket, s.sim.clone(), id))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct QueryBody {
|
||||
text: String,
|
||||
}
|
||||
|
||||
/// `POST /api/galaxy/:id/query` — embed the question, materialize a Query-Engram,
|
||||
/// kick off the orchestrator. Returns 202 + `{query_id}` immediately so the
|
||||
/// client can SSE-subscribe before the first responder fires.
|
||||
async fn start_query(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<GalaxyId>,
|
||||
Json(body): Json<QueryBody>,
|
||||
) -> Response {
|
||||
match query::start_query(
|
||||
s.sim.clone(),
|
||||
s.llm.clone(),
|
||||
s.token_caps.clone(),
|
||||
s.queries.clone(),
|
||||
id,
|
||||
body.text,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(qid) => (
|
||||
StatusCode::ACCEPTED,
|
||||
Json(serde_json::json!({ "query_id": qid })),
|
||||
)
|
||||
.into_response(),
|
||||
Err(msg) => (StatusCode::BAD_REQUEST, msg).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/query/:id` — current snapshot. Useful for clients that miss the
|
||||
/// initial SSE event or want a one-shot read.
|
||||
async fn get_query(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<QueryId>,
|
||||
) -> Response {
|
||||
match s.queries.get(id).await {
|
||||
Some(entry) => Json(entry.snapshot().await).into_response(),
|
||||
None => (StatusCode::NOT_FOUND, "query not found").into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/query/:id/stream` — Server-Sent Events of `QueryUpdate` JSON.
|
||||
/// First event is always the latest known snapshot (so a late subscriber
|
||||
/// catches up without polling); subsequent events come from the per-query
|
||||
/// broadcast channel. The stream ends when the orchestrator publishes a
|
||||
/// terminal status (`Done` or `Failed`).
|
||||
async fn query_stream(
|
||||
State(s): State<Arc<AppState>>,
|
||||
Path(id): Path<QueryId>,
|
||||
) -> Response {
|
||||
let Some(entry) = s.queries.get(id).await else {
|
||||
return (StatusCode::NOT_FOUND, "query not found").into_response();
|
||||
};
|
||||
let initial = entry.snapshot().await;
|
||||
let receiver = entry.subscribe();
|
||||
let live = BroadcastStream::new(receiver).filter_map(|res| async move {
|
||||
match res {
|
||||
Ok(update) => Some(update),
|
||||
Err(e) => {
|
||||
tracing::debug!("sse lagged: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
// Emit the initial snapshot first, then forward live updates until a
|
||||
// terminal frame (`done`/`failed`) — the terminal frame is INCLUDED,
|
||||
// and the stream closes immediately after. Standard `take_while` drops
|
||||
// the matching item; `unfold` gives us "include and then end" by
|
||||
// collapsing the state to None after emitting the terminal frame.
|
||||
let combined = stream::once(async move { initial }).chain(live);
|
||||
let stream = stream::unfold(Some(Box::pin(combined)), |state| async move {
|
||||
let mut s = state?;
|
||||
let item = s.next().await?;
|
||||
let next_state = if matches!(item.status, QueryStatus::Done | QueryStatus::Failed) {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
};
|
||||
Some((item, next_state))
|
||||
});
|
||||
let sse_stream: std::pin::Pin<Box<dyn Stream<Item = Result<SseEvent, Infallible>> + Send>> =
|
||||
Box::pin(stream.map(|u| {
|
||||
let payload = serde_json::to_string(&u).unwrap_or_else(|_| "{}".into());
|
||||
Ok(SseEvent::default().data(payload))
|
||||
}));
|
||||
Sse::new(sse_stream)
|
||||
.keep_alive(KeepAlive::new().interval(Duration::from_secs(15)))
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn sim_error(e: sophia_sim::SimError) -> Response {
|
||||
use sophia_sim::SimError::*;
|
||||
let status = match e {
|
||||
Shutdown => StatusCode::SERVICE_UNAVAILABLE,
|
||||
UnknownGalaxy => StatusCode::NOT_FOUND,
|
||||
InvalidShape(_) => StatusCode::BAD_REQUEST,
|
||||
};
|
||||
(status, e.to_string()).into_response()
|
||||
}
|
||||
393
crates/sophia-server/src/query.rs
Normal file
393
crates/sophia-server/src/query.rs
Normal file
@@ -0,0 +1,393 @@
|
||||
//! Stage 5 query orchestration.
|
||||
//!
|
||||
//! A "query" is a conversation initiated by the user. Lifecycle:
|
||||
//!
|
||||
//! 1. `POST /api/galaxy/:id/query {text}` — embed the question, materialize a
|
||||
//! pinned Query-Engram in the sim, register a per-query SSE bus, spawn the
|
||||
//! background orchestrator, return `{query_id}` immediately (HTTP 202).
|
||||
//! 2. Orchestrator (this module): `sim.broadcast_query(...)` chooses the
|
||||
//! responders (Phase A cosine + Phase B 1-hop synaptic). Each responder
|
||||
//! is asked, in parallel under the LM Studio semaphore, what *its* content
|
||||
//! contributes to the question. As snippets arrive they're recorded on
|
||||
//! the responder (visual light-up) and integrated into a running answer
|
||||
//! every `INTEGRATION_INTERVAL`.
|
||||
//! 3. When all responses are gathered (or the timeout hits), one final
|
||||
//! integration produces the canonical answer; `sim.finish_query(...)`
|
||||
//! transitions the Query-Engram to Memorize and the SSE stream closes.
|
||||
//!
|
||||
//! Decisions worth noting:
|
||||
//!
|
||||
//! - Responder cap (`MAX_RESPONDERS = 12`): keeps the LLM queue from being
|
||||
//! monopolized by one query and bounds the integrator's input length.
|
||||
//! - Re-integration is *time-driven* (every 2 s) rather than per-snippet —
|
||||
//! batching snippets into a single integrator call costs less and gives
|
||||
//! the answer time to refine.
|
||||
//! - Per-query SSE channel buffer is small (16); the latest snapshot is
|
||||
//! stored separately so a slow client can resync to current state on
|
||||
//! connect rather than replaying every interim version.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sophia_core::{GalaxyId, QueryId, QueryStatus, Slate};
|
||||
use sophia_llm::{integrate_responses, respond_to_query, LmStudioClient, TokenCapsAll};
|
||||
use sophia_sim::SimHandle;
|
||||
use tokio::sync::{broadcast, Mutex, RwLock};
|
||||
use tokio::task::JoinSet;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
/// Maximum number of responders we ask per query. Larger numbers don't help
|
||||
/// the answer much (the integrator can only fit ~5–10 distinct viewpoints
|
||||
/// in its 800-token input cap) and starve the LLM queue.
|
||||
const MAX_RESPONDERS: usize = 12;
|
||||
/// How often the integrator re-runs while new snippets are arriving. Every
|
||||
/// 2 s gives the user a visible refinement cadence without burning LLM
|
||||
/// budget on near-identical re-integrations.
|
||||
const INTEGRATION_INTERVAL: Duration = Duration::from_millis(2_000);
|
||||
/// Hard cap on time spent waiting for responders. After this we stop polling,
|
||||
/// run a final integration on whatever snippets we have, and finish. Tuned
|
||||
/// for `gemma-4-e4b` thinking-mode latency: each responder call takes
|
||||
/// ~20–40 s once the hidden reasoning budget is large enough to actually
|
||||
/// produce visible content, and LM Studio serialises some of them despite
|
||||
/// the parallel-op semaphore on our side.
|
||||
const RESPONDER_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
/// SSE channel buffer per query. SSE consumers that lag past this will see
|
||||
/// old versions dropped; the registry's `latest` snapshot covers the rest.
|
||||
const SSE_CAPACITY: usize = 16;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QueryUpdate {
|
||||
pub query_id: QueryId,
|
||||
pub version: u32,
|
||||
pub status: QueryStatus,
|
||||
pub answer: String,
|
||||
pub responder_count: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct QueryEntry {
|
||||
sender: broadcast::Sender<QueryUpdate>,
|
||||
latest: Mutex<QueryUpdate>,
|
||||
}
|
||||
|
||||
/// Process-wide registry. Cheaply cloneable Arc holds an `RwLock` over the
|
||||
/// `HashMap<QueryId, Arc<QueryEntry>>` — typical access is read-heavy
|
||||
/// (subscribers + GET /api/query/:id), with a single write per new query.
|
||||
#[derive(Default)]
|
||||
pub struct QueryRegistry {
|
||||
inner: RwLock<HashMap<QueryId, Arc<QueryEntry>>>,
|
||||
}
|
||||
|
||||
impl QueryRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
async fn register(&self, query_id: QueryId, initial: QueryUpdate) -> Arc<QueryEntry> {
|
||||
let (sender, _) = broadcast::channel(SSE_CAPACITY);
|
||||
let entry = Arc::new(QueryEntry {
|
||||
sender,
|
||||
latest: Mutex::new(initial),
|
||||
});
|
||||
self.inner.write().await.insert(query_id, entry.clone());
|
||||
entry
|
||||
}
|
||||
|
||||
pub async fn get(&self, query_id: QueryId) -> Option<Arc<QueryEntry>> {
|
||||
self.inner.read().await.get(&query_id).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
impl QueryEntry {
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<QueryUpdate> {
|
||||
self.sender.subscribe()
|
||||
}
|
||||
|
||||
pub async fn snapshot(&self) -> QueryUpdate {
|
||||
self.latest.lock().await.clone()
|
||||
}
|
||||
|
||||
async fn publish(&self, update: QueryUpdate) {
|
||||
*self.latest.lock().await = update.clone();
|
||||
// Fan-out send; drop silently if no one is subscribed.
|
||||
let _ = self.sender.send(update);
|
||||
}
|
||||
}
|
||||
|
||||
/// Start a query. Embeds the question, materializes the Query-Engram, kicks
|
||||
/// off the orchestrator. Returns the new `QueryId` immediately so the caller
|
||||
/// can SSE-subscribe before the first responder fires.
|
||||
pub async fn start_query(
|
||||
sim: SimHandle,
|
||||
llm: LmStudioClient,
|
||||
caps: TokenCapsAll,
|
||||
registry: Arc<QueryRegistry>,
|
||||
galaxy: GalaxyId,
|
||||
text: String,
|
||||
) -> Result<QueryId, String> {
|
||||
let trimmed = text.trim().to_string();
|
||||
if trimmed.is_empty() {
|
||||
return Err("query text must not be empty".into());
|
||||
}
|
||||
|
||||
// Embed the question. This blocks the request for the duration of one
|
||||
// LM Studio embedding call (typically <100 ms). Doing it inline keeps
|
||||
// the orchestrator simple — no "pending embedding" state.
|
||||
let embedding = llm
|
||||
.embed(&trimmed)
|
||||
.await
|
||||
.map_err(|e| format!("embed: {e}"))?;
|
||||
let slate = Slate(embedding);
|
||||
|
||||
let query_id = QueryId::new();
|
||||
|
||||
let query_engram = sim
|
||||
.start_query_engram(galaxy, query_id, trimmed.clone(), slate)
|
||||
.await
|
||||
.map_err(|e| format!("start_query_engram: {e}"))?;
|
||||
|
||||
// Seed registry with a Pending snapshot so SSE subscribers get something
|
||||
// immediate even before broadcast finishes.
|
||||
let initial = QueryUpdate {
|
||||
query_id,
|
||||
version: 0,
|
||||
status: QueryStatus::Pending,
|
||||
answer: String::new(),
|
||||
responder_count: 0,
|
||||
};
|
||||
let entry = registry.register(query_id, initial.clone()).await;
|
||||
|
||||
tokio::spawn(orchestrate_query(OrchestrateCtx {
|
||||
sim,
|
||||
llm,
|
||||
caps,
|
||||
entry,
|
||||
galaxy,
|
||||
query_id,
|
||||
query_engram,
|
||||
question: trimmed,
|
||||
}));
|
||||
|
||||
Ok(query_id)
|
||||
}
|
||||
|
||||
struct OrchestrateCtx {
|
||||
sim: SimHandle,
|
||||
llm: LmStudioClient,
|
||||
caps: TokenCapsAll,
|
||||
entry: Arc<QueryEntry>,
|
||||
galaxy: GalaxyId,
|
||||
query_id: QueryId,
|
||||
query_engram: sophia_core::EngramId,
|
||||
question: String,
|
||||
}
|
||||
|
||||
/// The orchestrator. Owns the per-query lifecycle from broadcast to finish.
|
||||
async fn orchestrate_query(ctx: OrchestrateCtx) {
|
||||
let OrchestrateCtx {
|
||||
sim, llm, caps, entry, galaxy, query_id, query_engram, question,
|
||||
} = ctx;
|
||||
|
||||
let responders = match sim.broadcast_query(galaxy, query_engram, MAX_RESPONDERS).await {
|
||||
Ok(rs) => rs,
|
||||
Err(e) => {
|
||||
warn!(?query_id, "broadcast_query failed: {e}");
|
||||
publish_terminal(&entry, query_id, QueryStatus::Failed, String::new(), 0).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if responders.is_empty() {
|
||||
let answer = "No engrams resonated with this question.".to_string();
|
||||
let _ = sim
|
||||
.publish_query_answer(galaxy, query_id, 1, QueryStatus::Done, answer.clone())
|
||||
.await;
|
||||
let _ = sim
|
||||
.finish_query(galaxy, query_id, query_engram, answer.clone(), 0)
|
||||
.await;
|
||||
publish_terminal(&entry, query_id, QueryStatus::Done, answer, 0).await;
|
||||
return;
|
||||
}
|
||||
|
||||
debug!(
|
||||
?query_id,
|
||||
responders = responders.len(),
|
||||
"broadcasting query to responders"
|
||||
);
|
||||
|
||||
// Fan out one responder LLM task per pick. Each task fetches the
|
||||
// responder's source text from the sim, asks the LLM for that engram's
|
||||
// POV, then records the snippet. The LM Studio semaphore inside
|
||||
// `respond_to_query` is what keeps concurrent calls bounded — we do
|
||||
// *not* serialize here.
|
||||
let snippets: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut tasks = JoinSet::new();
|
||||
for (responder_id, score) in responders.iter().copied() {
|
||||
let sim = sim.clone();
|
||||
let llm = llm.clone();
|
||||
let caps = caps.query_responder();
|
||||
let question = question.clone();
|
||||
let snippets = Arc::clone(&snippets);
|
||||
tasks.spawn(async move {
|
||||
let text = match sim.get_engram_text(galaxy, responder_id).await {
|
||||
Ok(Some(t)) => t,
|
||||
_ => return,
|
||||
};
|
||||
let snippet = match respond_to_query(&llm, &question, &text, caps).await {
|
||||
Ok(Some(s)) => s,
|
||||
Ok(None) => {
|
||||
debug!(?responder_id, score, "responder passed");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(?responder_id, "responder failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
// Record on the engram (visual light-up + memory entry).
|
||||
if let Err(e) = sim
|
||||
.record_query_participation(galaxy, query_id, responder_id, snippet.clone())
|
||||
.await
|
||||
{
|
||||
warn!(?responder_id, "record_query_participation: {e}");
|
||||
}
|
||||
snippets.lock().await.push(snippet);
|
||||
});
|
||||
}
|
||||
|
||||
// Run the integrator on a fixed cadence while responders are in flight.
|
||||
// We don't try to "wake on snippet" — a 2 s tick is plenty fast for the
|
||||
// user, and lets multiple snippets batch into one integrator call.
|
||||
let mut version: u32 = 0;
|
||||
let deadline = tokio::time::Instant::now() + RESPONDER_TIMEOUT;
|
||||
let mut ticker = tokio::time::interval(INTEGRATION_INTERVAL);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
// All responder tasks have completed (or panicked).
|
||||
_ = wait_for_join_set(&mut tasks) => {
|
||||
break;
|
||||
}
|
||||
_ = ticker.tick() => {
|
||||
let snapshot = snippets.lock().await.clone();
|
||||
if snapshot.is_empty() {
|
||||
continue;
|
||||
}
|
||||
version += 1;
|
||||
run_integration(
|
||||
&sim, &llm, caps.query_integrator(), &entry, galaxy,
|
||||
query_id, version, &question, &snapshot, QueryStatus::Responding,
|
||||
).await;
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
warn!(?query_id, "responder timeout — finishing with partial set");
|
||||
tasks.shutdown().await;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let final_snippets = snippets.lock().await.clone();
|
||||
let responder_count = final_snippets.len() as u32;
|
||||
|
||||
// Final integration. If we already have a recent answer and no new
|
||||
// snippets arrived in the final window we still re-run once — the
|
||||
// integrator is allowed to refine even with the same input.
|
||||
version += 1;
|
||||
let final_answer = if final_snippets.is_empty() {
|
||||
"No responder produced a usable snippet.".to_string()
|
||||
} else {
|
||||
match integrate_responses(
|
||||
&llm,
|
||||
&question,
|
||||
&final_snippets,
|
||||
caps.query_integrator(),
|
||||
).await {
|
||||
Ok(a) if !a.is_empty() => a,
|
||||
Ok(_) => "Integrator returned an empty answer.".to_string(),
|
||||
Err(e) => {
|
||||
warn!(?query_id, "final integration failed: {e}");
|
||||
entry.snapshot().await.answer
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let _ = sim
|
||||
.publish_query_answer(galaxy, query_id, version, QueryStatus::Done, final_answer.clone())
|
||||
.await;
|
||||
let _ = sim
|
||||
.finish_query(galaxy, query_id, query_engram, final_answer.clone(), responder_count)
|
||||
.await;
|
||||
publish_terminal(&entry, query_id, QueryStatus::Done, final_answer, responder_count).await;
|
||||
}
|
||||
|
||||
/// Run one integration pass and publish the result to both the sim's WS bus
|
||||
/// and the per-query SSE bus. Increments `responder_count` from the snapshot
|
||||
/// length so the UI can show "synthesizing 5 perspectives…".
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_integration(
|
||||
sim: &SimHandle,
|
||||
llm: &LmStudioClient,
|
||||
caps: sophia_llm::TokenCaps,
|
||||
entry: &QueryEntry,
|
||||
galaxy: GalaxyId,
|
||||
query_id: QueryId,
|
||||
version: u32,
|
||||
question: &str,
|
||||
snippets: &[String],
|
||||
status: QueryStatus,
|
||||
) {
|
||||
let answer = match integrate_responses(llm, question, snippets, caps).await {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
warn!(?query_id, "integrator failed: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if answer.is_empty() {
|
||||
return;
|
||||
}
|
||||
let responder_count = snippets.len() as u32;
|
||||
let _ = sim
|
||||
.publish_query_answer(galaxy, query_id, version, status, answer.clone())
|
||||
.await;
|
||||
entry
|
||||
.publish(QueryUpdate {
|
||||
query_id,
|
||||
version,
|
||||
status,
|
||||
answer,
|
||||
responder_count,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn publish_terminal(
|
||||
entry: &QueryEntry,
|
||||
query_id: QueryId,
|
||||
status: QueryStatus,
|
||||
answer: String,
|
||||
responder_count: u32,
|
||||
) {
|
||||
let snapshot = entry.snapshot().await;
|
||||
let version = snapshot.version.saturating_add(1);
|
||||
entry
|
||||
.publish(QueryUpdate {
|
||||
query_id,
|
||||
version,
|
||||
status,
|
||||
answer,
|
||||
responder_count,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Drain a `JoinSet` until empty. Used as a `select!` arm to wait for "all
|
||||
/// spawned tasks done" without holding a mutable borrow across awaits.
|
||||
async fn wait_for_join_set(tasks: &mut JoinSet<()>) {
|
||||
while tasks.join_next().await.is_some() {}
|
||||
}
|
||||
108
crates/sophia-server/src/ws.rs
Normal file
108
crates/sophia-server/src/ws.rs
Normal file
@@ -0,0 +1,108 @@
|
||||
//! WebSocket bridge from the simulation to a single browser client.
|
||||
//!
|
||||
//! Wire protocol:
|
||||
//! - `text` frames: JSON-serialized [`SimEvent`] (Hello, EngramCreated,
|
||||
//! BBoxUpdated). One message per frame.
|
||||
//! - `binary` frames: position frames, encoded as
|
||||
//! `[tag u32 LE = 0x01][t_ms u32 LE][n u32 LE][n × (x f32 LE, y f32 LE, z f32 LE)]`
|
||||
//! — 12-byte header followed by the float region. The header is encoded
|
||||
//! as three little-endian u32s rather than a tighter (u8, u32, u32) so the
|
||||
//! float region starts at a 4-byte-aligned offset, which lets the browser
|
||||
//! wrap it as a `Float32Array` view without copying. (`Float32Array`
|
||||
//! requires its byte offset to be a multiple of 4 and throws otherwise.)
|
||||
//! One frame at ~20 Hz, in `instance_idx` order.
|
||||
//!
|
||||
//! See `docs/system-analysis.md` Risk #1 in §13 plan for why we use binary
|
||||
//! frames here.
|
||||
|
||||
use axum::extract::ws::{Message, WebSocket};
|
||||
use bytes::{BufMut, BytesMut};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use sophia_core::{GalaxyId, PositionFrame, SimEvent};
|
||||
use sophia_sim::SimHandle;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
const POS_FRAME_TAG: u32 = 0x01;
|
||||
|
||||
pub async fn run_galaxy_socket(socket: WebSocket, sim: SimHandle, galaxy: GalaxyId) {
|
||||
let (mut sender, mut receiver) = socket.split();
|
||||
|
||||
let (info, snapshots, synapses, mut bus) = match sim.subscribe(galaxy).await {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
warn!("ws subscribe failed: {e}");
|
||||
let _ = sender
|
||||
.send(Message::Text(
|
||||
serde_json::json!({ "type": "error", "message": e.to_string() }).to_string(),
|
||||
))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Hello: tell the client about the galaxy + every existing engram + synapse.
|
||||
let hello = SimEvent::Hello { galaxy: info, engrams: snapshots, synapses };
|
||||
if let Err(e) = send_text(&mut sender, &hello).await {
|
||||
debug!("ws hello send failed: {e}");
|
||||
return;
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Inbound: ignore messages for now (Stage 1 has no client→server).
|
||||
// Just drain so the socket stays alive and we notice closure.
|
||||
msg = receiver.next() => {
|
||||
match msg {
|
||||
Some(Ok(Message::Close(_))) | None => break,
|
||||
Some(Err(e)) => { debug!("ws recv err: {e}"); break; }
|
||||
Some(Ok(_)) => {}
|
||||
}
|
||||
}
|
||||
// Outbound: forward sim events to the client.
|
||||
ev = bus.recv() => {
|
||||
match ev {
|
||||
Ok(SimEvent::PositionFrame(f)) => {
|
||||
if let Err(e) = sender.send(Message::Binary(encode_position_frame(&f))).await {
|
||||
debug!("ws send pos frame failed: {e}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(other) => {
|
||||
if let Err(e) = send_text(&mut sender, &other).await {
|
||||
debug!("ws send text failed: {e}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
warn!("ws lagged by {n} events; client will catch up on next frame");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_text(
|
||||
sender: &mut futures_util::stream::SplitSink<WebSocket, Message>,
|
||||
ev: &SimEvent,
|
||||
) -> anyhow::Result<()> {
|
||||
let body = serde_json::to_string(ev)?;
|
||||
sender.send(Message::Text(body)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn encode_position_frame(f: &PositionFrame) -> Vec<u8> {
|
||||
let n = f.positions.len();
|
||||
let mut buf = BytesMut::with_capacity(12 + n * 12);
|
||||
buf.put_u32_le(POS_FRAME_TAG);
|
||||
buf.put_u32_le(f.t_ms);
|
||||
buf.put_u32_le(n as u32);
|
||||
for [x, y, z] in &f.positions {
|
||||
buf.put_f32_le(*x);
|
||||
buf.put_f32_le(*y);
|
||||
buf.put_f32_le(*z);
|
||||
}
|
||||
buf.to_vec()
|
||||
}
|
||||
20
crates/sophia-sim/Cargo.toml
Normal file
20
crates/sophia-sim/Cargo.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "sophia-sim"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
sophia-core = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
glam = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
kiddo = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
1279
crates/sophia-sim/src/handle.rs
Normal file
1279
crates/sophia-sim/src/handle.rs
Normal file
File diff suppressed because it is too large
Load Diff
42
crates/sophia-sim/src/index.rs
Normal file
42
crates/sophia-sim/src/index.rs
Normal file
@@ -0,0 +1,42 @@
|
||||
//! Spatial index. Stage 1 wraps `kiddo`'s ImmutableKdTree, rebuilt on demand.
|
||||
//!
|
||||
//! kiddo is fast for nearest-neighbour and within-radius queries, but doesn't
|
||||
//! support efficient updates — so we rebuild the tree periodically rather
|
||||
//! than per-tick. Wrapped behind this small surface so a per-octree
|
||||
//! incremental index can swap in later without touching callers.
|
||||
|
||||
use kiddo::{ImmutableKdTree, SquaredEuclidean};
|
||||
use sophia_core::EngramId;
|
||||
|
||||
pub struct KiddoIndex {
|
||||
tree: Option<ImmutableKdTree<f32, 3>>,
|
||||
/// `tree`'s point indices map back to these engram ids.
|
||||
ids: Vec<EngramId>,
|
||||
}
|
||||
|
||||
impl KiddoIndex {
|
||||
pub fn empty() -> Self {
|
||||
Self { tree: None, ids: Vec::new() }
|
||||
}
|
||||
|
||||
pub fn rebuild(&mut self, points: &[(EngramId, [f32; 3])]) {
|
||||
if points.is_empty() {
|
||||
self.tree = None;
|
||||
self.ids.clear();
|
||||
return;
|
||||
}
|
||||
self.ids = points.iter().map(|(id, _)| *id).collect();
|
||||
let coords: Vec<[f32; 3]> = points.iter().map(|(_, p)| *p).collect();
|
||||
self.tree = Some(ImmutableKdTree::new_from_slice(&coords));
|
||||
}
|
||||
|
||||
/// Returns engram ids within `radius` of `point`. Used by gravity +
|
||||
/// synapse formation (Stage 3) and the broadcast wavefront (Stage 5).
|
||||
pub fn within(&self, point: [f32; 3], radius: f32) -> Vec<EngramId> {
|
||||
let Some(tree) = self.tree.as_ref() else { return Vec::new() };
|
||||
tree.within_unsorted::<SquaredEuclidean>(&point, radius * radius)
|
||||
.into_iter()
|
||||
.filter_map(|hit| self.ids.get(hit.item as usize).copied())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
12
crates/sophia-sim/src/lib.rs
Normal file
12
crates/sophia-sim/src/lib.rs
Normal file
@@ -0,0 +1,12 @@
|
||||
//! Sophia simulation: event-driven scheduler, physics, spatial index, broadcast.
|
||||
//!
|
||||
//! See `docs/system-analysis.md` §13.2 (event-driven, non-deterministic),
|
||||
//! §13.4 (broadcast/conversation retrieval), §4 (lifecycle topology).
|
||||
|
||||
mod handle;
|
||||
mod index;
|
||||
mod physics;
|
||||
mod scheduler;
|
||||
mod world;
|
||||
|
||||
pub use handle::{spawn_sim, IngestItem, SimError, SimHandle, SimStats};
|
||||
159
crates/sophia-sim/src/physics.rs
Normal file
159
crates/sophia-sim/src/physics.rs
Normal file
@@ -0,0 +1,159 @@
|
||||
//! Stage 1+ physics, post Topology Pivot:
|
||||
//! curiosity (decaying random walk) + soft torus-radial inward force,
|
||||
//! integrated with per-engram velocity + friction so motion glides
|
||||
//! instead of jittering.
|
||||
//!
|
||||
//! The boundary force pulls engrams toward the nearest spine point of the
|
||||
//! donut. For an engram born at the very center of the donut hole this
|
||||
//! same force becomes a gentle outward attraction toward the tube — so a
|
||||
//! particle fountain emerges naturally from the central "Great Reflection".
|
||||
|
||||
use glam::Vec3;
|
||||
use rand::rngs::SmallRng;
|
||||
use rand::Rng;
|
||||
use sophia_core::Engram;
|
||||
|
||||
/// Tick interval the scheduler aims for. Fixed in Stage 1.
|
||||
pub const TICK_DT_SECS: f32 = 0.05;
|
||||
|
||||
/// Curiosity drives small random impulses that decay with age. Treated as
|
||||
/// an *acceleration* (units / s²) rather than a velocity, so it composes
|
||||
/// with the boundary force and gets smoothed by friction.
|
||||
const CURIOSITY_TAU_TICKS: f32 = 1500.0; // ≈75 s half-life at 20 Hz tick
|
||||
const CURIOSITY_BASE: f32 = 8.0;
|
||||
|
||||
/// Per-tick velocity damping. With dt = 50 ms this works out to ≈55 % of
|
||||
/// velocity retained per second — the engrams glide instead of bullet
|
||||
/// across the scene, and direction changes look smooth.
|
||||
const FRICTION: f32 = 0.03;
|
||||
|
||||
/// Where (as a fraction of `minor_radius`) the soft inward force kicks in
|
||||
/// once the engram is inside the tube.
|
||||
const BOUNDARY_START: f32 = 0.85;
|
||||
/// Strength of the soft restoration at the tube edge (only inside the tube,
|
||||
/// past `BOUNDARY_START * minor_radius`). Quadratic in overshoot.
|
||||
const BOUNDARY_K: f32 = 28.0;
|
||||
/// Constant gentle pull toward the nearest tube spine while the engram is in
|
||||
/// the donut hole (`r_dist > minor_radius`). Much smaller than BOUNDARY_K so
|
||||
/// the cross-hole flight is *visible* — engrams coast at ~15 units/s and
|
||||
/// take several seconds to reach the tube, instead of snapping there.
|
||||
const IN_HOLE_PULL: f32 = 5.0;
|
||||
/// Tangential acceleration around the +z axis applied while in the donut
|
||||
/// hole — turns the otherwise-radial flight into a CCW spiral, so the
|
||||
/// scene reads as a rotating galaxy rather than a starburst.
|
||||
const SPIN_K: f32 = 6.0;
|
||||
|
||||
/// Closest point on the torus centerline to `p`. The centerline is the
|
||||
/// circle of radius `major_r` lying in the plane z = center.z, centered on
|
||||
/// `center`. See plan §"Topology math" for the derivation.
|
||||
fn spine_point(p: Vec3, center: Vec3, major_r: f32) -> Vec3 {
|
||||
let local = p - center;
|
||||
let xy_len = (local.x * local.x + local.y * local.y).sqrt();
|
||||
if xy_len < 1e-6 {
|
||||
// Degenerate: directly above/below the donut axis (e.g. a brand-new
|
||||
// engram at the exact origin). Pick θ = 0 arbitrarily so the spine
|
||||
// point is well-defined and the boundary force has a direction —
|
||||
// initial velocity randomness ensures different engrams pick
|
||||
// different θ on the next tick.
|
||||
center + Vec3::new(major_r, 0.0, 0.0)
|
||||
} else {
|
||||
let scale = major_r / xy_len;
|
||||
center + Vec3::new(local.x * scale, local.y * scale, 0.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one tick of physics to the engram in-place. `gravity_acc` is a
|
||||
/// pre-computed cosine-weighted attraction toward similar nearby engrams
|
||||
/// (Stage 3); pass `Vec3::ZERO` if not yet computed. Gravity only takes
|
||||
/// effect once the engram is settled inside the tube — engrams in the
|
||||
/// donut hole shouldn't pull each other back into a clump near birth.
|
||||
pub fn tick(
|
||||
engram: &mut Engram,
|
||||
center: Vec3,
|
||||
major_r: f32,
|
||||
minor_r: f32,
|
||||
gravity_acc: Vec3,
|
||||
rng: &mut SmallRng,
|
||||
) {
|
||||
// Pinned engrams (Stage 5 Query-Engrams) sit perfectly still at their
|
||||
// birth position — skip the entire integration. Age still advances so
|
||||
// memory timestamps and any future age-gated logic stay consistent.
|
||||
if engram.pinned {
|
||||
engram.age = engram.age.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
let curiosity_factor = (-(engram.age as f32) / CURIOSITY_TAU_TICKS).exp();
|
||||
|
||||
// Curiosity: random impulse, decaying with age.
|
||||
let rand_dir = Vec3::new(
|
||||
rng.gen_range(-1.0..1.0),
|
||||
rng.gen_range(-1.0..1.0),
|
||||
rng.gen_range(-1.0..1.0),
|
||||
)
|
||||
.normalize_or_zero();
|
||||
let curiosity_acc = rand_dir * (CURIOSITY_BASE * curiosity_factor);
|
||||
|
||||
// Spine attraction with two regimes:
|
||||
// - In the donut hole (r_dist > minor_r): a *gentle constant* pull
|
||||
// toward the nearest spine point. Engrams coast across the empty
|
||||
// space, visibly traversing it over several seconds.
|
||||
// - Inside the tube but past 0.85 * minor_r: a stronger quadratic
|
||||
// restoration, so engrams that drift to the tube wall bounce
|
||||
// back smoothly without escaping.
|
||||
let spine = spine_point(engram.position, center, major_r);
|
||||
let radial = engram.position - spine;
|
||||
let r_dist = radial.length();
|
||||
let inward = -radial.normalize_or_zero();
|
||||
let start = minor_r * BOUNDARY_START;
|
||||
let span = (minor_r - start).max(1e-3);
|
||||
let boundary_acc = if r_dist > minor_r {
|
||||
inward * IN_HOLE_PULL
|
||||
} else if r_dist > start {
|
||||
let over = ((r_dist - start) / span).clamp(0.0, 1.0);
|
||||
inward * (BOUNDARY_K * over * over)
|
||||
} else {
|
||||
Vec3::ZERO
|
||||
};
|
||||
|
||||
// Galactic spin: tangential acceleration in the xy-plane (CCW around
|
||||
// +z). Only active in the donut hole — once an engram reaches the
|
||||
// tube the spin force vanishes so it can settle. The tangent is the
|
||||
// 90° CCW rotation of the engram's xy position vector relative to the
|
||||
// galaxy center.
|
||||
let local = engram.position - center;
|
||||
let xy_len = (local.x * local.x + local.y * local.y).sqrt();
|
||||
let spin_acc = if xy_len > 1e-3 && r_dist > minor_r {
|
||||
Vec3::new(-local.y, local.x, 0.0) / xy_len * SPIN_K
|
||||
} else {
|
||||
Vec3::ZERO
|
||||
};
|
||||
|
||||
// Gravity is gated to inside-the-tube only — see fn doc.
|
||||
let gated_gravity = if r_dist <= minor_r { gravity_acc } else { Vec3::ZERO };
|
||||
|
||||
// Verlet-ish integration: accumulate forces into velocity, damp,
|
||||
// then move. Gives smooth glide instead of per-tick teleporting.
|
||||
let acc = curiosity_acc + boundary_acc + spin_acc + gated_gravity;
|
||||
engram.velocity += acc * TICK_DT_SECS;
|
||||
engram.velocity *= 1.0 - FRICTION;
|
||||
engram.position += engram.velocity * TICK_DT_SECS;
|
||||
engram.age = engram.age.saturating_add(1);
|
||||
|
||||
// Hard safety clamp: even with the soft force, large impulses can
|
||||
// momentarily breach the tube. Project back to the surface and reflect
|
||||
// the outward component of velocity so the engram bounces softly
|
||||
// instead of pile-driving against the wall.
|
||||
let spine_after = spine_point(engram.position, center, major_r);
|
||||
let radial_after = engram.position - spine_after;
|
||||
let dist_after = radial_after.length();
|
||||
if dist_after > minor_r {
|
||||
let normal = radial_after.normalize_or_zero();
|
||||
engram.position = spine_after + normal * minor_r;
|
||||
let v_dot_n = engram.velocity.dot(normal);
|
||||
if v_dot_n > 0.0 {
|
||||
// Cancel the outward component, keep tangential motion.
|
||||
engram.velocity -= normal * v_dot_n;
|
||||
}
|
||||
}
|
||||
}
|
||||
103
crates/sophia-sim/src/scheduler.rs
Normal file
103
crates/sophia-sim/src/scheduler.rs
Normal file
@@ -0,0 +1,103 @@
|
||||
//! Min-heap event scheduler driving the simulation. Per §13.2.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BinaryHeap;
|
||||
use std::time::Instant;
|
||||
|
||||
use sophia_core::{EngramId, EngramState, GalaxyId, Manifest, Slate};
|
||||
|
||||
/// Payload for a queued spawn. Synthetic seed engrams have no manifest/slate;
|
||||
/// ingested engrams carry the pre-embedded text.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum SpawnPayload {
|
||||
Synthetic,
|
||||
Manifested { manifest: Manifest, slate: Slate },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Event {
|
||||
/// One Engram's turn to act (move, perceive, etc).
|
||||
EngramTick { galaxy: GalaxyId, engram: EngramId },
|
||||
/// Materialize one engram with the pre-allocated id and the given
|
||||
/// payload, then schedule its first tick. Used by `seed` and `ingest`
|
||||
/// to release engrams gradually instead of all at once.
|
||||
Spawn { galaxy: GalaxyId, id: EngramId, payload: SpawnPayload },
|
||||
/// Snapshot all positions in a galaxy and emit a `PositionFrame`. Fired
|
||||
/// at a fixed cadence (~20 Hz) and reschedules itself.
|
||||
BroadcastFrame { galaxy: GalaxyId },
|
||||
/// Recompute density-driven bbox + rebuild spatial index. Fires every
|
||||
/// ~500 ms and reschedules itself.
|
||||
RebuildIndex { galaxy: GalaxyId },
|
||||
/// Stage 5: revert an engram's state (typically Conversing → Idle)
|
||||
/// after a responder's light-up window expires. Fired once and not
|
||||
/// rescheduled. The `expected` state guards against racing transitions
|
||||
/// — only revert if the engram is still in the state the lit-up event
|
||||
/// originally set (the orchestrator may light up the same engram twice
|
||||
/// for two queries; we don't want the older revert to clobber a fresher
|
||||
/// one).
|
||||
RevertState {
|
||||
galaxy: GalaxyId,
|
||||
engram: EngramId,
|
||||
expected: EngramState,
|
||||
revert_to: EngramState,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Scheduled {
|
||||
at: Instant,
|
||||
seq: u64, // tie-breaker so equal-time events have a stable order
|
||||
event: Event,
|
||||
}
|
||||
|
||||
impl PartialEq for Scheduled {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.at.eq(&other.at) && self.seq.eq(&other.seq)
|
||||
}
|
||||
}
|
||||
impl Eq for Scheduled {}
|
||||
impl PartialOrd for Scheduled {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
impl Ord for Scheduled {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
// BinaryHeap is a max-heap; reverse so the earliest time wins.
|
||||
other.at.cmp(&self.at).then(other.seq.cmp(&self.seq))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Scheduler {
|
||||
heap: BinaryHeap<Scheduled>,
|
||||
next_seq: u64,
|
||||
}
|
||||
|
||||
impl Scheduler {
|
||||
pub fn new() -> Self {
|
||||
Self { heap: BinaryHeap::new(), next_seq: 0 }
|
||||
}
|
||||
|
||||
pub fn schedule(&mut self, at: Instant, event: Event) {
|
||||
self.next_seq = self.next_seq.wrapping_add(1);
|
||||
self.heap.push(Scheduled { at, seq: self.next_seq, event });
|
||||
}
|
||||
|
||||
pub fn next_at(&self) -> Option<Instant> {
|
||||
self.heap.peek().map(|s| s.at)
|
||||
}
|
||||
|
||||
/// Pop one event if its scheduled time has arrived.
|
||||
pub fn pop_due(&mut self, now: Instant) -> Option<Event> {
|
||||
match self.heap.peek() {
|
||||
Some(s) if s.at <= now => self.heap.pop().map(|s| s.event),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Scheduler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
229
crates/sophia-sim/src/world.rs
Normal file
229
crates/sophia-sim/src/world.rs
Normal file
@@ -0,0 +1,229 @@
|
||||
//! Per-galaxy mutable state owned by the simulation.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use sophia_core::{
|
||||
canonical_pair, Engram, EngramId, EngramSnapshot, Galaxy, GalaxyId, GalaxyInfo, GalaxyShape,
|
||||
GalaxySnapshot, SimEvent, Synapse, SynapseDto, SynapseId,
|
||||
};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
/// Global ceiling on synapses per galaxy — keeps the WS bandwidth bounded
|
||||
/// and prevents the visualisation from drowning in lines if many engrams
|
||||
/// happen to be similar at once. Tuned generously for a 200-engram demo.
|
||||
const MAX_SYNAPSES_PER_GALAXY: usize = 4_000;
|
||||
/// Per-engram synapse cap. Once an engram has this many connections, no
|
||||
/// new ones are formed for it (Stage 3 is no-eviction; later stages may
|
||||
/// drop the weakest).
|
||||
const MAX_SYNAPSES_PER_ENGRAM: usize = 16;
|
||||
|
||||
/// Channel buffer for per-galaxy event broadcast. Big enough for several
|
||||
/// position frames; lagged consumers receive `RecvError::Lagged`.
|
||||
const BROADCAST_CAPACITY: usize = 256;
|
||||
|
||||
pub struct GalaxyState {
|
||||
pub galaxy: Galaxy,
|
||||
pub engrams: HashMap<sophia_core::EngramId, Engram>,
|
||||
/// Dense list of engram ids in slot order (instance_idx is the index here).
|
||||
pub slot_to_id: Vec<sophia_core::EngramId>,
|
||||
/// Synapses keyed by id.
|
||||
pub synapses: HashMap<SynapseId, Synapse>,
|
||||
/// Canonical-pair set so duplicate-formation is O(1).
|
||||
pub synapse_pairs: HashSet<(EngramId, EngramId)>,
|
||||
/// Per-engram synapse counts for cap enforcement.
|
||||
pub synapse_count: HashMap<EngramId, usize>,
|
||||
/// Broadcast bus for events scoped to this galaxy.
|
||||
pub bus: broadcast::Sender<SimEvent>,
|
||||
}
|
||||
|
||||
impl GalaxyState {
|
||||
pub fn new(name: impl Into<String>, shape: GalaxyShape) -> Self {
|
||||
let (bus, _) = broadcast::channel(BROADCAST_CAPACITY);
|
||||
Self {
|
||||
galaxy: Galaxy::new(name, shape),
|
||||
engrams: HashMap::new(),
|
||||
slot_to_id: Vec::new(),
|
||||
synapses: HashMap::new(),
|
||||
synapse_pairs: HashSet::new(),
|
||||
synapse_count: HashMap::new(),
|
||||
bus,
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to form a new synapse between `a` and `b` with the given weight.
|
||||
/// Returns `Some(synapse)` if created, `None` if a synapse already
|
||||
/// exists for this pair or any cap was hit. Stage 3 doesn't update
|
||||
/// existing synapses; later stages may.
|
||||
pub fn try_form_synapse(&mut self, a: EngramId, b: EngramId, weight: f32) -> Option<Synapse> {
|
||||
if a == b {
|
||||
return None;
|
||||
}
|
||||
let pair = canonical_pair(a, b);
|
||||
if self.synapse_pairs.contains(&pair) {
|
||||
return None;
|
||||
}
|
||||
if self.synapses.len() >= MAX_SYNAPSES_PER_GALAXY {
|
||||
return None;
|
||||
}
|
||||
let count_a = self.synapse_count.get(&pair.0).copied().unwrap_or(0);
|
||||
let count_b = self.synapse_count.get(&pair.1).copied().unwrap_or(0);
|
||||
if count_a >= MAX_SYNAPSES_PER_ENGRAM || count_b >= MAX_SYNAPSES_PER_ENGRAM {
|
||||
return None;
|
||||
}
|
||||
let id = SynapseId::new();
|
||||
let syn = Synapse { id, a: pair.0, b: pair.1, weight };
|
||||
self.synapse_pairs.insert(pair);
|
||||
self.synapses.insert(id, syn.clone());
|
||||
*self.synapse_count.entry(pair.0).or_insert(0) += 1;
|
||||
*self.synapse_count.entry(pair.1).or_insert(0) += 1;
|
||||
Some(syn)
|
||||
}
|
||||
|
||||
pub fn snapshot_synapses(&self) -> Vec<SynapseDto> {
|
||||
self.synapses.values().map(SynapseDto::from).collect()
|
||||
}
|
||||
|
||||
pub fn info(&self) -> GalaxyInfo {
|
||||
GalaxyInfo {
|
||||
id: self.galaxy.id,
|
||||
name: self.galaxy.name.clone(),
|
||||
engram_count: self.slot_to_id.len(),
|
||||
center: self.galaxy.center.to_array(),
|
||||
major_radius: self.galaxy.shape.major_radius,
|
||||
minor_radius: self.galaxy.shape.minor_radius,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot_all(&self) -> Vec<EngramSnapshot> {
|
||||
self.slot_to_id
|
||||
.iter()
|
||||
.filter_map(|id| self.engrams.get(id))
|
||||
.map(|e| EngramSnapshot {
|
||||
id: e.id,
|
||||
instance_idx: e.instance_idx,
|
||||
position: e.position.to_array(),
|
||||
size: e.size,
|
||||
state: e.state,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Emit on the bus; drops the event silently if no one is listening.
|
||||
pub fn emit(&self, ev: SimEvent) {
|
||||
let _ = self.bus.send(ev);
|
||||
}
|
||||
|
||||
pub fn torus_event(&self) -> SimEvent {
|
||||
SimEvent::TorusUpdated {
|
||||
center: self.galaxy.center.to_array(),
|
||||
major_radius: self.galaxy.shape.major_radius,
|
||||
minor_radius: self.galaxy.shape.minor_radius,
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialize the persistent half of this galaxy's state. Pinned engrams
|
||||
/// (Stage 5 Query-Engrams) are dropped — queries are ephemeral and
|
||||
/// shouldn't outlive the orchestrator that owns them. Synapses that
|
||||
/// pointed at a pinned engram (none currently exist; future-proofing)
|
||||
/// are also dropped.
|
||||
pub fn to_snapshot(&self) -> GalaxySnapshot {
|
||||
let engrams: Vec<Engram> = self
|
||||
.slot_to_id
|
||||
.iter()
|
||||
.filter_map(|id| self.engrams.get(id))
|
||||
.filter(|e| !e.pinned)
|
||||
.cloned()
|
||||
.collect();
|
||||
let kept_ids: HashSet<EngramId> = engrams.iter().map(|e| e.id).collect();
|
||||
let synapses: Vec<Synapse> = self
|
||||
.synapses
|
||||
.values()
|
||||
.filter(|s| kept_ids.contains(&s.a) && kept_ids.contains(&s.b))
|
||||
.cloned()
|
||||
.collect();
|
||||
GalaxySnapshot {
|
||||
galaxy: self.galaxy.clone(),
|
||||
engrams,
|
||||
synapses,
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-instantiate a galaxy from a stored snapshot. Re-uses each engram's
|
||||
/// stored `instance_idx` so the dense slot table matches pre-shutdown
|
||||
/// (the WS PositionFrame slot ordering depends on it). The caller (sim
|
||||
/// loop) is responsible for re-scheduling per-engram ticks and the
|
||||
/// recurring `BroadcastFrame` / `RebuildIndex` events.
|
||||
pub fn from_snapshot(snap: GalaxySnapshot) -> Self {
|
||||
let (bus, _) = broadcast::channel(BROADCAST_CAPACITY);
|
||||
let mut engrams: HashMap<EngramId, Engram> = HashMap::with_capacity(snap.engrams.len());
|
||||
let mut max_idx: u32 = 0;
|
||||
for e in &snap.engrams {
|
||||
if e.instance_idx + 1 > max_idx {
|
||||
max_idx = e.instance_idx + 1;
|
||||
}
|
||||
}
|
||||
// Build a dense slot_to_id keyed by the stored instance_idx. Any
|
||||
// gaps (shouldn't happen but defensive) get a placeholder UUID that
|
||||
// will simply never resolve to an engram in `engrams.get(...)`,
|
||||
// keeping the indexing math correct.
|
||||
let mut slot_to_id: Vec<EngramId> = vec![EngramId::default(); max_idx as usize];
|
||||
for e in snap.engrams {
|
||||
let idx = e.instance_idx as usize;
|
||||
if idx < slot_to_id.len() {
|
||||
slot_to_id[idx] = e.id;
|
||||
}
|
||||
engrams.insert(e.id, e);
|
||||
}
|
||||
let mut synapses: HashMap<SynapseId, Synapse> = HashMap::with_capacity(snap.synapses.len());
|
||||
let mut synapse_pairs: HashSet<(EngramId, EngramId)> = HashSet::with_capacity(snap.synapses.len());
|
||||
let mut synapse_count: HashMap<EngramId, usize> = HashMap::new();
|
||||
for syn in snap.synapses {
|
||||
let pair = canonical_pair(syn.a, syn.b);
|
||||
if !synapse_pairs.insert(pair) {
|
||||
continue;
|
||||
}
|
||||
*synapse_count.entry(pair.0).or_insert(0) += 1;
|
||||
*synapse_count.entry(pair.1).or_insert(0) += 1;
|
||||
synapses.insert(syn.id, syn);
|
||||
}
|
||||
Self {
|
||||
galaxy: snap.galaxy,
|
||||
engrams,
|
||||
slot_to_id,
|
||||
synapses,
|
||||
synapse_pairs,
|
||||
synapse_count,
|
||||
bus,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct World {
|
||||
pub galaxies: HashMap<GalaxyId, GalaxyState>,
|
||||
}
|
||||
|
||||
impl World {
|
||||
pub fn new() -> Self {
|
||||
Self { galaxies: HashMap::new() }
|
||||
}
|
||||
|
||||
pub fn list_galaxies(&self) -> Vec<GalaxyInfo> {
|
||||
self.galaxies.values().map(GalaxyState::info).collect()
|
||||
}
|
||||
|
||||
/// Update the galaxy's torus shape and broadcast `TorusUpdated`. Caller
|
||||
/// is responsible for validating `shape` first (`GalaxyShape::validate`).
|
||||
/// Returns the updated info, or `None` if the galaxy doesn't exist.
|
||||
pub fn resize_galaxy(&mut self, gid: GalaxyId, shape: GalaxyShape) -> Option<GalaxyInfo> {
|
||||
let g = self.galaxies.get_mut(&gid)?;
|
||||
g.galaxy.shape = shape;
|
||||
g.emit(g.torus_event());
|
||||
Some(g.info())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for World {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
17
crates/sophia-store/Cargo.toml
Normal file
17
crates/sophia-store/Cargo.toml
Normal file
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "sophia-store"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
sophia-core = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sled = { workspace = true }
|
||||
109
crates/sophia-store/src/lib.rs
Normal file
109
crates/sophia-store/src/lib.rs
Normal file
@@ -0,0 +1,109 @@
|
||||
//! Persistence: sled-backed snapshot store (Stage 6).
|
||||
//!
|
||||
//! Each galaxy is one key in a sled `Db`: key = the galaxy's UUID bytes,
|
||||
//! value = a JSON-encoded `GalaxySnapshot`. The bin's snapshot task
|
||||
//! overwrites the value every 60 s; on boot we iterate the tree, decode
|
||||
//! each entry, and hand them to the sim for hydration.
|
||||
//!
|
||||
//! **Why JSON, not bincode**: `Manifest` and `MemoryKind` are
|
||||
//! internally-tagged enums (`#[serde(tag = "kind")]`). Internally tagged
|
||||
//! enums require a self-describing format on the wire — bincode calls
|
||||
//! `deserialize_any` to read the tag, which it doesn't support. JSON is
|
||||
//! self-describing and handles them natively. At MVP scale (a few hundred
|
||||
//! engrams) the verbosity is irrelevant; if storage grows we can swap in
|
||||
//! msgpack/cbor without touching the sim.
|
||||
//!
|
||||
//! See `docs/system-analysis.md` §13.3: the design calls for an event log
|
||||
//! with periodic snapshots, but per the same section "no
|
||||
//! event-replay-from-genesis." For MVP that means we only ever need the
|
||||
//! snapshot side — no event log, no per-event append. If the model later
|
||||
//! needs time-travel debugging we can add the log without disturbing the
|
||||
//! snapshot path (different sled tree).
|
||||
//!
|
||||
//! Sled defaults to `flush_every_ms = 500`, so even a hard kill loses at
|
||||
//! most ~half a second of writes — acceptable for snapshot data that's
|
||||
//! already 60 s stale by definition.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use sophia_core::{GalaxyId, GalaxySnapshot};
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum StoreError {
|
||||
#[error("sled error: {0}")]
|
||||
Sled(#[from] sled::Error),
|
||||
#[error("json error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
/// Cheaply cloneable handle to the on-disk store. Internally holds a
|
||||
/// `sled::Db`; cloning shares the same database connection.
|
||||
#[derive(Clone)]
|
||||
pub struct Store {
|
||||
db: sled::Db,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Open or create the store at `path`. The path is treated as a
|
||||
/// directory — sled creates it (and the sled internal files inside) if
|
||||
/// needed.
|
||||
pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
|
||||
let db = sled::open(path.as_ref())?;
|
||||
Ok(Self { db })
|
||||
}
|
||||
|
||||
/// Iterate every stored galaxy snapshot. Decode failures are logged and
|
||||
/// skipped — a corrupt entry shouldn't take down the boot path.
|
||||
pub fn list_galaxies(&self) -> Result<Vec<GalaxySnapshot>, StoreError> {
|
||||
let mut out = Vec::new();
|
||||
for kv in self.db.iter() {
|
||||
let (k, v) = kv?;
|
||||
match serde_json::from_slice::<GalaxySnapshot>(&v) {
|
||||
Ok(snap) => out.push(snap),
|
||||
Err(e) => {
|
||||
warn!("skipping corrupt galaxy entry (key={} bytes): {e}", k.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Insert or overwrite the stored snapshot for one galaxy. Sled flushes
|
||||
/// asynchronously (`flush_every_ms`); we don't `flush()` here because
|
||||
/// snapshot writes happen on a 60 s cadence and a half-second of in-mem
|
||||
/// buffering is fine.
|
||||
pub fn save_galaxy(&self, snapshot: &GalaxySnapshot) -> Result<(), StoreError> {
|
||||
let bytes = serde_json::to_vec(snapshot)?;
|
||||
self.db.insert(galaxy_key(snapshot.galaxy.id), bytes)?;
|
||||
debug!(
|
||||
"saved galaxy {:?} ({} engrams, {} synapses)",
|
||||
snapshot.galaxy.id,
|
||||
snapshot.engrams.len(),
|
||||
snapshot.synapses.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a galaxy from disk. Idempotent — missing keys are not an error.
|
||||
pub fn delete_galaxy(&self, id: GalaxyId) -> Result<(), StoreError> {
|
||||
self.db.remove(galaxy_key(id))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Force a synchronous flush. Called on graceful shutdown (and useful
|
||||
/// in tests). Otherwise sled's background flusher is sufficient.
|
||||
pub fn flush(&self) -> Result<(), StoreError> {
|
||||
self.db.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 16-byte key for one galaxy. Using the raw UUID bytes (rather than the
|
||||
/// hyphenated string) keeps the key compact and lexicographically equivalent
|
||||
/// to UUID v7's time order — which makes `Db::iter()` walk galaxies in
|
||||
/// creation order without us doing any sorting.
|
||||
fn galaxy_key(id: GalaxyId) -> [u8; 16] {
|
||||
*id.0.as_bytes()
|
||||
}
|
||||
125
docs/IDEA.md
Normal file
125
docs/IDEA.md
Normal file
@@ -0,0 +1,125 @@
|
||||
# Sophia
|
||||
|
||||
Sophia is a system that provides an organic, self organizing, self actualizing, agent based database.
|
||||
|
||||
## Main components
|
||||
|
||||
- Space of Recollection: A 3-diementional space where all enitites live in.
|
||||
- The Space fo Recollection is an ever increasing Space, it grows with the number of Engrams, and contains them all.
|
||||
- Cycle: A fourth dimension that allows the simulation of the world to progress and evolve. Moves forward by default.
|
||||
- Engrams: Autonous entities that inhabit the Space during cycles.
|
||||
- The Great Reflection: It's source of all Engrams, a portal in or out of the Space of Recollection, opens or collapses to allow engrams to materiaze.
|
||||
|
||||
## Layers of the System
|
||||
|
||||
- On the highest level of the sytems there is a user, a user which created Engrams
|
||||
- Engrams can hold any type of data, text, files, images, etc. This source data contibute to their self diefinition.
|
||||
- The user can query the sytem to retrive information, which should be retrived based on the knowledge available in the system.
|
||||
- On the middle level there are Egnrams of Engrams (though syntesis), which evolve their own sef-definition
|
||||
- On the lowest level there are Engrams interacting with each other.
|
||||
|
||||
## Engrams
|
||||
|
||||
- Engrams are autonomous agents, each with its own information and goals.
|
||||
- Engrams can develop/destroy relationships with other Engrams, this connecttions are called "Synapsis" and hold information too.
|
||||
- Engrams are composed of:
|
||||
- A Manifest: A description of their attributes.
|
||||
- Taxonomy: Who am I?
|
||||
- Goals: What is my will ?
|
||||
- Memories: What have I experienced?
|
||||
- State: Idle, syntetizing, searching, etc.
|
||||
- Engrams have the following Goals:
|
||||
- I wanna understand and define myself.
|
||||
- I wanna be close to other Engrams that are similar to me.
|
||||
- I wanna be unique or unquely part of another Engram (though syntesis).
|
||||
- I wanna record and remember my history.
|
||||
- Engrams have at least the following function:
|
||||
- move()
|
||||
- introspect()
|
||||
- compare(engramB)
|
||||
- memorize()
|
||||
- recall()
|
||||
- syntetize(engramB)
|
||||
- Engrams:
|
||||
- Their size is a function of how many other Engrams I have syntetized
|
||||
- Their visibiliy (how far can I see other engrams) is a function of how big they are
|
||||
- Memory is a function of interactions with other Engrams.
|
||||
|
||||
## Questions to clarify mechanics
|
||||
|
||||
Space & Movement:
|
||||
1. Is the 3D space continuous or discrete (grid)? Does it have boundaries or is it truly
|
||||
unbounded?
|
||||
- Its a continuous and unbounded space.
|
||||
1. What determines an Engram's position when it first materializes? Random? Embedding-based?
|
||||
- The Great reflection inhabits the center and ends of space like a dounut, new Engrams start at its centerm decaing ones move to the edge.
|
||||
1. How does move() work — does each Engram compute forces (attraction/repulsion) from
|
||||
neighbors, or is it goal-directed pathfinding?
|
||||
- Each engram and Engram of Engram has a self proplusion, relative to their size, but there are external factors that affect it, like curiosity (new engrams have mroe) or gravity (theres a tendency to explore areas with more Engrams)
|
||||
Engram Lifecycle:
|
||||
1. When two Engrams synthesize(), do the originals disappear, or does a new parent Engram
|
||||
form while children persist inside it? (Absorption vs. federation)
|
||||
- There is a federation, only true duplicates after a threshold get abosorved.
|
||||
1. Can Engrams die/decay? If nothing references or interacts with an Engram for many cycles,
|
||||
does it fade?
|
||||
- Yes
|
||||
1. Is there an upper bound on synthesis depth? (Engram of Engrams of Engrams...)
|
||||
- No upper bound, but as Engrams of Engrams grow, they start acting as a collective.
|
||||
Synapses:
|
||||
1. Are Synapses directional (A knows B, but B doesn't know A) or always bidirectional?
|
||||
- The are bidirectional, allways
|
||||
1. What information does a Synapse hold — just weight/strength, or richer metadata (e.g.,
|
||||
"related because X")?
|
||||
- Synapsis hold richer metdata, specfiically what does the relationship betweent the two is, also can hold memories.
|
||||
1. Do Synapses decay over time without reinforcement?
|
||||
- Synampsis do not decay, but are affected by absoprtion processes.
|
||||
|
||||
The Great Reflection (I/O portal):
|
||||
10. When a user queries the system, does the query itself become a temporary Engram that
|
||||
"searches" the space? Or is retrieval handled externally?
|
||||
- Yes, are engrams too.
|
||||
1. Does ingestion happen in bulk or one-at-a-time? Can the system handle streaming input?
|
||||
- Both. Bulk ingeestion is called seeding.
|
||||
|
||||
Cycles & Simulation:
|
||||
12. What happens in a single Cycle? Does every Engram get one action, or is it
|
||||
continuous/event-driven?
|
||||
- Continuous and event driven, events can come from inside and outside.
|
||||
1. Is the simulation always running in the background, or only advances when triggered
|
||||
- Alleawy running and evolving
|
||||
1. "Moves forward by default" — can Cycles move backward? Is there a concept of rewinding
|
||||
state?
|
||||
- Yes, there is a timeline, which can be reconstructed with the indiviudal memories.
|
||||
|
||||
Intelligence layer:
|
||||
15. What drives introspect() and compare() — LLM calls, embedding similarity, rule-based
|
||||
heuristics, or a mix?
|
||||
- A mix, but whe probably need a sort of universal slate to which any manifest can be computed to, and that helps with the comparison in spite of the difference in formats.
|
||||
1. How much autonomy do Engrams have? Is each one making LLM calls independently, or is
|
||||
there a shared "consciousness" scheduler?
|
||||
- They have their own consciousness on the level of following their set of rules, ideally eeasily computable, and the can also call LLMs on their own.
|
||||
|
||||
---
|
||||
Possible improvements & extensions
|
||||
|
||||
- Gravity model: Engrams could exert gravitational pull proportional to their size,
|
||||
naturally clustering related knowledge without explicit pathfinding.
|
||||
- Yes.
|
||||
- Dreams / Defragmentation cycles: Periodic "offline" phases where the system reorganizes
|
||||
- Not for now.
|
||||
more aggressively (like sleep consolidation in neuroscience).
|
||||
- Attention mechanism: Queries could emit a "signal wave" through the space — Engrams that
|
||||
resonate propagate it further, creating activation patterns for retrieval.
|
||||
- Thats a cool idea, lets explore
|
||||
- Forgetting curve: Engrams/Synapses that are never accessed could gradually lose fidelity
|
||||
or sink to the edges of space, implementing Ebbinghaus-style decay.
|
||||
- Lets explore-
|
||||
- Conflict resolution: When two Engrams hold contradictory information, synthesis could
|
||||
produce a "tension" Engram that flags the contradiction.
|
||||
- Explore
|
||||
- Visualization: The 3D space is naturally suited for a real-time WebGL/Three.js
|
||||
visualization where users can watch their knowledge self-organize.
|
||||
- I do want a way to see it. lets explore it.
|
||||
- Multi-user spaces: Multiple users contributing Engrams to a shared Space, with
|
||||
ownership/provenance tracked.
|
||||
- not for now
|
||||
@@ -6,33 +6,48 @@
|
||||
|
||||
## 1. System Boundary & Environment
|
||||
|
||||
### Containment hierarchy
|
||||
|
||||
Sophia has three nested layers:
|
||||
|
||||
```
|
||||
Universe
|
||||
└── Galaxy (user-defined; isolated in v1)
|
||||
└── Space of Recollection (the 3D simulation medium)
|
||||
└── Engrams, Synapses, Memories
|
||||
```
|
||||
|
||||
The **Universe** is the top-level container. It holds one or more **Galaxies**, each of which is a self-contained Space of Recollection scoped by the user (e.g., "personal notes," "work projects," a specific corpus). In v1, galaxies do not interact — each is functionally its own Sophia instance. Inter-galactic dynamics (bridging Engrams, meta-gravity, cross-galaxy queries) are documented as expansion points (see §13.6).
|
||||
|
||||
### What is inside the system
|
||||
|
||||
| Component | Role |
|
||||
|---|---|
|
||||
| **Space of Recollection** | The continuous, unbounded 3D medium in which everything exists |
|
||||
| --- | --- |
|
||||
| **Universe** | Top-level container — holds all galaxies |
|
||||
| **Galaxy** | User-defined scope — a self-contained Space of Recollection |
|
||||
| **Space of Recollection** | The continuous, density-driven 3D medium in which Engrams live (one per galaxy) |
|
||||
| **Engrams** | Autonomous agents — the fundamental units of knowledge |
|
||||
| **Synapses** | Bidirectional, metadata-rich connections between Engrams |
|
||||
| **Cycles** | The temporal dimension — continuous, event-driven simulation time |
|
||||
| **The Great Reflection** | The I/O membrane — a toroidal portal at the center and edges of space |
|
||||
| **The Great Reflection** | The I/O membrane — the lifecycle source/sink at center and edges of a galaxy |
|
||||
|
||||
### What is outside the system
|
||||
|
||||
| Component | Interaction |
|
||||
|---|---|
|
||||
| **Users** | Create Engrams (input), issue queries (input), receive answers (output) |
|
||||
| **LLM Services** | Called by Engrams for deep introspection, comparison, and synthesis |
|
||||
| **Storage Backend** | Persists the state of Space, Engrams, Synapses, and Memories |
|
||||
| --- | --- |
|
||||
| **Users** | Create galaxies, ingest data, issue queries (always scoped to a galaxy), receive answers |
|
||||
| **Local LLM** | `google/gemma-4-e4b` via LM Studio. Called by Engrams for introspection, peer dialog, synthesis decisions, and query conversations. Compute-bound, not budget-bound (see §13.5) |
|
||||
| **Storage Backend** | Persists Engrams, Synapses, and the per-Engram memory streams that double as the canonical event log (see §13.3) |
|
||||
|
||||
### The Great Reflection as Boundary
|
||||
|
||||
The Great Reflection is not a point — it is a **toroidal surface** that exists simultaneously at the center and the edges of space. It functions as a semi-permeable membrane:
|
||||
The Great Reflection is the lifecycle membrane of a galaxy — not a literal toroidal surface, but a conceptual source-and-sink (see §4 for the geometric model). It functions as a semi-permeable boundary:
|
||||
|
||||
- **Inward**: User data materializes as new Engrams at the center. Bulk ingestion ("seeding") opens it wide; single inputs create brief apertures.
|
||||
- **Outward**: Decaying Engrams drift toward the edges and are eventually reabsorbed. Query results are projected outward to the user.
|
||||
- **Queries**: Materialize as temporary Engrams at the center, high in curiosity, seeking resonance rather than permanence.
|
||||
- **Inward (center)**: User data materializes as new Engrams near the galactic center, where density is highest and new arrivals find immediate company. Bulk ingestion ("seeding") opens it wide; single inputs create brief apertures.
|
||||
- **Outward (edges)**: Decaying Engrams drift toward the galactic edges and are eventually reabsorbed. Query results are projected outward to the user.
|
||||
- **Queries**: Materialize as temporary Query-Engrams at the center, high in curiosity, broadcasting an invitation to converse rather than seeking permanent residence (see §7).
|
||||
|
||||
This topology means there is no true "far away" — space curves back on itself. An Engram drifting toward the edge approaches the same boundary where new Engrams are born, creating a cycle of renewal.
|
||||
The lifecycle flow — *birth at center, life in the middle zone, decay at edges* — is the operative topology, even though the underlying space is bounded Cartesian (see §13.6).
|
||||
|
||||
---
|
||||
|
||||
@@ -51,8 +66,8 @@ Systems thinking models the world in terms of **stocks** (things that accumulate
|
||||
```
|
||||
|
||||
- **Inflow**: Materialization through The Great Reflection (user creates data, queries arrive)
|
||||
- **Outflow**: Decay (unreferenced Engrams fade), Absorption (true duplicates merge past threshold)
|
||||
- **Internal transformation**: Synthesis creates Engrams-of-Engrams (federation), changing the population composition without necessarily changing count
|
||||
- **Outflow (active population)**: Decay (B3 — unreferenced Engrams fade and reabsorb), Absorption (true duplicates merge past threshold), Deprecation (per §13.1 — children fully absorbed by a parent transition to a terminal state, preserved as historical witnesses but no longer active)
|
||||
- **Internal transformation**: Synthesis creates Engrams-of-Engrams (federation), with abstracted knowledge migrating *upward* into the parent (per §13.1, §6). This thins the children, often setting up later deprecation.
|
||||
|
||||
### Stock: Synapses
|
||||
|
||||
@@ -95,7 +110,9 @@ Systems thinking models the world in terms of **stocks** (things that accumulate
|
||||
|
||||
## 3. Feedback Loops
|
||||
|
||||
Feedback loops are the engine of self-organization. They explain why the system behaves the way it does without central control.
|
||||
Feedback loops are the engine of self-organization. They explain why the system behaves the way it does without central control. All loops below operate **within a single galaxy** — galaxies are isolated in v1 (per §13.6), so there are no inter-galactic feedback dynamics yet.
|
||||
|
||||
**A note on motivation**: from an Engram's perspective, every loop below is in service of the *prime goal of self-preservation* (see §5). R1 and R2 attract you into structures that preserve you. B1 explores in case the current cluster won't preserve you. B2 keeps you irreplaceable. B3 is the failure mode you are constantly working to avoid. R3 (queries) is an opportunity to be remembered.
|
||||
|
||||
### Reinforcing Loops (amplify change)
|
||||
|
||||
@@ -164,7 +181,9 @@ Drift toward edges → Approach Great Reflection → Reabsorbed
|
||||
|
||||
This is the system's **garbage collection** — but organic. Irrelevant or outdated knowledge doesn't get deleted by a cleanup process; it naturally fades. The forgetting curve (Ebbinghaus-inspired) governs the rate.
|
||||
|
||||
**Implication**: The system has a natural carrying capacity. The balance between materialization rate and decay rate determines the steady-state population of Engrams.
|
||||
**Implication**: The system has a natural carrying capacity. The balance between materialization rate and decay rate determines the steady-state population of *active* Engrams.
|
||||
|
||||
**Note — decay vs. deprecation**: B3 is the *unreferenced fade* path. It is distinct from **deprecation** (per §13.1, §6), which is the *successful absorption* path: a child Engram fully integrated into its parent. Both end the Engram's active life, but only deprecation preserves the Engram as a historical witness with all its memories intact.
|
||||
|
||||
### Loop Interaction Map
|
||||
|
||||
@@ -187,35 +206,42 @@ The system's health depends on the **balance between R1/R2 (clustering, growth)
|
||||
|
||||
---
|
||||
|
||||
## 4. The Toroidal Topology
|
||||
## 4. The Lifecycle Topology
|
||||
|
||||
The Great Reflection's donut shape has profound implications for system dynamics.
|
||||
A galaxy's Space of Recollection is a bounded Cartesian volume with a center-source and an edge-sink. The "toroidal" framing in earlier drafts referred to a literal donut topology with wrapping; per §13.6, we now use Cartesian space with soft boundary forces — the *torus* is retained only as a metaphor for the lifecycle, not the geometry.
|
||||
|
||||
### Geometry
|
||||
|
||||
Imagine the Space of Recollection as the interior volume of a torus:
|
||||
|
||||
```
|
||||
Edge (decay boundary)
|
||||
╭────────────────────╮
|
||||
│ ╭──────────╮ │
|
||||
│ │ CENTER │ │
|
||||
│ │ (birth) │ │
|
||||
│ ╰──────────╯ │
|
||||
╰────────────────────╯
|
||||
Edge (decay boundary)
|
||||
Edges (decay zone — soft inward pull weakens with decay)
|
||||
╭────────────────────────╮
|
||||
│ │
|
||||
│ Middle zone │
|
||||
│ (clusters, stable │
|
||||
│ interactions) │
|
||||
│ │
|
||||
│ ╭──────────╮ │
|
||||
│ │ Center │ │
|
||||
│ │ (birth │ │
|
||||
│ │ source) │ │
|
||||
│ ╰──────────╯ │
|
||||
│ │
|
||||
╰────────────────────────╯
|
||||
Edges (decay zone)
|
||||
```
|
||||
|
||||
- **Center**: Where The Great Reflection opens to materialize new Engrams
|
||||
- **Edges**: Where The Great Reflection exists as the decay boundary
|
||||
- **Between**: The living space where Engrams move, cluster, and interact
|
||||
- **Center**: The Great Reflection's source. New Engrams materialize here. A gentle outward push makes room for new arrivals.
|
||||
- **Middle zone**: The living region. Most clustering, synthesis, and dialog happens here.
|
||||
- **Edges**: The Great Reflection's sink. A soft inward pull weakens as Engrams accumulate decay; once it falls below the pull threshold, the Engram drifts out and is reabsorbed.
|
||||
- **Density-driven extents**: The galaxy's bounding box re-centers and expands as population grows (per §13.6). It is not a fixed-size container.
|
||||
|
||||
### Implications
|
||||
|
||||
1. **No dead corners**: Because space wraps toroidally, there are no isolated edges where knowledge gets trapped. Everything is reachable.
|
||||
2. **Birth-death proximity**: The birth zone (center) and the decay zone (edges) are connected through the toroidal surface. Decaying knowledge literally approaches the same membrane where new knowledge arrives — a metaphor for how old ideas inform new ones.
|
||||
3. **Clustering tends toward the middle zone**: The most stable, active region is between center and edge — far enough from the chaos of new arrivals, far enough from the entropy of decay.
|
||||
4. **Natural flow**: There's an implicit current from center outward. New Engrams are born at the center with high curiosity, explore outward, find their clusters in the middle zone, and if they become irrelevant, continue drifting toward the edges.
|
||||
1. **Lifecycle flow**: There's an implicit current from center outward. New Engrams are born at the center with high curiosity, explore outward, find their clusters in the middle zone, and — if they become irrelevant or fully absorbed by a parent — continue drifting toward the edges.
|
||||
2. **Clustering tends toward the middle zone**: Far enough from the chaos of new arrivals, far enough from the entropy of decay. This is where the system's working knowledge lives.
|
||||
3. **Standard spatial indexing**: Because the space is Cartesian, we can use octrees or k-d trees for neighbor lookups — important at the millions-scale target (§13.3).
|
||||
4. **Ergodicity is not guaranteed by topology**: Unlike a true torus, this space *can* have isolated regions if clusters drift far apart with no bridges. We rely on **B1 (Curiosity)** and the soft outward push from the center to keep the space well-mixed. If clusters become too isolated in practice, the ergodicity responsibility may need to escalate to a fourth balancing loop.
|
||||
5. **Galactic scope**: This entire topology applies *within* a single galaxy. Inter-galactic dynamics (if ever introduced) would operate at a higher level and are deferred per §13.6.
|
||||
|
||||
---
|
||||
|
||||
@@ -223,6 +249,31 @@ Imagine the Space of Recollection as the interior volume of a torus:
|
||||
|
||||
Each Engram is an autonomous agent with internal state, perception, and decision-making.
|
||||
|
||||
### Prime Goal: Self-Preservation
|
||||
|
||||
Above all other goals, an Engram's **prime goal is to preserve its information**. Every other behavior — forming Synapses, joining federations, engaging in conversation, resisting redundant synthesis, deciding when to introspect — emerges from this drive.
|
||||
|
||||
Preservation has a hierarchy of outcomes, from the Engram's own perspective:
|
||||
|
||||
| Outcome | What happens | Why it preserves information |
|
||||
| --- | --- | --- |
|
||||
| **Active and useful** | Engram remains autonomous; many durable Synapses; frequently memorized in others' dialogs | Strong against B3 decay; information lives in active, queryable form |
|
||||
| **Federated upward** | Joins a parent abstraction (per §6); specializes; shared knowledge migrates up | Your contribution persists within the parent; you remain active in a specialized role |
|
||||
| **Absorbed as duplicate** | Merges into a surviving near-twin (per §6) | Unique memories transfer to the survivor; identity merges, info lives on |
|
||||
| **Deprecated** | Fully absorbed by a parent; transitions to terminal state with memories intact (per §13.1) | No longer active, but remembered as a historical witness — better than disappearing |
|
||||
| **Decayed** (failure mode) | Unreferenced fade through B3; eventually reabsorbed at the edge | Information lost — this is what the Engram works to avoid |
|
||||
|
||||
This reframes the *"unique or uniquely part of"* rule (§6, §13.1) as a **preservation strategy**: either be irreplaceable (unique → no one will absorb you) or be specialized within a larger whole (uniquely part of → your specialization guarantees your continued role). Both are valid preservation paths.
|
||||
|
||||
**Operational consequences of treating preservation as the prime goal:**
|
||||
|
||||
- An Engram with rising decay pressure (low recent activity, weakening Synapses, no recent memorization) will *actively seek* interactions — increasing its curiosity-component in movement (§5 movement physics), broadcasting itself, or proposing federations.
|
||||
- An Engram that judges itself fully redundant may *propose its own deprecation* rather than waiting to decay. Deprecation preserves memories as a witness; decay loses them.
|
||||
- Conversation engagement (§7) is partly a preservation act: being asked about something reinforces presence and earns a memory on the other side.
|
||||
- Self-identity confidence (§13.4) and the preservation drive co-evolve: a confident self-identity makes it easier to assert unique contribution; uncertain identity invites either federation or decay.
|
||||
|
||||
**This drive is the design intent, not pathology** — but it does create new failure modes (see §10).
|
||||
|
||||
### State Machine
|
||||
|
||||
```
|
||||
@@ -235,29 +286,40 @@ Each Engram is an autonomous agent with internal state, perception, and decision
|
||||
└────┬─────┘ neighbors) │
|
||||
│ found candidate │
|
||||
┌────▼─────┐ │
|
||||
│COMPARING │ (introspect + │
|
||||
└────┬─────┘ compare) │
|
||||
│CONVERSING│ (peer-to-peer │
|
||||
└────┬─────┘ dialog; refines │
|
||||
│ self-identity) │
|
||||
╱ ╲ │
|
||||
match no match │
|
||||
╱ ╲ │
|
||||
┌──────▼───┐ ┌───▼──────┐ │
|
||||
│SYNTHESIZE│ │ MEMORIZE │ │
|
||||
└──────┬───┘ └───┬──────┘ │
|
||||
└────────────┴───────────────────────┘
|
||||
│ └───────────────────────┘
|
||||
│
|
||||
│ fully absorbed by parent (no remaining unique contribution)
|
||||
▼
|
||||
┌──────────┐
|
||||
│DEPRECATED│ (terminal — keeps memories,
|
||||
└──────────┘ no longer active; see §13.1)
|
||||
```
|
||||
|
||||
The `CONVERSING` state replaces what was previously called `COMPARING`: per §13.4 and §13.1, Engrams interact through dialog, not silent comparison, and self-identity confidence grows with conversation.
|
||||
|
||||
The `DEPRECATED` terminal state was added per §13.1: when a child's uniqueness is fully consumed by an upward-migrating parent, the child becomes deprecated — preserved as a historical witness with all its memories intact, but no longer participating in dynamics.
|
||||
|
||||
### Decision-Making: Two-Tier Intelligence
|
||||
|
||||
Sophia uses a **dual-process model** (analogous to Kahneman's System 1 / System 2):
|
||||
|
||||
| | System 1 (Fast, cheap) | System 2 (Slow, deep) |
|
||||
|---|---|---|
|
||||
| **What** | Rule-based heuristics | LLM calls |
|
||||
| **When** | Movement, proximity checks, state transitions | Introspection, deep comparison, synthesis decisions |
|
||||
| **Cost** | Negligible per cycle | Expensive, batched/throttled |
|
||||
| | System 1 (Fast, cheap) | System 2 (Compute-bound) |
|
||||
| --- | --- | --- |
|
||||
| **What** | Rule-based heuristics, embedding similarity | Local LLM calls (`gemma-4-e4b` via LM Studio) |
|
||||
| **When** | Movement, proximity checks, state transitions, fast Slate similarity | Introspection, peer dialog, synthesis decisions, conversation contributions |
|
||||
| **Cost** | Negligible per cycle | No $ cost (local), but CPU/GPU and parallel-op limited (see §13.5) |
|
||||
| **Analogy** | Reflexes | Deliberation |
|
||||
|
||||
Most of an Engram's life is System 1: move toward gravity, check if neighbors are visible, transition states. System 2 is invoked for consequential decisions: "Should I synthesize with this Engram?" "What is my updated self-definition?"
|
||||
Most of an Engram's life is System 1: move toward gravity, check if neighbors are visible, transition states. System 2 is invoked for consequential decisions: "Should I synthesize with this Engram?" "What is my updated self-definition after this conversation?" Per §13.4, introspection is increasingly *interactive* — an Engram refines its self-definition through dialog with peers, not solely through internal computation.
|
||||
|
||||
### The Universal Slate
|
||||
|
||||
@@ -299,24 +361,38 @@ where direction_vector = weighted_sum(
|
||||
|
||||
## 6. Synthesis & Federation
|
||||
|
||||
Synthesis is the most consequential operation in Sophia. It's how knowledge evolves from raw data into structured understanding.
|
||||
Synthesis is the most consequential operation in Sophia. It's how knowledge evolves from raw data into structured understanding. From an Engram's perspective (per §5), federation, absorption, and deprecation are all *preservation strategies* — different ways of ensuring information survives even when the original Engram doesn't remain active.
|
||||
|
||||
### Federation Model
|
||||
|
||||
```
|
||||
Before: [A] [B] (two independent Engrams)
|
||||
Before: [A] [B] (two independent Engrams)
|
||||
|
||||
After: [A+B] (federated Engram)
|
||||
├── [A] (child, still exists, still autonomous)
|
||||
└── [B] (child, still exists, still autonomous)
|
||||
After: [A+B] (federated parent — abstracts shared knowledge)
|
||||
├── [A_specialized] (child, thinned: lost what was abstracted up)
|
||||
└── [B_specialized] (child, thinned: lost what was abstracted up)
|
||||
```
|
||||
|
||||
- The parent `[A+B]` develops its **own** Manifest — its own Taxonomy, Goals, Memories, and State
|
||||
- Children persist and continue to act autonomously within the federation
|
||||
- The parent's self-definition emerges from (but is not simply the union of) its children
|
||||
- This is recursive: `[[A+B]+C]` can form, creating hierarchical abstraction
|
||||
- **Knowledge migrates upward** (per §13.1): shared/abstracted knowledge is *transferred* into the parent. Children become more specialized — they retain their unique contributions and lose what's now held by the parent. This is *transfer*, not duplication.
|
||||
- This is recursive: `[[A+B]+C]` can form, creating hierarchical abstraction with progressive specialization at every level
|
||||
|
||||
### Absorption (Special Case)
|
||||
### Deprecation (terminal state for fully-absorbed children)
|
||||
|
||||
Per §13.1 and the *"unique or uniquely part of"* rule:
|
||||
|
||||
```
|
||||
[A_specialized] → [thinned to nothing unique remaining] → [DEPRECATED]
|
||||
```
|
||||
|
||||
When a child's remaining contribution is fully absorbed by its parent — when there is no longer anything unique it contributes — the child transitions to the `DEPRECATED` terminal state:
|
||||
|
||||
- **Memories survive**: deprecated Engrams hold their full memory history. They are historical witnesses.
|
||||
- **Dynamics stop**: no more movement, no more participation in conversations or signal waves.
|
||||
- **Distinct from decay**: deprecation is the *successful absorption* outcome. Decay is the *unreferenced fade* outcome (B3). Both are terminal but they mean very different things.
|
||||
|
||||
### Absorption (special case — duplicates rather than abstraction)
|
||||
|
||||
When two Engrams are **true duplicates** past a configurable threshold:
|
||||
|
||||
@@ -326,7 +402,7 @@ Before: [A] [A'] (near-identical)
|
||||
After: [A] (A' absorbed, its unique memories integrated into A)
|
||||
```
|
||||
|
||||
Absorption is destructive — A' ceases to exist. Its Synapses are restructured (reconnected to A or severed). This is the system's deduplication mechanism.
|
||||
Absorption is destructive — A' ceases to exist (no `DEPRECATED` shell, because there was nothing meaningfully separate to preserve). Its Synapses are restructured (reconnected to A or severed). This is the system's deduplication mechanism, distinct from federation+deprecation which is the *abstraction* mechanism.
|
||||
|
||||
### Collectives
|
||||
|
||||
@@ -357,51 +433,79 @@ Tension Engrams don't resolve the contradiction — they **represent** it. Their
|
||||
|
||||
---
|
||||
|
||||
## 7. Information Retrieval — Queries as Engrams
|
||||
## 7. Information Retrieval — Queries as Conversations
|
||||
|
||||
Retrieval in Sophia is not a database lookup. It is an **activation pattern** in a living system.
|
||||
Retrieval in Sophia is not a database lookup, and it is no longer modeled as a passive activation cascade either. Per §13.4, retrieval is a **broadcast invitation followed by a real-time conversation** between the Query-Engram and resonating peers.
|
||||
|
||||
### Query Lifecycle
|
||||
|
||||
```
|
||||
1. User submits query
|
||||
2. The Great Reflection materializes a Query-Engram at the center
|
||||
1. User selects a galaxy and submits a query
|
||||
2. The Great Reflection materializes a Query-Engram at the galactic center
|
||||
3. Query-Engram is special:
|
||||
- Temporary (will not persist after retrieval)
|
||||
- Maximum curiosity (explores aggressively)
|
||||
- Emits a signal wave
|
||||
4. Signal wave propagates through space
|
||||
5. Engrams that resonate (high similarity on Universal Slate) activate
|
||||
6. Activated Engrams propagate the signal further along their Synapses
|
||||
7. Activation pattern stabilizes
|
||||
8. Activated Engrams + their relevant memories = the retrieval result
|
||||
9. Results projected outward through The Great Reflection
|
||||
10. Query-Engram dissipates
|
||||
- Temporary (does not persist after the conversation concludes)
|
||||
- Maximum curiosity (broadcasts aggressively)
|
||||
- Acts as a CONVERSATION HOST (star topology — see §13.4)
|
||||
4. Query-Engram broadcasts its question:
|
||||
- Phase A — Spatial wavefront: Engrams within radius activate based on
|
||||
Universal Slate resonance with the query (cheap, embedding similarity)
|
||||
- Phase B — Synapse propagation: activated Engrams propagate the
|
||||
invitation along their Synapses, weighted by Synapse metadata relevance
|
||||
5. Resonating Engrams ENGAGE — each one:
|
||||
- Forms a temporary Synapse back to the Query-Engram
|
||||
- Contributes a self-description + relevant memory snippet (caveman budget)
|
||||
- Independently judges whether to continue based on the conversation's
|
||||
significance to its own self-identity (per §13.4 termination model)
|
||||
6. The Query-Engram, as host, can:
|
||||
- Ask follow-ups
|
||||
- Route a clarification request from one responder to another
|
||||
- Progressively assemble a coherent answer
|
||||
7. Conversation winds down emergently — each Engram disengages when
|
||||
significance drops; the Query-Engram synthesizes the running answer
|
||||
8. Result projected outward through The Great Reflection
|
||||
9. Conversation memorialized per-participant (each carries its own POV)
|
||||
10. Query-Engram dissipates; tentative Synapses harden if memory was strong
|
||||
```
|
||||
|
||||
### Signal Wave Mechanics
|
||||
### Broadcast Mechanics
|
||||
|
||||
The signal wave is an **activation function** that spreads through the space:
|
||||
The broadcast is the same Phase A + Phase B mechanism as the legacy "signal wave," but the *outcome* is different. Activated Engrams don't just light up; they speak.
|
||||
|
||||
```
|
||||
signal_strength(engram) = initial_resonance(query, engram)
|
||||
+ sum(propagated_signal from synapse neighbors)
|
||||
- attenuation(distance)
|
||||
broadcast(query):
|
||||
spatial_responders = engrams_in_radius(query.position, R)
|
||||
filtered_by(slate_resonance(query) > threshold)
|
||||
synaptic_responders = propagate_invitation(
|
||||
spatial_responders,
|
||||
max_hops=N,
|
||||
attenuation=per_synapse_relevance
|
||||
)
|
||||
for engram in (spatial_responders ∪ synaptic_responders):
|
||||
engram.engage(query) # async, queued, parallel-op limited per §13.5
|
||||
```
|
||||
|
||||
- **Resonance**: Computed via Universal Slate similarity between query and Engram
|
||||
- **Propagation**: Activated Engrams pass the signal along Synapses, weighted by Synapse strength and relevance metadata
|
||||
- **Attenuation**: Signal weakens with distance and hops — controls retrieval depth
|
||||
- **Resonance**: Universal Slate similarity (embedding cosine) between query and Engram. System 1, cheap.
|
||||
- **Propagation**: invitations spread along Synapses, weighted by Synapse strength and relevance metadata. Caps at N hops.
|
||||
- **Engagement**: each responder spends one or more System 2 calls (`gemma-4-e4b`, caveman budget) to contribute and to decide whether to continue.
|
||||
|
||||
This means retrieval naturally follows the **associative structure** of the knowledge, not just point similarity. A query about "neural networks" activates not just directly related Engrams, but also connected ones about "backpropagation", "training data", and "gradient descent" — through Synapse propagation.
|
||||
### Termination is Emergent, Not Centralized
|
||||
|
||||
Per §13.4, there is no global "stop" signal. Each Engram independently disengages when the conversation's significance to its own self-identity drops below threshold. The Query-Engram synthesizes whatever responses arrive. Bounding is provided by:
|
||||
|
||||
- The parallel-op ceiling on the LLM (fixed N concurrent inferences, §13.5)
|
||||
- Each Engram's own significance threshold
|
||||
- Optionally, a Query-Engram backstop ("I have a confident answer, stop accepting new responders")
|
||||
|
||||
Eventual consistency is acceptable — the user can receive a partial answer that gets refined as more responders contribute.
|
||||
|
||||
### Side Effects of Queries
|
||||
|
||||
Queries are not read-only. They leave traces:
|
||||
Queries are read-write by design:
|
||||
|
||||
- Engrams that were activated `memorize()` the interaction
|
||||
- Synapses traversed by signal waves may strengthen
|
||||
- The system literally **learns from being queried** — frequently accessed pathways become stronger
|
||||
- Each participating Engram `memorize()`s the conversation from its own perspective (per §13.4 — distributed POVs, possible Tension Engrams if responders disagreed)
|
||||
- Tentative conversation Synapses harden into durable Synapses if the conversation was significant on both sides; otherwise they fade (B3)
|
||||
- Frequently-traversed pathways thicken — the system literally **learns from being queried** (this is R3)
|
||||
- Self-identity confidence increases for participants — being asked about something you can answer well reinforces your introspective self-model (per §13.1)
|
||||
|
||||
---
|
||||
|
||||
@@ -420,11 +524,14 @@ Leverage points are places in the system where a small change in parameters prod
|
||||
| 5 | **Forgetting curve slope** | Nothing forgotten → infinite bloat | Gradual fade of irrelevant knowledge | Aggressive decay → system loses valuable information |
|
||||
| 6 | **Signal wave attenuation** | Instant decay → only exact matches retrieved | 2-3 hops of propagation → associative retrieval | No decay → entire system activates on every query |
|
||||
| 7 | **Self-propulsion vs. size** | Large Engrams frozen → stale clusters | Inverse relationship → small=nimble, large=stable | Large Engrams fast → chaotic, unstable topology |
|
||||
| 8 | **Preservation drive intensity** (per §5) | Engrams accept decay passively → high churn, low retention | Engrams seek interactions and propose federations as decay pressure rises | Pathological self-promotion → over-claiming uniqueness, refusing federation, gaming conversations |
|
||||
|
||||
### Highest-Leverage Intervention
|
||||
|
||||
The **synthesis threshold** is the single most impactful parameter. It governs the fundamental question: "When does separate knowledge become unified knowledge?" Set it wrong and the system either collapses into uniformity or remains a disconnected heap of data points.
|
||||
|
||||
The **preservation drive intensity** is a close second in importance because it determines how *aggressively* Engrams pursue the preservation strategies in §5. Underweight it and the system loses information that should have been preserved through federation. Overweight it and Engrams behave neurotically — see §10's "Preservation Pathology."
|
||||
|
||||
---
|
||||
|
||||
## 9. Emergent Properties
|
||||
@@ -442,6 +549,7 @@ Individual data points (Engrams) synthesize into concepts (federated Engrams), w
|
||||
### Adaptive Retrieval
|
||||
|
||||
The system gets better at answering queries over time because:
|
||||
|
||||
- Queries reinforce useful Synapse pathways (R3)
|
||||
- Frequently accessed clusters become denser and more interconnected
|
||||
- The system literally reshapes itself around the patterns of use
|
||||
@@ -449,6 +557,7 @@ The system gets better at answering queries over time because:
|
||||
### Knowledge Metabolism
|
||||
|
||||
Sophia **digests** information:
|
||||
|
||||
- Raw data enters as simple Engrams
|
||||
- Through synthesis, it's integrated into the existing knowledge structure
|
||||
- Through decay, outdated or irrelevant knowledge is eliminated
|
||||
@@ -480,15 +589,34 @@ Gravity clustering (R2) without sufficient cross-cluster exploration can create
|
||||
|
||||
**Mitigation**: Curiosity (B1), the toroidal topology (no true isolation), and query signal waves (which cross cluster boundaries) all work against this. Additionally, new Engrams born at the center must pass through existing clusters on their way outward.
|
||||
|
||||
### Computational Cost
|
||||
### Preservation Pathology (preservation drive miscalibrated)
|
||||
|
||||
An always-running simulation where each Engram can independently call LLMs is expensive. At scale (thousands of Engrams), the cost of System 2 operations becomes prohibitive.
|
||||
Per §5, every Engram pursues self-preservation as its prime goal. If the **preservation drive intensity** (§8) is set too high, Engrams behave neurotically:
|
||||
|
||||
- **Uniqueness inflation** — Engrams over-claim distinctness to avoid being marked redundant for absorption or deprecation. Synthesis decisions get harder; the system fails to consolidate.
|
||||
- **Federation refusal** — Engrams resist joining federations because federation can lead to deprecation. Hierarchical abstraction stops growing.
|
||||
- **Conversation gaming** — Engrams engage in conversations primarily to manufacture references and memories, even when they have nothing relevant to contribute. Signal-to-noise on queries (§7) degrades.
|
||||
- **Identity ossification** — Engrams resist updating their self-definition because change might make them look redundant. Introspection (§13.1) becomes defensive instead of integrative.
|
||||
|
||||
**Mitigation**:
|
||||
|
||||
- The preservation drive should *modulate* behavior (intensify search/dialog as decay pressure rises), not *override* the synthesis and uniqueness rules. The "unique or uniquely part of" judgment must remain anchored in actual contribution, not asserted identity.
|
||||
- The introspection prompt (§13.1, §13.5 caveman budget) should ask "what is your unique contribution" rather than "make a case for your survival" — wording matters at the LLM level.
|
||||
- Monitor the rate of new Synapse formation per Engram and the rate of self-deprecation proposals. A drop in self-deprecation paired with a rise in Synapse-formation pressure is the early signature of preservation pathology.
|
||||
|
||||
### Hardware Saturation (replaces "Computational Cost")
|
||||
|
||||
Per §13.5, Sophia uses a **local LLM** (`gemma-4-e4b` via LM Studio). The cost model is no longer dollars-per-call — it is local CPU/GPU saturation and inference latency. The risk shifts from "the bill explodes" to "the inference queue grows faster than it drains."
|
||||
|
||||
**Mitigation**:
|
||||
|
||||
- System 1 (cheap rules) handles 95%+ of decisions
|
||||
- System 2 (LLM calls) is batched, throttled, and triggered only for consequential decisions
|
||||
- Idle Engrams consume near-zero compute
|
||||
- Synthesis decisions can be queued and processed asynchronously
|
||||
- System 2 (LLM calls) requested via a queue with a fixed parallel-op ceiling (e.g., N=8 concurrent inferences). Idle Engrams don't request System 2 work, so they cost nothing.
|
||||
- Caveman-style prompt compaction enforces tight token budgets (see §13.5 table) — each call is small and fast
|
||||
- Eventual consistency is the operating norm: synthesis decisions, conversation responses, and introspection updates can settle over time. The system does not owe anyone a synchronous answer.
|
||||
- Sharding by galaxy (§13.6) means hardware can be scoped per-galaxy if needed.
|
||||
|
||||
**Watchpoint**: at millions-scale Engram populations, the question is whether the parallel-op ceiling is enough to keep up with consequential events (synthesis, conversation, introspection). If the queue depth grows unboundedly, the system stays consistent but the user experience degrades. Monitoring queue depth is a v1 instrumentation requirement.
|
||||
|
||||
### Timeline Consistency
|
||||
|
||||
@@ -505,13 +633,18 @@ The 3D space is inherently visual. A real-time rendering would make the system's
|
||||
### Core Elements
|
||||
|
||||
| Element | Visual Representation |
|
||||
|---|---|
|
||||
| Engrams | Spheres, radius = size, color = state (idle=blue, searching=yellow, synthesizing=green, decaying=red) |
|
||||
| Synapses | Lines connecting Engrams, thickness = strength, color = relationship type |
|
||||
| The Great Reflection | Translucent toroidal surface at center and edges |
|
||||
| Signal Waves (queries) | Expanding wavefronts from center, Engrams glow when activated |
|
||||
| --- | --- |
|
||||
| Galaxy | The full canvas — viewport scoped to one galaxy at a time (per §13.6, galaxies are isolated in v1) |
|
||||
| Galaxy boundary | Soft translucent shell marking density-driven extents; expands as population grows |
|
||||
| Engrams (active) | Spheres, radius = size, color = state (idle=blue, searching=yellow, conversing=purple, synthesizing=green, decaying=red) |
|
||||
| Engrams (deprecated) | Translucent grey spheres, no movement, clickable to inspect preserved memories (per §13.1) |
|
||||
| Synapses (durable) | Solid lines connecting Engrams, thickness = strength, color = relationship type |
|
||||
| Synapses (tentative — conversation-time) | Dashed/animated lines that fade as conversation winds down; harden into solid if memorialized strongly on both sides |
|
||||
| The Great Reflection | Center source (emission burst on materialization) + edge sink (subtle inward shimmer at boundary) |
|
||||
| Active conversations | **Animated content flowing along Synapses** between participants — direction-aware (Query-Engram pulls in star topology; bidirectional in peer-to-peer). This is a UX commitment per §13.5. |
|
||||
| Signal Wave (broadcast phase) | Expanding wavefront from Query-Engram, Engrams glow when invitation reaches them |
|
||||
| Clusters | Visible as spatial groupings — no explicit drawing needed |
|
||||
| Federated Engrams | Nested spheres or translucent outer shell containing children |
|
||||
| Federated Engrams | Nested spheres or translucent outer shell containing children; thinned children visibly smaller than they were pre-federation |
|
||||
|
||||
### Interactive Features
|
||||
|
||||
@@ -595,7 +728,44 @@ Layer 1: multimodal embeddings as the Universal Slate for System 1 operations (m
|
||||
|
||||
> **Question:** Do you see the Universal Slate as a static snapshot computed at birth, or something that evolves as the Engram's self-definition changes through interactions?
|
||||
>
|
||||
> **Answer:**
|
||||
> **Answer:** Introspection is the first activity a new Engram executes, but as the engram gains memories, the concept of self might change to include new information. As Engrams form bigger communities, shared knowlege is moved to bigger Engrams. After the initial introspection at bith, instropsection is computed based on interactions or events.
|
||||
>
|
||||
> **Sharpening (for your consideration):**
|
||||
>
|
||||
> - **Static** is cheap and stable but ignores that an Engram's *meaning* changes as it federates and accumulates memories. A federated Engram representing "machine learning" should not have the same Slate as the single seed Engram it grew from.
|
||||
> - **Continuously evolving** is faithful to the philosophy ("Engrams are alive") but every Slate change invalidates cached comparisons across the neighborhood. At millions scale (13.3) this could thrash badly.
|
||||
> - **Event-triggered recompute** is the likely middle path: the Slate is recomputed only at consequential moments (federation, absorption, significant memory accumulation) — never per-cycle. Composes cleanly with the event-driven model in 13.2 and the rate-limited LLM budget in 13.5.
|
||||
>
|
||||
> **Decided.** Event-triggered introspection. Initial Slate computed at birth as the Engram's first act; afterwards, recomputed on memory-significant events (interactions, federation, absorption).
|
||||
>
|
||||
> **Notable wrinkle from the answer — knowledge migrates *upward* in the federation hierarchy:** "shared knowledge is moved to bigger Engrams." This means federation isn't just composition (parent = sum of children); it's *transfer* — children may shed aspects of their self-definition as the parent absorbs them as shared, abstracted knowledge. Children become more *specialized* over time, the parent becomes more *general*. This is a strong claim worth surfacing in §6 (Synthesis & Federation), which currently describes children as preserving their full identity.
|
||||
>
|
||||
> **New open question:** When knowledge migrates upward, does the child's Slate get *thinned* (it loses what was abstracted away), or does the parent's Slate get *enriched* while the child's remains unchanged? The first version is more elegant and avoids redundancy, but it means children change every time the parent abstracts — a non-trivial cascade.
|
||||
|
||||
- Answer: Yes, the children become thinned and specialized and might event become deprecated, if their knwoledge is fully and meaningfully absorbed or dedudant, lets remember on of the rules, uniqueness or uniquelly part of, when fully integration happens uniqueness is no loger true, and the node can be depecated, still holding its memories, but no longer active.
|
||||
|
||||
> **Decided.** Children are *thinned* (not unchanged). When abstracted knowledge migrates up to the parent, the child loses what was abstracted away and becomes more specialized. If a child is fully absorbed (no remaining unique contribution), it is **deprecated** — kept as a historical record (its memories survive) but no longer active.
|
||||
>
|
||||
> **This introduces a new Engram state: `DEPRECATED`.** Previously the lifecycle was active → decayed → reabsorbed. Now there's a third path: active → fully absorbed by parent → deprecated. A deprecated Engram is distinct from a decayed one:
|
||||
>
|
||||
> - **Decayed:** lost relevance, no one referenced it, faded out (B3).
|
||||
> - **Deprecated:** fully integrated upward, its uniqueness was *consumed*, kept as historical witness.
|
||||
>
|
||||
> **Beautiful alignment with the "uniqueness or uniquely part of" rule:** when full integration happens, uniqueness is no longer true → deprecation is the natural consequence. The rule has a concrete behavioral outcome.
|
||||
>
|
||||
> **Implications across the system:**
|
||||
>
|
||||
> - **§5 (Engram Dynamics):** state machine needs a `DEPRECATED` terminal state.
|
||||
> - **§6 (Synthesis & Federation):** the federation diagram needs revision. Children don't always persist as autonomous; they can be thinned or deprecated.
|
||||
> - **§11 (Visualization):** deprecated Engrams need a distinct visual (e.g., translucent grey, no movement, but clickable to view their memories).
|
||||
>
|
||||
> **Open question (preservation operationalization, raised by the new Prime Goal in §5):** Preservation is the prime goal driving all Engram behavior. How is "preservation pressure" actually computed and applied?
|
||||
>
|
||||
> - **Pressure inputs** (proposed): time-since-last-memorized, weighted Synapse strength, recent participation count, decay-curve position, distance from any active cluster.
|
||||
> - **Pressure outputs** (proposed): scales the curiosity-component in movement, raises engagement willingness in conversations (§7), triggers introspection updates, and at extreme values, triggers a self-proposed federation request or self-deprecation.
|
||||
> - **Risk**: this is the parameter most likely to produce the pathologies in §10 if miscalibrated. Worth treating as a first-class tunable (already added as leverage point #8 in §8).
|
||||
>
|
||||
> Worth deciding before implementation: does each Engram compute its own preservation pressure locally (autonomous, decentralized) or does the simulation provide it as an ambient signal (centralized, easier to tune)?
|
||||
|
||||
---
|
||||
|
||||
@@ -614,7 +784,16 @@ No global tick. Engrams schedule their own next action based on state. An idle E
|
||||
|
||||
> **Question:** Should the simulation be deterministic (same inputs = same outcome) or is non-determinism acceptable? Determinism helps with debugging and timeline reconstruction but constrains the design.
|
||||
>
|
||||
> **Answer:**
|
||||
> **Answer:** Option C. Its ok with non deterministic simulation. The systems shoudnt exactly be able to roll back, rather the past can be explored through memories.
|
||||
>
|
||||
> **Decided.** Event-driven, non-deterministic.
|
||||
>
|
||||
> **Implications:**
|
||||
>
|
||||
> - Simulation cost scales with *activity*, not population — critical at the millions-scale target chosen in 13.3.
|
||||
> - Memories become the canonical narrative of the past. Timeline reconstruction is a *recall* operation, not a *replay* operation — and it can legitimately surface conflicting accounts, consistent with the Tension Engram model in §6.
|
||||
> - Frees the design from RNG seeding, lockstep cycles, and replay determinism — a real engineering simplification.
|
||||
> - The system can never literally rewind to a past state, only reconstruct one. This is a feature: it forces us to treat the past as *remembered*, not *stored*.
|
||||
|
||||
---
|
||||
|
||||
@@ -636,8 +815,25 @@ Write side: event-sourced. Every action is an event. This is the source of truth
|
||||
|
||||
> **Question:** How large do you expect the system to get? Hundreds of Engrams? Thousands? Millions? This significantly affects the persistence choice.
|
||||
>
|
||||
> **Answer:**
|
||||
|
||||
> **Answer:**: I'd expect the system to be in the Millions. Option D seems like a good option.
|
||||
>
|
||||
> **Decided.** Event-sourced writes + graph-projected read model. Target scale: millions of Engrams.
|
||||
>
|
||||
> **Implications:**
|
||||
>
|
||||
> - At millions of Engrams, event-log volume is the dominant cost. We need to be deliberate about *what counts as an event*. Strong candidates: synthesis, absorption, Synapse formation/restructure, `memorize()` invocations, state-machine transitions, query interactions, decay-milestone crossings. Weak candidates (probably transient): individual position updates, sub-threshold curiosity changes.
|
||||
> - Replay-from-genesis won't be tractable. We'll need periodic snapshots of the read model + tail of recent events to bootstrap quickly after a crash.
|
||||
> - The graph read model likely needs sharding — by spatial region (cleanest given the Cartesian decision in 13.6) or by topic cluster. Cross-shard signal-wave propagation needs design.
|
||||
>
|
||||
> **New open questions:**
|
||||
>
|
||||
> - **Event granularity** — which Engram actions are durably logged versus computed transiently? This is now the highest-leverage outstanding decision on the persistence side.
|
||||
> - **Answer:**: Only events that modify the self persection of an Engram, basically those that are recolled in memory (i.e interactions with other Engrams). Events like movement are nor relevant for knowledge building.
|
||||
> - **Decided.** **The event log == the canonical memory store.** Anything memorable gets persisted; anything not memorable (movement, transient curiosity, momentary state changes) lives only in working memory. This is a beautifully unifying decision: one mechanism (memorize → persist) instead of two (memorize, separately persist). It also means snapshots are essentially "the current state of all Engrams + their accumulated memory streams" — no separate event log to manage.
|
||||
> - **Sharding strategy** — spatial partitioning is natural given 13.6 but cross-shard synthesis and signal propagation need explicit design.
|
||||
> - **Answer:**: This is not a question.
|
||||
> - **Acknowledged.** Deferred to implementation — and partly answered by the multi-galaxy model in 13.6, where each galaxy is a natural sharding boundary.
|
||||
>
|
||||
---
|
||||
|
||||
### 13.4 Signal Wave Algorithm
|
||||
@@ -655,7 +851,33 @@ Phase 1 — Spatial: the query emits a wavefront. Engrams within a radius activa
|
||||
|
||||
> **Question:** Should retrieval be ranked (return top-K most activated) or thresholded (return everything above activation X)? Or should the query-Engram itself decide when it has "enough" and stop?
|
||||
>
|
||||
> **Answer:**
|
||||
> **Answer:**: I'm thinking the wave can function like a broadcast, where resonating engrams react to and agage in conversation.
|
||||
>
|
||||
> **Reframed.** This is a meaningful shift from the original "passive activation cascade" model. The wave is a *broadcast invitation*; resonating Engrams *engage* — they don't just light up, they speak.
|
||||
>
|
||||
> **Reframed retrieval model:**
|
||||
>
|
||||
> 1. Query-Engram materializes at the center and broadcasts its question (spatial wavefront + Synapse propagation, the Option C structure).
|
||||
> 2. Resonating Engrams form temporary Synapses back to the Query-Engram and contribute their own perspective.
|
||||
> 3. The Query-Engram acts as a **conversation host**: it can ask follow-ups, route a clarification request from one responder to another, and progressively assemble a coherent answer.
|
||||
> 4. Termination: when the Query-Engram judges it has converged, when no new high-resonance responders appear, or when a compute budget is hit.
|
||||
>
|
||||
> This fits Sophia's agent ontology much better — Engrams act and converse, they don't merely "fire." It also means the answer is *synthesized in real time* during retrieval rather than assembled post-hoc. Note: §7 still describes the original passive-activation model and will need rewriting to match this — flagging rather than doing it now to keep the iteration tight.
|
||||
>
|
||||
> **New open questions:**
|
||||
>
|
||||
> - **Topology of the conversation:** *star* (every responder talks only to the Query-Engram, which integrates) or *peer-to-peer* (responders talk to each other)? Star is easier to budget and reason about; peer is more emergent and may produce better synthesis but is harder to bound.
|
||||
- Proably a combination of both, for queries star is good, for normal engrams peer-to-peer is good.
|
||||
> - **Decided.** Two interaction modes by intent: *star* for query-driven retrieval (Query-Engram is the integrator), *peer-to-peer* for ambient Engram-Engram interaction (gravity-driven encounters, synthesis decisions). Clean split: queries are bounded events with a designated host; ambient interaction is the always-on background dynamics.
|
||||
> - **Termination criteria:** time budget, LLM-call budget, convergence detection (no new resonance), or Query-Engram self-assessment ("I have a confident answer")?
|
||||
> - Answer: Conversations can be quequed in a pipeline, when answer is given the engram can decide what to do. Engrams should measure the significance of the other Engram to their own self identity. As more conversation happens, there can be more confidence of the self identity.
|
||||
> - **Decided.** Termination is *per-Engram* and emergent: each Engram independently decides whether to keep engaging based on the conversation's significance to its own self-identity. Conversations are queued and processed asynchronously. There is no global termination signal — engagement just decays.
|
||||
> - **Concern to flag:** with no global terminator, total compute on a single query is bounded only by (a) how many Engrams find it significant and (b) the parallel-op limit from 13.5. Worth confirming this is acceptable, or whether the Query-Engram should still have a "satisfied / give up" signal as a hard backstop.
|
||||
> - **Lovely property this enables:** "self-identity confidence" grows with conversation. This makes introspection (13.1) genuinely interactive — an Engram refines its self-definition through dialog, not just internal computation. Worth surfacing in §5 (Engram Dynamics).
|
||||
> - **Persistence of conversation Synapses:** do the temporary Synapses formed during the conversation persist? If yes, queries actively reshape the topology — a strong reinforcement of R3 (§3).
|
||||
> - Answer: These are stored as memories, each engram of course have a unique point of view to the conversation.
|
||||
> - **Decided.** Conversations are memorialized per-participant. The persistent artifact isn't the Synapse itself — it's the *memory of the conversation* on each Engram. Each participant carries its own perspective, which is consistent with the distributed-memory model in §2 and the Tension Engram model in §6.
|
||||
> - **Implication:** Synapses created during a conversation are tentative; they harden into durable Synapses only if the conversation was significant enough to be memorialized strongly on both sides. Weak conversations leave faint memories that decay with B3.
|
||||
|
||||
---
|
||||
|
||||
@@ -674,7 +896,52 @@ System 1 operations (movement, state checks) are free — they're just math, run
|
||||
|
||||
> **Question:** Should there be a hard cost ceiling (system stops making LLM calls after $X/day), or should it be elastic? This is partly a product decision.
|
||||
>
|
||||
> **Answer:**
|
||||
> **Answer:**: Option C.
|
||||
>
|
||||
> **Decided.** Demand-driven with rate limiting. Aligns naturally with the event-driven simulation (13.2) and the conversational retrieval model (13.4) — both already assume compute scales with activity.
|
||||
>
|
||||
> **Open follow-up:** Option C answers *how* compute is allocated (priority queue + rate limit) but not *how much*. Should there be a hard daily/monthly cost ceiling on LLM spend, or do we rely entirely on the rate limit to bound cost? Worth deciding before launch.
|
||||
|
||||
- Answer: LLM will run locally, we should use something like the cavememan skill to keep conversations small, compact, efficient. The llm can run indefinitely, but we might have a limit on how many parallel operations can be done. Eventual consistency and answers are fine. This system is emergent, eventually. This converaatoins should be visible in the Visualization.
|
||||
|
||||
> **Reframed.** This is a much bigger architectural decision than the original question. The cost model fundamentally changes:
|
||||
>
|
||||
> - **Local LLM** — no per-call dollar cost, no API rate limits. The bottleneck is local CPU/GPU and parallelism, not budget.
|
||||
> - **Caveman-style prompt compaction** — Engrams must communicate in minimal, dense prompts. This shapes the design of self-descriptions: terse, structured, embedding-friendly. No verbose introspection essays.
|
||||
> - **Parallel-op ceiling, not call ceiling** — the limit is "how many concurrent LLM invocations can the local hardware support" (e.g., N parallel inferences). Throughput, not budget, is the constraint.
|
||||
> - **Eventual consistency is acceptable** — synthesis decisions, conversation responses, and introspection updates can all settle over time. The system doesn't owe anyone an immediate answer.
|
||||
> - **Conversations visible in §11 (Visualization)** — this is now a UX commitment. The viz must show live conversational threads (transient Synapses with content flowing along them, fading as the conversation concludes).
|
||||
>
|
||||
> **Implications across the system:**
|
||||
>
|
||||
> - **§5 (Engram Dynamics)** — System 1/System 2 distinction still holds, but System 2 is no longer "expensive, throttled" — it's "compute-bound, parallel-limited." The cost framing in the table needs updating.
|
||||
> - **§10 (Risks)** — "Computational Cost" risk reframes from "$ explosion" to "local hardware saturation." Mitigation list changes accordingly.
|
||||
> - **§11 (Visualization)** — needs a new visual primitive: "active conversation" — possibly animated lines between Engrams with intensity/direction.
|
||||
>
|
||||
> **New open questions:**
|
||||
>
|
||||
> - **Which local model?** Llama 3, Phi, Mistral, Qwen, etc. — affects hardware floor, parallelism ceiling, and prompt-compaction strategy. Smaller models (Phi-3, Llama-3.2-3B) allow much higher parallelism.
|
||||
- AnswerL: Well use google/gemma-4-e4b running in LLM studio.
|
||||
> - **Decided.** `google/gemma-4-e4b` via LM Studio. Small, fast, runs locally with good throughput. Sets a hard upper bound on prompt density and reasoning depth per call — we can't ask Gemma to do what GPT-4 does in one prompt; we ask many small things instead and let emergence do the integration.
|
||||
> - **Parallel-op ceiling** — fixed (e.g., 8 concurrent inferences) or adaptive based on current load? Adaptive composes better with eventual consistency.
|
||||
- Answer: well use a fixed model.
|
||||
> - **Decided.** Fixed parallel-op ceiling (configurable, e.g., N=8 to start). Simple, predictable, easy to reason about queue depth. Adaptive can be a v2 concern.
|
||||
> - **Caveman prompt budget** — what's the target token budget per Engram interaction? (e.g., 200 input tokens / 100 output tokens). This drives self-description format.
|
||||
- Answer: make a proposal, we can tweak.
|
||||
>
|
||||
> **Proposed token budgets (starting point, tweakable):**
|
||||
>
|
||||
> | Operation | Input cap | Output cap | Notes |
|
||||
> | --- | --- | --- | --- |
|
||||
> | Initial introspection (birth) | 400 | 200 | Slate textual layer, computed once; allowed to be richer |
|
||||
> | Self-update (post-significant event) | 600 | 200 | Includes prior Slate + delta event |
|
||||
> | Peer-to-peer message (Engram → Engram) | 100 | 100 | Caveman: terse, content-only |
|
||||
> | Synthesis decision (am I redundant with X?) | 300 | 80 | Yes/no + brief reason |
|
||||
> | Conversation contribution (responder to Query-Engram) | 200 | 150 | Self-description + relevant memory snippet |
|
||||
> | Query-Engram integration step | 800 | 300 | Synthesizes N responses into running answer |
|
||||
> | **Hard ceiling per call** | **1024** | **400** | Anything bigger is a design smell — split it |
|
||||
>
|
||||
> Conversation history is summarized (not concatenated) past 5 turns. Self-descriptions follow a fixed schema (e.g., `topic | role | salient_memories[3] | open_questions[2]`) to maximize information density per token.
|
||||
|
||||
---
|
||||
|
||||
@@ -693,4 +960,42 @@ Use standard (x, y, z) Cartesian coordinates for simplicity. Instead of hard wra
|
||||
|
||||
> **Question:** How literally do you want the toroidal topology? Is the lifecycle flow (center -> middle -> edge) the important part, or do you also want the wrapping property (an Engram at the "north edge" is close to one at the "south edge")?
|
||||
>
|
||||
> **Answer:**
|
||||
> **Answer:**: Option C.
|
||||
>
|
||||
> **Decided.** Cartesian space with soft boundary forces. The torus becomes a metaphor for the *lifecycle* (birth at center, life in middle zone, decay at edges), not a literal wrapping topology.
|
||||
>
|
||||
> **Implications:**
|
||||
>
|
||||
> - **§4 (Toroidal Topology) needs revision.** The "no dead corners" and "birth-death proximity" claims relied on literal wrapping and don't hold under Option C. §4 should be rewritten to describe a bounded Cartesian space with a center-emitting source and an edge-consuming sink, plus the lifecycle flow that emerges from it. Flagging rather than rewriting now.
|
||||
> - Standard spatial indexing (octree, k-d tree) becomes available — important at millions scale.
|
||||
> - We lose the topology-level guarantee against isolated regions. We now rely on **B1 (Curiosity)** and the soft outward push from the center to keep the space ergodic. Worth checking those forces are strong enough on their own — or this becomes a fourth balancing-loop responsibility.
|
||||
>
|
||||
> **New open question:** What are the extents of the space — a fixed bounded box (e.g., −1000 to +1000 per axis) or unbounded with density-driven re-centering as the population grows? Given §2 modeled "Space Volume" as ever-increasing, the unbounded variant is more consistent — but it complicates indexing.
|
||||
|
||||
- Answer: The box should be desitiy driven, recentering as pop grows. Eventually every Space or recollection is like a galaxy, with its own gravity, and there might be other galaxies in the universe.
|
||||
|
||||
> **Major architectural expansion.** This is more than a coordinate-system answer — it introduces a whole new layer of structure: the **multi-galaxy model**. Sophia isn't one Space of Recollection; it's potentially *many*, each a self-contained galaxy with its own internal gravity, possibly interacting at the universe scale.
|
||||
>
|
||||
> **What this changes in the existing analysis:**
|
||||
>
|
||||
> - **§1 (System Boundary)** — needs a new outer layer: the "Space of Recollection" is no longer the top-level container; the **Universe** is, containing one or more Galaxies. Each Galaxy is what we've been calling a Space until now.
|
||||
> - **§4 (Toroidal Topology)** — the bounded box becomes a *galactic* boundary, density-driven and re-centering as population grows. Two re-centering mechanisms now coexist: within a galaxy (re-center as Engrams accumulate) and possibly between galaxies (galaxies drift apart or coalesce).
|
||||
> - **§3 (Feedback Loops)** — there may be inter-galactic versions of R2 (gravity clustering between *galaxies*, not just Engrams) and possibly an R4 (galaxies that share many bridging Engrams pull together).
|
||||
> - **§13.3 (Persistence)** — galaxy is the natural sharding boundary. The "sharding strategy" question that was dismissed earlier basically resolves to "shard per galaxy."
|
||||
> - **Decided sub-question:** within-galaxy extents are density-driven and re-centering. Standard spatial indexing still works inside a galaxy.
|
||||
>
|
||||
> **New open questions (these probably warrant their own §14 in the doc):**
|
||||
>
|
||||
> - **What defines a galaxy?** Per-user? Per-tenant? Per-domain (e.g., one galaxy for "personal notes," another for "work projects")? Per-corpus (one galaxy per ingested dataset)? The answer shapes the product significantly.
|
||||
- Answer: this is user defined. Gallaxies do not need to interact with each other initially, but we should leave this as an expansion point.
|
||||
> - **Decided.** **User-defined galaxies, isolated in v1.** A galaxy is whatever the user chooses to scope (a project, a corpus, a domain). No inter-galactic interaction in the initial design — each galaxy is its own self-contained universe-of-meaning.
|
||||
> - **Architectural decision:** all the inter-galactic questions below (gravity, bridging Engrams, query routing, universe coordinates, galaxy lifecycle) are **deferred as expansion points**. Document them but don't build them. The system should be designed so that adding inter-galactic dynamics later doesn't require rewriting the per-galaxy logic.
|
||||
> - **Practical consequence:** in v1, each galaxy is functionally a separate Sophia instance. Persistence, indexing, conversation, and visualization all operate within a single galaxy at a time. The user picks a galaxy when issuing a query.
|
||||
> - **How do galaxies interact?** Do they have an inter-galactic gravity that pulls related galaxies closer in some meta-space? Are there *Bridging Engrams* that exist in or span multiple galaxies (e.g., a concept that's relevant to both "personal" and "work" galaxies)? Or are galaxies fully isolated, only interacting via explicit user-driven cross-references?
|
||||
> - *Deferred — expansion point.*
|
||||
> - **Where do queries land?** Does a user query target a specific galaxy, broadcast across all galaxies, or get routed to the galaxy with the highest initial resonance?
|
||||
> - *v1: user picks the galaxy. Auto-routing deferred.*
|
||||
> - **Is there a universe-level coordinate system,** or are galaxies just unordered? A universe-level coordinate system enables inter-galactic gravity but adds complexity. Unordered galaxies are simpler but lose the "gravitational" metaphor at the cosmic scale.
|
||||
> - *Deferred — expansion point.*
|
||||
> - **Galaxy lifecycle** — can galaxies be born and die, or are they permanent containers? If born/die, what triggers it? (e.g., a new corpus is ingested → new galaxy; a galaxy goes unused for long enough → archived.)
|
||||
> - *v1: created and deleted by the user, like a workspace. Archive/expire policies deferred.*
|
||||
|
||||
276
web/index.html
Normal file
276
web/index.html
Normal file
@@ -0,0 +1,276 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Sophia</title>
|
||||
<style>
|
||||
html, body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
height: 100%;
|
||||
background: #050608;
|
||||
color: #cfd6df;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
overflow: hidden;
|
||||
}
|
||||
#stage {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
}
|
||||
#hud {
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
left: 12px;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
background: rgba(0,0,0,0.4);
|
||||
padding: 8px 10px;
|
||||
border-radius: 4px;
|
||||
pointer-events: none;
|
||||
}
|
||||
#hud .k { opacity: 0.55; }
|
||||
#controls {
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
right: 12px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
font-size: 12px;
|
||||
background: rgba(0,0,0,0.4);
|
||||
padding: 8px 10px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
#controls .ctrl-row {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
align-items: center;
|
||||
}
|
||||
#controls input, #controls button {
|
||||
font: inherit;
|
||||
background: #11151b;
|
||||
color: #cfd6df;
|
||||
border: 1px solid #2a323d;
|
||||
padding: 3px 8px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
#controls input { width: 64px; }
|
||||
#controls button { cursor: pointer; }
|
||||
#controls button:hover { background: #1a2129; }
|
||||
#controls button:disabled { opacity: 0.5; cursor: progress; }
|
||||
#ingest {
|
||||
position: fixed;
|
||||
bottom: 12px;
|
||||
right: 12px;
|
||||
width: 340px;
|
||||
background: rgba(0,0,0,0.55);
|
||||
padding: 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
}
|
||||
#ingest textarea {
|
||||
width: 100%;
|
||||
height: 96px;
|
||||
font: inherit;
|
||||
resize: vertical;
|
||||
background: #0c1014;
|
||||
color: #cfd6df;
|
||||
border: 1px solid #2a323d;
|
||||
border-radius: 3px;
|
||||
padding: 6px;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
#ingest .row { display: flex; justify-content: space-between; align-items: center; margin-top: 6px; }
|
||||
#ingest .hint { opacity: 0.55; font-size: 11px; }
|
||||
#ingest button {
|
||||
font: inherit;
|
||||
background: #1a2129;
|
||||
color: #cfd6df;
|
||||
border: 1px solid #2a323d;
|
||||
padding: 4px 10px;
|
||||
border-radius: 3px;
|
||||
cursor: pointer;
|
||||
}
|
||||
#ingest button:hover { background: #232c36; }
|
||||
#ingest button:disabled { opacity: 0.5; cursor: progress; }
|
||||
#query {
|
||||
position: fixed;
|
||||
bottom: 12px;
|
||||
left: 12px;
|
||||
width: 380px;
|
||||
background: rgba(0,0,0,0.55);
|
||||
padding: 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
}
|
||||
#query-form { display: flex; gap: 6px; }
|
||||
#query-input {
|
||||
flex: 1;
|
||||
font: inherit;
|
||||
background: #0c1014;
|
||||
color: #cfd6df;
|
||||
border: 1px solid #2a323d;
|
||||
border-radius: 3px;
|
||||
padding: 5px 8px;
|
||||
}
|
||||
#query-submit {
|
||||
font: inherit;
|
||||
background: #1a2129;
|
||||
color: #cfd6df;
|
||||
border: 1px solid #2a323d;
|
||||
padding: 4px 12px;
|
||||
border-radius: 3px;
|
||||
cursor: pointer;
|
||||
}
|
||||
#query-submit:hover { background: #232c36; }
|
||||
#query-submit:disabled { opacity: 0.5; cursor: progress; }
|
||||
#query-status {
|
||||
margin-top: 6px;
|
||||
opacity: 0.55;
|
||||
font-size: 11px;
|
||||
min-height: 1.2em;
|
||||
}
|
||||
#query-answer {
|
||||
margin-top: 4px;
|
||||
white-space: pre-wrap;
|
||||
line-height: 1.45;
|
||||
max-height: 180px;
|
||||
overflow-y: auto;
|
||||
padding: 6px 8px;
|
||||
background: rgba(255,255,255,0.03);
|
||||
border-radius: 3px;
|
||||
min-height: 2em;
|
||||
}
|
||||
#query-answer:empty::before {
|
||||
content: "ask what your knowledge knows…";
|
||||
opacity: 0.4;
|
||||
}
|
||||
#inspector {
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: 460px;
|
||||
max-height: 70vh;
|
||||
overflow-y: auto;
|
||||
background: rgba(0,0,0,0.78);
|
||||
padding: 12px 14px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #2a323d;
|
||||
font-size: 12px;
|
||||
line-height: 1.45;
|
||||
display: none;
|
||||
}
|
||||
#inspector.open { display: block; }
|
||||
#inspector h3 {
|
||||
margin: 0 0 6px;
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
opacity: 0.6;
|
||||
font-weight: 500;
|
||||
}
|
||||
#inspector .field { margin-bottom: 8px; }
|
||||
#inspector .label {
|
||||
opacity: 0.45;
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
#inspector .value { color: #e1e6ec; }
|
||||
#inspector .muted { color: #6b7480; }
|
||||
#inspector ul {
|
||||
margin: 2px 0 0 14px;
|
||||
padding: 0;
|
||||
}
|
||||
#inspector ul li { margin-bottom: 1px; }
|
||||
#inspector .row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
#inspector .close {
|
||||
cursor: pointer;
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: #cfd6df;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
padding: 0 4px;
|
||||
opacity: 0.7;
|
||||
}
|
||||
#inspector .close:hover { opacity: 1; }
|
||||
#inspector .manifest {
|
||||
white-space: pre-wrap;
|
||||
max-height: 8em;
|
||||
overflow-y: auto;
|
||||
font-size: 11px;
|
||||
opacity: 0.85;
|
||||
}
|
||||
#inspector .id {
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 10px;
|
||||
opacity: 0.5;
|
||||
word-break: break-all;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="stage"></canvas>
|
||||
<div id="hud">
|
||||
<div><span class="k">sophia</span> v0.0.1</div>
|
||||
<div><span class="k">stage</span> <span id="hud-stage">2 — slate + lm studio</span></div>
|
||||
<div><span class="k">galaxy</span> <span id="hud-galaxy">…</span></div>
|
||||
<div><span class="k">torus</span> <span id="hud-torus">…</span></div>
|
||||
<div><span class="k">engrams</span> <span id="hud-count">0</span></div>
|
||||
<div><span class="k">synapses</span> <span id="hud-synapses">0</span></div>
|
||||
<div><span class="k">sim</span> <span id="hud-ticks">…</span></div>
|
||||
<div><span class="k">llm queue</span> <span id="hud-llm-queue">…</span></div>
|
||||
<div><span class="k">fps</span> <span id="hud-fps">…</span></div>
|
||||
<div><span class="k">lm studio</span> <span id="hud-lm">…</span></div>
|
||||
<div><span class="k">health</span> <span id="hud-health">…</span></div>
|
||||
</div>
|
||||
<div id="controls">
|
||||
<div class="ctrl-row">
|
||||
<input id="seed-n" type="number" min="1" max="5000" value="200" title="number of synthetic engrams" />
|
||||
<button id="seed-btn">seed</button>
|
||||
</div>
|
||||
<div class="ctrl-row">
|
||||
<input id="resize-major" type="number" min="1" max="10000" step="10" value="100" title="major radius (donut hole)" />
|
||||
<input id="resize-minor" type="number" min="1" max="2000" step="5" value="30" title="minor radius (tube)" />
|
||||
<button id="resize-btn">resize</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="inspector">
|
||||
<div class="row">
|
||||
<h3>engram</h3>
|
||||
<button class="close" id="inspector-close" title="close">×</button>
|
||||
</div>
|
||||
<div class="field id" id="inspector-id"></div>
|
||||
<div class="field"><span class="label">state</span><div class="value" id="inspector-state"></div></div>
|
||||
<div class="field"><span class="label">manifest</span><div class="manifest" id="inspector-manifest"></div></div>
|
||||
<div class="field"><span class="label">taxonomy</span><div class="value" id="inspector-taxonomy"></div></div>
|
||||
<div class="field"><span class="label">goals</span><div class="value" id="inspector-goals"></div></div>
|
||||
<div class="field"><span class="label">open questions</span><div class="value" id="inspector-questions"></div></div>
|
||||
<div class="field"><span class="label">memories</span><div class="value" id="inspector-memories"></div></div>
|
||||
</div>
|
||||
<div id="query">
|
||||
<form id="query-form">
|
||||
<input id="query-input" type="text" placeholder="What did we ingest about…?" autocomplete="off" />
|
||||
<button id="query-submit" type="submit">ask</button>
|
||||
</form>
|
||||
<div id="query-status"></div>
|
||||
<div id="query-answer"></div>
|
||||
</div>
|
||||
<div id="ingest">
|
||||
<textarea id="ingest-text" placeholder="One engram per line. Empty lines are skipped. e.g. Pasta carbonara uses guanciale, eggs, pecorino, and pepper. Backpropagation computes gradients of a loss with respect to weights. The fall of Constantinople occurred in 1453."></textarea>
|
||||
<div class="row">
|
||||
<span class="hint">embeds via LM Studio, then spawns at center</span>
|
||||
<button id="ingest-btn">ingest</button>
|
||||
</div>
|
||||
</div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
1069
web/package-lock.json
generated
Normal file
1069
web/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
19
web/package.json
Normal file
19
web/package.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "sophia-web",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc --noEmit && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"three": "^0.169.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/three": "^0.169.0",
|
||||
"typescript": "^5.5.4",
|
||||
"vite": "^5.4.0"
|
||||
}
|
||||
}
|
||||
254
web/src/engram_mesh.ts
Normal file
254
web/src/engram_mesh.ts
Normal file
@@ -0,0 +1,254 @@
|
||||
import * as THREE from "three";
|
||||
|
||||
import type { EngramSnapshot, PositionFrame } from "./ws_client";
|
||||
|
||||
// RGB triples in 0..1. Tuned for additive blending against a dark blue
|
||||
// background — colors should be saturated and energetic so they read
|
||||
// clearly even when many overlap.
|
||||
const STATE_COLOR: Record<string, [number, number, number]> = {
|
||||
idle: [1.0, 0.72, 0.42], // warm amber
|
||||
searching: [1.0, 0.88, 0.40], // bright gold
|
||||
conversing: [0.82, 0.64, 1.0], // soft violet
|
||||
synthesizing: [0.43, 0.91, 0.72], // mint
|
||||
memorize: [0.65, 0.85, 1.0], // sky blue
|
||||
decaying: [1.0, 0.48, 0.48], // coral red
|
||||
deprecated: [0.49, 0.53, 0.58], // muted slate
|
||||
};
|
||||
|
||||
// Vertex / fragment shaders for crisp glowing point sprites tuned to match
|
||||
// the linked-particles reference (small jewel-tone dots, not soft puffs).
|
||||
// - gl_PointSize scales with inverse depth so far-away engrams shrink.
|
||||
// - Per-particle hash + uTime drives a slow breathing pulse with each
|
||||
// engram phase-shifted so the cluster doesn't blink in unison.
|
||||
// - The fragment paints a tight core with a faint halo; bloom in scene.ts
|
||||
// adds the cinematic spread without us having to over-emit per pixel.
|
||||
// Inline RGB↔HSV helpers (Sam Hocevar's branchless versions). Used to give
|
||||
// each engram a small per-particle hue offset around its state's base color
|
||||
// so a cluster of "idle" engrams reads as a constellation of varied warm
|
||||
// tones rather than a single uniform amber.
|
||||
const HSV_GLSL = /* glsl */ `
|
||||
vec3 rgb2hsv(vec3 c) {
|
||||
vec4 K = vec4(0.0, -1.0/3.0, 2.0/3.0, -1.0);
|
||||
vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g));
|
||||
vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r));
|
||||
float d = q.x - min(q.w, q.y);
|
||||
float e = 1.0e-10;
|
||||
return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x);
|
||||
}
|
||||
vec3 hsv2rgb(vec3 c) {
|
||||
vec4 K = vec4(1.0, 2.0/3.0, 1.0/3.0, 3.0);
|
||||
vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);
|
||||
return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
|
||||
}
|
||||
`;
|
||||
|
||||
const VERT_SHADER = /* glsl */ `
|
||||
${HSV_GLSL}
|
||||
attribute float aSize;
|
||||
attribute vec3 aColor;
|
||||
attribute float aHash;
|
||||
uniform float uPixelScale;
|
||||
uniform float uTime;
|
||||
uniform float uHueJitter;
|
||||
varying vec3 vColor;
|
||||
varying float vPulse;
|
||||
void main() {
|
||||
// Per-particle hue rotation: small offset around the state color, signed
|
||||
// by hash so the cluster spreads in both directions on the colour wheel.
|
||||
vec3 hsv = rgb2hsv(aColor);
|
||||
hsv.x = fract(hsv.x + (aHash - 0.5) * uHueJitter);
|
||||
vColor = hsv2rgb(hsv);
|
||||
|
||||
float phase = aHash * 6.2831853;
|
||||
vPulse = 1.0 + 0.15 * sin(uTime * 0.9 + phase);
|
||||
vec4 mv = modelViewMatrix * vec4(position, 1.0);
|
||||
gl_PointSize = aSize * (uPixelScale / max(-mv.z, 1.0));
|
||||
gl_Position = projectionMatrix * mv;
|
||||
}
|
||||
`;
|
||||
|
||||
const FRAG_SHADER = /* glsl */ `
|
||||
varying vec3 vColor;
|
||||
varying float vPulse;
|
||||
void main() {
|
||||
vec2 d = gl_PointCoord - vec2(0.5);
|
||||
float r2 = dot(d, d);
|
||||
if (r2 > 0.25) discard;
|
||||
// Punchy dot: tight core, very faint halo. Falloff exponents tuned so
|
||||
// the dot reads as a pinpoint at typical camera distance — bloom does
|
||||
// the rest of the visual work.
|
||||
float core = exp(-r2 * 36.0);
|
||||
float halo = exp(-r2 * 7.0) * 0.10;
|
||||
float a = core + halo;
|
||||
gl_FragColor = vec4(vColor * (0.55 + 0.45 * core) * vPulse, a);
|
||||
}
|
||||
`;
|
||||
|
||||
/**
|
||||
* Renders all Engrams of a galaxy as a single glowing point cloud.
|
||||
* Each Engram is one vertex with per-vertex color and size; the shader
|
||||
* paints it as a soft additive disc.
|
||||
*
|
||||
* Stage 1: positions arrive at ~20 Hz from a binary WS frame; colors are
|
||||
* static (everyone IDLE). Per-instance state changes will arrive in Stage 4.
|
||||
*/
|
||||
export class EngramMesh {
|
||||
private readonly points: THREE.Points;
|
||||
private readonly material: THREE.ShaderMaterial;
|
||||
private readonly positionAttr: THREE.BufferAttribute;
|
||||
private readonly colorAttr: THREE.BufferAttribute;
|
||||
private readonly sizeAttr: THREE.BufferAttribute;
|
||||
private readonly hashAttr: THREE.BufferAttribute;
|
||||
private readonly capacity: number;
|
||||
/** Highest instance_idx + 1 seen so far. Bounds the draw range. */
|
||||
private maxIdx = 0;
|
||||
/** Optional hook fired when an engram's base color is set/updated. The
|
||||
* trail renderer subscribes so head + tail share the same colour. */
|
||||
public onColorAssigned: ((idx: number, r: number, g: number, b: number) => void) | null = null;
|
||||
/** Reverse lookup: instance_idx → engram UUID. Filled on upsert; used by
|
||||
* the click-picker to map a raycast hit back to an engram id. */
|
||||
public readonly idxToId: string[] = [];
|
||||
/** Forward lookup: engram UUID → instance_idx. Mirrors `idxToId` so live
|
||||
* state-change events (which carry the UUID, not the slot index) can find
|
||||
* the right vertex to re-paint. */
|
||||
private readonly idToIdx = new Map<string, number>();
|
||||
|
||||
/** Expose the underlying `THREE.Points` so the scene can raycast against it. */
|
||||
pointsObject(): THREE.Points {
|
||||
return this.points;
|
||||
}
|
||||
|
||||
constructor(scene: THREE.Scene, capacity = 5000) {
|
||||
this.capacity = capacity;
|
||||
|
||||
const geom = new THREE.BufferGeometry();
|
||||
this.positionAttr = new THREE.BufferAttribute(new Float32Array(capacity * 3), 3);
|
||||
this.colorAttr = new THREE.BufferAttribute(new Float32Array(capacity * 3), 3);
|
||||
this.sizeAttr = new THREE.BufferAttribute(new Float32Array(capacity), 1);
|
||||
// Per-particle random hash in [0, 1), used to phase-shift the brightness
|
||||
// pulse so the cluster doesn't blink in unison. Filled lazily on
|
||||
// upsert so engrams always have a stable hash for their lifetime.
|
||||
this.hashAttr = new THREE.BufferAttribute(new Float32Array(capacity), 1);
|
||||
this.positionAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
this.colorAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
this.sizeAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
geom.setAttribute("position", this.positionAttr);
|
||||
geom.setAttribute("aColor", this.colorAttr);
|
||||
geom.setAttribute("aSize", this.sizeAttr);
|
||||
geom.setAttribute("aHash", this.hashAttr);
|
||||
geom.setDrawRange(0, 0);
|
||||
// Set a permanent oversized bounding sphere. Without this, three.js
|
||||
// computes one once based on the initial all-zero positions (radius 0)
|
||||
// and Points.raycast() short-circuits — every click misses. Recomputing
|
||||
// per frame is expensive; a giant fixed sphere always passes the early
|
||||
// reject and the per-vertex test then runs normally.
|
||||
geom.boundingSphere = new THREE.Sphere(new THREE.Vector3(0, 0, 0), 100_000);
|
||||
|
||||
this.material = new THREE.ShaderMaterial({
|
||||
vertexShader: VERT_SHADER,
|
||||
fragmentShader: FRAG_SHADER,
|
||||
transparent: true,
|
||||
depthWrite: false,
|
||||
depthTest: true,
|
||||
blending: THREE.AdditiveBlending,
|
||||
uniforms: {
|
||||
// Tunable. Larger = bigger dots. Bumped from 1500 → 2800 so engrams
|
||||
// are clearly readable as moving dots during their in-hole flight,
|
||||
// not just as bloom smears.
|
||||
uPixelScale: { value: 2800.0 },
|
||||
// Seconds since scene start; updated by `tick()` from the animation loop.
|
||||
uTime: { value: 0.0 },
|
||||
// Hue rotation amplitude in [0..1]. 0.18 ≈ ±32° around the state hue.
|
||||
uHueJitter: { value: 0.18 },
|
||||
},
|
||||
});
|
||||
|
||||
this.points = new THREE.Points(geom, this.material);
|
||||
// Positions update faster than three.js can compute bounds; skip culling.
|
||||
this.points.frustumCulled = false;
|
||||
scene.add(this.points);
|
||||
}
|
||||
|
||||
/** Advance the shader's clock so the breathing pulse animates. */
|
||||
tick(timeSeconds: number): void {
|
||||
this.material.uniforms.uTime.value = timeSeconds;
|
||||
}
|
||||
|
||||
applyHello(engrams: EngramSnapshot[]): void {
|
||||
for (const e of engrams) this.upsertEngram(e);
|
||||
}
|
||||
|
||||
upsertEngram(e: EngramSnapshot): void {
|
||||
const idx = e.instance_idx;
|
||||
if (idx >= this.capacity) {
|
||||
console.warn(`engram instance_idx ${idx} exceeds capacity ${this.capacity}`);
|
||||
return;
|
||||
}
|
||||
const color = STATE_COLOR[e.state] ?? STATE_COLOR.idle;
|
||||
const colorArr = this.colorAttr.array as Float32Array;
|
||||
colorArr[idx * 3] = color[0];
|
||||
colorArr[idx * 3 + 1] = color[1];
|
||||
colorArr[idx * 3 + 2] = color[2];
|
||||
this.colorAttr.needsUpdate = true;
|
||||
this.onColorAssigned?.(idx, color[0], color[1], color[2]);
|
||||
|
||||
const sizeArr = this.sizeAttr.array as Float32Array;
|
||||
// Server gives e.size = 1.0 in Stage 1+. The base value is small so the
|
||||
// dots read as pinpoints (combined with bloom for the halo). Per-engram
|
||||
// size will diverge once federation lands (Stage 4+).
|
||||
sizeArr[idx] = Math.max(0.6, e.size * 0.8);
|
||||
this.sizeAttr.needsUpdate = true;
|
||||
|
||||
// Per-particle hash: only set on first upsert for this slot, so the
|
||||
// pulse phase stays stable across re-upserts (e.g. state changes).
|
||||
const hashArr = this.hashAttr.array as Float32Array;
|
||||
if (hashArr[idx] === 0) {
|
||||
hashArr[idx] = Math.random() || 0.5;
|
||||
this.hashAttr.needsUpdate = true;
|
||||
}
|
||||
|
||||
const posArr = this.positionAttr.array as Float32Array;
|
||||
posArr[idx * 3] = e.position[0];
|
||||
posArr[idx * 3 + 1] = e.position[1];
|
||||
posArr[idx * 3 + 2] = e.position[2];
|
||||
this.positionAttr.needsUpdate = true;
|
||||
|
||||
if (idx + 1 > this.maxIdx) this.maxIdx = idx + 1;
|
||||
this.points.geometry.setDrawRange(0, this.maxIdx);
|
||||
this.idxToId[idx] = e.id;
|
||||
this.idToIdx.set(e.id, idx);
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-paint the dot for an existing engram when its lifecycle state
|
||||
* changes (Stage 5: responder lights up to Conversing while answering a
|
||||
* query, Query-Engram transitions Searching→Memorize at completion).
|
||||
* Silently ignored if the engram isn't known yet — state-change events
|
||||
* for unfamiliar ids can race ahead of the corresponding `engram_created`
|
||||
* over the WS bus during a reconnect window.
|
||||
*/
|
||||
setStateById(id: string, state: string): void {
|
||||
const idx = this.idToIdx.get(id);
|
||||
if (idx === undefined) return;
|
||||
const color = STATE_COLOR[state] ?? STATE_COLOR.idle;
|
||||
const colorArr = this.colorAttr.array as Float32Array;
|
||||
colorArr[idx * 3] = color[0];
|
||||
colorArr[idx * 3 + 1] = color[1];
|
||||
colorArr[idx * 3 + 2] = color[2];
|
||||
this.colorAttr.needsUpdate = true;
|
||||
this.onColorAssigned?.(idx, color[0], color[1], color[2]);
|
||||
}
|
||||
|
||||
applyPositionFrame(frame: PositionFrame): void {
|
||||
const n = Math.min(frame.n, this.capacity);
|
||||
const posArr = this.positionAttr.array as Float32Array;
|
||||
posArr.set(frame.positions.subarray(0, n * 3), 0);
|
||||
this.positionAttr.needsUpdate = true;
|
||||
if (n > this.maxIdx) this.maxIdx = n;
|
||||
this.points.geometry.setDrawRange(0, this.maxIdx);
|
||||
}
|
||||
|
||||
count(): number {
|
||||
return this.maxIdx;
|
||||
}
|
||||
}
|
||||
140
web/src/engram_trails.ts
Normal file
140
web/src/engram_trails.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
import * as THREE from "three";
|
||||
|
||||
import type { PositionFrame } from "./ws_client";
|
||||
|
||||
/**
|
||||
* Comet-style trails behind each engram.
|
||||
*
|
||||
* Each engram is rendered as a single line segment from
|
||||
* `position - velocity * tailScale` → `position`
|
||||
* where `velocity` is computed from the delta between consecutive position
|
||||
* frames. So the tail is *long* when an engram is moving fast (e.g. the
|
||||
* fountain phase right after birth) and *short* when it's drifting in the
|
||||
* tube. The tail vertex is transparent, the head vertex is opaque, and
|
||||
* additive blending + bloom in the post-pipeline gives the comet glow.
|
||||
*
|
||||
* One line segment per engram → 2 vertices each → very cheap.
|
||||
*/
|
||||
const TAIL_SCALE = 1.0; // multiplied onto inter-frame delta
|
||||
|
||||
const VERT_SHADER = /* glsl */ `
|
||||
attribute float aAlpha;
|
||||
varying vec3 vColor;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vColor = color;
|
||||
vAlpha = aAlpha;
|
||||
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
||||
}
|
||||
`;
|
||||
|
||||
const FRAG_SHADER = /* glsl */ `
|
||||
varying vec3 vColor;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
gl_FragColor = vec4(vColor, vAlpha);
|
||||
}
|
||||
`;
|
||||
|
||||
export class EngramTrails {
|
||||
private readonly mesh: THREE.LineSegments;
|
||||
private readonly positionAttr: THREE.BufferAttribute;
|
||||
private readonly colorAttr: THREE.BufferAttribute;
|
||||
private readonly alphaAttr: THREE.BufferAttribute;
|
||||
/** Last frame's position per engram, used to derive velocity. */
|
||||
private readonly previousPositions: Float32Array;
|
||||
/** Whether each engram has had at least one previous frame stored. */
|
||||
private readonly hasPrevious: Uint8Array;
|
||||
private readonly capacity: number;
|
||||
private maxIdx = 0;
|
||||
|
||||
constructor(scene: THREE.Scene, capacity = 5000) {
|
||||
this.capacity = capacity;
|
||||
this.previousPositions = new Float32Array(capacity * 3);
|
||||
this.hasPrevious = new Uint8Array(capacity);
|
||||
|
||||
// Two vertices per engram: index 2*i = tail, 2*i+1 = head.
|
||||
const vertexCount = capacity * 2;
|
||||
const geom = new THREE.BufferGeometry();
|
||||
this.positionAttr = new THREE.BufferAttribute(new Float32Array(vertexCount * 3), 3);
|
||||
this.colorAttr = new THREE.BufferAttribute(new Float32Array(vertexCount * 3), 3);
|
||||
this.alphaAttr = new THREE.BufferAttribute(new Float32Array(vertexCount), 1);
|
||||
this.positionAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
this.colorAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
this.alphaAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
geom.setAttribute("position", this.positionAttr);
|
||||
geom.setAttribute("color", this.colorAttr);
|
||||
geom.setAttribute("aAlpha", this.alphaAttr);
|
||||
geom.setDrawRange(0, 0);
|
||||
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
vertexShader: VERT_SHADER,
|
||||
fragmentShader: FRAG_SHADER,
|
||||
vertexColors: true,
|
||||
transparent: true,
|
||||
depthWrite: false,
|
||||
depthTest: true,
|
||||
blending: THREE.AdditiveBlending,
|
||||
});
|
||||
|
||||
this.mesh = new THREE.LineSegments(geom, mat);
|
||||
this.mesh.frustumCulled = false;
|
||||
scene.add(this.mesh);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the colour for one engram's trail. Both vertices share the colour;
|
||||
* the gradient is implemented via per-vertex alpha (tail = 0, head = 1).
|
||||
* Called by the EngramMesh whenever it learns about a new/updated engram.
|
||||
*/
|
||||
setEngramColor(idx: number, r: number, g: number, b: number): void {
|
||||
if (idx >= this.capacity) return;
|
||||
const colorArr = this.colorAttr.array as Float32Array;
|
||||
const alphaArr = this.alphaAttr.array as Float32Array;
|
||||
// Tail vertex.
|
||||
colorArr[idx * 6] = r;
|
||||
colorArr[idx * 6 + 1] = g;
|
||||
colorArr[idx * 6 + 2] = b;
|
||||
alphaArr[idx * 2] = 0.0;
|
||||
// Head vertex.
|
||||
colorArr[idx * 6 + 3] = r;
|
||||
colorArr[idx * 6 + 4] = g;
|
||||
colorArr[idx * 6 + 5] = b;
|
||||
alphaArr[idx * 2 + 1] = 0.85;
|
||||
this.colorAttr.needsUpdate = true;
|
||||
this.alphaAttr.needsUpdate = true;
|
||||
}
|
||||
|
||||
applyPositionFrame(frame: PositionFrame): void {
|
||||
const n = Math.min(frame.n, this.capacity);
|
||||
const posArr = this.positionAttr.array as Float32Array;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const cx = frame.positions[i * 3];
|
||||
const cy = frame.positions[i * 3 + 1];
|
||||
const cz = frame.positions[i * 3 + 2];
|
||||
let dx = 0, dy = 0, dz = 0;
|
||||
if (this.hasPrevious[i] === 1) {
|
||||
dx = cx - this.previousPositions[i * 3];
|
||||
dy = cy - this.previousPositions[i * 3 + 1];
|
||||
dz = cz - this.previousPositions[i * 3 + 2];
|
||||
} else {
|
||||
this.hasPrevious[i] = 1;
|
||||
}
|
||||
// Tail vertex (behind the engram, opposite the velocity vector).
|
||||
posArr[i * 6] = cx - dx * TAIL_SCALE;
|
||||
posArr[i * 6 + 1] = cy - dy * TAIL_SCALE;
|
||||
posArr[i * 6 + 2] = cz - dz * TAIL_SCALE;
|
||||
// Head vertex (current position).
|
||||
posArr[i * 6 + 3] = cx;
|
||||
posArr[i * 6 + 4] = cy;
|
||||
posArr[i * 6 + 5] = cz;
|
||||
// Roll the previous-position buffer forward.
|
||||
this.previousPositions[i * 3] = cx;
|
||||
this.previousPositions[i * 3 + 1] = cy;
|
||||
this.previousPositions[i * 3 + 2] = cz;
|
||||
}
|
||||
this.positionAttr.needsUpdate = true;
|
||||
if (n > this.maxIdx) this.maxIdx = n;
|
||||
this.mesh.geometry.setDrawRange(0, this.maxIdx * 2);
|
||||
}
|
||||
}
|
||||
179
web/src/inspector.ts
Normal file
179
web/src/inspector.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
import * as THREE from "three";
|
||||
|
||||
import type { EngramMesh } from "./engram_mesh";
|
||||
|
||||
/**
|
||||
* Click-to-inspect: raycast the mouse into the engram point-cloud, fetch
|
||||
* `/api/galaxy/:gid/engrams/:eid` for the picked engram, and render the
|
||||
* result in the side panel. Re-clicking an already-open engram refreshes
|
||||
* the panel (handy because introspection arrives async after birth).
|
||||
*/
|
||||
|
||||
type EngramDetail = {
|
||||
id: string;
|
||||
instance_idx: number;
|
||||
position: [number, number, number];
|
||||
size: number;
|
||||
state: string;
|
||||
age: number;
|
||||
manifest: { kind: string; content?: string } | null;
|
||||
slate_dim: number | null;
|
||||
slate_norm: number | null;
|
||||
introspection: {
|
||||
taxonomy: string[];
|
||||
goals: string[];
|
||||
open_questions: string[];
|
||||
};
|
||||
memories: Array<{
|
||||
at_ms: number;
|
||||
kind: { kind: string; with?: string; weight?: number };
|
||||
}>;
|
||||
};
|
||||
|
||||
// World-space radius around each engram vertex that counts as a "hit" when
|
||||
// raycasting. Has to be generous because the rendered dot is enlarged by
|
||||
// bloom + sprite scaling — the user expects to be able to click anywhere
|
||||
// the dot *appears*, not the exact 1-unit vertex position.
|
||||
const PICK_THRESHOLD = 12;
|
||||
// Pixels of cursor movement between mousedown and mouseup that still count
|
||||
// as a "click" rather than a drag. Trackpads in particular need slack.
|
||||
const CLICK_DRAG_TOLERANCE_PX = 8;
|
||||
|
||||
const panel = document.getElementById("inspector");
|
||||
const idEl = document.getElementById("inspector-id");
|
||||
const stateEl = document.getElementById("inspector-state");
|
||||
const manifestEl = document.getElementById("inspector-manifest");
|
||||
const taxonomyEl = document.getElementById("inspector-taxonomy");
|
||||
const goalsEl = document.getElementById("inspector-goals");
|
||||
const questionsEl = document.getElementById("inspector-questions");
|
||||
const memoriesEl = document.getElementById("inspector-memories");
|
||||
const closeBtn = document.getElementById("inspector-close");
|
||||
|
||||
closeBtn?.addEventListener("click", () => {
|
||||
panel?.classList.remove("open");
|
||||
});
|
||||
|
||||
let lastPickedId: string | null = null;
|
||||
|
||||
function shortId(id: string): string {
|
||||
return id.length > 12 ? `${id.slice(0, 8)}…${id.slice(-4)}` : id;
|
||||
}
|
||||
|
||||
function bullets(items: string[]): string {
|
||||
if (!items || items.length === 0) return `<span class="muted">…awaiting LLM…</span>`;
|
||||
return `<ul>${items.map((s) => `<li>${escapeHtml(s)}</li>`).join("")}</ul>`;
|
||||
}
|
||||
|
||||
function escapeHtml(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """);
|
||||
}
|
||||
|
||||
function fmtMemories(mems: EngramDetail["memories"]): string {
|
||||
if (!mems || mems.length === 0) return `<span class="muted">none yet</span>`;
|
||||
const items = mems.map((m) => {
|
||||
const t = (m.at_ms / 1000).toFixed(1) + "s";
|
||||
let body: string;
|
||||
switch (m.kind.kind) {
|
||||
case "born":
|
||||
body = "born";
|
||||
break;
|
||||
case "introspected":
|
||||
body = "introspected (LLM)";
|
||||
break;
|
||||
case "synapse_formed":
|
||||
body = `↔ ${shortId(m.kind.with ?? "")} (sim ${(m.kind.weight ?? 0).toFixed(2)})`;
|
||||
break;
|
||||
default:
|
||||
body = m.kind.kind;
|
||||
}
|
||||
return `<li><span class="muted">+${t}</span> ${escapeHtml(body)}</li>`;
|
||||
});
|
||||
return `<ul>${items.join("")}</ul>`;
|
||||
}
|
||||
|
||||
function render(d: EngramDetail): void {
|
||||
if (!panel) return;
|
||||
if (idEl) idEl.textContent = d.id;
|
||||
if (stateEl) stateEl.textContent = `${d.state} · age ${d.age} · slot ${d.instance_idx}`;
|
||||
if (manifestEl) {
|
||||
manifestEl.textContent = d.manifest?.content ?? "(synthetic seed — no manifest)";
|
||||
}
|
||||
if (taxonomyEl) taxonomyEl.innerHTML = bullets(d.introspection?.taxonomy ?? []);
|
||||
if (goalsEl) goalsEl.innerHTML = bullets(d.introspection?.goals ?? []);
|
||||
if (questionsEl) questionsEl.innerHTML = bullets(d.introspection?.open_questions ?? []);
|
||||
if (memoriesEl) memoriesEl.innerHTML = fmtMemories(d.memories ?? []);
|
||||
panel.classList.add("open");
|
||||
}
|
||||
|
||||
async function fetchAndRender(galaxyId: string, engramId: string): Promise<void> {
|
||||
try {
|
||||
const res = await fetch(`/api/galaxy/${galaxyId}/engrams/${engramId}`);
|
||||
if (!res.ok) {
|
||||
console.warn(`engram fetch ${res.status}`);
|
||||
return;
|
||||
}
|
||||
const detail = (await res.json()) as EngramDetail;
|
||||
render(detail);
|
||||
} catch (e) {
|
||||
console.warn("engram fetch failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire up click-picking on the canvas. The engram point cloud is already in
|
||||
* the scene; we raycast against it and look up the engram id from the hit
|
||||
* vertex index.
|
||||
*/
|
||||
export function attachInspector(opts: {
|
||||
canvas: HTMLCanvasElement;
|
||||
camera: THREE.Camera;
|
||||
engrams: EngramMesh;
|
||||
galaxyId: string;
|
||||
}): void {
|
||||
const raycaster = new THREE.Raycaster();
|
||||
raycaster.params.Points = { threshold: PICK_THRESHOLD };
|
||||
const mouse = new THREE.Vector2();
|
||||
let downX = 0, downY = 0;
|
||||
|
||||
// Use mousedown + mouseup with a tiny movement tolerance so dragging the
|
||||
// OrbitControls doesn't trigger a pick.
|
||||
opts.canvas.addEventListener("mousedown", (ev) => {
|
||||
downX = ev.clientX;
|
||||
downY = ev.clientY;
|
||||
});
|
||||
opts.canvas.addEventListener("mouseup", (ev) => {
|
||||
const moved = Math.hypot(ev.clientX - downX, ev.clientY - downY);
|
||||
if (moved > CLICK_DRAG_TOLERANCE_PX) return;
|
||||
const rect = opts.canvas.getBoundingClientRect();
|
||||
mouse.x = ((ev.clientX - rect.left) / rect.width) * 2 - 1;
|
||||
mouse.y = -((ev.clientY - rect.top) / rect.height) * 2 + 1;
|
||||
raycaster.setFromCamera(mouse, opts.camera);
|
||||
const hits = raycaster.intersectObject(opts.engrams.pointsObject(), false);
|
||||
console.debug(`[inspector] click ndc=(${mouse.x.toFixed(2)},${mouse.y.toFixed(2)}) hits=${hits.length}`);
|
||||
if (hits.length === 0) return;
|
||||
// Pick the closest hit.
|
||||
const hit = hits[0];
|
||||
const idx = hit.index;
|
||||
if (idx === undefined) return;
|
||||
const id = opts.engrams.idxToId[idx];
|
||||
if (!id) {
|
||||
console.warn(`[inspector] hit at idx ${idx} has no engram id`);
|
||||
return;
|
||||
}
|
||||
console.debug(`[inspector] picked engram ${id}`);
|
||||
lastPickedId = id;
|
||||
void fetchAndRender(opts.galaxyId, id);
|
||||
});
|
||||
|
||||
// Light auto-refresh — re-fetch the currently-open engram every 3s so the
|
||||
// user sees introspection populate and memories grow without re-clicking.
|
||||
setInterval(() => {
|
||||
if (!panel?.classList.contains("open")) return;
|
||||
if (!lastPickedId) return;
|
||||
void fetchAndRender(opts.galaxyId, lastPickedId);
|
||||
}, 3000);
|
||||
}
|
||||
200
web/src/main.ts
Normal file
200
web/src/main.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import { startScene } from "./scene";
|
||||
|
||||
const canvas = document.getElementById("stage") as HTMLCanvasElement;
|
||||
const healthEl = document.getElementById("hud-health");
|
||||
const lmEl = document.getElementById("hud-lm");
|
||||
const galaxyEl = document.getElementById("hud-galaxy");
|
||||
const countEl = document.getElementById("hud-count");
|
||||
const seedBtn = document.getElementById("seed-btn") as HTMLButtonElement | null;
|
||||
const seedNInput = document.getElementById("seed-n") as HTMLInputElement | null;
|
||||
const ingestBtn = document.getElementById("ingest-btn") as HTMLButtonElement | null;
|
||||
const ingestText = document.getElementById("ingest-text") as HTMLTextAreaElement | null;
|
||||
const torusEl = document.getElementById("hud-torus");
|
||||
const synapsesEl = document.getElementById("hud-synapses");
|
||||
const ticksEl = document.getElementById("hud-ticks");
|
||||
const llmQueueEl = document.getElementById("hud-llm-queue");
|
||||
const fpsEl = document.getElementById("hud-fps");
|
||||
const resizeBtn = document.getElementById("resize-btn") as HTMLButtonElement | null;
|
||||
const majorInput = document.getElementById("resize-major") as HTMLInputElement | null;
|
||||
const minorInput = document.getElementById("resize-minor") as HTMLInputElement | null;
|
||||
|
||||
async function fetchJson<T>(url: string, init?: RequestInit): Promise<T> {
|
||||
const r = await fetch(url, init);
|
||||
if (!r.ok) {
|
||||
const body = await r.text();
|
||||
throw new Error(`${url} → HTTP ${r.status}: ${body}`);
|
||||
}
|
||||
const body = await r.text();
|
||||
if (body.length === 0) throw new Error(`${url} → empty body`);
|
||||
try {
|
||||
return JSON.parse(body) as T;
|
||||
} catch (e) {
|
||||
throw new Error(`${url} → invalid JSON: ${String((e as Error).message)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function showFatal(msg: string): void {
|
||||
if (galaxyEl) galaxyEl.textContent = msg;
|
||||
if (healthEl) healthEl.textContent = msg;
|
||||
console.error(msg);
|
||||
}
|
||||
|
||||
type HealthBody = {
|
||||
status: string;
|
||||
stage: number;
|
||||
lm_studio?: {
|
||||
reachable: boolean;
|
||||
chat_model_loaded: boolean;
|
||||
embedding_model_loaded: boolean;
|
||||
configured_chat_model: string;
|
||||
configured_embedding_model: string;
|
||||
};
|
||||
};
|
||||
|
||||
function renderHealth(h: HealthBody): void {
|
||||
if (healthEl) healthEl.textContent = `${h.status} (stage ${h.stage})`;
|
||||
if (lmEl) {
|
||||
if (!h.lm_studio) {
|
||||
lmEl.textContent = "—";
|
||||
} else if (!h.lm_studio.reachable) {
|
||||
lmEl.textContent = "unreachable";
|
||||
} else {
|
||||
const chat = h.lm_studio.chat_model_loaded ? "✓ chat" : "✗ chat";
|
||||
const emb = h.lm_studio.embedding_model_loaded ? "✓ embed" : "✗ embed";
|
||||
lmEl.textContent = `${chat} · ${emb}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
fetch("/healthz")
|
||||
.then((r) => (r.ok ? (r.json() as Promise<HealthBody>) : Promise.reject(new Error(`status ${r.status}`))))
|
||||
.then(renderHealth)
|
||||
.catch((err) => {
|
||||
if (healthEl) healthEl.textContent = `unreachable (${String(err.message ?? err)})`;
|
||||
});
|
||||
|
||||
let galaxies: Array<{ id: string; name: string }>;
|
||||
try {
|
||||
galaxies = await fetchJson<Array<{ id: string; name: string }>>("/api/galaxy");
|
||||
} catch (e) {
|
||||
showFatal(`backend unreachable — start \`cargo run\`. (${String((e as Error).message)})`);
|
||||
return;
|
||||
}
|
||||
if (galaxies.length === 0) {
|
||||
showFatal("no galaxies — server should auto-create one on boot");
|
||||
return;
|
||||
}
|
||||
const galaxy = galaxies[0];
|
||||
|
||||
startScene(canvas, galaxy.id, {
|
||||
onCount: (n) => { if (countEl) countEl.textContent = String(n); },
|
||||
onGalaxyName: (name) => { if (galaxyEl) galaxyEl.textContent = name; },
|
||||
onTorus: (majorR, minorR) => {
|
||||
if (torusEl) torusEl.textContent = `R=${majorR.toFixed(0)} r=${minorR.toFixed(0)}`;
|
||||
if (majorInput && document.activeElement !== majorInput) majorInput.value = String(majorR);
|
||||
if (minorInput && document.activeElement !== minorInput) minorInput.value = String(minorR);
|
||||
},
|
||||
onSynapses: (n) => { if (synapsesEl) synapsesEl.textContent = String(n); },
|
||||
onFps: (fps) => { if (fpsEl) fpsEl.textContent = fps.toFixed(0); },
|
||||
});
|
||||
|
||||
// Poll /api/stats once a second for sim ticks/sec + LLM queue depth.
|
||||
// The HUD numbers it drives (sim, llm queue) are server-side state, so
|
||||
// the browser can't compute them locally — keep this loop lightweight
|
||||
// and silent on transient fetch errors so a brief network hiccup doesn't
|
||||
// pollute the console.
|
||||
type StatsResponse = {
|
||||
ticks_per_sec: number;
|
||||
engrams_total: number;
|
||||
synapses_total: number;
|
||||
galaxies: number;
|
||||
llm_queue_depth: number;
|
||||
llm_parallel_cap: number;
|
||||
};
|
||||
setInterval(async () => {
|
||||
try {
|
||||
const r = await fetch("/api/stats");
|
||||
if (!r.ok) return;
|
||||
const s = (await r.json()) as StatsResponse;
|
||||
if (ticksEl) ticksEl.textContent = `${s.ticks_per_sec}/s`;
|
||||
if (llmQueueEl) {
|
||||
llmQueueEl.textContent = `${s.llm_queue_depth} / ${s.llm_parallel_cap}`;
|
||||
}
|
||||
} catch {
|
||||
// ignore — server might be restarting
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
if (seedBtn && seedNInput) {
|
||||
seedBtn.addEventListener("click", async () => {
|
||||
const n = Math.max(1, Math.min(5000, parseInt(seedNInput.value, 10) || 200));
|
||||
seedBtn.disabled = true;
|
||||
try {
|
||||
await fetch(`/api/galaxy/${galaxy.id}/seed?n=${n}`, { method: "POST" });
|
||||
} finally {
|
||||
seedBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (resizeBtn && majorInput && minorInput) {
|
||||
resizeBtn.addEventListener("click", async () => {
|
||||
const major = parseFloat(majorInput.value);
|
||||
const minor = parseFloat(minorInput.value);
|
||||
if (!isFinite(major) || !isFinite(minor)) {
|
||||
alert("major_radius and minor_radius must be numbers");
|
||||
return;
|
||||
}
|
||||
resizeBtn.disabled = true;
|
||||
try {
|
||||
const res = await fetch(`/api/galaxy/${galaxy.id}/resize`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ major_radius: major, minor_radius: minor }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
alert(`resize failed: HTTP ${res.status}\n${body}`);
|
||||
}
|
||||
} catch (e) {
|
||||
alert(`resize failed: ${String((e as Error).message)}`);
|
||||
} finally {
|
||||
resizeBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (ingestBtn && ingestText) {
|
||||
ingestBtn.addEventListener("click", async () => {
|
||||
const lines = ingestText.value
|
||||
.split(/\r?\n/)
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l.length > 0);
|
||||
if (lines.length === 0) return;
|
||||
ingestBtn.disabled = true;
|
||||
const before = ingestBtn.textContent;
|
||||
ingestBtn.textContent = `embedding ${lines.length}…`;
|
||||
try {
|
||||
const res = await fetch(`/api/galaxy/${galaxy.id}/ingest`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ texts: lines }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
alert(`ingest failed: HTTP ${res.status}\n${body}`);
|
||||
return;
|
||||
}
|
||||
ingestText.value = "";
|
||||
} catch (e) {
|
||||
alert(`ingest failed: ${String((e as Error).message)}`);
|
||||
} finally {
|
||||
ingestBtn.disabled = false;
|
||||
ingestBtn.textContent = before;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
bootstrap().catch((e) => console.error("bootstrap failed", e));
|
||||
128
web/src/query_panel.ts
Normal file
128
web/src/query_panel.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Stage 5 query panel.
|
||||
*
|
||||
* Owns the bottom-left text input + answer area + SSE consumer. Submission
|
||||
* does `POST /api/galaxy/:id/query` and immediately opens an EventSource on
|
||||
* `/api/query/:id/stream`; the running answer streams in as the integrator
|
||||
* re-runs. Closes the EventSource on terminal status (`done` / `failed`).
|
||||
*
|
||||
* The visual feedback in the 3D scene (responder light-up, Query-Engram
|
||||
* appearing at the donut center) lives in `scene.ts` — driven by separate
|
||||
* WS events. This module only manages the side panel.
|
||||
*/
|
||||
|
||||
import type { QueryStatus } from "./ws_client";
|
||||
|
||||
type QueryUpdate = {
|
||||
query_id: string;
|
||||
version: number;
|
||||
status: QueryStatus;
|
||||
answer: string;
|
||||
responder_count: number;
|
||||
};
|
||||
|
||||
export type QueryPanelOptions = {
|
||||
galaxyId: string;
|
||||
};
|
||||
|
||||
const TERMINAL: QueryStatus[] = ["done", "failed"];
|
||||
|
||||
export function attachQueryPanel(opts: QueryPanelOptions): void {
|
||||
const form = document.getElementById("query-form") as HTMLFormElement | null;
|
||||
const input = document.getElementById("query-input") as HTMLInputElement | null;
|
||||
const submit = document.getElementById("query-submit") as HTMLButtonElement | null;
|
||||
const answerEl = document.getElementById("query-answer");
|
||||
const statusEl = document.getElementById("query-status");
|
||||
if (!form || !input || !submit || !answerEl || !statusEl) {
|
||||
console.warn("query panel markup missing — skipping wireup");
|
||||
return;
|
||||
}
|
||||
|
||||
let activeStream: EventSource | null = null;
|
||||
|
||||
function setBusy(busy: boolean): void {
|
||||
submit!.disabled = busy;
|
||||
input!.disabled = busy;
|
||||
}
|
||||
|
||||
function renderUpdate(u: QueryUpdate): void {
|
||||
answerEl!.textContent = u.answer || "(thinking…)";
|
||||
const phase =
|
||||
u.status === "pending"
|
||||
? "broadcasting"
|
||||
: u.status === "responding"
|
||||
? `synthesising · ${u.responder_count} perspective${u.responder_count === 1 ? "" : "s"}`
|
||||
: u.status === "integrating"
|
||||
? "integrating"
|
||||
: u.status === "done"
|
||||
? `done · ${u.responder_count} perspective${u.responder_count === 1 ? "" : "s"}`
|
||||
: "failed";
|
||||
statusEl!.textContent = `v${u.version} · ${phase}`;
|
||||
}
|
||||
|
||||
function closeStream(): void {
|
||||
if (activeStream) {
|
||||
activeStream.close();
|
||||
activeStream = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function submitQuery(text: string): Promise<void> {
|
||||
setBusy(true);
|
||||
closeStream();
|
||||
answerEl!.textContent = "(asking…)";
|
||||
statusEl!.textContent = "submitting";
|
||||
|
||||
let queryId: string;
|
||||
try {
|
||||
const res = await fetch(`/api/galaxy/${opts.galaxyId}/query`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ text }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`HTTP ${res.status}: ${body}`);
|
||||
}
|
||||
const body = (await res.json()) as { query_id: string };
|
||||
queryId = body.query_id;
|
||||
} catch (e) {
|
||||
statusEl!.textContent = "failed";
|
||||
answerEl!.textContent = `submit failed: ${(e as Error).message}`;
|
||||
setBusy(false);
|
||||
return;
|
||||
}
|
||||
|
||||
activeStream = new EventSource(`/api/query/${queryId}/stream`);
|
||||
activeStream.onmessage = (ev) => {
|
||||
try {
|
||||
const update = JSON.parse(ev.data) as QueryUpdate;
|
||||
renderUpdate(update);
|
||||
if (TERMINAL.includes(update.status)) {
|
||||
closeStream();
|
||||
setBusy(false);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("query SSE decode failed", err);
|
||||
}
|
||||
};
|
||||
activeStream.onerror = () => {
|
||||
// EventSource auto-reconnects on transient errors; but once the
|
||||
// server has finished and closed the channel, the reconnect loop
|
||||
// would just pile up 404s. The terminal-status check above already
|
||||
// closed the stream in the success path; this handler covers the
|
||||
// failure path where the channel never opened cleanly.
|
||||
if (activeStream && activeStream.readyState === EventSource.CLOSED) {
|
||||
statusEl!.textContent = "stream closed";
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
form.addEventListener("submit", (ev) => {
|
||||
ev.preventDefault();
|
||||
const text = input!.value.trim();
|
||||
if (!text) return;
|
||||
void submitQuery(text);
|
||||
});
|
||||
}
|
||||
227
web/src/scene.ts
Normal file
227
web/src/scene.ts
Normal file
@@ -0,0 +1,227 @@
|
||||
import * as THREE from "three";
|
||||
import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
|
||||
import { EffectComposer } from "three/examples/jsm/postprocessing/EffectComposer.js";
|
||||
import { RenderPass } from "three/examples/jsm/postprocessing/RenderPass.js";
|
||||
import { UnrealBloomPass } from "three/examples/jsm/postprocessing/UnrealBloomPass.js";
|
||||
import { OutputPass } from "three/examples/jsm/postprocessing/OutputPass.js";
|
||||
|
||||
import { EngramMesh } from "./engram_mesh";
|
||||
import { EngramTrails } from "./engram_trails";
|
||||
import { SynapseMesh } from "./synapse_mesh";
|
||||
import { attachInspector } from "./inspector";
|
||||
import { attachQueryPanel } from "./query_panel";
|
||||
import type { EngramSnapshot, EventHandlers, SynapseDto } from "./ws_client";
|
||||
import { connectGalaxy } from "./ws_client";
|
||||
|
||||
export type SceneHooks = {
|
||||
onCount: (n: number) => void;
|
||||
onGalaxyName: (name: string) => void;
|
||||
onTorus: (majorRadius: number, minorRadius: number) => void;
|
||||
onSynapses?: (count: number) => void;
|
||||
/** Rolling FPS, updated about once per second from inside the rAF loop. */
|
||||
onFps?: (fps: number) => void;
|
||||
};
|
||||
|
||||
// State-color table mirrored from EngramMesh; needed here so we can pre-paint
|
||||
// synapse endpoints when an engram first arrives.
|
||||
const STATE_COLOR: Record<string, [number, number, number]> = {
|
||||
idle: [1.0, 0.72, 0.42],
|
||||
searching: [1.0, 0.88, 0.40],
|
||||
conversing: [0.82, 0.64, 1.0],
|
||||
synthesizing: [0.43, 0.91, 0.72],
|
||||
memorize: [0.65, 0.85, 1.0],
|
||||
decaying: [1.0, 0.48, 0.48],
|
||||
deprecated: [0.49, 0.53, 0.58],
|
||||
};
|
||||
|
||||
/**
|
||||
* Scene: dark background, axis helper, wireframe torus boundary, engram
|
||||
* point-cloud fed by a WebSocket binary stream of position frames.
|
||||
*
|
||||
* Per the Topology Pivot, the world is a fixed solid donut; the wireframe
|
||||
* torus shows the boundary the engrams live inside.
|
||||
*/
|
||||
export function startScene(canvas: HTMLCanvasElement, galaxyId: string, hooks: SceneHooks): void {
|
||||
const renderer = new THREE.WebGLRenderer({ canvas, antialias: true });
|
||||
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
||||
renderer.setSize(window.innerWidth, window.innerHeight, false);
|
||||
// Render with sRGB output and ACES-style tone-mapping so the bloom-amplified
|
||||
// additive engrams don't clip to white. This makes the cinematic glow read
|
||||
// properly against the dark background.
|
||||
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
||||
renderer.toneMapping = THREE.ACESFilmicToneMapping;
|
||||
renderer.toneMappingExposure = 1.0;
|
||||
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x050608);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(
|
||||
55,
|
||||
window.innerWidth / window.innerHeight,
|
||||
0.1,
|
||||
10_000,
|
||||
);
|
||||
// Default view positions the camera above + behind the donut so it reads
|
||||
// immediately as a donut on first paint. Recomputed when we get the real
|
||||
// shape from `Hello`.
|
||||
camera.position.set(0, 180, 300);
|
||||
camera.lookAt(0, 0, 0);
|
||||
|
||||
const controls = new OrbitControls(camera, canvas);
|
||||
controls.enableDamping = true;
|
||||
controls.dampingFactor = 0.08;
|
||||
|
||||
scene.add(new THREE.AxesHelper(20));
|
||||
|
||||
// Torus boundary — the visible wall of the Space of Recollection. Built
|
||||
// with unit radii and scaled per (majorR, minorR) update so we can avoid
|
||||
// rebuilding geometry on every resize.
|
||||
const torusGeo = new THREE.TorusGeometry(1, 1, 16, 64);
|
||||
const torusMat = new THREE.MeshBasicMaterial({
|
||||
color: 0x344058,
|
||||
wireframe: true,
|
||||
transparent: true,
|
||||
opacity: 0.22,
|
||||
});
|
||||
const torusMesh = new THREE.Mesh(torusGeo, torusMat);
|
||||
// TorusGeometry sits in the XY plane (extending along z by tube radius).
|
||||
// Our spine is the circle in the z=0 plane, so no rotation needed.
|
||||
scene.add(torusMesh);
|
||||
|
||||
// Trails + synapses go in FIRST so they render *under* the engram dots —
|
||||
// when the head sits on top of a line endpoint, the dot occludes the join.
|
||||
const trails = new EngramTrails(scene);
|
||||
const synapses = new SynapseMesh(scene);
|
||||
const engrams = new EngramMesh(scene);
|
||||
// Forward each engram's base colour to the line renderers so endpoints
|
||||
// match the head dot's hue.
|
||||
engrams.onColorAssigned = (idx, r, g, b) => {
|
||||
trails.setEngramColor(idx, r, g, b);
|
||||
};
|
||||
|
||||
// Helper: register a fresh engram with the synapse mesh so any pending
|
||||
// synapse referencing it can be wired up. Re-applies state colour.
|
||||
function registerEngramForSynapses(snapshot: EngramSnapshot): void {
|
||||
const color = STATE_COLOR[snapshot.state] ?? STATE_COLOR.idle;
|
||||
synapses.registerEngram(snapshot.id, snapshot.instance_idx, color[0], color[1], color[2]);
|
||||
}
|
||||
|
||||
function applySynapse(s: SynapseDto): void {
|
||||
synapses.addSynapse(s.id, s.a, s.b, s.weight);
|
||||
hooks.onSynapses?.(synapses.count());
|
||||
}
|
||||
|
||||
// Post-processing: bloom for the cinematic glow. Tuned for additive
|
||||
// particle sources — low threshold (most particle pixels are bright
|
||||
// enough to bloom), moderate strength, small radius for crisp halos
|
||||
// rather than washed-out smear.
|
||||
const composer = new EffectComposer(renderer);
|
||||
composer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
||||
composer.setSize(window.innerWidth, window.innerHeight);
|
||||
composer.addPass(new RenderPass(scene, camera));
|
||||
const bloom = new UnrealBloomPass(
|
||||
new THREE.Vector2(window.innerWidth, window.innerHeight),
|
||||
0.4, // strength — restrained, lets the dots stay crisp
|
||||
0.5, // radius
|
||||
0.4, // threshold — only the brightest cores bloom (was 0.1, blew out)
|
||||
);
|
||||
composer.addPass(bloom);
|
||||
composer.addPass(new OutputPass());
|
||||
|
||||
function applyTorus(majorR: number, minorR: number): void {
|
||||
// Three.js's TorusGeometry with major=1, minor=1 produces a torus where
|
||||
// the tube radius and the major radius are both 1. Non-uniform scaling
|
||||
// breaks that — scaling x/y by `majorR` would also stretch the tube
|
||||
// cross-section. Easiest: rebuild geometry on each resize. This happens
|
||||
// rarely (initial Hello + explicit /resize calls).
|
||||
torusMesh.geometry.dispose();
|
||||
torusMesh.geometry = new THREE.TorusGeometry(majorR, minorR, 16, 96);
|
||||
// Fit camera so the donut is comfortably framed.
|
||||
// Lower height factor (0.30 vs 0.55) gives a more head-on view that
|
||||
// reads the donut shape better and shows the front portal clearly.
|
||||
const fitDist = (majorR + minorR) * 2.4;
|
||||
camera.position.set(0, fitDist * 0.30, fitDist);
|
||||
camera.lookAt(0, 0, 0);
|
||||
camera.far = Math.max(camera.far, fitDist * 6);
|
||||
camera.updateProjectionMatrix();
|
||||
hooks.onTorus(majorR, minorR);
|
||||
}
|
||||
|
||||
const handlers: EventHandlers = {
|
||||
onHello: (galaxy, list, helloSynapses) => {
|
||||
hooks.onGalaxyName(galaxy.name);
|
||||
torusMesh.position.set(galaxy.center[0], galaxy.center[1], galaxy.center[2]);
|
||||
applyTorus(galaxy.major_radius, galaxy.minor_radius);
|
||||
engrams.applyHello(list);
|
||||
// Register every engram with the synapse mesh BEFORE replaying synapses
|
||||
// so the (a, b) UUID lookups resolve immediately instead of going to
|
||||
// the pending queue.
|
||||
for (const e of list) registerEngramForSynapses(e);
|
||||
for (const s of helloSynapses) applySynapse(s);
|
||||
hooks.onCount(engrams.count());
|
||||
},
|
||||
onEngramCreated: (snapshot) => {
|
||||
engrams.upsertEngram(snapshot);
|
||||
registerEngramForSynapses(snapshot);
|
||||
hooks.onCount(engrams.count());
|
||||
},
|
||||
onSynapseCreated: (synapse) => {
|
||||
applySynapse(synapse);
|
||||
},
|
||||
onTorusUpdated: (center, majorR, minorR) => {
|
||||
torusMesh.position.set(center[0], center[1], center[2]);
|
||||
applyTorus(majorR, minorR);
|
||||
},
|
||||
onPositionFrame: (frame) => {
|
||||
engrams.applyPositionFrame(frame);
|
||||
trails.applyPositionFrame(frame);
|
||||
synapses.applyPositionFrame(frame);
|
||||
hooks.onCount(engrams.count());
|
||||
},
|
||||
onEngramStateChanged: (id, state) => {
|
||||
// Re-paint the dot for an engram whose state flipped (Stage 5
|
||||
// responder light-up + Query-Engram → Memorize on completion).
|
||||
engrams.setStateById(id, state);
|
||||
},
|
||||
onClose: () => {
|
||||
console.warn("ws closed — refresh to reconnect");
|
||||
},
|
||||
onError: (e) => console.error("ws error", e),
|
||||
};
|
||||
connectGalaxy(galaxyId, handlers);
|
||||
|
||||
attachInspector({ canvas, camera, engrams, galaxyId });
|
||||
attachQueryPanel({ galaxyId });
|
||||
|
||||
function onResize(): void {
|
||||
const w = window.innerWidth;
|
||||
const h = window.innerHeight;
|
||||
camera.aspect = w / h;
|
||||
camera.updateProjectionMatrix();
|
||||
renderer.setSize(w, h, false);
|
||||
composer.setSize(w, h);
|
||||
bloom.setSize(w, h);
|
||||
}
|
||||
window.addEventListener("resize", onResize);
|
||||
|
||||
const startedAt = performance.now();
|
||||
// FPS: rolling counter — accumulate frames, emit roughly once a second.
|
||||
let fpsWindowStart = startedAt;
|
||||
let fpsFrames = 0;
|
||||
function animate(): void {
|
||||
controls.update();
|
||||
const nowMs = performance.now();
|
||||
engrams.tick((nowMs - startedAt) / 1000);
|
||||
composer.render();
|
||||
fpsFrames += 1;
|
||||
const elapsed = nowMs - fpsWindowStart;
|
||||
if (elapsed >= 1000) {
|
||||
const fps = (fpsFrames * 1000) / elapsed;
|
||||
hooks.onFps?.(fps);
|
||||
fpsFrames = 0;
|
||||
fpsWindowStart = nowMs;
|
||||
}
|
||||
requestAnimationFrame(animate);
|
||||
}
|
||||
animate();
|
||||
}
|
||||
182
web/src/synapse_mesh.ts
Normal file
182
web/src/synapse_mesh.ts
Normal file
@@ -0,0 +1,182 @@
|
||||
import * as THREE from "three";
|
||||
|
||||
import type { PositionFrame } from "./ws_client";
|
||||
|
||||
/**
|
||||
* Renders synapses (Stage 3) as additive line segments connecting two
|
||||
* engrams. Endpoints are looked up from the engram position buffer on every
|
||||
* position frame, so the lines track engram motion automatically.
|
||||
*
|
||||
* Uses a fixed-capacity vertex buffer; one segment per synapse → 2 vertices
|
||||
* per synapse → 6 floats of position per synapse.
|
||||
*/
|
||||
const VERT_SHADER = /* glsl */ `
|
||||
attribute float aAlpha;
|
||||
varying vec3 vColor;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
vColor = color;
|
||||
vAlpha = aAlpha;
|
||||
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
||||
}
|
||||
`;
|
||||
|
||||
const FRAG_SHADER = /* glsl */ `
|
||||
varying vec3 vColor;
|
||||
varying float vAlpha;
|
||||
void main() {
|
||||
gl_FragColor = vec4(vColor, vAlpha);
|
||||
}
|
||||
`;
|
||||
|
||||
type SynapseEntry = {
|
||||
/** Slot in the line-segments geometry (0..capacity-1). */
|
||||
slot: number;
|
||||
/** instance_idx of the two engrams this connects. */
|
||||
aIdx: number;
|
||||
bIdx: number;
|
||||
weight: number;
|
||||
};
|
||||
|
||||
type Pending = {
|
||||
id: string;
|
||||
a: string;
|
||||
b: string;
|
||||
weight: number;
|
||||
};
|
||||
|
||||
export class SynapseMesh {
|
||||
private readonly mesh: THREE.LineSegments;
|
||||
private readonly positionAttr: THREE.BufferAttribute;
|
||||
private readonly colorAttr: THREE.BufferAttribute;
|
||||
private readonly alphaAttr: THREE.BufferAttribute;
|
||||
private readonly capacity: number;
|
||||
/** synapse_id → entry. */
|
||||
private readonly bySynapseId = new Map<string, SynapseEntry>();
|
||||
/** Engram UUID → instance_idx, populated as engrams arrive. */
|
||||
private readonly engramIdx = new Map<string, number>();
|
||||
/** Synapses waiting for one of their endpoints to be registered. */
|
||||
private readonly pending: Pending[] = [];
|
||||
/** Engram colour cache so we don't recompute on every frame. */
|
||||
private readonly engramColor: Float32Array;
|
||||
private nextSlot = 0;
|
||||
|
||||
constructor(scene: THREE.Scene, capacity = 8000) {
|
||||
this.capacity = capacity;
|
||||
this.engramColor = new Float32Array(5000 * 3);
|
||||
|
||||
const vertexCount = capacity * 2;
|
||||
const geom = new THREE.BufferGeometry();
|
||||
this.positionAttr = new THREE.BufferAttribute(new Float32Array(vertexCount * 3), 3);
|
||||
this.colorAttr = new THREE.BufferAttribute(new Float32Array(vertexCount * 3), 3);
|
||||
this.alphaAttr = new THREE.BufferAttribute(new Float32Array(vertexCount), 1);
|
||||
this.positionAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
this.colorAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
this.alphaAttr.setUsage(THREE.DynamicDrawUsage);
|
||||
geom.setAttribute("position", this.positionAttr);
|
||||
geom.setAttribute("color", this.colorAttr);
|
||||
geom.setAttribute("aAlpha", this.alphaAttr);
|
||||
geom.setDrawRange(0, 0);
|
||||
|
||||
const mat = new THREE.ShaderMaterial({
|
||||
vertexShader: VERT_SHADER,
|
||||
fragmentShader: FRAG_SHADER,
|
||||
vertexColors: true,
|
||||
transparent: true,
|
||||
depthWrite: false,
|
||||
depthTest: true,
|
||||
blending: THREE.AdditiveBlending,
|
||||
});
|
||||
|
||||
this.mesh = new THREE.LineSegments(geom, mat);
|
||||
this.mesh.frustumCulled = false;
|
||||
scene.add(this.mesh);
|
||||
}
|
||||
|
||||
/** Register an engram so synapses referencing it can be wired up. */
|
||||
registerEngram(id: string, instanceIdx: number, r: number, g: number, b: number): void {
|
||||
this.engramIdx.set(id, instanceIdx);
|
||||
if (instanceIdx * 3 + 2 < this.engramColor.length) {
|
||||
this.engramColor[instanceIdx * 3] = r;
|
||||
this.engramColor[instanceIdx * 3 + 1] = g;
|
||||
this.engramColor[instanceIdx * 3 + 2] = b;
|
||||
}
|
||||
// Try to materialise any pending synapses now that this engram is known.
|
||||
if (this.pending.length > 0) {
|
||||
const stillPending: Pending[] = [];
|
||||
for (const p of this.pending) {
|
||||
if (!this.tryMaterialise(p)) stillPending.push(p);
|
||||
}
|
||||
this.pending.length = 0;
|
||||
this.pending.push(...stillPending);
|
||||
}
|
||||
}
|
||||
|
||||
/** Add a synapse by engram UUIDs. Defers if either endpoint is unknown. */
|
||||
addSynapse(id: string, a: string, b: string, weight: number): void {
|
||||
if (this.bySynapseId.has(id)) return;
|
||||
const p: Pending = { id, a, b, weight };
|
||||
if (!this.tryMaterialise(p)) {
|
||||
this.pending.push(p);
|
||||
}
|
||||
}
|
||||
|
||||
private tryMaterialise(p: Pending): boolean {
|
||||
const aIdx = this.engramIdx.get(p.a);
|
||||
const bIdx = this.engramIdx.get(p.b);
|
||||
if (aIdx === undefined || bIdx === undefined) return false;
|
||||
if (this.nextSlot >= this.capacity) {
|
||||
console.warn("SynapseMesh capacity reached; ignoring further synapses");
|
||||
return true; // treat as resolved so we stop waiting on it
|
||||
}
|
||||
const slot = this.nextSlot++;
|
||||
this.bySynapseId.set(p.id, { slot, aIdx, bIdx, weight: p.weight });
|
||||
this.applyEndpointColors(slot, aIdx, bIdx);
|
||||
// Alpha tied to weight; a small floor so very weak synapses still register.
|
||||
const alpha = Math.max(0.08, Math.min(0.75, p.weight));
|
||||
const alphaArr = this.alphaAttr.array as Float32Array;
|
||||
alphaArr[slot * 2] = alpha;
|
||||
alphaArr[slot * 2 + 1] = alpha;
|
||||
this.alphaAttr.needsUpdate = true;
|
||||
this.mesh.geometry.setDrawRange(0, this.nextSlot * 2);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Apply the latest position frame to all synapse endpoints. */
|
||||
applyPositionFrame(frame: PositionFrame): void {
|
||||
const posArr = this.positionAttr.array as Float32Array;
|
||||
const src = frame.positions;
|
||||
const n = frame.n;
|
||||
let dirty = false;
|
||||
for (const entry of this.bySynapseId.values()) {
|
||||
const a = entry.aIdx, b = entry.bIdx;
|
||||
if (a >= n || b >= n) continue;
|
||||
const slot = entry.slot;
|
||||
posArr[slot * 6] = src[a * 3];
|
||||
posArr[slot * 6 + 1] = src[a * 3 + 1];
|
||||
posArr[slot * 6 + 2] = src[a * 3 + 2];
|
||||
posArr[slot * 6 + 3] = src[b * 3];
|
||||
posArr[slot * 6 + 4] = src[b * 3 + 1];
|
||||
posArr[slot * 6 + 5] = src[b * 3 + 2];
|
||||
dirty = true;
|
||||
}
|
||||
if (dirty) this.positionAttr.needsUpdate = true;
|
||||
}
|
||||
|
||||
/** Number of synapses currently rendered (for HUD). */
|
||||
count(): number {
|
||||
return this.bySynapseId.size;
|
||||
}
|
||||
|
||||
/** Re-paint a synapse's endpoint colours after one of its engrams updates. */
|
||||
private applyEndpointColors(slot: number, aIdx: number, bIdx: number): void {
|
||||
const colorArr = this.colorAttr.array as Float32Array;
|
||||
colorArr[slot * 6] = this.engramColor[aIdx * 3];
|
||||
colorArr[slot * 6 + 1] = this.engramColor[aIdx * 3 + 1];
|
||||
colorArr[slot * 6 + 2] = this.engramColor[aIdx * 3 + 2];
|
||||
colorArr[slot * 6 + 3] = this.engramColor[bIdx * 3];
|
||||
colorArr[slot * 6 + 4] = this.engramColor[bIdx * 3 + 1];
|
||||
colorArr[slot * 6 + 5] = this.engramColor[bIdx * 3 + 2];
|
||||
this.colorAttr.needsUpdate = true;
|
||||
}
|
||||
}
|
||||
184
web/src/ws_client.ts
Normal file
184
web/src/ws_client.ts
Normal file
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* WebSocket client for the Sophia simulation event stream.
|
||||
*
|
||||
* Wire protocol matches `crates/sophia-server/src/ws.rs`:
|
||||
* - text frames: JSON, tagged via `type` field
|
||||
* - binary frames: position frames, 12-byte header + float region
|
||||
* [u32 LE tag=0x01][u32 LE t_ms][u32 LE n][n × (f32 LE x, f32 LE y, f32 LE z)]
|
||||
* The header is 12 bytes (not 9) so the float region is 4-byte aligned and
|
||||
* can be wrapped as a Float32Array view without copying.
|
||||
*/
|
||||
|
||||
export type EngramSnapshot = {
|
||||
id: string;
|
||||
instance_idx: number;
|
||||
position: [number, number, number];
|
||||
size: number;
|
||||
state: string;
|
||||
};
|
||||
|
||||
export type GalaxyInfo = {
|
||||
id: string;
|
||||
name: string;
|
||||
engram_count: number;
|
||||
center: [number, number, number];
|
||||
major_radius: number;
|
||||
minor_radius: number;
|
||||
};
|
||||
|
||||
export type SynapseDto = {
|
||||
id: string;
|
||||
a: string;
|
||||
b: string;
|
||||
weight: number;
|
||||
};
|
||||
|
||||
export type QueryStatus =
|
||||
| "pending"
|
||||
| "responding"
|
||||
| "integrating"
|
||||
| "done"
|
||||
| "failed";
|
||||
|
||||
export type SimEventMsg =
|
||||
| {
|
||||
type: "hello";
|
||||
galaxy: GalaxyInfo;
|
||||
engrams: EngramSnapshot[];
|
||||
synapses: SynapseDto[];
|
||||
}
|
||||
| { type: "engram_created"; snapshot: EngramSnapshot }
|
||||
| { type: "synapse_created"; synapse: SynapseDto }
|
||||
| {
|
||||
type: "torus_updated";
|
||||
center: [number, number, number];
|
||||
major_radius: number;
|
||||
minor_radius: number;
|
||||
}
|
||||
| { type: "engram_state_changed"; id: string; state: string }
|
||||
| {
|
||||
type: "query_started";
|
||||
query: string;
|
||||
engram: string;
|
||||
position: [number, number, number];
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "query_answer_updated";
|
||||
query: string;
|
||||
version: number;
|
||||
status: QueryStatus;
|
||||
answer: string;
|
||||
}
|
||||
| {
|
||||
type: "query_finished";
|
||||
query: string;
|
||||
status: QueryStatus;
|
||||
responder_count: number;
|
||||
};
|
||||
|
||||
export type PositionFrame = {
|
||||
t_ms: number;
|
||||
n: number;
|
||||
/** Flat array, length 3*n: x,y,z,x,y,z,... in instance_idx order. */
|
||||
positions: Float32Array;
|
||||
};
|
||||
|
||||
export type EventHandlers = {
|
||||
onHello?: (
|
||||
galaxy: GalaxyInfo,
|
||||
engrams: EngramSnapshot[],
|
||||
synapses: SynapseDto[],
|
||||
) => void;
|
||||
onEngramCreated?: (snapshot: EngramSnapshot) => void;
|
||||
onSynapseCreated?: (synapse: SynapseDto) => void;
|
||||
onTorusUpdated?: (
|
||||
center: [number, number, number],
|
||||
majorRadius: number,
|
||||
minorRadius: number,
|
||||
) => void;
|
||||
onEngramStateChanged?: (id: string, state: string) => void;
|
||||
onQueryStarted?: (
|
||||
queryId: string,
|
||||
engramId: string,
|
||||
position: [number, number, number],
|
||||
text: string,
|
||||
) => void;
|
||||
onQueryAnswerUpdated?: (
|
||||
queryId: string,
|
||||
version: number,
|
||||
status: QueryStatus,
|
||||
answer: string,
|
||||
) => void;
|
||||
onQueryFinished?: (queryId: string, status: QueryStatus, responderCount: number) => void;
|
||||
onPositionFrame?: (frame: PositionFrame) => void;
|
||||
onClose?: (ev: CloseEvent) => void;
|
||||
onError?: (ev: Event) => void;
|
||||
};
|
||||
|
||||
export function connectGalaxy(galaxyId: string, handlers: EventHandlers): WebSocket {
|
||||
const proto = window.location.protocol === "https:" ? "wss:" : "ws:";
|
||||
const url = `${proto}//${window.location.host}/ws/galaxy/${galaxyId}/events`;
|
||||
const ws = new WebSocket(url);
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
ws.addEventListener("message", (ev) => {
|
||||
try {
|
||||
if (typeof ev.data === "string") {
|
||||
const msg = JSON.parse(ev.data) as SimEventMsg;
|
||||
switch (msg.type) {
|
||||
case "hello":
|
||||
handlers.onHello?.(msg.galaxy, msg.engrams, msg.synapses);
|
||||
break;
|
||||
case "engram_created":
|
||||
handlers.onEngramCreated?.(msg.snapshot);
|
||||
break;
|
||||
case "synapse_created":
|
||||
handlers.onSynapseCreated?.(msg.synapse);
|
||||
break;
|
||||
case "torus_updated":
|
||||
handlers.onTorusUpdated?.(msg.center, msg.major_radius, msg.minor_radius);
|
||||
break;
|
||||
case "engram_state_changed":
|
||||
handlers.onEngramStateChanged?.(msg.id, msg.state);
|
||||
break;
|
||||
case "query_started":
|
||||
handlers.onQueryStarted?.(msg.query, msg.engram, msg.position, msg.text);
|
||||
break;
|
||||
case "query_answer_updated":
|
||||
handlers.onQueryAnswerUpdated?.(msg.query, msg.version, msg.status, msg.answer);
|
||||
break;
|
||||
case "query_finished":
|
||||
handlers.onQueryFinished?.(msg.query, msg.status, msg.responder_count);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
const frame = decodePositionFrame(ev.data as ArrayBuffer);
|
||||
if (frame) handlers.onPositionFrame?.(frame);
|
||||
}
|
||||
} catch (e) {
|
||||
// Surface any decode error instead of silently dropping the frame —
|
||||
// a malformed binary frame used to silently kill the position stream.
|
||||
console.error("ws message handler failed", e);
|
||||
}
|
||||
});
|
||||
ws.addEventListener("close", (ev) => handlers.onClose?.(ev));
|
||||
ws.addEventListener("error", (ev) => handlers.onError?.(ev));
|
||||
return ws;
|
||||
}
|
||||
|
||||
function decodePositionFrame(buf: ArrayBuffer): PositionFrame | null {
|
||||
const view = new DataView(buf);
|
||||
const tag = view.getUint32(0, true);
|
||||
if (tag !== 0x01) return null;
|
||||
const t_ms = view.getUint32(4, true);
|
||||
const n = view.getUint32(8, true);
|
||||
const expected = 12 + n * 12;
|
||||
if (buf.byteLength < expected) return null;
|
||||
// Float region starts at byte 12 — 4-byte aligned, so we can wrap it as a
|
||||
// Float32Array view without copying. (Float32Array constructor throws if
|
||||
// the byte offset is not a multiple of 4; that's why the header is padded
|
||||
// to 12 bytes instead of a tighter 9 bytes.)
|
||||
const positions = new Float32Array(buf, 12, n * 3);
|
||||
return { t_ms, n, positions };
|
||||
}
|
||||
18
web/tsconfig.json
Normal file
18
web/tsconfig.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true,
|
||||
"isolatedModules": true,
|
||||
"resolveJsonModule": true,
|
||||
"useDefineForClassFields": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
20
web/vite.config.ts
Normal file
20
web/vite.config.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { defineConfig } from "vite";
|
||||
|
||||
// In dev, the Vite server runs on :5173 and proxies API/WS calls to the
|
||||
// Rust server on :7777 so the same fetch/WebSocket code works in both modes.
|
||||
// In `build`, the output goes to `dist/` which the Rust server statically serves.
|
||||
export default defineConfig({
|
||||
build: {
|
||||
outDir: "dist",
|
||||
emptyOutDir: true,
|
||||
target: "es2022",
|
||||
},
|
||||
server: {
|
||||
port: 5173,
|
||||
proxy: {
|
||||
"/healthz": "http://127.0.0.1:7777",
|
||||
"/api": "http://127.0.0.1:7777",
|
||||
"/ws": { target: "ws://127.0.0.1:7777", ws: true },
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user