From 9607ff3478a77e4a00a42705d0cdb40c5e1dcb1a Mon Sep 17 00:00:00 2001 From: dtoro Date: Sun, 29 Mar 2026 01:22:33 +0100 Subject: [PATCH] feat: runnersok --- .gitignore | 6 + PERFORMANCE_IMPROVEMENTS.md | 108 ----- backend/package-lock.json | 438 ++++++++++++++++++ backend/package.json | 4 +- backend/src/db/connection.ts | 69 +++ backend/src/index.ts | 21 + backend/src/models/run.ts | 67 +++ backend/src/repositories/runRepository.ts | 62 +++ backend/src/routes/runRoutes.ts | 127 +++++ backend/src/services/graphRunnerService.ts | 237 ++++++++++ frontend/src/app/canvas/CanvasPage.tsx | 31 ++ .../recollections/katalogos/KatalogosPage.tsx | 27 +- .../app/recollections/katalogos/RunsTab.tsx | 135 ++++++ .../layout/RecollectionActionsContext.tsx | 1 + .../layout/RecollectionEditViewMenus.tsx | 60 ++- frontend/src/components/graph/BaseNode.tsx | 2 + .../graph/NodeFooterEdgeIndicators.tsx | 2 + .../components/graph/NodeRunStatusOverlay.tsx | 84 ++++ .../components/nodes/render/RenderingNode.tsx | 101 +--- .../nodes/render/useRenderingNodeState.ts | 10 +- frontend/src/hooks/useRunStream.ts | 109 +++++ frontend/src/lib/graph/abstractNode.ts | 26 +- frontend/src/lib/graph/runStore.ts | 99 ++++ frontend/vite.config.ts | 8 + 24 files changed, 1613 insertions(+), 221 deletions(-) delete mode 100644 PERFORMANCE_IMPROVEMENTS.md create mode 100644 backend/src/db/connection.ts create mode 100644 backend/src/models/run.ts create mode 100644 backend/src/repositories/runRepository.ts create mode 100644 backend/src/routes/runRoutes.ts create mode 100644 backend/src/services/graphRunnerService.ts create mode 100644 frontend/src/app/recollections/katalogos/RunsTab.tsx create mode 100644 frontend/src/components/graph/NodeRunStatusOverlay.tsx create mode 100644 frontend/src/hooks/useRunStream.ts create mode 100644 frontend/src/lib/graph/runStore.ts diff --git a/.gitignore b/.gitignore index e22bbf6..7997d90 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,12 @@ Thumbs.db .cache .parcel-cache +# SQLite data +data/ +*.db +*.db-wal +*.db-shm + # Docker (optional local overrides) docker-compose.override.yml docker-compose.override.yaml diff --git a/PERFORMANCE_IMPROVEMENTS.md b/PERFORMANCE_IMPROVEMENTS.md deleted file mode 100644 index 744071c..0000000 --- a/PERFORMANCE_IMPROVEMENTS.md +++ /dev/null @@ -1,108 +0,0 @@ -# Performance Improvements Plan - -## 1. Current State - -The core rendering and state management architecture is already well-structured for performance: - -- **Command/reducer store** — all mutations go through a pure reducer; individual selectors can prevent unnecessary re-renders. -- **Delta-based undo/redo** — only diffs are stored, not full graph snapshots (max 100 entries). -- **Plugin node registry** — node types are loaded once at app init, not dynamically on each render. -- **Streaming AI responses** — `POST /api/agent/stream` uses SSE so the UI updates incrementally. - -The sections below identify remaining bottlenecks and concrete next steps. - ---- - -## 2. Known Bottlenecks - -| # | Issue | Location | Impact | Root Cause | -| --- | --- | --- | --- | --- | -| 1 | Unguarded re-renders on canvas | `CanvasPage.tsx` | Frame drops on large graphs | Components not subscribed to granular store slices | -| 2 | Nunjucks template resolution on every render | `rendering.ts` resolve step | Slow Config node updates | No memoization of template output keyed to input hash | -| 3 | Kroki SVG requests not deduplicated | `rendering.ts` render step | Redundant network calls | No in-flight request deduplication or client-side cache | -| 4 | Sidebar tree renders all items | `KosmosPage` recollection tree | Scrolling lag with many workspaces | No list virtualization | -| 5 | Backend cache not wired to agent routes | `agentRoutes.ts` | Repeated identical LLM calls | `InMemoryCache` and `rateLimiter` exist but are unused | -| 6 | Full graph serialized to localStorage on every change | `useGraphStateWithHistory` | Storage I/O on every keypress | No debounce on the persistence write | -| 7 | Initial bundle size | Vite build | Slow first load | Heavy deps (BlockNote, React Flow, Nunjucks) loaded eagerly | - ---- - -## 3. Recommended Improvements - -### 3.1 Granular Store Subscriptions - -Zustand supports slice-level subscriptions. Node components should select only their own data slice: - -```ts -// Instead of subscribing to the entire graph: -const node = useCanvasStore(s => s.graph.nodes.find(n => n.id === id)) -``` - -This prevents all nodes from re-rendering when a single node changes. - -### 3.2 Memoize Template Resolution - -Cache the Nunjucks resolution output keyed to a hash of the template source plus variable inputs. Invalidate only when those inputs change: - -```ts -const resolved = useMemo( - () => resolveTemplate(template, variables), - [templateHash, variableHash] -) -``` - -### 3.3 Deduplicate Kroki Requests - -Add a simple in-flight map in the render step: if a request for the same PlantUML source is already pending, reuse its promise. Cache successful responses keyed to the source string with a short TTL (e.g. 5 minutes). - -### 3.4 Wire Backend Cache and Rate Limiter - -`InMemoryCache` and `rateLimiter` middleware are implemented in `backend/src/`. Connect them to `agentRoutes.ts`: - -1. Add cache lookup before calling the AI service. -2. Store the response on cache miss. -3. Apply rate limiting per IP to prevent abuse. - -### 3.5 Debounce localStorage Writes - -Wrap the graph persistence call in a debounce (e.g., 300 ms) to avoid a write on every keystroke or node drag. The delta-based history already computes minimal diffs; the bottleneck is the frequency of writes. - -### 3.6 Virtualize the Sidebar Tree - -Integrate `react-arborist` (already installed) with virtualization enabled for the recollection sidebar when item count exceeds a threshold (~50). - -### 3.7 Code Split Heavy Routes - -Add lazy imports for the three heavy route components so the initial bundle only loads what the user navigates to: - -```ts -const FluxRoute = lazy(() => import('./app/recollections/flux/FluxRoute')) -const LogosPage = lazy(() => import('./app/recollections/logos/LogosPage')) -const KatalogosPage = lazy(() => import('./app/recollections/katalogos/KatalogosPage')) -``` - -### 3.8 Enable Brotli Compression in Nginx - -Add brotli/gzip compression to `frontend/nginx.conf` for JS, CSS, and SVG assets. This can cut transfer size by 60–70% for the JS bundle. - ---- - -## 4. Success Metrics - -| Metric | Current (estimated) | Target | -| --- | --- | --- | -| Frame time on 50-node canvas | ~16 ms | < 10 ms | -| Initial JS bundle (gzipped) | ~800 KB | < 600 KB | -| Repeated identical LLM calls | uncached | 0 network round-trips | -| localStorage write frequency | every change | debounced 300 ms | - ---- - -## 5. Contribution Path - -1. Read [ARCHITECTURE.md](ARCHITECTURE.md) to understand the module you're optimizing. -2. Pick one item from section 2. -3. Add a Vitest benchmark (`performance.now()` before/after) alongside your change. -4. Submit a PR with the benchmark results in the description and update this file's "Current" column. - -*This plan is a living document; update the metrics table when improvements land.* diff --git a/backend/package-lock.json b/backend/package-lock.json index ba26e85..4fd147b 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,10 +10,12 @@ "dependencies": { "@ai-sdk/openai": "^1.0.0", "ai": "^4.0.0", + "better-sqlite3": "^12.8.0", "cors": "^2.8.5", "express": "^4.21.0" }, "devDependencies": { + "@types/better-sqlite3": "^7.6.13", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^22.0.0", @@ -561,6 +563,16 @@ "node": ">=8.0.0" } }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/body-parser": { "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", @@ -720,6 +732,60 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "12.8.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.8.0.tgz", + "integrity": "sha512-RxD2Vd96sQDjQr20kdP+F+dK/1OUNiVOl200vKBZY8u0vTwysfolF6Hq+3ZK2+h8My9YvZhHsF+RSGZW2VYrPQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/body-parser": { "version": "1.20.4", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", @@ -744,6 +810,30 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -794,6 +884,12 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -856,6 +952,30 @@ "ms": "2.0.0" } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -884,6 +1004,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/diff-match-patch": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", @@ -919,6 +1048,15 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -1006,6 +1144,15 @@ "node": ">= 0.6" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.22.1", "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", @@ -1052,6 +1199,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/finalhandler": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", @@ -1088,6 +1241,12 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1162,6 +1321,12 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -1230,12 +1395,38 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1337,6 +1528,33 @@ "node": ">= 0.6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -1361,6 +1579,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -1370,6 +1594,18 @@ "node": ">= 0.6" } }, + "node_modules/node-abi": { + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -1403,6 +1639,15 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1418,6 +1663,33 @@ "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1431,6 +1703,16 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/qs": { "version": "6.14.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", @@ -1470,6 +1752,21 @@ "node": ">= 0.8" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, "node_modules/react": { "version": "19.2.4", "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", @@ -1480,6 +1777,20 @@ "node": ">=0.10.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -1522,6 +1833,18 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", "license": "BSD-3-Clause" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/send": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", @@ -1645,6 +1968,51 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -1654,6 +2022,24 @@ "node": ">= 0.8" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/swr": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/swr/-/swr-2.4.1.tgz", @@ -1667,6 +2053,34 @@ "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/throttleit": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", @@ -1708,6 +2122,18 @@ "fsevents": "~2.3.3" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -1760,6 +2186,12 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -1778,6 +2210,12 @@ "node": ">= 0.8" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/backend/package.json b/backend/package.json index 0bd6bec..14b9e33 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,12 +14,14 @@ "node": ">=20" }, "dependencies": { - "ai": "^4.0.0", "@ai-sdk/openai": "^1.0.0", + "ai": "^4.0.0", + "better-sqlite3": "^12.8.0", "cors": "^2.8.5", "express": "^4.21.0" }, "devDependencies": { + "@types/better-sqlite3": "^7.6.13", "@types/cors": "^2.8.17", "@types/express": "^5.0.0", "@types/node": "^22.0.0", diff --git a/backend/src/db/connection.ts b/backend/src/db/connection.ts new file mode 100644 index 0000000..92d7490 --- /dev/null +++ b/backend/src/db/connection.ts @@ -0,0 +1,69 @@ +/** + * SQLite database connection singleton. + * Uses better-sqlite3 for synchronous, fast access. + * DB_PATH env var controls location; defaults to ./data/zui.db + */ + +import Database from 'better-sqlite3' +import path from 'node:path' +import fs from 'node:fs' + +const DB_PATH = process.env.DB_PATH || path.resolve('data', 'zui.db') + +let db: Database.Database | null = null + +export function getDb(): Database.Database { + if (db) return db + + // Ensure parent directory exists + const dir = path.dirname(DB_PATH) + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) + + db = new Database(DB_PATH) + db.pragma('journal_mode = WAL') + db.pragma('foreign_keys = ON') + + initSchema(db) + return db +} + +function initSchema(db: Database.Database): void { + db.exec(` + CREATE TABLE IF NOT EXISTS runs ( + id TEXT PRIMARY KEY, + recollectionId TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + graphSnapshot TEXT NOT NULL, + createdAt TEXT NOT NULL DEFAULT (datetime('now')), + updatedAt TEXT NOT NULL DEFAULT (datetime('now')), + error TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_runs_recollection + ON runs(recollectionId, createdAt DESC); + + CREATE TABLE IF NOT EXISTS run_steps ( + id TEXT PRIMARY KEY, + runId TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE, + nodeId TEXT NOT NULL, + nodeType TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + input TEXT, + output TEXT, + error TEXT, + startedAt TEXT, + endedAt TEXT, + sortOrder INTEGER NOT NULL DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_run_steps_run + ON run_steps(runId, sortOrder); + `) +} + +export function closeDb(): void { + if (db) { + db.close() + db = null + } +} diff --git a/backend/src/index.ts b/backend/src/index.ts index bafb249..46bd71a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -61,6 +61,8 @@ app.use(express.json()) // --------------------------------------------------------------------------- import { handleAgentRequest, handleAgentStreamRequest } from './routes/agentRoutes.js' +import { handleCreateRun, handleRunStream, handleGetRun, handleListRuns } from './routes/runRoutes.js' +import { getDb } from './db/connection.js' /** POST /api/agent - Run AI agent */ app.post('/api/agent', handleAgentRequest) @@ -68,6 +70,22 @@ app.post('/api/agent', handleAgentRequest) /** POST /api/agent/stream - Stream AI agent response */ app.post('/api/agent/stream', handleAgentStreamRequest) +// --------------------------------------------------------------------------- +// Run Routes +// --------------------------------------------------------------------------- + +/** POST /api/runs - Create and queue a graph execution run */ +app.post('/api/runs', handleCreateRun) + +/** GET /api/runs/:id/stream - SSE stream of run execution events */ +app.get('/api/runs/:id/stream', handleRunStream) + +/** GET /api/runs/:id - Get run details with steps */ +app.get('/api/runs/:id', handleGetRun) + +/** GET /api/recollections/:recollectionId/runs - List runs for a recollection */ +app.get('/api/recollections/:recollectionId/runs', handleListRuns) + // --------------------------------------------------------------------------- // Health Check Endpoint // --------------------------------------------------------------------------- @@ -91,6 +109,9 @@ app.use((err: Error, req: express.Request, res: express.Response, next: express. // Start Server // --------------------------------------------------------------------------- +// Initialize database on startup +getDb() + app.listen(PORT, '0.0.0.0', () => { console.log(`Backend listening on port ${PORT} (CORS: ${CORS_ORIGIN})`) }) diff --git a/backend/src/models/run.ts b/backend/src/models/run.ts new file mode 100644 index 0000000..08e9145 --- /dev/null +++ b/backend/src/models/run.ts @@ -0,0 +1,67 @@ +/** + * Run and RunStep domain types for graph execution. + */ + +export type RunStatus = 'pending' | 'running' | 'completed' | 'failed' + +export type Run = { + id: string + recollectionId: string + status: RunStatus + graphSnapshot: string + createdAt: string + updatedAt: string + error?: string | null +} + +export type RunStep = { + id: string + runId: string + nodeId: string + nodeType: string + status: RunStatus + input?: string | null + output?: string | null + error?: string | null + startedAt?: string | null + endedAt?: string | null + sortOrder: number +} + +/** SSE event types for run streaming */ +export type RunEventType = + | 'connected' + | 'run/started' + | 'run/completed' + | 'run/failed' + | 'step/started' + | 'step/completed' + | 'step/failed' + | 'step/chunk' + | 'ping' + +export type RunEvent = { + type: RunEventType + data: Record +} + +/** Graph snapshot types (subset of frontend StoredGraphState) */ +export type GraphNode = { + id: string + type?: string + data?: Record + position?: { x: number; y: number } + [key: string]: unknown +} + +export type GraphEdge = { + id: string + source: string + target: string + [key: string]: unknown +} + +export type GraphSnapshot = { + nodes: GraphNode[] + edges: GraphEdge[] +} diff --git a/backend/src/repositories/runRepository.ts b/backend/src/repositories/runRepository.ts new file mode 100644 index 0000000..fb313dc --- /dev/null +++ b/backend/src/repositories/runRepository.ts @@ -0,0 +1,62 @@ +/** + * Data access layer for runs and run_steps. + */ + +import { getDb } from '../db/connection.js' +import type { Run, RunStep, RunStatus } from '../models/run.js' + +export function createRun(run: Run): void { + const db = getDb() + db.prepare(` + INSERT INTO runs (id, recollectionId, status, graphSnapshot, createdAt, updatedAt, error) + VALUES (@id, @recollectionId, @status, @graphSnapshot, @createdAt, @updatedAt, @error) + `).run(run) +} + +export function getRun(id: string): Run | undefined { + const db = getDb() + return db.prepare('SELECT * FROM runs WHERE id = ?').get(id) as Run | undefined +} + +export function updateRunStatus(id: string, status: RunStatus, error?: string): void { + const db = getDb() + db.prepare(` + UPDATE runs SET status = ?, error = ?, updatedAt = datetime('now') WHERE id = ? + `).run(status, error ?? null, id) +} + +export function getRunsByRecollection(recollectionId: string, limit = 50, offset = 0): Run[] { + const db = getDb() + return db.prepare(` + SELECT * FROM runs WHERE recollectionId = ? ORDER BY createdAt DESC LIMIT ? OFFSET ? + `).all(recollectionId, limit, offset) as Run[] +} + +export function createRunStep(step: RunStep): void { + const db = getDb() + db.prepare(` + INSERT INTO run_steps (id, runId, nodeId, nodeType, status, input, output, error, startedAt, endedAt, sortOrder) + VALUES (@id, @runId, @nodeId, @nodeType, @status, @input, @output, @error, @startedAt, @endedAt, @sortOrder) + `).run(step) +} + +export function updateRunStep(id: string, updates: Partial>): void { + const db = getDb() + const fields: string[] = [] + const values: unknown[] = [] + + if (updates.status !== undefined) { fields.push('status = ?'); values.push(updates.status) } + if (updates.output !== undefined) { fields.push('output = ?'); values.push(updates.output) } + if (updates.error !== undefined) { fields.push('error = ?'); values.push(updates.error) } + if (updates.startedAt !== undefined) { fields.push('startedAt = ?'); values.push(updates.startedAt) } + if (updates.endedAt !== undefined) { fields.push('endedAt = ?'); values.push(updates.endedAt) } + + if (fields.length === 0) return + values.push(id) + db.prepare(`UPDATE run_steps SET ${fields.join(', ')} WHERE id = ?`).run(...values) +} + +export function getRunSteps(runId: string): RunStep[] { + const db = getDb() + return db.prepare('SELECT * FROM run_steps WHERE runId = ? ORDER BY sortOrder').all(runId) as RunStep[] +} diff --git a/backend/src/routes/runRoutes.ts b/backend/src/routes/runRoutes.ts new file mode 100644 index 0000000..8d230e9 --- /dev/null +++ b/backend/src/routes/runRoutes.ts @@ -0,0 +1,127 @@ +/** + * Run routes: create runs, stream execution, list history. + */ + +import crypto from 'node:crypto' +import type { Request, Response } from 'express' +import type { Run, RunEvent } from '../models/run.js' +import * as runRepo from '../repositories/runRepository.js' +import { executeRun } from '../services/graphRunnerService.js' + +/** + * POST /api/runs + * Body: { recollectionId, graph: { nodes, edges } } + * Creates a new run and starts execution, returning the run ID immediately. + * Client should connect to GET /api/runs/:id/stream for live updates. + */ +export async function handleCreateRun(req: Request, res: Response): Promise { + try { + const { recollectionId, graph } = req.body + + if (!recollectionId || typeof recollectionId !== 'string') { + res.status(400).json({ error: 'recollectionId is required' }) + return + } + if (!graph || !Array.isArray(graph.nodes) || !Array.isArray(graph.edges)) { + res.status(400).json({ error: 'graph with nodes and edges is required' }) + return + } + + const run: Run = { + id: crypto.randomUUID(), + recollectionId, + status: 'pending', + graphSnapshot: JSON.stringify(graph), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + error: null, + } + + runRepo.createRun(run) + res.status(201).json({ id: run.id, status: run.status }) + } catch (err) { + console.error('Create run error:', err) + res.status(500).json({ error: (err as Error).message ?? 'Failed to create run' }) + } +} + +/** + * GET /api/runs/:id/stream + * SSE endpoint that executes the run and streams events in real time. + */ +export async function handleRunStream(req: Request, res: Response): Promise { + const { id } = req.params + + const run = runRepo.getRun(id) + if (!run) { + res.status(404).json({ error: 'Run not found' }) + return + } + + if (run.status !== 'pending') { + res.status(409).json({ error: `Run already ${run.status}` }) + return + } + + // Set up SSE headers + res.setHeader('Content-Type', 'text/event-stream') + res.setHeader('Cache-Control', 'no-cache') + res.setHeader('Connection', 'keep-alive') + res.setHeader('X-Accel-Buffering', 'no') + res.flushHeaders() + + const emit = (event: RunEvent) => { + res.write(`event: ${event.type}\ndata: ${JSON.stringify(event.data)}\n\n`) + } + + emit({ type: 'connected', data: { runId: id } }) + + // Keepalive ping + const pingInterval = setInterval(() => { + emit({ type: 'ping', data: { timestamp: Date.now() } }) + }, 15_000) + + req.on('close', () => { + clearInterval(pingInterval) + }) + + try { + await executeRun(id, emit) + } catch (err) { + const errorMsg = err instanceof Error ? err.message : String(err) + emit({ type: 'run/failed', data: { runId: id, error: errorMsg } }) + runRepo.updateRunStatus(id, 'failed', errorMsg) + } finally { + clearInterval(pingInterval) + res.end() + } +} + +/** + * GET /api/runs/:id + * Returns run details with steps. + */ +export async function handleGetRun(req: Request, res: Response): Promise { + const { id } = req.params + const run = runRepo.getRun(id) + if (!run) { + res.status(404).json({ error: 'Run not found' }) + return + } + + const steps = runRepo.getRunSteps(id) + res.json({ ...run, steps }) +} + +/** + * GET /api/recollections/:recollectionId/runs + * Returns run history for a recollection. + */ +export async function handleListRuns(req: Request, res: Response): Promise { + const { recollectionId } = req.params + const limit = Math.min(Number(req.query.limit) || 50, 200) + const offset = Number(req.query.offset) || 0 + + const runs = runRepo.getRunsByRecollection(recollectionId, limit, offset) + res.json({ runs }) +} diff --git a/backend/src/services/graphRunnerService.ts b/backend/src/services/graphRunnerService.ts new file mode 100644 index 0000000..32397e2 --- /dev/null +++ b/backend/src/services/graphRunnerService.ts @@ -0,0 +1,237 @@ +/** + * Graph execution engine. + * Topologically sorts graph nodes (Kahn's algorithm) and executes them in order. + * Emits SSE events via a callback for real-time streaming to clients. + */ + +import crypto from 'node:crypto' +import type { GraphSnapshot, GraphNode, GraphEdge, RunEvent, RunStep, RunStatus } from '../models/run.js' +import * as runRepo from '../repositories/runRepository.js' +import { buildAgentRequest } from './agentService.js' + +type EmitFn = (event: RunEvent) => void + +/** + * Topological sort using Kahn's algorithm. + * Returns ordered node IDs or throws on cycle detection. + */ +function topologicalSort(nodes: GraphNode[], edges: GraphEdge[]): string[] { + const nodeIds = new Set(nodes.map((n) => n.id)) + const inDegree = new Map() + const adjacency = new Map() + + for (const id of nodeIds) { + inDegree.set(id, 0) + adjacency.set(id, []) + } + + for (const edge of edges) { + if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) continue + adjacency.get(edge.source)!.push(edge.target) + inDegree.set(edge.target, (inDegree.get(edge.target) ?? 0) + 1) + } + + const queue: string[] = [] + for (const [id, deg] of inDegree) { + if (deg === 0) queue.push(id) + } + + const sorted: string[] = [] + while (queue.length > 0) { + const current = queue.shift()! + sorted.push(current) + for (const neighbor of adjacency.get(current) ?? []) { + const newDeg = (inDegree.get(neighbor) ?? 1) - 1 + inDegree.set(neighbor, newDeg) + if (newDeg === 0) queue.push(neighbor) + } + } + + if (sorted.length !== nodeIds.size) { + throw new Error('Cycle detected in graph') + } + + return sorted +} + +/** Node types that produce output when executed */ +const EXECUTABLE_TYPES = new Set(['agent', 'config', 'render']) + +/** Node types that are pure inputs (no execution needed) */ +const INPUT_TYPES = new Set(['variable', 'data', 'function']) + +/** + * Execute a single node based on its type. + * Returns the output string or null for input-only nodes. + */ +async function executeNode( + node: GraphNode, + _edges: GraphEdge[], + _nodes: GraphNode[], + _outputs: Map, + emit: EmitFn +): Promise { + const nodeType = node.type ?? 'unknown' + + if (INPUT_TYPES.has(nodeType)) { + // Input nodes: extract their value as output + const data = node.data ?? {} + if (nodeType === 'variable') { + const v = data.value + return v === undefined || v === null ? '' : String(v) + } + if (nodeType === 'data') { + const rows = (data.rows as Record[]) ?? [] + return JSON.stringify(rows) + } + if (nodeType === 'function') { + return (data.body as string) ?? '' + } + return null + } + + if (nodeType === 'agent') { + // Agent nodes: call the AI service + const configContents: string[] = [] + for (const edge of _edges) { + if (edge.target !== node.id) continue + const output = _outputs.get(edge.source) + if (output) configContents.push(output) + } + + const prompt = configContents.join('\n\n---\n\n') || 'No prompt provided.' + const connection = (node.data?.connection as Record) ?? undefined + + const built = buildAgentRequest({ + prompt, + connection: connection as any, + }) + + if ('error' in built) { + throw new Error(built.error) + } + + // Use streaming for agent nodes + const { streamText } = await import('ai') + const result = streamText({ + model: built.openai as any, + prompt: built.fullPrompt, + }) + + let fullText = '' + for await (const chunk of (await result).textStream) { + fullText += chunk + emit({ + type: 'step/chunk', + data: { nodeId: node.id, chunk }, + }) + } + + return fullText + } + + if (nodeType === 'config') { + // Config nodes: return their template content as-is during execution + // (Nunjucks resolution happens on the frontend; backend treats as passthrough) + const content = (node.data?.content as string) ?? '' + return content + } + + if (nodeType === 'render') { + // Render nodes: collect input from connected source nodes + const inputs: string[] = [] + for (const edge of _edges) { + if (edge.target !== node.id) continue + const output = _outputs.get(edge.source) + if (output) inputs.push(output) + } + return inputs.join('\n\n') + } + + return null +} + +/** + * Run the full graph execution for a given run ID. + * Emits SSE events for each step. + */ +export async function executeRun(runId: string, emit: EmitFn): Promise { + const run = runRepo.getRun(runId) + if (!run) throw new Error(`Run not found: ${runId}`) + + let graph: GraphSnapshot + try { + graph = JSON.parse(run.graphSnapshot) as GraphSnapshot + } catch { + throw new Error('Invalid graph snapshot') + } + + const { nodes, edges } = graph + + // Sort nodes topologically + const sortedIds = topologicalSort(nodes, edges) + const nodeMap = new Map(nodes.map((n) => [n.id, n])) + const outputs = new Map() + + // Create run_steps records + const steps: RunStep[] = sortedIds.map((nodeId, i) => { + const node = nodeMap.get(nodeId)! + return { + id: crypto.randomUUID(), + runId, + nodeId, + nodeType: node.type ?? 'unknown', + status: 'pending' as RunStatus, + input: null, + output: null, + error: null, + startedAt: null, + endedAt: null, + sortOrder: i, + } + }) + + for (const step of steps) { + runRepo.createRunStep(step) + } + + // Mark run as running + runRepo.updateRunStatus(runId, 'running') + const stepNodeIds = steps.map((s) => s.nodeId) + emit({ type: 'run/started', data: { runId, totalSteps: steps.length, nodeIds: stepNodeIds } }) + + // Execute each node in order + for (const step of steps) { + const node = nodeMap.get(step.nodeId) + if (!node) continue + + const now = new Date().toISOString() + runRepo.updateRunStep(step.id, { status: 'running', startedAt: now }) + emit({ type: 'step/started', data: { stepId: step.id, nodeId: step.nodeId, nodeType: step.nodeType } }) + + try { + const output = await executeNode(node, edges, nodes, outputs, emit) + if (output !== null) { + outputs.set(node.id, output) + } + + const endedAt = new Date().toISOString() + runRepo.updateRunStep(step.id, { status: 'completed', output: output ?? '', endedAt }) + emit({ type: 'step/completed', data: { stepId: step.id, nodeId: step.nodeId, output: output?.slice(0, 500) ?? '' } }) + } catch (err) { + const endedAt = new Date().toISOString() + const errorMsg = err instanceof Error ? err.message : String(err) + runRepo.updateRunStep(step.id, { status: 'failed', error: errorMsg, endedAt }) + emit({ type: 'step/failed', data: { stepId: step.id, nodeId: step.nodeId, error: errorMsg } }) + + // Fail the entire run + runRepo.updateRunStatus(runId, 'failed', errorMsg) + emit({ type: 'run/failed', data: { runId, error: errorMsg } }) + return + } + } + + // Mark run as completed + runRepo.updateRunStatus(runId, 'completed') + emit({ type: 'run/completed', data: { runId } }) +} diff --git a/frontend/src/app/canvas/CanvasPage.tsx b/frontend/src/app/canvas/CanvasPage.tsx index 204ab02..749705a 100644 --- a/frontend/src/app/canvas/CanvasPage.tsx +++ b/frontend/src/app/canvas/CanvasPage.tsx @@ -68,6 +68,8 @@ import { import type { AppNode, AppEdge } from '@/lib/graph/nodeTypes' import { toast } from 'sonner' import { RECOLLECTION_VERSION } from '@/app/recollections/state/recollectionGraphStorage' +import { useRunStream, createAndStreamRun } from '@/hooks/useRunStream' +import { useRunStore } from '@/lib/graph/runStore' const SNAP_GRID: [number, number] = [15, 15] const DUPLICATE_OFFSET = { x: 30, y: 30 } @@ -257,6 +259,33 @@ export function CanvasPage({ recollectionId, focusNodeId }: CanvasPageProps) { const connectionPath = useCanvasConnectionPathFromStore() + // --- Run execution --- + const { connectToRun } = useRunStream() + const runStatus = useRunStore((s) => s.status) + const resetRun = useRunStore((s) => s.reset) + const markDirty = useRunStore((s) => s.markDirty) + + // Mark run state as dirty when graph structure or data changes + const prevNodesLenRef = useRef(nodes.length) + const prevEdgesLenRef = useRef(edges.length) + useEffect(() => { + // Skip the initial render + if (prevNodesLenRef.current === nodes.length && prevEdgesLenRef.current === edges.length) return + prevNodesLenRef.current = nodes.length + prevEdgesLenRef.current = edges.length + markDirty() + }, [nodes.length, edges.length, markDirty]) + const handleRun = useCallback(() => { + if (!recollectionId) return + if (runStatus === 'running' || runStatus === 'pending') return + // Reset previous run state, save, then run + resetRun() + save() + createAndStreamRun(recollectionId, { nodes, edges }, connectToRun).catch((err) => { + toast.error(`Run failed: ${err.message}`) + }) + }, [recollectionId, nodes, edges, connectToRun, save, runStatus, resetRun]) + const nodesRef = useRef(nodes) nodesRef.current = nodes const graphRef = useRef<{ nodes: AppNode[]; edges: AppEdge[] }>({ nodes: [], edges: [] }) @@ -464,6 +493,7 @@ export function CanvasPage({ recollectionId, focusNodeId }: CanvasPageProps) { canDuplicate: selectedNodes.length > 0, canCopy: selectedNodes.length === 1, onFitView: () => flowActionsRef.current?.fitView?.(), + onRun: recollectionId ? handleRun : undefined, } setFluxSlot(slot) return () => setFluxSlot(null) @@ -481,6 +511,7 @@ export function CanvasPage({ recollectionId, focusNodeId }: CanvasPageProps) { handleCopy, handlePaste, selectedNodes.length, + handleRun, ]) const graphContextValue = useMemo( diff --git a/frontend/src/app/recollections/katalogos/KatalogosPage.tsx b/frontend/src/app/recollections/katalogos/KatalogosPage.tsx index e12de46..860439f 100644 --- a/frontend/src/app/recollections/katalogos/KatalogosPage.tsx +++ b/frontend/src/app/recollections/katalogos/KatalogosPage.tsx @@ -3,17 +3,19 @@ * Shows live \"Artifacts\" from Flux rendering nodes in a card grid. */ -import React, { useMemo } from 'react' +import React, { useMemo, useState } from 'react' import { useNavigate, useParams } from 'react-router-dom' import { usePlatform } from '@/app/kosmos/KosmosContext' import { getRenderOutputCache, formatTimeSinceLastUpdate } from '../state/recollectionStore' import { Button } from '@/components/ui/button' +import { RunsTab } from './RunsTab' export function KatalogosPage() { const { recollectionId } = useParams<{ recollectionId: string }>() const { recollections } = usePlatform() const navigate = useNavigate() + const [activeTab, setActiveTab] = useState<'artifacts' | 'runs'>('artifacts') const recollection = recollectionId ? recollections.find((p) => p.id === recollectionId) : null const title = recollection?.name ?? 'Untitled' @@ -33,10 +35,25 @@ export function KatalogosPage() {

{title}

-

- Katalogos · Live artifacts produced by Flux rendering nodes for this recollection. -

- {artifacts.length === 0 ? ( +
+ + +
+ {activeTab === 'runs' ? ( + + ) : artifacts.length === 0 ? (
No artifacts yet. In Flux, run a graph with a rendering node; its output will be cached as an artifact and appear here, as well as in Logos blocks that insert artifacts. diff --git a/frontend/src/app/recollections/katalogos/RunsTab.tsx b/frontend/src/app/recollections/katalogos/RunsTab.tsx new file mode 100644 index 0000000..4b735c8 --- /dev/null +++ b/frontend/src/app/recollections/katalogos/RunsTab.tsx @@ -0,0 +1,135 @@ +/** + * Runs tab for Katalogos: shows execution history for the current recollection. + */ + +import React, { useEffect, useState, useCallback } from 'react' +import { useParams } from 'react-router-dom' +import { CheckCircle2, XCircle, Clock, Loader2 } from 'lucide-react' + +type RunSummary = { + id: string + status: string + createdAt: string + updatedAt: string + error?: string | null +} + +type RunDetail = RunSummary & { + steps: Array<{ + id: string + nodeId: string + nodeType: string + status: string + error?: string | null + startedAt?: string | null + endedAt?: string | null + }> +} + +const statusIcon: Record = { + pending: , + running: , + completed: , + failed: , +} + +export function RunsTab() { + const { recollectionId } = useParams<{ recollectionId: string }>() + const [runs, setRuns] = useState([]) + const [loading, setLoading] = useState(true) + const [expandedRunId, setExpandedRunId] = useState(null) + const [runDetail, setRunDetail] = useState(null) + + const fetchRuns = useCallback(async () => { + if (!recollectionId) return + setLoading(true) + try { + const res = await fetch(`/api/recollections/${recollectionId}/runs`) + if (res.ok) { + const data = await res.json() + setRuns(data.runs) + } + } catch { + // silently fail + } finally { + setLoading(false) + } + }, [recollectionId]) + + useEffect(() => { fetchRuns() }, [fetchRuns]) + + const toggleExpand = async (runId: string) => { + if (expandedRunId === runId) { + setExpandedRunId(null) + setRunDetail(null) + return + } + setExpandedRunId(runId) + try { + const res = await fetch(`/api/runs/${runId}`) + if (res.ok) { + const data = await res.json() + setRunDetail(data) + } + } catch { + // silently fail + } + } + + if (loading) { + return ( +
+ + Loading runs… +
+ ) + } + + if (runs.length === 0) { + return ( +
+ No runs yet. Use the Run button in Flux to execute your graph. +
+ ) + } + + return ( +
+ {runs.map((run) => ( +
+ + {expandedRunId === run.id && runDetail && ( +
+ {run.error && ( +

{run.error}

+ )} +
+ {runDetail.steps.map((step) => ( +
+ {statusIcon[step.status] ?? statusIcon.pending} + {step.nodeId.slice(0, 8)} + {step.nodeType} + {step.status} + {step.error && {step.error}} +
+ ))} +
+
+ )} +
+ ))} +
+ ) +} diff --git a/frontend/src/app/recollections/layout/RecollectionActionsContext.tsx b/frontend/src/app/recollections/layout/RecollectionActionsContext.tsx index 45979cb..7e3d320 100644 --- a/frontend/src/app/recollections/layout/RecollectionActionsContext.tsx +++ b/frontend/src/app/recollections/layout/RecollectionActionsContext.tsx @@ -48,6 +48,7 @@ export type FluxSlot = { canDuplicate?: boolean canCopy?: boolean onFitView?: () => void + onRun?: () => void } export type LogosSlot = { diff --git a/frontend/src/app/recollections/layout/RecollectionEditViewMenus.tsx b/frontend/src/app/recollections/layout/RecollectionEditViewMenus.tsx index bc4237f..a18136d 100644 --- a/frontend/src/app/recollections/layout/RecollectionEditViewMenus.tsx +++ b/frontend/src/app/recollections/layout/RecollectionEditViewMenus.tsx @@ -14,7 +14,8 @@ import { } from '@/components/ui/menubar' import { Kbd, KbdGroup } from '@/components/ui/kbd' import { useRecollectionActions } from './RecollectionActionsContext' -import { ClipboardPaste, Copy, CopyPlus, Redo2, Undo2 } from 'lucide-react' +import { ClipboardPaste, Copy, CopyPlus, Redo2, Undo2, Play, Square } from 'lucide-react' +import { useRunStore } from '@/lib/graph/runStore' const UNDO_KEYS = { key: 'z', shiftKey: false } const REDO_KEYS = { key: 'z', shiftKey: true } @@ -26,8 +27,28 @@ function matchKey(ev: KeyboardEvent, want: { key: string; shiftKey: boolean }) { export function RecollectionEditViewMenus() { const { activeSlot, flux, isFluxActive } = useRecollectionActions() + const runStatus = useRunStore((s) => s.status) + const resetRun = useRunStore((s) => s.reset) + const isDirty = useRunStore((s) => s.dirty) const fluxSlot = isFluxActive ? flux : null + const isRunning = runStatus === 'running' || runStatus === 'pending' + const hasFinished = runStatus === 'completed' || runStatus === 'failed' + + // Keyboard shortcut: Cmd+Enter to run + useEffect(() => { + if (!fluxSlot?.onRun) return + const onKeyDown = (ev: KeyboardEvent) => { + const mod = ev.ctrlKey || ev.metaKey + if (mod && ev.key === 'Enter' && !isRunning) { + ev.preventDefault() + ev.stopPropagation() + fluxSlot.onRun?.() + } + } + window.addEventListener('keydown', onKeyDown, true) + return () => window.removeEventListener('keydown', onKeyDown, true) + }, [fluxSlot?.onRun, isRunning]) useEffect(() => { if (!activeSlot) return @@ -150,5 +171,40 @@ export function RecollectionEditViewMenus() { [activeSlot, fluxSlot, hasFluxOnly] ) - return menus + return ( +
+ {menus} + {fluxSlot?.onRun && ( + + )} +
+ ) } diff --git a/frontend/src/components/graph/BaseNode.tsx b/frontend/src/components/graph/BaseNode.tsx index 319e821..0e42c30 100644 --- a/frontend/src/components/graph/BaseNode.tsx +++ b/frontend/src/components/graph/BaseNode.tsx @@ -5,6 +5,7 @@ import { useContext } from "react"; import { FlowUIContext } from "@/lib/graph/flowContext"; import { useConnectionPathRoleFromStore } from "@/app/canvas/useCanvasConnectionPathFromStore"; import { cn } from "@/lib/utils"; +import { NodeRunStatusOverlay } from "./NodeRunStatusOverlay"; /** Default min size for resizable nodes (used by NodeResizer). */ export const RESIZE_MIN_WIDTH = 120; @@ -102,6 +103,7 @@ export function BaseNode({ {!isFullscreenInstance && (
{handles}
)} + {nodeId && }
); } diff --git a/frontend/src/components/graph/NodeFooterEdgeIndicators.tsx b/frontend/src/components/graph/NodeFooterEdgeIndicators.tsx index f663cb5..f60b188 100644 --- a/frontend/src/components/graph/NodeFooterEdgeIndicators.tsx +++ b/frontend/src/components/graph/NodeFooterEdgeIndicators.tsx @@ -3,6 +3,7 @@ import { GraphContext } from '@/lib/graph/flowContext' import { ArrowDownLeft, ArrowUpRight } from 'lucide-react' import { NodeHelpPopover } from '@/components/graph/NodeHelpPopover' import { getNodeType, getNodeClassificationLabel } from '@/lib/graph/nodeRegistry' +import { NodeRunStatusBadge } from './NodeRunStatusOverlay' type Props = { nodeId: string @@ -55,6 +56,7 @@ export function NodeFooterEdgeIndicators({ nodeId, nodeType, children }: Props) {classificationLabel} )} + diff --git a/frontend/src/components/graph/NodeRunStatusOverlay.tsx b/frontend/src/components/graph/NodeRunStatusOverlay.tsx new file mode 100644 index 0000000..a7f8a45 --- /dev/null +++ b/frontend/src/components/graph/NodeRunStatusOverlay.tsx @@ -0,0 +1,84 @@ +/** + * Run status indicator for nodes. Two exports: + * - NodeRunStatusBadge: inline badge for node footers (primary integration point) + * - NodeRunStatusOverlay: subtle border overlay for running/pending/failed states + */ + +import React from 'react' +import { useRunStore, type NodeRunStatus } from '@/lib/graph/runStore' +import { cn } from '@/lib/utils' +import { CheckCircle2, Loader2, XCircle, Clock } from 'lucide-react' + +const badgeConfig: Record = { + pending: { + icon: , + label: 'Pending', + className: 'text-muted-foreground', + }, + running: { + icon: , + label: 'Running', + className: 'text-blue-500', + }, + completed: { + icon: , + label: 'Done', + className: 'text-emerald-500', + }, + failed: { + icon: , + label: 'Failed', + className: 'text-destructive', + }, +} + +/** Inline badge for node footers — shows run status next to edge indicators. */ +export function NodeRunStatusBadge({ nodeId }: { nodeId: string }) { + const nodeState = useRunStore((s) => s.nodeStates[nodeId]) + const runStatus = useRunStore((s) => s.status) + + if (runStatus === 'idle') return null + if (!nodeState) return null + + const { icon, label, className } = badgeConfig[nodeState.status] + + return ( + + {icon} + {label} + + ) +} + +const overlayBorder: Record = { + pending: 'border-muted-foreground/20', + running: 'border-blue-500/40', + completed: 'border-transparent', + failed: 'border-destructive/40', +} + +/** Subtle border overlay — only visible for running/pending/failed. */ +export function NodeRunStatusOverlay({ nodeId }: { nodeId: string }) { + const nodeState = useRunStore((s) => s.nodeStates[nodeId]) + const runStatus = useRunStore((s) => s.status) + + if (runStatus === 'idle') return null + if (!nodeState) return null + // No overlay needed for completed — the footer badge is enough + if (nodeState.status === 'completed') return null + + return ( +
+ {nodeState.error && ( +
+ {nodeState.error} +
+ )} +
+ ) +} diff --git a/frontend/src/components/nodes/render/RenderingNode.tsx b/frontend/src/components/nodes/render/RenderingNode.tsx index 6a8a76c..c0b0b87 100644 --- a/frontend/src/components/nodes/render/RenderingNode.tsx +++ b/frontend/src/components/nodes/render/RenderingNode.tsx @@ -22,17 +22,9 @@ import { MenubarSubContent, MenubarSubTrigger, } from '@/components/ui/menubar' -import { Sparkles, Play, ChevronDown, Loader2, RotateCw } from 'lucide-react' +import { Sparkles, Loader2, RotateCw } from 'lucide-react' import { InputHandle, OutputHandle } from '@/components/graph/NodeHandles' import { Button } from '@/components/ui/button' -import { ButtonGroup } from '@/components/ui/button-group' -import { - DropdownMenu, - DropdownMenuCheckboxItem, - DropdownMenuContent, - DropdownMenuLabel, - DropdownMenuTrigger, -} from '@/components/ui/dropdown-menu' import { useTheme } from '@/lib/themeContext' import { useRenderingNodeState, @@ -165,95 +157,8 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { : undefined } right={ - state.incomingIds.length > 0 ? ( -
- - {state.effectiveUpdateMode === 'manual' ? ( - - ) : state.loading ? ( - - ) : null} - - - - - e.stopPropagation()}> - - When to re-render - - - checked && state.setUpdateMode('auto') - } - className="flex flex-col items-start gap-0.5 py-2" - > - Auto - - Re-renders when upstream content changes - - - - checked && state.setUpdateMode('manual') - } - className="flex flex-col items-start gap-0.5 py-2" - > - Manual - - Re-renders only when you click Run - - - - - -
+ state.incomingIds.length > 0 && state.loading ? ( + ) : undefined } /> diff --git a/frontend/src/components/nodes/render/useRenderingNodeState.ts b/frontend/src/components/nodes/render/useRenderingNodeState.ts index a4e7d37..1e18f31 100644 --- a/frontend/src/components/nodes/render/useRenderingNodeState.ts +++ b/frontend/src/components/nodes/render/useRenderingNodeState.ts @@ -76,6 +76,7 @@ import { import { buildSourceSignatures, type NodeLike, type EdgeLike } from '@/lib/graph/renderingSignatures' import { getNodeDisplayStatus, type NodeDisplayStatus } from '@/lib/graph/state' import type { ConfigTypeId, SourceRenderingLogicContext } from '@/lib/graph/rendering' +import { useRunStore } from '@/lib/graph/runStore' export type OutputMode = 'image' | 'string' @@ -207,8 +208,11 @@ export function useRenderingNodeState( () => (srcNode?.type ? getSourceRenderingLogic(srcNode.type as string) : null), [srcNode?.type] ) - const effectiveUpdateMode = (data?.updateMode ?? sourceLogic?.defaultUpdateMode ?? 'auto') as 'auto' | 'manual' - const runTrigger = data?.runTrigger ?? 0 + // All rendering is now triggered by the global Run button — no auto/manual distinction. + const effectiveUpdateMode: 'auto' | 'manual' = 'manual' + // Use global run trigger from runStore instead of per-node trigger + const globalRunTrigger = useRunStore((s) => s.globalRunTrigger) + const runTrigger = globalRunTrigger const outputMode: OutputMode = (data?.outputMode ?? 'image') as OutputMode const setOutputMode = useCallback( (mode: OutputMode) => updateData({ outputMode: mode }), @@ -330,7 +334,7 @@ export function useRenderingNodeState( if (!hasCachedOutput) { setRenderedContent(null) setResolvedContent(null) - setError({ kind: 'no-content', message: 'Click Run to render.' }) + setError({ kind: 'no-content', message: 'Press Run (⌘↵) to render.' }) setLoading(false) } return diff --git a/frontend/src/hooks/useRunStream.ts b/frontend/src/hooks/useRunStream.ts new file mode 100644 index 0000000..b8d2d8b --- /dev/null +++ b/frontend/src/hooks/useRunStream.ts @@ -0,0 +1,109 @@ +/** + * SSE subscriber hook for graph run execution. + * Connects to GET /api/runs/:id/stream and updates runStore. + */ + +import { useEffect, useRef, useCallback } from 'react' +import { useRunStore } from '@/lib/graph/runStore' +import { dispatchCanvasCommand } from '@/app/canvas/canvasStore' + +export function useRunStream() { + const eventSourceRef = useRef(null) + const { startRun, setRunStatus, setNodeStatus, appendChunk } = useRunStore() + + const disconnect = useCallback(() => { + if (eventSourceRef.current) { + eventSourceRef.current.close() + eventSourceRef.current = null + } + }, []) + + const connectToRun = useCallback( + (runId: string) => { + disconnect() + startRun(runId) + + const es = new EventSource(`/api/runs/${runId}/stream`) + eventSourceRef.current = es + + es.addEventListener('run/started', (e) => { + const data = JSON.parse(e.data) + setRunStatus('running') + // Initialize all nodes as pending so overlays appear immediately + const nodeIds = data.nodeIds as string[] | undefined + if (nodeIds) { + for (const nodeId of nodeIds) { + setNodeStatus(nodeId, { status: 'pending' }) + } + } + }) + + es.addEventListener('run/completed', () => { + setRunStatus('completed') + es.close() + }) + + es.addEventListener('run/failed', (e) => { + const data = JSON.parse(e.data) + setRunStatus('failed', data.error) + es.close() + }) + + es.addEventListener('step/started', (e) => { + const data = JSON.parse(e.data) + setNodeStatus(data.nodeId, { status: 'running' }) + // Fire trail animation along edges leading to this node + dispatchCanvasCommand({ type: 'path/addTrigger', payload: data.nodeId }) + }) + + es.addEventListener('step/completed', (e) => { + const data = JSON.parse(e.data) + setNodeStatus(data.nodeId, { status: 'completed', output: data.output }) + }) + + es.addEventListener('step/failed', (e) => { + const data = JSON.parse(e.data) + setNodeStatus(data.nodeId, { status: 'failed', error: data.error }) + }) + + es.addEventListener('step/chunk', (e) => { + const data = JSON.parse(e.data) + appendChunk(data.nodeId, data.chunk) + }) + + es.onerror = () => { + setRunStatus('failed', 'Connection lost') + es.close() + } + }, + [disconnect, startRun, setRunStatus, setNodeStatus, appendChunk] + ) + + // Cleanup on unmount + useEffect(() => disconnect, [disconnect]) + + return { connectToRun, disconnect } +} + +/** + * Trigger a new run: POST the graph, then connect SSE. + */ +export async function createAndStreamRun( + recollectionId: string, + graph: { nodes: unknown[]; edges: unknown[] }, + connectToRun: (runId: string) => void +): Promise { + const res = await fetch('/api/runs', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ recollectionId, graph }), + }) + + if (!res.ok) { + const err = await res.json().catch(() => ({ error: 'Failed to create run' })) + throw new Error(err.error ?? 'Failed to create run') + } + + const { id } = await res.json() + connectToRun(id) +} diff --git a/frontend/src/lib/graph/abstractNode.ts b/frontend/src/lib/graph/abstractNode.ts index d5f9a23..0216a02 100644 --- a/frontend/src/lib/graph/abstractNode.ts +++ b/frontend/src/lib/graph/abstractNode.ts @@ -4,7 +4,7 @@ * - **AbstractNodeProps** — Typed props (id, data, width?, height?, selected?) for your node. * - **useAbstractNode(id, data)** — Flow context plus helpers: nodes, edges, setNodes, setEdges, * updateData(partial), incomingEdges, outgoingEdges, sourceIds, targetIds. Calling updateData() - * also marks this node as a connection-path trigger so edges update on data changes. + * also marks the run state as dirty so the Run button shows pending changes. * - **createAbstractNodeComponent(displayName, Component)** — Wraps with memo + nodePropsAreEqual. * * **Node lifecycle / connection status:** Nodes that can be updating, paused, or in error should @@ -17,9 +17,23 @@ import React, { useCallback, useContext, useMemo } from 'react' import { GraphContext } from './flowContext' -import { dispatchCanvasCommand } from '@/app/canvas/canvasStore' import { nodePropsAreEqual } from './flowUtils' import type { AppNode } from './nodeTypes' +import { useRunStore } from './runStore' + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** Data keys written by the render pipeline cache — these should NOT mark the run state as dirty. */ +const CACHE_DATA_KEYS = new Set([ + 'cachedRenderedContent', + 'cachedResolvedContent', + 'cachedReasoningContent', + 'cachedOutputValue', + 'lastRunSourceSignature', + 'outputMarkdown', +]) // --------------------------------------------------------------------------- // Types @@ -87,6 +101,7 @@ export function useAbstractNode>( const setNodes = graphCtx?.setNodes const setEdges = graphCtx?.setEdges + const markDirty = useRunStore((s) => s.markDirty) const updateData = useCallback( (partial: Partial) => { if (!setNodes) return @@ -95,9 +110,12 @@ export function useAbstractNode>( n.id === id ? { ...n, data: { ...(n.data as object), ...partial } } : n ) as AppNode[] ) - dispatchCanvasCommand({ type: 'path/addTrigger', payload: id }) + // Only mark dirty for user-facing changes, not internal cache writes from the render pipeline + const keys = Object.keys(partial) + const isCacheOnly = keys.length > 0 && keys.every((k) => CACHE_DATA_KEYS.has(k)) + if (!isCacheOnly) markDirty() }, - [id, setNodes] + [id, setNodes, markDirty] ) const incomingEdges = useMemo( diff --git a/frontend/src/lib/graph/runStore.ts b/frontend/src/lib/graph/runStore.ts new file mode 100644 index 0000000..33c0935 --- /dev/null +++ b/frontend/src/lib/graph/runStore.ts @@ -0,0 +1,99 @@ +/** + * Run execution state (Zustand). Tracks active run status and per-node execution state. + * Separate from canvasStore to avoid coupling graph editing with run state. + */ + +import { create } from 'zustand' + +export type NodeRunStatus = 'pending' | 'running' | 'completed' | 'failed' +export type RunStatus = 'idle' | 'pending' | 'running' | 'completed' | 'failed' + +export type NodeStepState = { + status: NodeRunStatus + output?: string + error?: string + chunk?: string +} + +export type RunState = { + /** Current run ID (null when no run is active) */ + activeRunId: string | null + /** Overall run status */ + status: RunStatus + /** Per-node execution state, keyed by node ID */ + nodeStates: Record + /** Error message if the run failed */ + error: string | null + /** Whether the graph has changed since the last run */ + dirty: boolean + /** Global trigger counter — render nodes subscribe to this to trigger their pipeline */ + globalRunTrigger: number +} + +type RunActions = { + startRun: (runId: string) => void + setRunStatus: (status: RunStatus, error?: string) => void + setNodeStatus: (nodeId: string, state: Partial) => void + appendChunk: (nodeId: string, chunk: string) => void + markDirty: () => void + reset: () => void +} + +const initialState: RunState = { + activeRunId: null, + status: 'idle', + nodeStates: {}, + error: null, + dirty: true, + globalRunTrigger: 0, +} + +export const useRunStore = create((set) => ({ + ...initialState, + + startRun: (runId) => + set((s) => ({ + activeRunId: runId, + status: 'pending', + nodeStates: {}, + error: null, + dirty: false, + globalRunTrigger: s.globalRunTrigger + 1, + })), + + setRunStatus: (status, error) => + set({ status, error: error ?? null }), + + setNodeStatus: (nodeId, partial) => + set((s) => ({ + nodeStates: { + ...s.nodeStates, + [nodeId]: { ...s.nodeStates[nodeId], ...partial } as NodeStepState, + }, + })), + + appendChunk: (nodeId, chunk) => + set((s) => { + const prev = s.nodeStates[nodeId] + return { + nodeStates: { + ...s.nodeStates, + [nodeId]: { + ...prev, + chunk: (prev?.chunk ?? '') + chunk, + }, + }, + } + }), + + markDirty: () => set((s) => { + // Clear completed/failed overlays when graph changes — stale results no longer meaningful + const isFinished = s.status === 'completed' || s.status === 'failed' + return { + dirty: true, + ...(isFinished ? { nodeStates: {}, status: 'idle' as RunStatus } : {}), + } + }), + + reset: () => set((s) => ({ ...initialState, globalRunTrigger: s.globalRunTrigger })), +})) diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 10b5fab..5e8350d 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -16,6 +16,14 @@ export default defineConfig({ target: 'http://localhost:8080', changeOrigin: true, }, + '/api/runs': { + target: 'http://localhost:8080', + changeOrigin: true, + }, + '/api/recollections': { + target: 'http://localhost:8080', + changeOrigin: true, + }, // Kroki diagram service '/api/kroki': { target: 'https://kroki.io',