From 38e1ae369265aa41af91fac06c52c0ba5a27087e Mon Sep 17 00:00:00 2001 From: dtoro Date: Tue, 10 Mar 2026 23:49:28 +0100 Subject: [PATCH] feat: add initial data node --- frontend/package-lock.json | 34 ++ frontend/package.json | 1 + frontend/src/components/base/NodeMenubar.tsx | 14 +- frontend/src/components/nodes/ConfigNode.tsx | 31 +- frontend/src/components/nodes/DataNode.tsx | 354 ++++++++++++++++++ .../src/components/nodes/RenderingNode.tsx | 26 +- frontend/src/lib/csvParse.ts | 54 +++ frontend/src/lib/flowUtils.ts | 1 + frontend/src/lib/nodeHelp.tsx | 15 +- frontend/src/lib/nodeTypes.ts | 3 +- frontend/src/lib/nunjucksAutocomplete.ts | 6 + frontend/src/lib/registerBuiltinNodes.tsx | 21 +- 12 files changed, 550 insertions(+), 10 deletions(-) create mode 100644 frontend/src/components/nodes/DataNode.tsx create mode 100644 frontend/src/lib/csvParse.ts diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ccca5f0..8e2b347 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -24,6 +24,7 @@ "@radix-ui/react-toggle": "^1.1.10", "@radix-ui/react-toggle-group": "^1.1.11", "@radix-ui/react-tooltip": "^1.2.8", + "@tanstack/react-table": "^8.21.3", "@uiw/react-codemirror": "^4.25.7", "@wireweave/core": "^2.6.0", "@xyflow/react": "^12.10.1", @@ -3272,6 +3273,39 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@tanstack/react-table": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.21.3.tgz", + "integrity": "sha512-5nNMTSETP4ykGegmVkhjcS8tTLW6Vl4axfEGQN3v0zdHYbK4UfoqfPChclTrJ4EoK9QynqAu9oUf8VEmrpZ5Ww==", + "license": "MIT", + "dependencies": { + "@tanstack/table-core": "8.21.3" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, + "node_modules/@tanstack/table-core": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.21.3.tgz", + "integrity": "sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@ts-morph/common": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.27.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index ae0cf6d..0e18528 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -24,6 +24,7 @@ "@radix-ui/react-toggle": "^1.1.10", "@radix-ui/react-toggle-group": "^1.1.11", "@radix-ui/react-tooltip": "^1.2.8", + "@tanstack/react-table": "^8.21.3", "@uiw/react-codemirror": "^4.25.7", "@wireweave/core": "^2.6.0", "@xyflow/react": "^12.10.1", diff --git a/frontend/src/components/base/NodeMenubar.tsx b/frontend/src/components/base/NodeMenubar.tsx index ab91205..2ed754e 100644 --- a/frontend/src/components/base/NodeMenubar.tsx +++ b/frontend/src/components/base/NodeMenubar.tsx @@ -35,9 +35,11 @@ type Props = { insertTagsLabel?: string /** Extra content in Node menu (e.g. Export submenu for render nodes), before the separator */ nodeMenuExtraContent?: React.ReactNode + /** When set, renders a top-level "Data" menu with this content (e.g. data nodes) */ + dataMenuContent?: React.ReactNode } -export function NodeMenubar({ nodeId, nodeType, editInputsContent, inputsMenuContent, insertTagsContent, insertMenuLabel = 'Insert', insertContentDirect, insertTagsLabel = 'Tags', nodeMenuExtraContent }: Props) { +export function NodeMenubar({ nodeId, nodeType, editInputsContent, inputsMenuContent, insertTagsContent, insertMenuLabel = 'Insert', insertContentDirect, insertTagsLabel = 'Tags', nodeMenuExtraContent, dataMenuContent }: Props) { const ctx = useContext(FlowContext) const nodes = ctx?.nodes ?? [] const setNodes = ctx?.setNodes @@ -140,6 +142,16 @@ export function NodeMenubar({ nodeId, nodeType, editInputsContent, inputsMenuCon + {dataMenuContent != null && ( + + + Data + + + {dataMenuContent} + + + )} {inputsMenuContent != null && ( diff --git a/frontend/src/components/nodes/ConfigNode.tsx b/frontend/src/components/nodes/ConfigNode.tsx index 275571c..34224c5 100644 --- a/frontend/src/components/nodes/ConfigNode.tsx +++ b/frontend/src/components/nodes/ConfigNode.tsx @@ -27,7 +27,7 @@ import { BaseNodeFooter, BaseNodeHeaderRow, } from '../base/BaseNode' -import { Code2, ScrollText, Variable } from 'lucide-react' +import { Code2, Database, ScrollText, Variable } from 'lucide-react' import { MenubarItem, MenubarSeparator, @@ -67,7 +67,11 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { () => (nodes as FlowNode[]).filter((n) => sourceIds.includes(n.id) && n.type === 'function'), [nodes, sourceIds] ) - const hasDependencies = connectedConfigNodes.length > 0 || connectedVariableNodes.length > 0 || connectedFunctionNodes.length > 0 + const connectedDataNodes = useMemo( + () => (nodes as FlowNode[]).filter((n) => sourceIds.includes(n.id) && n.type === 'data'), + [nodes, sourceIds] + ) + const hasDependencies = connectedConfigNodes.length > 0 || connectedVariableNodes.length > 0 || connectedFunctionNodes.length > 0 || connectedDataNodes.length > 0 const onChange = useCallback( (val: string) => updateData({ content: val, configType: configTypeId }), @@ -150,12 +154,20 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { [insertAt] ) + const insertDataReference = useCallback( + (sourceNode: any, mode: 'prepend' | 'append' | 'cursor') => { + insertAt(`{% for row in ${sourceNode.id} %}\n \n{% endfor %}`, mode) + }, + [insertAt] + ) + const variableIds = useMemo(() => connectedVariableNodes.map((n: any) => n.id), [connectedVariableNodes]) const functionIds = useMemo(() => connectedFunctionNodes.map((n: any) => n.id), [connectedFunctionNodes]) const configTitles = useMemo( () => connectedConfigNodes.map((n: any) => n.data?.title ?? n.id), [connectedConfigNodes], ) + const dataIds = useMemo(() => connectedDataNodes.map((n: any) => n.id), [connectedDataNodes]) const extensions = useMemo(() => { const lang = configTypeId === 'wireframe' @@ -166,11 +178,11 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { return [ lang, autocompletion({ - override: [nunjucksCompletionSource(variableIds, configTitles, functionIds)], + override: [nunjucksCompletionSource(variableIds, configTitles, functionIds, dataIds)], activateOnTyping: true, }), ] - }, [configTypeId, configType.language, variableIds, functionIds, configTitles]) + }, [configTypeId, configType.language, variableIds, functionIds, configTitles, dataIds]) const [editorHeight, editorContainerRef] = useResizeHeight(180) const insertBlocksContent = useMemo(() => { @@ -313,6 +325,17 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { Insert ))} + {connectedDataNodes.map((n: any) => ( + insertDataReference(n, 'cursor')} + > + + {n.id} + Insert + + ))} ) : ( Connect nodes to insert references diff --git a/frontend/src/components/nodes/DataNode.tsx b/frontend/src/components/nodes/DataNode.tsx new file mode 100644 index 0000000..69aac51 --- /dev/null +++ b/frontend/src/components/nodes/DataNode.tsx @@ -0,0 +1,354 @@ +import React, { useCallback, useMemo, useRef, useState } from 'react' +import { + AbstractNodeProps, + createAbstractNodeComponent, + useAbstractNode, +} from '../../lib/abstractNode' +import { + BaseNode, + BaseNodeContent, + BaseNodeFooter, + BaseNodeHeaderRow, +} from '../base/BaseNode' +import { NodeFooterEdgeIndicators } from '../base/NodeFooterEdgeIndicators' +import { NodeHeaderTitle } from '../base/NodeHeaderTitle' +import { NodeMenubar } from '../base/NodeMenubar' +import { OutputHandle } from '../base/NodeHandles' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '../ui/table' +import { parseCsvToRows } from '../../lib/csvParse' +import { ArrowDown, ArrowUp, ArrowUpDown, Database, FileUp } from 'lucide-react' +import { + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, + type ColumnDef, + type SortingState, +} from '@tanstack/react-table' +import { Button } from '../ui/button' +import { Input } from '@/components/ui/input' +import { ContextMenuCheckboxItem } from '../ui/context-menu' +import { + MenubarCheckboxItem, + MenubarItem, + MenubarSeparator, + MenubarSub, + MenubarSubContent, + MenubarSubTrigger, +} from '../ui/menubar' + +export type DataNodeData = { + rows?: Record[] + columns?: string[] + fileName?: string + /** Column ids (header names) that are hidden; hidden columns are excluded from template output */ + hiddenColumns?: string[] +} + +type Props = AbstractNodeProps + +function DataNodeComponent({ id, data, width, height, selected }: Props) { + const { updateData } = useAbstractNode(id, data ?? {}) + const rows = data?.rows ?? [] + const columns = data?.columns ?? [] + const hiddenColumns = data?.hiddenColumns ?? [] + const [isDragging, setIsDragging] = useState(false) + const [globalFilter, setGlobalFilter] = useState('') + const [sorting, setSorting] = useState([]) + const fileInputRef = useRef(null) + const hasData = rows.length > 0 && columns.length > 0 + + const onFile = useCallback( + (file: File) => { + if (!file.name.toLowerCase().endsWith('.csv')) return + const reader = new FileReader() + reader.onload = () => { + try { + const text = (reader.result as string) ?? '' + const { columns: cols, rows: parsed } = parseCsvToRows(text) + updateData({ rows: parsed, columns: cols, fileName: file.name, hiddenColumns: [] }) + } catch { + updateData({ rows: [], columns: [], fileName: '', hiddenColumns: [] }) + } + } + reader.readAsText(file) + }, + [updateData] + ) + + const onDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault() + setIsDragging(false) + const file = e.dataTransfer?.files?.[0] + if (file) onFile(file) + }, + [onFile] + ) + + const onDragOver = useCallback((e: React.DragEvent) => { + e.preventDefault() + e.dataTransfer.dropEffect = 'copy' + setIsDragging(true) + }, []) + + const onDragLeave = useCallback(() => setIsDragging(false), []) + + const onInputChange = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0] + if (file) onFile(file) + e.target.value = '' + }, + [onFile] + ) + + const openFileDialog = useCallback(() => { + fileInputRef.current?.click() + }, []) + + const clearData = useCallback(() => { + updateData({ rows: [], columns: [], fileName: '', hiddenColumns: [] }) + }, [updateData]) + + const columnVisibility = useMemo( + () => Object.fromEntries(columns.map((col) => [col, !hiddenColumns.includes(col)])), + [columns, hiddenColumns] + ) + + const onColumnVisibilityChange = useCallback( + (updater: (prev: Record) => Record) => { + const next = updater(columnVisibility) + const newHidden = columns.filter((col) => !next[col]) + updateData({ hiddenColumns: newHidden }) + }, + [columnVisibility, columns, updateData] + ) + + const tableColumns = useMemo>[]>(() => { + return columns.map((key) => ({ + accessorKey: key, + header: ({ column }: { column: { getIsSorted: () => false | 'asc' | 'desc'; toggleSorting: (desc: boolean) => void } }) => ( + + ), + cell: ({ row }: { row: { getValue: (key: string) => unknown } }) => ( + + {String(row.getValue(key) ?? '')} + + ), + })) + }, [columns]) + + const table = useReactTable({ + data: rows, + columns: tableColumns, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onSortingChange: setSorting, + onColumnVisibilityChange, + onGlobalFilterChange: setGlobalFilter, + state: { + sorting, + columnVisibility, + globalFilter, + }, + globalFilterFn: (row, _columnIds, filterValue) => { + const str = String(filterValue ?? '').toLowerCase() + if (!str) return true + const obj = row.original as Record + for (const v of Object.values(obj)) { + if (String(v ?? '').toLowerCase().includes(str)) return true + } + return false + }, + }) + + const fileInputId = `data-node-file-${id}` + + const dataMenuContent = ( + <> + + Upload new dataset + + + Clear dataset + + {hasData && ( + <> + + + Columns + + {table + .getAllColumns() + .filter((col) => typeof col.accessorFn !== 'undefined' && col.getCanHide()) + .map((col) => ( + col.toggleVisibility(!!value)} + > + {col.id} + + ))} + + + + Rows per page + + {[10, 20, 30, 50].map((size) => ( + table.setPageSize(size)} + > + {size} + + ))} + + + + )} + + ) + + const dimensions = + width != null && height != null && width > 0 && height > 0 + ? { width, height } + : undefined + + return ( + }> + + } title={} /> + + +
+ +
+ {!hasData ? ( + + ) : ( +
+
+ setGlobalFilter(e.target.value)} + className="h-8 max-w-[180px] text-xs" + /> +
+ + {table.getState().pagination.pageIndex + 1} of {table.getPageCount()} + + + +
+
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {flexRender(header.column.columnDef.header, header.getContext())} + + ))} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ))} + + )) + ) : ( + + + No results. + + + )} + +
+
+
+ )} + + + + + {hasData + ? `${table.getFilteredRowModel().rows.length} of ${rows.length} rows ยท ${table.getVisibleLeafColumns().length} cols` + : 'Drop CSV'} + + + + ) +} + +export const DataNode = createAbstractNodeComponent('DataNode', DataNodeComponent) +export default DataNode diff --git a/frontend/src/components/nodes/RenderingNode.tsx b/frontend/src/components/nodes/RenderingNode.tsx index 5529907..9a4f044 100644 --- a/frontend/src/components/nodes/RenderingNode.tsx +++ b/frontend/src/components/nodes/RenderingNode.tsx @@ -168,6 +168,16 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { [nodes, connectedNodeIds] ) + const dataSignature = useMemo( + () => + nodes + .filter((n: any) => n.type === 'data' && connectedNodeIds.has(n.id)) + .map((n: any) => `${n.id}:${JSON.stringify(n.data?.rows ?? [])}:${JSON.stringify(n.data?.hiddenColumns ?? [])}`) + .sort() + .join('|'), + [nodes, connectedNodeIds] + ) + const RENDER_DEBOUNCE_MS = 250 useEffect(() => { @@ -273,6 +283,20 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { if (!configIdsUsed.has(e.target)) continue const src = nodes.find((n: any) => n.id === e.source) if (src?.type === 'variable') setVarInContext(src) + if (src?.type === 'data') { + const rows = (src.data?.rows as Record[] | undefined) ?? [] + const hidden = (src.data?.hiddenColumns as string[] | undefined) ?? [] + const visibleCols = (src.data?.columns as string[] | undefined) ?? [] + .filter((c) => !hidden.includes(c)) + const filteredRows = rows.map((row) => { + const out: Record = {} + for (const col of visibleCols) { + if (col in row) out[col] = row[col] + } + return out + }) + nunjucksContext[src.id] = filteredRows + } } // All function node ids that feed (directly or transitively) into config โ€” need to register them and collect their variables const functionIdsToRegister = new Set() @@ -505,7 +529,7 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { } } // Only re-run when inputs that affect the resolved output change (signatures + source). Debounced to avoid excessive re-renders while typing. retryCount triggers re-run on Retry. - }, [id, sourceContent, configTypeId, configSignature, edgesSignature, variablesSignature, functionsSignature, viewportWidth, viewportHeight, retryCount]) + }, [id, sourceContent, configTypeId, configSignature, edgesSignature, variablesSignature, functionsSignature, dataSignature, viewportWidth, viewportHeight, retryCount]) const dimensions = width != null && height != null && width > 0 && height > 0 diff --git a/frontend/src/lib/csvParse.ts b/frontend/src/lib/csvParse.ts new file mode 100644 index 0000000..c0b6bc4 --- /dev/null +++ b/frontend/src/lib/csvParse.ts @@ -0,0 +1,54 @@ +/** + * Parse a CSV string into an array of row objects (first row = headers). + * Handles quoted fields with commas. + */ +export function parseCsvToRows(csvText: string): { columns: string[]; rows: Record[] } { + const lines = csvText.trim().split(/\r?\n/) + if (lines.length === 0) return { columns: [], rows: [] } + + const parseRow = (line: string): string[] => { + const out: string[] = [] + let i = 0 + while (i < line.length) { + if (line[i] === '"') { + let cell = '' + i += 1 + while (i < line.length) { + if (line[i] === '"') { + i += 1 + if (line[i] === '"') { + cell += '"' + i += 1 + } else break + } else { + cell += line[i] + i += 1 + } + } + out.push(cell) + if (line[i] === ',') i += 1 + } else { + const comma = line.indexOf(',', i) + const value = comma === -1 ? line.slice(i) : line.slice(i, comma) + out.push(value.trim()) + i = comma === -1 ? line.length : comma + 1 + } + } + return out + } + + const headers = parseRow(lines[0]) + const columns = headers.map((h) => h.trim() || `Column${headers.indexOf(h) + 1}`) + const rows: Record[] = [] + + for (let r = 1; r < lines.length; r++) { + const cells = parseRow(lines[r]) + const row: Record = {} + for (let c = 0; c < columns.length; c++) { + row[columns[c]] = cells[c] ?? '' + } + rows.push(row) + } + + return { columns, rows } +} diff --git a/frontend/src/lib/flowUtils.ts b/frontend/src/lib/flowUtils.ts index c6a5fb3..c35ff70 100644 --- a/frontend/src/lib/flowUtils.ts +++ b/frontend/src/lib/flowUtils.ts @@ -70,6 +70,7 @@ export const DEFAULT_NODE_STYLE: Record = { ), }, + data: { + title: 'Data node', + content: ( + <> +
+

Data nodes hold CSV data. Drop a .csv file onto the node to load it; the table is shown in the node. Connect this node to a Config node to use the data in Nunjucks templates.

+
+
+

In a Config template connected to this data node, the variable {'{{ '}nodeId{' }}'} is an array of row objects (one per CSV row). Each row has keys from the CSV header. Example: {'{% for row in data_001 %}{{ row.name }}, {{ row.value }}{% endfor %}'}

+
+ + ), + }, } export function getNodeHelp(nodeType: NodeType): NodeHelpEntry { diff --git a/frontend/src/lib/nodeTypes.ts b/frontend/src/lib/nodeTypes.ts index 942c370..791d7aa 100644 --- a/frontend/src/lib/nodeTypes.ts +++ b/frontend/src/lib/nodeTypes.ts @@ -7,7 +7,8 @@ import type { ConfigNodeData } from '@/components/nodes/ConfigNode' import type { RenderingNodeData } from '@/components/nodes/RenderingNode' import type { VariableNodeData } from '@/components/nodes/VariableNode' import type { FunctionNodeData } from '@/components/nodes/FunctionNode' +import type { DataNodeData } from '@/components/nodes/DataNode' -export type AppNodeData = ConfigNodeData | RenderingNodeData | VariableNodeData | FunctionNodeData +export type AppNodeData = ConfigNodeData | RenderingNodeData | VariableNodeData | FunctionNodeData | DataNodeData export type AppNode = Node export type AppEdge = Edge diff --git a/frontend/src/lib/nunjucksAutocomplete.ts b/frontend/src/lib/nunjucksAutocomplete.ts index dfffa1d..57e6ece 100644 --- a/frontend/src/lib/nunjucksAutocomplete.ts +++ b/frontend/src/lib/nunjucksAutocomplete.ts @@ -43,6 +43,7 @@ export function nunjucksCompletionSource( variableIds: string[], configTitles?: string[], functionIds?: string[], + dataIds?: string[], ): (context: CompletionContext) => CompletionResult | null { return (context: CompletionContext) => { const { state, pos } = context @@ -81,6 +82,11 @@ export function nunjucksCompletionSource( } } } + for (const dataId of dataIds ?? []) { + if (!word || dataId.toLowerCase().startsWith(word.toLowerCase())) { + options.push({ label: dataId, type: 'variable', info: 'Data (array of rows)' }) + } + } if (options.length === 0) return null return { diff --git a/frontend/src/lib/registerBuiltinNodes.tsx b/frontend/src/lib/registerBuiltinNodes.tsx index 7686ee7..ca460d1 100644 --- a/frontend/src/lib/registerBuiltinNodes.tsx +++ b/frontend/src/lib/registerBuiltinNodes.tsx @@ -4,13 +4,14 @@ */ import React from 'react' -import { ScrollText, Sparkles, Variable, Code2 } from 'lucide-react' +import { ScrollText, Sparkles, Variable, Code2, Database } from 'lucide-react' import { registerNodeType } from './nodeRegistry' import { NODE_HELP } from './nodeHelp' import ConfigNode from '../components/nodes/ConfigNode' import RenderingNode from '../components/nodes/RenderingNode' import VariableNode from '../components/nodes/VariableNode' import FunctionNode from '../components/nodes/FunctionNode' +import DataNode from '../components/nodes/DataNode' const ICON_CLASS = 'mr-2 h-4 w-4' @@ -24,7 +25,7 @@ export function registerBuiltinNodes(): void { hasInput: true, hasOutput: true, classification: 'psyche', - allowedSourceTypes: ['config', 'variable', 'function'], + allowedSourceTypes: ['config', 'variable', 'function', 'data'], allowedTargetTypes: ['config', 'render'], help: NODE_HELP.config, menuLabel: 'Config', @@ -73,6 +74,22 @@ export function registerBuiltinNodes(): void { menuIcon: , }) + registerNodeType({ + id: 'data', + component: DataNode, + defaultStyle: { width: 360, height: 280 }, + defaultData: { rows: [], columns: [], fileName: '' }, + idPrefix: 'data_', + hasInput: false, + hasOutput: true, + classification: 'physis', + allowedTargetTypes: ['config'], + help: NODE_HELP.data, + menuLabel: 'Data', + menuIcon: , + connectionLabel: 'data source', + }) + registerNodeType({ id: 'function', component: FunctionNode,