From 2326bbe035f98bb13c8497136c5acff61e5b64b0 Mon Sep 17 00:00:00 2001 From: dtoro Date: Sat, 14 Mar 2026 22:58:57 +0100 Subject: [PATCH] feat: render image in markdown --- .../src/components/nodes/agent/descriptor.tsx | 2 +- .../components/nodes/config/ConfigNode.tsx | 60 ++++++++++++++++++- .../components/nodes/config/descriptor.tsx | 2 +- .../components/nodes/config/renderingLogic.ts | 4 ++ .../components/nodes/render/RenderingNode.tsx | 35 ++++++----- .../components/nodes/render/descriptor.tsx | 3 +- .../nodes/render/useRenderingNodeState.ts | 28 +++++++++ 7 files changed, 113 insertions(+), 21 deletions(-) diff --git a/frontend/src/components/nodes/agent/descriptor.tsx b/frontend/src/components/nodes/agent/descriptor.tsx index 50065f4..ca8ecb8 100644 --- a/frontend/src/components/nodes/agent/descriptor.tsx +++ b/frontend/src/components/nodes/agent/descriptor.tsx @@ -13,7 +13,7 @@ export function getAgentNodeDescriptor(): NodeTypeDescriptor { .idPrefix('agt_') .withInputOutput(true, true) .classification('archon') - .allowedSourceTypes(['config', 'variable', 'data']) + .allowedSourceTypes(['config', 'variable', 'data', 'render']) .allowedTargetTypes(['render']) .help(NODE_HELP.agent) .menu('Agent', ) diff --git a/frontend/src/components/nodes/config/ConfigNode.tsx b/frontend/src/components/nodes/config/ConfigNode.tsx index c9097ae..66e5bdd 100644 --- a/frontend/src/components/nodes/config/ConfigNode.tsx +++ b/frontend/src/components/nodes/config/ConfigNode.tsx @@ -25,7 +25,7 @@ import { BaseNodeFooter, BaseNodeHeaderRow, } from '@/components/graph/BaseNode' -import { Code2, Database, ScrollText, Variable } from 'lucide-react' +import { Code2, Database, ScrollText, Sparkles, Variable } from 'lucide-react' import { MenubarItem, MenubarSeparator, @@ -34,6 +34,7 @@ import { MenubarSubContent, MenubarSubTrigger, } from '@/components/ui/menubar' +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' import { Kbd } from '@/components/ui/kbd' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { InputHandle, OutputHandle } from '@/components/graph/NodeHandles' @@ -78,7 +79,16 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { () => getConnectedNodesByType(nodes, sourceIds, 'data'), [nodes, sourceIds] ) - const hasDependencies = connectedConfigNodes.length > 0 || connectedVariableNodes.length > 0 || connectedFunctionNodes.length > 0 || connectedDataNodes.length > 0 + const connectedRenderNodes = useMemo( + () => getConnectedNodesByType(nodes, sourceIds, 'render'), + [nodes, sourceIds] + ) + const hasDependencies = + connectedConfigNodes.length > 0 || + connectedVariableNodes.length > 0 || + connectedFunctionNodes.length > 0 || + connectedDataNodes.length > 0 || + connectedRenderNodes.length > 0 const setConfigType = useCallback( (newTypeId: ConfigTypeId) => { @@ -133,6 +143,18 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { [insertAt] ) + const insertRenderReference = useCallback( + (sourceNode: any, mode: 'prepend' | 'append' | 'cursor') => { + const isMarkdownImage = + configTypeId === 'markdown' && (sourceNode.data?.outputMode ?? 'image') === 'image' + const snippet = isMarkdownImage + ? `![Rendered]({{ ${sourceNode.id} }})` + : `{{ ${sourceNode.id} }}` + insertAt(snippet, mode) + }, + [insertAt, configTypeId] + ) + const variableIds = useMemo(() => connectedVariableNodes.map((n: any) => n.id), [connectedVariableNodes]) const functionIds = useMemo(() => connectedFunctionNodes.map((n: any) => n.id), [connectedFunctionNodes]) const configTitles = useMemo( @@ -296,6 +318,40 @@ function ConfigNodeComponent({ id, data, width, height, selected }: Props) { Insert ))} + {connectedRenderNodes.map((n: any) => { + const renderInputDisabled = + configTypeId === 'plantuml' || configTypeId === 'wireframe' + const item = ( + + !renderInputDisabled && insertRenderReference(n, 'cursor') + } + > + + {n.id} + + Insert + + + ) + return renderInputDisabled ? ( + + + {item} + + Rendering output is not available for Diagram and + Wireframe configs. Use a Markdown config to embed + render output. + + + + ) : ( + item + ) + })} ) : ( Connect nodes to insert references diff --git a/frontend/src/components/nodes/config/descriptor.tsx b/frontend/src/components/nodes/config/descriptor.tsx index e745c5f..720f17b 100644 --- a/frontend/src/components/nodes/config/descriptor.tsx +++ b/frontend/src/components/nodes/config/descriptor.tsx @@ -68,7 +68,7 @@ export function getConfigNodeDescriptor(): NodeTypeDescriptor { .idPrefix('cfg_') .withInputOutput(true, true) .classification('psyche') - .allowedSourceTypes(['config', 'variable', 'function', 'data']) + .allowedSourceTypes(['config', 'variable', 'function', 'data', 'render']) .allowedTargetTypes(['config', 'render', 'agent']) .help(NODE_HELP.config) .menu('Config', ) diff --git a/frontend/src/components/nodes/config/renderingLogic.ts b/frontend/src/components/nodes/config/renderingLogic.ts index 0b7a9b6..c8ef0ad 100644 --- a/frontend/src/components/nodes/config/renderingLogic.ts +++ b/frontend/src/components/nodes/config/renderingLogic.ts @@ -74,6 +74,10 @@ export async function getResolvedContentForConfig(context: SourceRenderingLogicC }) nunjucksContext[src.id] = filteredRows } + if (src?.type === 'render') { + nunjucksContext[src.id] = + ((src.data as Record)?.cachedOutputValue as string | undefined) ?? '' + } } const functionIdsToRegister = new Set() diff --git a/frontend/src/components/nodes/render/RenderingNode.tsx b/frontend/src/components/nodes/render/RenderingNode.tsx index f7c7224..6a8a76c 100644 --- a/frontend/src/components/nodes/render/RenderingNode.tsx +++ b/frontend/src/components/nodes/render/RenderingNode.tsx @@ -23,7 +23,7 @@ import { MenubarSubTrigger, } from '@/components/ui/menubar' import { Sparkles, Play, ChevronDown, Loader2, RotateCw } from 'lucide-react' -import { InputHandle } from '@/components/graph/NodeHandles' +import { InputHandle, OutputHandle } from '@/components/graph/NodeHandles' import { Button } from '@/components/ui/button' import { ButtonGroup } from '@/components/ui/button-group' import { @@ -50,16 +50,14 @@ export type { RenderingNodeData } type Props = AbstractNodeProps -type ViewMode = 'preview' | 'raw' - function RenderingNodeComponent({ id, data, width, height, selected }: Props) { const flowUIContext = useContext(FlowUIContext) const setFullscreenNodeId = flowUIContext?.setFullscreenNodeId const supportsFullscreen = getNodeType('render')?.supportsFullscreen const state = useRenderingNodeState(id, data) + const outputMode = state.outputMode - const [viewMode, setViewMode] = useState('preview') const [viewportFocused, setViewportFocused] = useState(false) const dimensions = @@ -68,7 +66,7 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { : undefined const { theme } = useTheme() - const [rawEditorHeight, rawEditorContainerRef] = useResizeHeight(180, [viewMode]) + const [rawEditorHeight, rawEditorContainerRef] = useResizeHeight(180, [outputMode]) const showEmpty = state.incomingIds.length === 0 && @@ -151,7 +149,12 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { resizable nodeId={id} selected={selected} - handles={} + handles={ + <> + + + + } > } @@ -264,17 +267,17 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { <> checked && setViewMode('preview')} + checked={outputMode === 'image'} + onCheckedChange={(checked) => checked && state.setOutputMode('image')} > - Preview + Image checked && setViewMode('raw')} + checked={outputMode === 'string'} + onCheckedChange={(checked) => checked && state.setOutputMode('string')} > - Raw + String {getNodeType(state.sourceNodeType ?? '')?.getOutputMenuContent?.(state.rawLanguage, { state, nodeId: id })} @@ -306,7 +309,7 @@ function RenderingNodeComponent({ id, data, width, height, selected }: Props) { ) : state.error ? ( errorUi - ) : viewMode === 'raw' ? ( + ) : outputMode === 'string' ? ( - {viewMode === 'raw' + {outputMode === 'string' ? state.rawDisplayContent - ? `Raw · ${state.rawDisplayContent.length} chars` + ? `String · ${state.rawDisplayContent.length} chars` : '—' : state.renderedContent - ? `${state.outputLabel} · ${state.renderedContent.length} chars` + ? `Image · ${state.renderedContent.length} chars` : state.error ? 'Error' : '—'} diff --git a/frontend/src/components/nodes/render/descriptor.tsx b/frontend/src/components/nodes/render/descriptor.tsx index e756823..1fc42f4 100644 --- a/frontend/src/components/nodes/render/descriptor.tsx +++ b/frontend/src/components/nodes/render/descriptor.tsx @@ -15,9 +15,10 @@ export function getRenderNodeDescriptor(): NodeTypeDescriptor { { viewportWidth: 1200, viewportHeight: 800 } ) .idPrefix('rnd_') - .withInputOutput(true, false) + .withInputOutput(true, true) .classification('pneuma') .allowedSourceTypes(['config', 'agent']) + .allowedTargetTypes(['config', 'agent']) .help(NODE_HELP.render) .menu('Renderer', ) .connectionLabelByStatus({ default: 'listening', updating: 'giving life', paused: 'pending', error: 'corrupted' }) diff --git a/frontend/src/components/nodes/render/useRenderingNodeState.ts b/frontend/src/components/nodes/render/useRenderingNodeState.ts index 68116c9..fe8fcba 100644 --- a/frontend/src/components/nodes/render/useRenderingNodeState.ts +++ b/frontend/src/components/nodes/render/useRenderingNodeState.ts @@ -24,15 +24,21 @@ import { buildSourceSignatures, type NodeLike, type EdgeLike } from '@/lib/graph import { getNodeDisplayStatus, type NodeDisplayStatus } from '@/lib/graph/state' import type { ConfigTypeId, SourceRenderingLogicContext } from '@/lib/graph/rendering' +export type OutputMode = 'image' | 'string' + export type RenderingNodeData = { viewportWidth?: number viewportHeight?: number updateMode?: 'auto' | 'manual' runTrigger?: number lastRunSourceSignature?: string + /** Controls which view is shown and what the node emits to downstream (Image = markdown embed, String = resolved text). */ + outputMode?: OutputMode cachedRenderedContent?: string cachedResolvedContent?: string cachedReasoningContent?: string + /** Value exposed to config/agent when they reference this node (e.g. {{ renderId }}). Set on pipeline completion. */ + cachedOutputValue?: string } const DEFAULT_VIEWPORT_WIDTH = 1200 @@ -96,6 +102,10 @@ export type RenderingNodeState = { /** Source node type id (e.g. 'config', 'agent') for Output menu from descriptor. */ sourceNodeType: string | null + + /** Output mode (Image vs String); controls view and emitted cachedOutputValue. */ + outputMode: OutputMode + setOutputMode: (mode: OutputMode) => void } export function useRenderingNodeState( @@ -140,6 +150,11 @@ export function useRenderingNodeState( ) const effectiveUpdateMode = (data?.updateMode ?? sourceLogic?.defaultUpdateMode ?? 'auto') as 'auto' | 'manual' const runTrigger = data?.runTrigger ?? 0 + const outputMode: OutputMode = (data?.outputMode ?? 'image') as OutputMode + const setOutputMode = useCallback( + (mode: OutputMode) => updateData({ outputMode: mode }), + [updateData] + ) const isAgentSource = srcNode?.type === 'agent' const agentOutputMarkdown = isAgentSource ? ((srcNode?.data as { outputMarkdown?: string })?.outputMarkdown ?? '') @@ -201,6 +216,8 @@ export function useRenderingNodeState( const updateDataRef = useRef(updateData) updateDataRef.current = updateData + const outputModeRef = useRef(outputMode) + outputModeRef.current = outputMode const setNodesRef = useRef(setNodes) setNodesRef.current = setNodes const aiConnectionRef = useRef(aiConnection) @@ -294,6 +311,7 @@ export function useRenderingNodeState( cachedRenderedContent: undefined, cachedResolvedContent: undefined, cachedReasoningContent: undefined, + cachedOutputValue: undefined, }) try { // Pipeline: 1) Resolve (source) → 2) Render (output type) → 3) Cache @@ -324,10 +342,18 @@ export function useRenderingNodeState( if (thisRunId !== runIdRef.current) return setRenderedContent(htmlOrSvg) setError(null) + const mode = outputModeRef.current + const cachedOutputValue = + mode === 'string' + ? resolved + : mode === 'image' && htmlOrSvg?.trim() && /]/i.test(htmlOrSvg.trim()) + ? `data:image/svg+xml;charset=utf-8,${encodeURIComponent(htmlOrSvg)}` + : '' updateDataRef.current({ cachedRenderedContent: htmlOrSvg, cachedResolvedContent: resolved, cachedReasoningContent: reasoning ?? '', + cachedOutputValue, ...(isManualMode ? { lastRunSourceSignature: signatureForThisRun } : {}), }) } catch (err: unknown) { @@ -546,5 +572,7 @@ export function useRenderingNodeState( emptyStateAction, sourceData, sourceNodeType, + outputMode, + setOutputMode, } }