prototype / refactor

- introspection protocol
- more nodes
- arrows into sub canvas
- library build
- factor out experimental dependency bloat
- add library build target

packages/canvas-dev-server/package.json

@@ -21,7 +21,7 @@
     "color": "^4.2.3",
     "postcss": "^8.4.32",
     "postcss-load-config": "^5.0.2",
-    "svelte": "^4.2.7",
+    "svelte": "^4.2.14",
     "svelte-check": "^3.6.0",
     "tailwindcss": "^3.3.6",
     "tslib": "^2.4.1",

packages/canvas-dev-server/src/lib/store.ts

@@ -16,6 +16,7 @@ export const nodes = writable<Node<ONode>[]>([]);
 export const edges = writable<Edge[]>([]);
 export const stack = writable<zRFrame[]>([])
 export const this_step_frame = writable<zRFrame | null>(null)
+export const messages = writable<MsgRunner2Inspector[]>([])
 
 let ws: WebSocket
 
@@ -45,6 +46,9 @@ function startClient() {
                 console.log('Runner state:', data.state)
             } else if (data.type === 'frame-step') {
                 this_step_frame.set(get(stack).find((f) => f.id === data.frame_id) || null)
+            } else {
+                // push on stack
+                messages.update((s) => [...s, data])
             }
 
             return

packages/canvas-dev-server/src/routes/+page.svelte

@@ -3,7 +3,7 @@
     import {Background, Controls, type FitViewOptions, MiniMap, type Node, SvelteFlow} from '@xyflow/svelte';
     import '@xyflow/svelte/dist/style.css';
 
-    import {edges, last_message, nodes, sendToRunner} from '$lib/store';
+    import {edges, last_message, messages, nodes, sendToRunner, stack} from '$lib/store';
     import FNode from "$lib/client/FNode.svelte"
     import {get} from "svelte/store"
     import type {ONode} from "canvas-engine/src/compile/canvas-node-transform"
@@ -107,8 +107,13 @@
                         <p>Select a node</p>
                     {/if}
                 {:else if tabSet === 2}
-                    TODO: show logs
-                    {JSON.stringify($last_message)}
+                    <div class="">
+                    {#each $messages as msg}
+                        {#if msg.type === 'llm-chunk'}
+                            <span>{msg.chunk}</span>
+                        {/if}
+                    {/each}
+                    </div>
                 {:else if tabSet === 3}
                     TODO: show input
                 {/if}

packages/canvas-engine/package.json

@@ -32,7 +32,10 @@
     "unist-util-visit": "^5.0.0",
     "yaml": "^2.4.1",
     "yargs": "^17.7.2",
-    "zod": "^3.22.4"
+    "zod": "^3.22.4",
+    "@langchain/community": "^0.0.44",
+    "@langchain/openai": "^0.0.26",
+    "openai": "^4.29.2"
   },
   "devDependencies": {
     "@swc/cli": "^0.3.10",

packages/canvas-engine/src/compile/file-loader.ts

@@ -23,9 +23,11 @@ export async function loadFileNode(node: FileNode, ectx: ExecutionContext, gctx:
     if (file.ext === '.canvas') {
         const canvas_blueprint = new ExecutableCanvas(node.file, await parseCanvas(node.file, gctx));
         const fn: Fn = async (top_ctx: CTX) => {
-            let sub_canvas: ExecutableCanvas
+            let sub_canvas: ExecutableCanvas = top_ctx._this._canvas_instance
+            console.log('canvas call')
             if (!top_ctx._this._canvas_instance) {
                 sub_canvas = top_ctx._this._canvas_instance = _.cloneDeep(canvas_blueprint)
+
                 sub_canvas.nodes.filter((node) => node.type === 'start').forEach((node) => {
                     top_ctx.injectFrame({
                         node,
@@ -57,6 +59,25 @@ export async function loadFileNode(node: FileNode, ectx: ExecutionContext, gctx:
                     }
                 })
             }
+
+            const label = top_ctx.frame.edge?.label
+            if (label) {
+                sub_canvas.nodes.filter((node) => {
+                    return node.type === 'code' && node.lang === 'on'
+                }).forEach((node) => {
+                    top_ctx.injectFrame({
+                        node,
+                        input: top_ctx.input,
+                        state: top_ctx.state,
+                        edge: top_ctx.frame.edge, // pass top edge so that the on node can look at the label
+                        is_aggregating: false,
+                        chart: sub_canvas,
+                        // id and ctx will be added by the engine later
+                    })
+                })
+            }
+
+
             throw new NodeReturnNotIntendedByDesign()
         }
         result_node = {

packages/canvas-engine/src/compile/parse-canvas.ts

@@ -72,8 +72,6 @@ export async function parseCanvas(canvas_path: string, config: GlobalContext): P
                 const magic_word = magic_word_check[2].toLowerCase()
                 const text = onode.code.substring(magic_word_check[1].length)
 
-                logger.debug('magic word:', magic_word, 'text:', text)
-
                 for (const comp of code_node_compilers) {
                     if (comp.magic_word && comp.lang === magic_word) {
                         onode = preParseNode({

packages/canvas-engine/src/lib.ts

@@ -0,0 +1,16 @@
+import {GlobalContext} from "./types"
+import {parseCanvas} from "./compile/parse-canvas"
+import {execCanvas} from "./runtime/exec-canvas"
+import {ExecutableCanvas} from "./runtime/ExecutableCanvas"
+import {Introspection} from "./runtime/runtime-types"
+
+
+export const createCanvasEngine = async (vault_dir: string, canvas_path: string, introspection: Introspection) => {
+    let global_context: GlobalContext = {
+        vault_dir,
+        loaded_files: {}
+    }
+    global_context.introspection = introspection
+    let node_data = await parseCanvas(canvas_path, global_context)
+    await execCanvas(new ExecutableCanvas(canvas_path, node_data), global_context)
+}

packages/canvas-engine/src/node_library/lang/yaml/node-yaml-gpt-hepler.ts

@@ -4,7 +4,6 @@ import {CTX} from "../../../runtime/runtime-types"
 import {logger} from "../../../globals"
 import chalk from "chalk"
 
-
 const openai = new OpenAI({
     apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted,
 });
@@ -26,8 +25,6 @@ export async function gpt_runner_yaml(data: z.output<typeof ZSchemaGPT>, ctx:  C
 }
 
 
-
-
 export async function gpt_runner_generic(data: z.output<typeof ZSchemaGPT>, ctx:  CTX) {
     const chatCompletion = await openai.chat.completions.create({
         ...data

packages/canvas-engine/src/node_library/magic-word/canvas-io/index.ts

@@ -1,7 +1,9 @@
 import emitInput from "./emit-input"
 import emitState from "./emit-state"
+import on from "./on"
 
 export default [
     emitInput,
-    emitState
+    emitState,
+    on
 ]

packages/canvas-engine/src/node_library/magic-word/canvas-io/on.ts

@@ -0,0 +1,16 @@
+import {NodeCompiler} from "../../../compile/template"
+import {NodeReturnNotIntendedByDesign} from "../../../runtime/errors"
+
+
+export default {
+    magic_word: true,
+    lang: 'on',
+    compile: async (code) => {
+        return (ctx) => {
+            if (ctx.frame.edge?.label?.trim() === code.trim()) {
+                return ctx.input
+            }
+            throw new NodeReturnNotIntendedByDesign()
+        }
+    }
+} satisfies NodeCompiler

packages/canvas-engine/src/node_library/magic-word/index.ts

@@ -3,6 +3,7 @@ import nodeInlineDecide from './decide'
 import log from './log'
 import llmPrompt from './llm-prompt'
 import canvasIo from "./canvas-io"
+import llm from "./llm"
 
 import {NodeCompiler} from "../../compile/template"
 
@@ -12,5 +13,6 @@ export default [
     question,
     log,
     llmPrompt,
-    ...canvasIo
+    ...canvasIo,
+    ...llm
 ] as NodeCompiler[]

packages/canvas-engine/src/node_library/magic-word/llm/index.ts

@@ -0,0 +1,6 @@
+import lchain from "./lchain"
+
+
+export default [
+    lchain
+]

packages/canvas-engine/src/node_library/magic-word/llm/lchain.ts

@@ -0,0 +1,48 @@
+import {NodeCompiler} from "../../../compile/template"
+import {logger} from "../../../globals"
+import {template_render} from "../../lang/yaml"
+import chalk from "chalk"
+import {ChatOllama} from "@langchain/community/chat_models/ollama";
+import {StringOutputParser} from "@langchain/core/output_parsers";
+import {HumanMessage} from "@langchain/core/messages";
+import {zRLLMChunk, zRUpdate} from "../../../runtime/inspection/protocol"
+import {z} from "zod"
+
+
+const model_ollama_gemma = new ChatOllama({
+    baseUrl: "http://localhost:11434", // Default value
+    model: "gemma", // Default value
+});
+
+
+
+export default {
+    lang: 'lchain',
+    magic_word: true,
+    compile: async (code) => {
+        return async (ctx) => {
+            const final_prompt = template_render(code, ctx).trim()
+            logger.debug(`${chalk.green('llm-prompt')}: `, chalk.gray(final_prompt))
+
+            const stream = await model_ollama_gemma
+                .pipe(new StringOutputParser())
+                .stream([
+                    new HumanMessage(final_prompt)
+                ])
+
+
+            const chunks: string[] = []
+            for await (const message of stream) {
+                process.stdout.write(message)
+                chunks.push(message)
+                ctx.gctx.introspection?.inform({
+                    type: 'llm-chunk',
+                    chunk: message,
+                    frame_id: ctx.frame.id
+                } satisfies z.input<typeof zRLLMChunk>)
+            }
+
+            return chunks.join('')
+        }
+    }
+} satisfies NodeCompiler

packages/canvas-engine/src/node_library/magic-word/log.ts

@@ -1,5 +1,7 @@
 import {NodeCompiler} from "../../compile/template"
-import {nunjucks_env, template_render} from "../lang/yaml"
+import {template_render} from "../lang/yaml"
+import {z} from "zod"
+import {zRLog} from "../../runtime/inspection/protocol"
 
 
 export default {
@@ -9,6 +11,13 @@ export default {
         return (ctx) => {
             const render = template_render(code, ctx)
             console.log(render)
+
+            ctx.gctx.introspection?.inform({
+                type: 'log',
+                content: render,
+                frame_id: ctx.frame.id
+            } satisfies z.input<typeof zRLog>)
+
             return ctx.input
         }
     }

packages/canvas-engine/src/runtime/exec-canvas.ts

@@ -117,10 +117,7 @@ export async function execCanvas(inital_canvas: ExecutableCanvas, gctx: GlobalCo
 
 
         const {fn, ...node_debug} = node
-        logger.debug('executing node: ', {
-            ...node_debug,
-            edges: node_debug.edges.length
-        })
+        logger.debug('executing node: ', node.type, (node as any)?.code ?? '')
 
         await gctx.introspection?.inform({
             type: 'frame-step',
@@ -129,7 +126,24 @@ export async function execCanvas(inital_canvas: ExecutableCanvas, gctx: GlobalCo
 
 
         const edges_default_out = node.edges.filter((edge) => {
-            return edge.direction === 'forward' && edge.from === node.id && (edge.label?.trim() || '').length === 0
+            if (edge.direction !== 'forward') {
+                return false
+            }
+
+            if (edge.from === node.id && (edge.label?.trim() || '').length === 0) {
+                return true
+            }
+
+            const target_node = frame!.chart.node_map.get(edge.to)
+            if (!target_node) {
+                return false
+            }
+            if (target_node.type === 'file' && target_node.file.endsWith('.canvas')) {
+                return true
+            }
+
+
+            return false
         })
 
 
@@ -153,7 +167,8 @@ export async function execCanvas(inital_canvas: ExecutableCanvas, gctx: GlobalCo
                     })
                     emit_along_edges(frame!, edges_label_out, emission)
                 },
-                gctx: gctx
+                gctx: gctx,
+                frame
             }
             frame.ctx = ctx
 

packages/canvas-engine/src/runtime/inspection/protocol.ts

@@ -40,6 +40,20 @@ export const zRFrameStep = z.object({
 })
 
 
+export const zRLLMChunk = z.object({
+    type: z.literal('llm-chunk'),
+    chunk: z.string(),
+    frame_id: z.number().optional()
+})
+
+export const zRLog = z.object({
+    type: z.literal('log'),
+    content: z.string(),
+    frame_id: z.number().optional()
+})
+
+export const zRUpdate = z.union([zRLLMChunk, zRLog])
+
 const zRRunnerState = z.object({
     type: z.literal('runner-state'),
     state: z.enum(['stepping', 'running'])
@@ -63,7 +77,8 @@ export const runner2inspector = z.union([
     zRFrameNew,
     zRFrameComplete,
     zRFrameStep,
-    zRRunnerState
+    zRRunnerState,
+    zRUpdate
 ])
 
 export const inspector2runner = zIDebugAction

packages/canvas-engine/src/runtime/runtime-types.ts

@@ -18,7 +18,8 @@ export type CTX = {
     updateInput: (new_input: any) => void
     updateState: (new_state: any) => void,
     injectFrame: (frame: StackFrame) => void
-    gctx: GlobalContext
+    gctx: GlobalContext,
+    frame: StackFrame
 } & Record<string, any>
 
 

packages/extensions/example-call-canvas/README.md

@@ -0,0 +1,15 @@
+# example-call-canvas
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.0.26. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

packages/extensions/example-call-canvas/index.ts

@@ -0,0 +1 @@
+console.log("Hello via Bun!");