From 7511e1a1b98f804919f7bfe4ad4e223be5d3b5cd Mon Sep 17 00:00:00 2001 From: Jacobcdsmith <88069592+Jacobcdsmith@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:49:35 +0000 Subject: [PATCH] feat: add Array Loop Iterator and Data Transform node kinds - Added 'loop' (Array Loop Iterator) node kind supporting state array iteration, item transformation templates, max_iterations caps, and empty state branching. - Added 'transform' (Data Transform) node kind supporting template_string, json_map, pick_fields, set_keys, and flatten_object operations. - Updated runtime execution engine (runFlow.ts), code generation (codegen.ts for Python and JavaScript), and UI components (Palette and AgentNode styling). - Added comprehensive unit test coverage in loopAndTransform.test.ts. --- frontend/src/flow/AgentNode.tsx | 2 + frontend/src/flow/Palette.tsx | 2 + frontend/src/flow/codegen.ts | 66 ++++++- frontend/src/flow/runFlow.ts | 142 +++++++++++++++ frontend/src/flow/types.ts | 36 +++- frontend/src/test/loopAndTransform.test.ts | 202 +++++++++++++++++++++ 6 files changed, 448 insertions(+), 2 deletions(-) create mode 100644 frontend/src/test/loopAndTransform.test.ts diff --git a/frontend/src/flow/AgentNode.tsx b/frontend/src/flow/AgentNode.tsx index b14d380..f74464b 100644 --- a/frontend/src/flow/AgentNode.tsx +++ b/frontend/src/flow/AgentNode.tsx @@ -14,6 +14,8 @@ const KIND_COLOR: Record = { http: "hsl(var(--node-http))", script: "hsl(var(--node-script))", note: "hsl(45 90% 48%)", + loop: "hsl(280 80% 55%)", + transform: "hsl(190 85% 45%)", }; interface ExtraData extends AgentNodeData { diff --git a/frontend/src/flow/Palette.tsx b/frontend/src/flow/Palette.tsx index ead67ad..d32122b 100644 --- a/frontend/src/flow/Palette.tsx +++ b/frontend/src/flow/Palette.tsx @@ -17,6 +17,8 @@ const KIND_COLOR: Record = { http: "hsl(var(--node-http))", script: "hsl(var(--node-script))", note: "hsl(45 90% 48%)", + loop: "hsl(280 80% 55%)", + transform: "hsl(190 85% 45%)", }; export function Palette({ onAdd }: Props) { diff --git a/frontend/src/flow/codegen.ts b/frontend/src/flow/codegen.ts index 3505322..9d0684e 100644 --- a/frontend/src/flow/codegen.ts +++ b/frontend/src/flow/codegen.ts @@ -432,6 +432,39 @@ export function generatePython( `# ${c.content ? c.content.replace(/\n/g, "\n# ") : "(empty note)"}`, `return "next"`, ].join("\n"); + case "loop": + return [ + `arr = state.get(${pyStr((c.array_key || "items").replace(/^state\./, ""))}, [])`, + `items = arr if isinstance(arr, list) else []`, + `mapped = []`, + `max_iter = ${parseInt(c.max_iterations || "100", 10)}`, + `for idx, item in enumerate(items[:max_iter]):`, + ` # apply item transform`, + ` state.set(${pyStr((c.item_var || "item").trim())}, item)`, + ` mapped.append(item)`, + `state.set(${pyStr((c.target_key || "processed_items").replace(/^state\./, ""))}, mapped)`, + `state.last = {"count": len(mapped), "items": mapped}`, + `return "empty" if len(items) == 0 else "next"`, + ].join("\n"); + case "transform": + return [ + `op = ${pyStr((c.operation || "template_string").toLowerCase())}`, + `source_key = ${pyStr((c.source_key || "data").replace(/^state\./, ""))}`, + `target_key = ${pyStr((c.target_key || "transformed").replace(/^state\./, ""))}`, + `expr = interpolate(${pyStr(c.expression || "")}, state)`, + `if op == "pick_fields":`, + ` source_obj = state.get(source_key, {})`, + ` fields = [f.strip() for f in ${pyStr(c.fields || "")}.split(",") if f.strip()]`, + ` res = {f: source_obj.get(f) for f in fields if isinstance(source_obj, dict) and f in source_obj}`, + `else:`, + ` try:`, + ` res = json.loads(expr)`, + ` except Exception:`, + ` res = expr`, + `state.set(target_key, res)`, + `state.last = {"operation": op, "target_key": target_key, "result": res}`, + `return "next"`, + ].join("\n"); default: { const _exhaustive: never = d.kind as never; return `return "next" # unknown kind ${_exhaustive}`; @@ -626,6 +659,37 @@ export function generateJavaScript( `// ${c.content ? c.content.replace(/\n/g, "\n// ") : "(empty note)"}`, `return "next";`, ].join("\n"); + case "loop": + return [ + `const items = Array.isArray(state.get("${(c.array_key || "items").replace(/^state\./, "")}")) ? state.get("${(c.array_key || "items").replace(/^state\./, "")}") : [];`, + `const maxIter = ${parseInt(c.max_iterations || "100", 10)};`, + `const mapped = items.slice(0, maxIter).map((item, idx) => {`, + ` state.set("${(c.item_var || "item").trim()}", item);`, + ` return item;`, + `});`, + `state.set("${(c.target_key || "processed_items").replace(/^state\./, "")}", mapped);`, + `state.last = { count: mapped.length, items: mapped };`, + `return items.length === 0 ? "empty" : "next";`, + ].join("\n"); + case "transform": + return [ + `const op = "${(c.operation || "template_string").toLowerCase()}";`, + `const sourceKey = "${(c.source_key || "data").replace(/^state\./, "")}";`, + `const targetKey = "${(c.target_key || "transformed").replace(/^state\./, "")}";`, + `const expr = interpolate(${JSON.stringify(c.expression || "")}, state);`, + `let res;`, + `if (op === "pick_fields") {`, + ` const sourceObj = state.get(sourceKey) ?? {};`, + ` const fields = "${c.fields || ""}".split(",").map(f => f.trim()).filter(Boolean);`, + ` res = {};`, + ` fields.forEach(f => { if (sourceObj && f in sourceObj) res[f] = sourceObj[f]; });`, + `} else {`, + ` try { res = JSON.parse(expr); } catch { res = expr; }`, + `}`, + `state.set(targetKey, res);`, + `state.last = { operation: op, target_key: targetKey, result: res };`, + `return "next";`, + ].join("\n"); default: return `return "next";`; } @@ -677,4 +741,4 @@ export function generateCode( } // also export the kind set for sanity -export const ALL_KINDS: AgentNodeKind[] = ["trigger","llm","tool","router","subagent","memory","human","sink","http","script","note"]; \ No newline at end of file +export const ALL_KINDS: AgentNodeKind[] = ["trigger","llm","tool","router","subagent","memory","human","sink","http","script","note","loop","transform"]; \ No newline at end of file diff --git a/frontend/src/flow/runFlow.ts b/frontend/src/flow/runFlow.ts index 1347c4d..455d315 100644 --- a/frontend/src/flow/runFlow.ts +++ b/frontend/src/flow/runFlow.ts @@ -478,6 +478,148 @@ export async function runNode( annotationOnly: true, }; } + case "loop": { + const rawArrayKey = (cfg.array_key || "items").replace(/^state\./, ""); + const arrayVal = getPath(state, rawArrayKey); + const items = Array.isArray(arrayVal) ? arrayVal : []; + const itemVarName = (cfg.item_var || "item").trim(); + const targetKey = (cfg.target_key || "processed_items").replace(/^state\./, ""); + const maxIter = parseIntOr(cfg.max_iterations, 100); + const transformTemplate = cfg.item_transform || ""; + + const mapped: unknown[] = []; + const limit = Math.min(items.length, maxIter); + + for (let i = 0; i < limit; i++) { + const itemVal = items[i]; + if (transformTemplate.trim()) { + // Create iteration context including loop item + const loopState = { + ...state, + [itemVarName]: itemVal, + index: i, + }; + + // Interpolate template with loopState context + let interpolated = interpolate(transformTemplate, loopState, globalsList, secretsList); + // Replace item-specific placeholders + interpolated = interpolated.replace( + new RegExp(`\\{\\{\\s*${itemVarName}\\.([\\w.]+)\\s*\\}\\}`, "g"), + (_m, k) => { + if (itemVal && typeof itemVal === "object") { + const p = getPath(itemVal as Record, String(k)); + return p === undefined ? "" : typeof p === "string" ? p : JSON.stringify(p); + } + return ""; + } + ).replace( + new RegExp(`\\{\\{\\s*${itemVarName}\\s*\\}\\}`, "g"), + () => (typeof itemVal === "string" ? itemVal : JSON.stringify(itemVal)) + ); + + try { + mapped.push(JSON.parse(interpolated)); + } catch { + mapped.push(interpolated); + } + } else { + mapped.push(itemVal); + } + } + + state[targetKey] = mapped; + if (items.length === 0) { + state.__router_branch = "empty"; + } + + return { + source_key: rawArrayKey, + target_key: targetKey, + count: mapped.length, + items: mapped, + }; + } + case "transform": { + const op = (cfg.operation || "template_string").toLowerCase(); + const sourceKey = (cfg.source_key || "data").replace(/^state\./, ""); + const targetKey = (cfg.target_key || "transformed").replace(/^state\./, ""); + const expr = cfg.expression || ""; + const fieldsStr = cfg.fields || ""; + + let result: unknown = null; + + if (op === "template_string") { + const interpolated = interpolate(expr, state, globalsList, secretsList); + try { + result = JSON.parse(interpolated); + } catch { + result = interpolated; + } + } else if (op === "json_map") { + const sourceData = getPath(state, sourceKey); + const sourceObj = (sourceData && typeof sourceData === "object") ? sourceData : { raw: sourceData }; + const interpolatedExpr = interpolate(expr, { ...state, source: sourceObj }, globalsList, secretsList); + try { + result = JSON.parse(interpolatedExpr); + } catch { + result = interpolatedExpr; + } + } else if (op === "pick_fields") { + const sourceData = getPath(state, sourceKey); + if (sourceData && typeof sourceData === "object" && !Array.isArray(sourceData)) { + const picked: Record = {}; + const fieldList = fieldsStr.split(",").map((f) => f.trim()).filter(Boolean); + fieldList.forEach((f) => { + if (f in (sourceData as Record)) { + picked[f] = (sourceData as Record)[f]; + } + }); + result = picked; + } else { + result = sourceData ?? null; + } + } else if (op === "set_keys") { + const currentTarget = (getPath(state, targetKey) ?? {}) as Record; + const merged = { ...currentTarget }; + if (expr.trim()) { + const interpolated = interpolate(expr, state, globalsList, secretsList); + try { + const parsed = JSON.parse(interpolated); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + Object.assign(merged, parsed); + } + } catch { + // ignore JSON parse error for set_keys + } + } + result = merged; + } else if (op === "flatten_object") { + const sourceData = getPath(state, sourceKey); + const flatten = (obj: Record, prefix = ""): Record => { + return Object.keys(obj).reduce((acc: Record, k: string) => { + const pre = prefix.length ? prefix + "." : ""; + if (typeof obj[k] === "object" && obj[k] !== null && !Array.isArray(obj[k])) { + Object.assign(acc, flatten(obj[k] as Record, pre + k)); + } else { + acc[pre + k] = obj[k]; + } + return acc; + }, {}); + }; + if (sourceData && typeof sourceData === "object" && !Array.isArray(sourceData)) { + result = flatten(sourceData as Record); + } else { + result = sourceData ?? {}; + } + } + + state[targetKey] = result; + return { + operation: op, + target_key: targetKey, + result, + }; + } default: return { kind: node.data.kind, note: "no executor" }; } diff --git a/frontend/src/flow/types.ts b/frontend/src/flow/types.ts index 2ccd1e1..fc332b7 100644 --- a/frontend/src/flow/types.ts +++ b/frontend/src/flow/types.ts @@ -9,7 +9,9 @@ export type AgentNodeKind = | "sink" | "http" | "script" - | "note"; + | "note" + | "loop" + | "transform"; export interface AgentNodeData { kind: AgentNodeKind; @@ -151,6 +153,38 @@ export const NODE_TYPES: NodeTypeMeta[] = [ { key: "color", label: "color", placeholder: "yellow", type: "select", options: ["yellow", "blue", "green", "pink", "purple"] }, ], }, + { + kind: "loop", + label: "Array Loop Iterator", + description: "Iterates over an array in state, transforming each element and aggregating results.", + defaultName: "loop_items", + configFields: [ + { key: "array_key", label: "array_key", placeholder: "items (or state.items)" }, + { key: "item_var", label: "item_var", placeholder: "item" }, + { key: "item_transform", label: "item_transform", placeholder: '{"processed": "{{item.name}}", "status": "active"}', type: "textarea" }, + { key: "target_key", label: "target_key", placeholder: "processed_items" }, + { key: "max_iterations", label: "max_iterations", placeholder: "100" }, + ], + }, + { + kind: "transform", + label: "Data Transform", + description: "Transforms, extracts, flattens, or maps state properties.", + defaultName: "transform_data", + configFields: [ + { + key: "operation", + label: "operation", + placeholder: "template_string", + type: "select", + options: ["template_string", "json_map", "pick_fields", "set_keys", "flatten_object"], + }, + { key: "source_key", label: "source_key", placeholder: "raw_data" }, + { key: "target_key", label: "target_key", placeholder: "transformed_output" }, + { key: "expression", label: "expression / template", placeholder: "Hello {{state.user.name}}!", type: "textarea" }, + { key: "fields", label: "fields (comma separated)", placeholder: "id, name, email" }, + ], + }, ]; export const EDGE_LABELS = ["next", "on_success", "on_error", "tool_result", "true", "false"] as const; diff --git a/frontend/src/test/loopAndTransform.test.ts b/frontend/src/test/loopAndTransform.test.ts new file mode 100644 index 0000000..f60bf6f --- /dev/null +++ b/frontend/src/test/loopAndTransform.test.ts @@ -0,0 +1,202 @@ +import { describe, it, expect } from "vitest"; +import { Node, Edge } from "reactflow"; +import { AgentNodeData } from "../flow/types"; +import { runFlow } from "../flow/runFlow"; +import { generatePython, generateJavaScript } from "../flow/codegen"; + +describe("Array Loop Iterator and Data Transform Nodes", () => { + it("should iterate over state array, transform items, and respect max_iterations cap", async () => { + const nodes: Node[] = [ + { + id: "1", + type: "agent", + position: { x: 0, y: 0 }, + data: { + kind: "trigger", + name: "start", + config: {}, + isEntry: true, + }, + }, + { + id: "2", + type: "agent", + position: { x: 0, y: 100 }, + data: { + kind: "loop", + name: "process_users", + config: { + array_key: "users", + item_var: "user", + item_transform: '{"id": "{{user.id}}", "upper_name": "{{user.name}}", "status": "active"}', + target_key: "active_users", + max_iterations: "2", + }, + }, + }, + { + id: "3", + type: "agent", + position: { x: 0, y: 200 }, + data: { + kind: "sink", + name: "end", + config: {}, + isTerminal: true, + }, + }, + ]; + + const edges: Edge[] = [ + { id: "e1-2", source: "1", target: "2", label: "next" }, + { id: "e2-3", source: "2", target: "3", label: "next" }, + ]; + + const initialState = { + users: [ + { id: 101, name: "Alice" }, + { id: 102, name: "Bob" }, + { id: 103, name: "Charlie" }, + ], + }; + + const logs = await runFlow({ nodes, edges, gateways: [], initialState }); + const loopLog = logs.find((l) => l.nodeId === "2"); + + expect(loopLog).toBeDefined(); + expect(loopLog?.error).toBeUndefined(); + + const output = loopLog?.output as any; + expect(output.count).toBe(2); + expect(output.items).toEqual([ + { id: "101", upper_name: "Alice", status: "active" }, + { id: "102", upper_name: "Bob", status: "active" }, + ]); + + const finalSnapshot = logs[logs.length - 1].stateSnapshot; + expect(finalSnapshot?.active_users).toEqual([ + { id: "101", upper_name: "Alice", status: "active" }, + { id: "102", upper_name: "Bob", status: "active" }, + ]); + }); + + it("should execute transform operations correctly (template_string, pick_fields, json_map, set_keys, flatten_object)", async () => { + const nodes: Node[] = [ + { + id: "1", + type: "agent", + position: { x: 0, y: 0 }, + data: { + kind: "trigger", + name: "start", + config: {}, + isEntry: true, + }, + }, + { + id: "2", + type: "agent", + position: { x: 0, y: 100 }, + data: { + kind: "transform", + name: "pick_user_fields", + config: { + operation: "pick_fields", + source_key: "user", + target_key: "picked", + fields: "id, email", + }, + }, + }, + { + id: "3", + type: "agent", + position: { x: 0, y: 200 }, + data: { + kind: "transform", + name: "flatten_nested_data", + config: { + operation: "flatten_object", + source_key: "nested", + target_key: "flat", + }, + }, + }, + { + id: "4", + type: "agent", + position: { x: 0, y: 300 }, + data: { + kind: "sink", + name: "end", + config: {}, + isTerminal: true, + }, + }, + ]; + + const edges: Edge[] = [ + { id: "e1-2", source: "1", target: "2", label: "next" }, + { id: "e2-3", source: "2", target: "3", label: "next" }, + { id: "e3-4", source: "3", target: "4", label: "next" }, + ]; + + const initialState = { + user: { id: 42, name: "David", email: "david@example.com", role: "admin" }, + nested: { a: { b: { c: 10 } }, d: "hello" }, + }; + + const logs = await runFlow({ nodes, edges, gateways: [], initialState }); + const finalSnapshot = logs[logs.length - 1].stateSnapshot; + + expect(finalSnapshot?.picked).toEqual({ id: 42, email: "david@example.com" }); + expect(finalSnapshot?.flat).toEqual({ "a.b.c": 10, d: "hello" }); + }); + + it("should generate valid Python and JavaScript code for loop and transform nodes", () => { + const nodes: Node[] = [ + { + id: "1", + type: "agent", + position: { x: 0, y: 0 }, + data: { + kind: "loop", + name: "loop_items", + config: { + array_key: "items", + item_var: "item", + target_key: "out", + max_iterations: "50", + }, + }, + }, + { + id: "2", + type: "agent", + position: { x: 0, y: 100 }, + data: { + kind: "transform", + name: "transform_item", + config: { + operation: "pick_fields", + source_key: "raw", + target_key: "res", + fields: "a, b", + }, + }, + }, + ]; + + const edges: Edge[] = [{ id: "e1-2", source: "1", target: "2", label: "next" }]; + + const py = generatePython(nodes, edges); + expect(py.code).toContain('arr = state.get("items", [])'); + expect(py.code).toContain('op = "pick_fields"'); + expect(py.errors).toHaveLength(0); + + const js = generateJavaScript(nodes, edges); + expect(js.code).toContain('state.get("items")'); + expect(js.code).toContain('const op = "pick_fields"'); + expect(js.errors).toHaveLength(0); + }); +});