Add Data Transform and Array Loop nodes with Performance Analytics Modal

dev_server.log

@@ -1,6 +1,6 @@
 $ vite --host 0.0.0.0 --port 3000
 
-  VITE v5.4.21  ready in 312 ms
+  VITE v5.4.21  ready in 329 ms
 
   ➜  Local:   http://localhost:3000/
   ➜  Network: http://192.168.0.2:3000/

frontend/src/flow/AgentNode.tsx

@@ -14,6 +14,8 @@ const KIND_COLOR: Record<string, string> = {
   http: "hsl(var(--node-http))",
   script: "hsl(var(--node-script))",
   note: "hsl(45 90% 48%)",
+  transform: "hsl(var(--node-transform, 280 75% 55%))",
+  loop: "hsl(var(--node-loop, 160 70% 40%))",
 };
 
 interface ExtraData extends AgentNodeData {

frontend/src/flow/Palette.tsx

@@ -17,6 +17,8 @@ const KIND_COLOR: Record<string, string> = {
   http: "hsl(var(--node-http))",
   script: "hsl(var(--node-script))",
   note: "hsl(45 90% 48%)",
+  transform: "hsl(var(--node-transform, 280 75% 55%))",
+  loop: "hsl(var(--node-loop, 160 70% 40%))",
 };
 
 export function Palette({ onAdd }: Props) {

frontend/src/flow/codegen.ts

@@ -432,6 +432,64 @@ export function generatePython(
           `# ${c.content ? c.content.replace(/\n/g, "\n# ") : "(empty note)"}`,
           `return "next"`,
         ].join("\n");
+      case "transform": {
+        const op = c.operation || "json_map";
+        const target = c.target_key || "transformed";
+        const inputKey = c.input_key || "";
+        const expr = c.expression || "";
+        return [
+          `# Data Transform op=${pyStr(op)} target=${pyStr(target)}`,
+          `input_val = state.get(${pyStr(inputKey || "last")}, state.last)`,
+          `expr_val = interpolate(${pyStr(expr)}, state)`,
+          `if ${pyStr(op)} == "template_string":`,
+          `    res = expr_val`,
+          `elif ${pyStr(op)} == "pick_fields":`,
+          `    fields = [f.strip() for f in expr_val.replace("\\n", ",").split(",") if f.strip()]`,
+          `    res = {f: input_val.get(f) for f in fields if isinstance(input_val, dict) and f in input_val}`,
+          `elif ${pyStr(op)} == "flatten_object":`,
+          `    def _flatten(o, prefix=""):`,
+          `        acc = {}`,
+          `        for k, v in o.items():`,
+          `            pre = f"{prefix}.{k}" if prefix else k`,
+          `            if isinstance(v, dict): acc.update(_flatten(v, pre))`,
+          `            else: acc[pre] = v`,
+          `        return acc`,
+          `    res = _flatten(input_val) if isinstance(input_val, dict) else input_val`,
+          `else:`,
+          `    try:`,
+          `        res = json.loads(expr_val) if expr_val.strip() else input_val`,
+          `    except Exception:`,
+          `        res = expr_val`,
+          `state.set(${pyStr(target)}, res)`,
+          `state.last = res`,
+          `return "next"`,
+        ].join("\n");
+      }
+      case "loop": {
+        const arrKey = c.array_key || "items";
+        const itemVar = c.item_var || "item";
+        const targetKey = c.target_key || "processed";
+        const tmpl = c.transform_template || "";
+        const maxIter = parseInt(c.max_iterations || "100", 10) || 100;
+        return [
+          `# Array Loop over state.${arrKey}`,
+          `raw_arr = state.get(${pyStr(arrKey)}, [])`,
+          `arr = raw_arr if isinstance(raw_arr, list) else ([raw_arr] if raw_arr is not None else [])`,
+          `res_list = []`,
+          `for idx, item in enumerate(arr[:${maxIter}]):`,
+          `    state.set(${pyStr(itemVar)}, item)`,
+          `    tmpl_val = interpolate(${pyStr(tmpl)}, state)`,
+          `    if tmpl_val.strip():`,
+          `        try: val = json.loads(tmpl_val)`,
+          `        except Exception: val = tmpl_val`,
+          `    else:`,
+          `        val = item`,
+          `    res_list.append(val)`,
+          `state.set(${pyStr(targetKey)}, res_list)`,
+          `state.last = res_list`,
+          `return "next"`,
+        ].join("\n");
+      }
       default: {
         const _exhaustive: never = d.kind as never;
         return `return "next"  # unknown kind ${_exhaustive}`;
@@ -626,6 +684,55 @@ export function generateJavaScript(
           `// ${c.content ? c.content.replace(/\n/g, "\n// ") : "(empty note)"}`,
           `return "next";`,
         ].join("\n");
+      case "transform": {
+        const op = c.operation || "json_map";
+        const target = c.target_key || "transformed";
+        const inputKey = c.input_key || "";
+        const expr = c.expression || "";
+        return [
+          `const op = ${JSON.stringify(op)};`,
+          `const inputVal = state.get(${JSON.stringify(inputKey || "last")}) ?? state.last;`,
+          `const exprVal = interpolate(${JSON.stringify(expr)}, state);`,
+          `let res;`,
+          `if (op === "template_string") { res = exprVal; }`,
+          `else if (op === "pick_fields") {`,
+          `  const fields = exprVal.split(/[\\n,]+/).map(f => f.trim()).filter(Boolean);`,
+          `  res = {};`,
+          `  if (typeof inputVal === "object" && inputVal !== null) {`,
+          `    for (const f of fields) { if (f in inputVal) res[f] = inputVal[f]; }`,
+          `  }`,
+          `} else {`,
+          `  try { res = JSON.parse(exprVal); } catch { res = exprVal; }`,
+          `}`,
+          `state.set(${JSON.stringify(target)}, res);`,
+          `state.last = res;`,
+          `return "next";`,
+        ].join("\n");
+      }
+      case "loop": {
+        const arrKey = c.array_key || "items";
+        const itemVar = c.item_var || "item";
+        const targetKey = c.target_key || "processed";
+        const tmpl = c.transform_template || "";
+        const maxIter = parseInt(c.max_iterations || "100", 10) || 100;
+        return [
+          `const rawArr = state.get(${JSON.stringify(arrKey)}) ?? [];`,
+          `const arr = Array.isArray(rawArr) ? rawArr : (rawArr ? [rawArr] : []);`,
+          `const resList = [];`,
+          `for (const item of arr.slice(0, ${maxIter})) {`,
+          `  state.set(${JSON.stringify(itemVar)}, item);`,
+          `  const tmplVal = interpolate(${JSON.stringify(tmpl)}, state);`,
+          `  let val = item;`,
+          `  if (tmplVal.trim()) {`,
+          `    try { val = JSON.parse(tmplVal); } catch { val = tmplVal; }`,
+          `  }`,
+          `  resList.push(val);`,
+          `}`,
+          `state.set(${JSON.stringify(targetKey)}, resList);`,
+          `state.last = resList;`,
+          `return "next";`,
+        ].join("\n");
+      }
       default:
         return `return "next";`;
     }
@@ -677,4 +784,4 @@ export function generateCode(
 }
 
 // also export the kind set for sanity
-export const ALL_KINDS: AgentNodeKind[] = ["trigger","llm","tool","router","subagent","memory","human","sink","http","script","note"];
+export const ALL_KINDS: AgentNodeKind[] = ["trigger","llm","tool","router","subagent","memory","human","sink","http","script","note","transform","loop"];

frontend/src/flow/runFlow.ts

@@ -77,7 +77,12 @@ export function interpolate(
       const v = getPath(state, String(k));
       return v === undefined ? "" : typeof v === "string" ? v : JSON.stringify(v);
     })
-    .replace(/\{\{?\s*query\s*\}?\}/g, () => String(state.query ?? ""));
+    .replace(/\{\{?\s*query\s*\}?\}/g, () => String(state.query ?? ""))
+    .replace(/\{\{?\s*([\w.]+)\s*\}?\}/g, (_m, k) => {
+      const key = String(k);
+      const v = getPath(state, key);
+      return v === undefined ? _m : typeof v === "string" ? v : JSON.stringify(v);
+    });
 }
 
 /**
@@ -478,6 +483,153 @@ export async function runNode(
         annotationOnly: true,
       };
     }
+    case "transform": {
+      const op = (cfg.operation || "json_map").toLowerCase();
+      const inputKey = (cfg.input_key || "").trim();
+      const targetKey = (cfg.target_key || "transformed").trim();
+      const expr = cfg.expression || "";
+
+      let rawInput: unknown = state;
+      if (inputKey) {
+        if (inputKey.startsWith("state.")) {
+          rawInput = getPath(state, inputKey.slice(6));
+        } else if (inputKey in state) {
+          rawInput = state[inputKey];
+        } else if (inputKey === "last_output") {
+          rawInput = state.last_output;
+        } else {
+          rawInput = getPath(state, inputKey);
+        }
+      } else if (state.last_output !== undefined) {
+        rawInput = state.last_output;
+      }
+
+      let result: unknown;
+
+      if (op === "template_string") {
+        result = interpolate(expr, state, globalsList, secretsList);
+      } else if (op === "pick_fields") {
+        const fields = expr
+          .split(/[\n,]+/)
+          .map((f) => f.trim())
+          .filter(Boolean);
+        if (typeof rawInput === "object" && rawInput !== null) {
+          const picked: Record<string, unknown> = {};
+          for (const f of fields) {
+            const val = getPath(rawInput as Record<string, unknown>, f) ?? (rawInput as Record<string, unknown>)[f];
+            if (val !== undefined) picked[f] = val;
+          }
+          result = picked;
+        } else {
+          result = {};
+        }
+      } else if (op === "flatten_object") {
+        const flatten = (obj: Record<string, unknown>, prefix = ""): Record<string, unknown> => {
+          return Object.keys(obj).reduce((acc: Record<string, unknown>, k: string) => {
+            const pre = prefix ? `${prefix}.${k}` : k;
+            if (typeof obj[k] === "object" && obj[k] !== null && !Array.isArray(obj[k])) {
+              Object.assign(acc, flatten(obj[k] as Record<string, unknown>, pre));
+            } else {
+              acc[pre] = obj[k];
+            }
+            return acc;
+          }, {});
+        };
+        result = typeof rawInput === "object" && rawInput !== null && !Array.isArray(rawInput)
+          ? flatten(rawInput as Record<string, unknown>)
+          : rawInput;
+      } else if (op === "set_keys") {
+        let parsedKeys: Record<string, unknown> = {};
+        if (expr.trim()) {
+          const interpolated = interpolate(expr, state, globalsList, secretsList);
+          try {
+            parsedKeys = JSON.parse(interpolated);
+          } catch {
+            expr.split("\n").forEach((line) => {
+              const idx = line.indexOf("=");
+              if (idx > -1) {
+                const k = line.slice(0, idx).trim();
+                const v = line.slice(idx + 1).trim();
+                parsedKeys[k] = interpolate(v, state, globalsList, secretsList);
+              }
+            });
+          }
+        }
+        if (typeof rawInput === "object" && rawInput !== null && !Array.isArray(rawInput)) {
+          result = { ...rawInput, ...parsedKeys };
+        } else {
+          result = parsedKeys;
+        }
+      } else {
+        // default "json_map"
+        if (expr.trim()) {
+          const interpolated = interpolate(expr, state, globalsList, secretsList);
+          try {
+            result = JSON.parse(interpolated);
+          } catch {
+            result = interpolated;
+          }
+        } else {
+          result = rawInput;
+        }
+      }
+
+      state[targetKey] = result;
+      return { operation: op, targetKey, result };
+    }
+    case "loop": {
+      const arrayKey = (cfg.array_key || "items").trim();
+      const itemVar = (cfg.item_var || "item").trim();
+      const targetKey = (cfg.target_key || "processed").trim();
+      const template = cfg.transform_template || "";
+      const maxIter = parseInt(cfg.max_iterations || "100", 10) || 100;
+
+      let arr: unknown[];
+      let rawVal: unknown;
+      if (arrayKey.startsWith("state.")) {
+        rawVal = getPath(state, arrayKey.slice(6));
+      } else if (arrayKey in state) {
+        rawVal = state[arrayKey];
+      } else {
+        rawVal = getPath(state, arrayKey);
+      }
+
+      if (Array.isArray(rawVal)) {
+        arr = rawVal;
+      } else if (rawVal !== undefined && rawVal !== null) {
+        arr = [rawVal];
+      } else {
+        arr = [];
+      }
+
+      const boundedArr = arr.slice(0, maxIter);
+      const results: unknown[] = [];
+
+      for (const item of boundedArr) {
+        const itemCtx = {
+          ...state,
+          [itemVar]: item,
+          item: item,
+        };
+        if (template.trim()) {
+          const interpolated = interpolate(template, itemCtx, globalsList, secretsList);
+          try {
+            results.push(JSON.parse(interpolated));
+          } catch {
+            results.push(interpolated);
+          }
+        } else {
+          results.push(item);
+        }
+      }
+
+      state[targetKey] = results;
+      return {
+        total: boundedArr.length,
+        items: results,
+        target_key: targetKey,
+      };
+    }
     default:
       return { kind: node.data.kind, note: "no executor" };
   }

frontend/src/flow/types.ts

@@ -9,7 +9,9 @@ export type AgentNodeKind =
   | "sink"
   | "http"
   | "script"
-  | "note";
+  | "note"
+  | "transform"
+  | "loop";
 
 export interface AgentNodeData {
   kind: AgentNodeKind;
@@ -151,6 +153,37 @@ export const NODE_TYPES: NodeTypeMeta[] = [
       { key: "color", label: "color", placeholder: "yellow", type: "select", options: ["yellow", "blue", "green", "pink", "purple"] },
     ],
   },
+  {
+    kind: "transform",
+    label: "Data Transform",
+    description: "Transforms state variables (json_map, pick_fields, template_string, set_keys, flatten_object).",
+    defaultName: "transform_data",
+    configFields: [
+      {
+        key: "operation",
+        label: "operation",
+        placeholder: "json_map | pick_fields | template_string | set_keys | flatten_object",
+        type: "select",
+        options: ["json_map", "pick_fields", "template_string", "set_keys", "flatten_object"],
+      },
+      { key: "input_key", label: "input_key", placeholder: "state.items or last_output" },
+      { key: "target_key", label: "target_key", placeholder: "transformed_result" },
+      { key: "expression", label: "expression", placeholder: '{"key": "{{state.val}}"} or "id, name"', type: "textarea" },
+    ],
+  },
+  {
+    kind: "loop",
+    label: "Array Loop",
+    description: "Iterates over an array in state, evaluating transform templates per item.",
+    defaultName: "loop_array",
+    configFields: [
+      { key: "array_key", label: "array_key", placeholder: "state.items" },
+      { key: "item_var", label: "item_var", placeholder: "item" },
+      { key: "target_key", label: "target_key", placeholder: "processed_list" },
+      { key: "transform_template", label: "transform_template", placeholder: '{"item": "{{item}}", "status": "processed"}', type: "textarea" },
+      { key: "max_iterations", label: "max_iterations", placeholder: "100" },
+    ],
+  },
 ];
 
 export const EDGE_LABELS = ["next", "on_success", "on_error", "tool_result", "true", "false"] as const;