Add Data Transform and Array Loop Iterator Nodes

frontend/src/flow/AgentNode.tsx

@@ -13,6 +13,8 @@ const KIND_COLOR: Record<string, string> = {
   sink: "hsl(var(--node-sink))",
   http: "hsl(var(--node-http))",
   script: "hsl(var(--node-script))",
+  transform: "hsl(280 75% 55%)",
+  loop: "hsl(190 85% 45%)",
   note: "hsl(45 90% 48%)",
 };
 

frontend/src/flow/Inspector.tsx

@@ -64,6 +64,21 @@ export function Inspector({
         />
       </label>
 
+      {node.data.kind === "transform" && (
+        <div className="p-2 border border-dashed border-[hsl(var(--grid-line))] bg-[hsl(var(--ink)/0.015)] space-y-1">
+          <span className="text-[9px] uppercase tracking-wider text-[hsl(var(--ink-faint))] font-semibold block">
+            Operation Preset Guide
+          </span>
+          <p className="text-[10px] text-[hsl(var(--ink-soft))] leading-snug">
+            {node.data.config?.operation === "pick_fields" && "Pick Fields: extracts listed properties (e.g. 'id, title, status') from input object."}
+            {node.data.config?.operation === "template_string" && "Template String: interpolates '{{state.val}}', '{{global.KEY}}', and '{{secret.KEY}}'."}
+            {node.data.config?.operation === "set_keys" && "Set Keys: assigns JSON key-value pairs directly onto execution state."}
+            {node.data.config?.operation === "flatten_object" && "Flatten Object: flattens nested JSON objects into top-level dot-notation keys."}
+            {(!node.data.config?.operation || node.data.config?.operation === "json_map") && "JSON Map: renames input object keys based on JSON mapping params."}
+          </p>
+        </div>
+      )}
+
       {meta.configFields.map((f) => (
         <label key={f.key} className="block">
           <span className="text-[10px] text-[hsl(var(--ink-faint))]">{f.label}</span>

frontend/src/flow/Palette.tsx

@@ -16,6 +16,8 @@ const KIND_COLOR: Record<string, string> = {
   sink: "hsl(var(--node-sink))",
   http: "hsl(var(--node-http))",
   script: "hsl(var(--node-script))",
+  transform: "hsl(280 75% 55%)",
+  loop: "hsl(190 85% 45%)",
   note: "hsl(45 90% 48%)",
 };
 

frontend/src/flow/codegen.ts

@@ -426,6 +426,47 @@ export function generatePython(
           `state.last = result`,
           `return "next"`,
         ].join("\n");
+      case "transform":
+        return [
+          `op = ${pyStr(c.operation || "json_map")}`,
+          `input_path = ${pyStr(c.input_path || "state.last_output")}`,
+          `output_key = ${pyStr(c.output_key || "transformed_data")}`,
+          `raw_params = ${pyStr(c.params || "")}`,
+          `input_val = state.get(input_path.replace("state.", ""), state.last)`,
+          `# transform operation: ${c.operation || "json_map"}`,
+          `if op == "template_string":`,
+          `    res = interpolate(raw_params or str(input_val or ""), state)`,
+          `elif op == "pick_fields":`,
+          `    fields = [f.strip() for f in raw_params.split(",") if f.strip()]`,
+          `    res = {k: v for k, v in input_val.items() if k in fields} if isinstance(input_val, dict) else input_val`,
+          `elif op == "set_keys":`,
+          `    keys = json.loads(interpolate(raw_params, state)) if raw_params.strip() else {}`,
+          `    for k, v in keys.items(): state.set(k, v)`,
+          `    res = keys`,
+          `else:`,
+          `    res = input_val`,
+          `state.set(output_key, res)`,
+          `state.last = res`,
+          `return "next"`,
+        ].join("\n");
+      case "loop":
+        return [
+          `arr_path = ${pyStr(c.array_path || "state.items")}`,
+          `item_var = ${pyStr(c.item_var || "item")}`,
+          `tmpl = ${pyStr(c.transform_template || "")}`,
+          `max_iter = ${parseInt(c.max_iterations || "100", 10)}`,
+          `out_key = ${pyStr(c.output_key || "loop_results")}`,
+          `arr = state.get(arr_path.replace("state.", ""), [])`,
+          `arr = arr if isinstance(arr, list) else [arr] if arr is not None else []`,
+          `results = []`,
+          `for item in arr[:max_iter]:`,
+          `    state.set(item_var, item)`,
+          `    mapped = interpolate(tmpl, state) if tmpl else item`,
+          `    results.append(mapped)`,
+          `state.set(out_key, results)`,
+          `state.last = results`,
+          `return "next"`,
+        ].join("\n");
       case "note":
         return [
           `# Sticky Note Annotation:`,
@@ -620,6 +661,45 @@ export function generateJavaScript(
           `state.last = await runJsScript(${JSON.stringify(c.code || "")}, state);`,
           `return "next";`,
         ].join("\n");
+      case "transform":
+        return [
+          `const op = ${JSON.stringify(c.operation || "json_map")};`,
+          `const inputPath = ${JSON.stringify(c.input_path || "state.last_output")};`,
+          `const outputKey = ${JSON.stringify(c.output_key || "transformed_data")};`,
+          `const rawParams = ${JSON.stringify(c.params || "")};`,
+          `const inputVal = state.get(inputPath.replace("state.", "")) ?? state.last;`,
+          `let res = inputVal;`,
+          `if (op === "template_string") res = interpolate(rawParams || String(inputVal ?? ""), state);`,
+          `else if (op === "pick_fields") {`,
+          `  const fields = rawParams.split(",").map(s => s.trim());`,
+          `  if (inputVal && typeof inputVal === "object") { res = {}; fields.forEach(f => { if (f in inputVal) res[f] = inputVal[f]; }); }`,
+          `} else if (op === "set_keys") {`,
+          `  const keys = JSON.parse(interpolate(rawParams, state) || "{}");`,
+          `  Object.entries(keys).forEach(([k, v]) => state.set(k, v));`,
+          `  res = keys;`,
+          `}`,
+          `state.set(outputKey, res);`,
+          `state.last = res;`,
+          `return "next";`,
+        ].join("\n");
+      case "loop":
+        return [
+          `const arrPath = ${JSON.stringify(c.array_path || "state.items")};`,
+          `const itemVar = ${JSON.stringify(c.item_var || "item")};`,
+          `const tmpl = ${JSON.stringify(c.transform_template || "")};`,
+          `const maxIter = ${parseInt(c.max_iterations || "100", 10)};`,
+          `const outKey = ${JSON.stringify(c.output_key || "loop_results")};`,
+          `const rawArr = state.get(arrPath.replace("state.", "")) ?? [];`,
+          `const arr = Array.isArray(rawArr) ? rawArr : (rawArr != null ? [rawArr] : []);`,
+          `const results = [];`,
+          `for (let i = 0; i < Math.min(arr.length, maxIter); i++) {`,
+          `  state.set(itemVar, arr[i]);`,
+          `  results.push(tmpl ? interpolate(tmpl, state) : arr[i]);`,
+          `}`,
+          `state.set(outKey, results);`,
+          `state.last = results;`,
+          `return "next";`,
+        ].join("\n");
       case "note":
         return [
           `// Sticky Note Annotation:`,
@@ -677,4 +757,4 @@ export function generateCode(
 }
 
 // also export the kind set for sanity
-export const ALL_KINDS: AgentNodeKind[] = ["trigger","llm","tool","router","subagent","memory","human","sink","http","script","note"];
+export const ALL_KINDS: AgentNodeKind[] = ["trigger","llm","tool","router","subagent","memory","human","sink","http","script","transform","loop","note"];

frontend/src/flow/runFlow.ts

@@ -77,7 +77,16 @@ export function interpolate(
       const v = getPath(state, String(k));
       return v === undefined ? "" : typeof v === "string" ? v : JSON.stringify(v);
     })
-    .replace(/\{\{?\s*query\s*\}?\}/g, () => String(state.query ?? ""));
+    .replace(/\{\{?\s*query\s*\}?\}/g, () => String(state.query ?? ""))
+    .replace(/\{\{?\s*([\w.]+)\s*\}?\}/g, (_m, k) => {
+      const keyStr = String(k);
+      const rootKey = keyStr.split(".")[0];
+      if (rootKey in state) {
+        const v = getPath(state, keyStr);
+        return v === undefined ? "" : typeof v === "string" ? v : JSON.stringify(v);
+      }
+      return _m;
+    });
 }
 
 /**
@@ -465,6 +474,157 @@ export async function runNode(
         throw new Error(`Script execution failed: ${e instanceof Error ? e.message : String(e)}`);
       }
     }
+    case "transform": {
+      const op = cfg.operation || "json_map";
+      const inputPath = cfg.input_path || "state.last_output";
+      const outputKey = cfg.output_key || "transformed_data";
+      const rawParams = cfg.params || "";
+
+      let inputVal: unknown;
+      if (inputPath.startsWith("state.")) {
+        inputVal = getPath(state, inputPath.slice(6));
+      } else {
+        inputVal = getPath(state, inputPath) ?? state.last_output ?? inputPath;
+      }
+
+      let transformed: unknown;
+
+      if (op === "json_map") {
+        let parsedInput = inputVal;
+        if (typeof inputVal === "string") {
+          try {
+            parsedInput = JSON.parse(inputVal);
+          } catch {
+            parsedInput = inputVal;
+          }
+        }
+        let mapping: Record<string, string> = {};
+        if (rawParams.trim()) {
+          try {
+            mapping = JSON.parse(interpolate(rawParams, state, globalsList, secretsList));
+          } catch {
+            mapping = {};
+          }
+        }
+        if (parsedInput && typeof parsedInput === "object" && !Array.isArray(parsedInput)) {
+          const mappedObj: Record<string, unknown> = {};
+          for (const [k, v] of Object.entries(parsedInput as Record<string, unknown>)) {
+            const targetKey = mapping[k] ?? k;
+            mappedObj[targetKey] = v;
+          }
+          transformed = mappedObj;
+        } else {
+          transformed = parsedInput;
+        }
+      } else if (op === "pick_fields") {
+        let fieldsToPick: string[] = [];
+        if (rawParams.trim()) {
+          try {
+            const parsed = JSON.parse(rawParams);
+            fieldsToPick = Array.isArray(parsed) ? parsed : String(parsed).split(",").map((s) => s.trim());
+          } catch {
+            fieldsToPick = rawParams.split(",").map((s) => s.trim());
+          }
+        }
+        if (inputVal && typeof inputVal === "object" && !Array.isArray(inputVal)) {
+          const pickedObj: Record<string, unknown> = {};
+          for (const f of fieldsToPick) {
+            if (f in (inputVal as Record<string, unknown>)) {
+              pickedObj[f] = (inputVal as Record<string, unknown>)[f];
+            }
+          }
+          transformed = pickedObj;
+        } else {
+          transformed = inputVal;
+        }
+      } else if (op === "template_string") {
+        const template = rawParams || String(inputVal ?? "");
+        transformed = interpolate(template, state, globalsList, secretsList);
+      } else if (op === "set_keys") {
+        let keysToSet: Record<string, unknown> = {};
+        if (rawParams.trim()) {
+          try {
+            const interpolatedParams = interpolate(rawParams, state, globalsList, secretsList);
+            keysToSet = JSON.parse(interpolatedParams);
+          } catch {
+            keysToSet = {};
+          }
+        }
+        for (const [k, v] of Object.entries(keysToSet)) {
+          state[k] = v;
+        }
+        transformed = keysToSet;
+      } else if (op === "flatten_object") {
+        const flatten = (obj: Record<string, unknown>, prefix = ""): Record<string, unknown> => {
+          return Object.keys(obj).reduce((acc: Record<string, unknown>, k: string) => {
+            const pre = prefix.length ? prefix + "." : "";
+            if (obj[k] && typeof obj[k] === "object" && !Array.isArray(obj[k])) {
+              Object.assign(acc, flatten(obj[k] as Record<string, unknown>, pre + k));
+            } else {
+              acc[pre + k] = obj[k];
+            }
+            return acc;
+          }, {});
+        };
+        if (inputVal && typeof inputVal === "object" && !Array.isArray(inputVal)) {
+          transformed = flatten(inputVal as Record<string, unknown>);
+        } else {
+          transformed = inputVal;
+        }
+      } else {
+        transformed = inputVal;
+      }
+
+      state[outputKey] = transformed;
+      return { operation: op, output_key: outputKey, result: transformed };
+    }
+    case "loop": {
+      const arrayPath = cfg.array_path || "state.items";
+      const itemVar = cfg.item_var || "item";
+      const transformTemplate = cfg.transform_template || "";
+      const maxIterations = parseIntOr(cfg.max_iterations, 100);
+      const outputKey = cfg.output_key || "loop_results";
+
+      let rawArray: unknown;
+      if (arrayPath.startsWith("state.")) {
+        rawArray = getPath(state, arrayPath.slice(6));
+      } else {
+        rawArray = getPath(state, arrayPath) ?? state.last_output;
+      }
+
+      let arrayToIterate: unknown[] = [];
+      if (Array.isArray(rawArray)) {
+        arrayToIterate = rawArray;
+      } else if (rawArray !== undefined && rawArray !== null) {
+        arrayToIterate = [rawArray];
+      }
+
+      const results: unknown[] = [];
+      const iterateCount = Math.min(arrayToIterate.length, maxIterations);
+
+      for (let i = 0; i < iterateCount; i++) {
+        const currentItem = arrayToIterate[i];
+        state[itemVar] = currentItem;
+
+        if (!transformTemplate.trim()) {
+          results.push(currentItem);
+        } else if (transformTemplate.includes("{{")) {
+          results.push(interpolate(transformTemplate, state, globalsList, secretsList));
+        } else if (transformTemplate.startsWith(`${itemVar}.`)) {
+          const subProp = transformTemplate.slice(itemVar.length + 1);
+          if (currentItem && typeof currentItem === "object") {
+            results.push(getPath(currentItem as Record<string, unknown>, subProp));
+          } else {
+            results.push(undefined);
+          }
+        } else {
+          results.push(interpolate(transformTemplate, state, globalsList, secretsList));
+        }
+      }
+
+      state[outputKey] = results;
+      return { count: results.length, output_key: outputKey, results };
+    }
     case "sink": {
       return {
         target: cfg.target || "response",

frontend/src/flow/types.ts

@@ -9,6 +9,8 @@ export type AgentNodeKind =
   | "sink"
   | "http"
   | "script"
+  | "transform"
+  | "loop"
   | "note";
 
 export interface AgentNodeData {
@@ -141,6 +143,31 @@ export const NODE_TYPES: NodeTypeMeta[] = [
       { key: "code", label: "code", placeholder: "state.query = state.query.toUpperCase();\nreturn state;", type: "textarea" },
     ],
   },
+  {
+    kind: "transform",
+    label: "Data Transform",
+    description: "Performs zero-code state mappings, field picking, template interpolation, or key setting.",
+    defaultName: "transform_data",
+    configFields: [
+      { key: "operation", label: "operation", placeholder: "json_map", type: "select", options: ["json_map", "pick_fields", "template_string", "set_keys", "flatten_object"] },
+      { key: "input_path", label: "input_path", placeholder: "state.last_output  (or state property path)" },
+      { key: "output_key", label: "output_key", placeholder: "transformed_data  (key to save in state)" },
+      { key: "params", label: "params", placeholder: '{"key": "value"} or ["field1", "field2"] or template string', type: "textarea" },
+    ],
+  },
+  {
+    kind: "loop",
+    label: "Array Loop Iterator",
+    description: "Iterates over an array in state, mapping items and storing outputs in state.",
+    defaultName: "loop_items",
+    configFields: [
+      { key: "array_path", label: "array_path", placeholder: "state.items  (array path in state)" },
+      { key: "item_var", label: "item_var", placeholder: "item  (context variable name, default 'item')" },
+      { key: "transform_template", label: "transform_template", placeholder: "{{item.name}} - {{item.status}} or item.id", type: "textarea" },
+      { key: "max_iterations", label: "max_iterations", placeholder: "100  (safety iteration limit)" },
+      { key: "output_key", label: "output_key", placeholder: "loop_results  (key to save mapped array in state)" },
+    ],
+  },
   {
     kind: "note",
     label: "Sticky Note",

frontend/src/flow/validate.ts

@@ -49,6 +49,24 @@ export function validateGraph(
         });
       }
     }
+    if (n.data.kind === "transform") {
+      if (!n.data.config?.output_key?.trim()) {
+        issues.push({
+          nodeId: n.id,
+          kind: "orphan",
+          message: `Transform "${n.data.name}" requires an output_key`,
+        });
+      }
+    }
+    if (n.data.kind === "loop") {
+      if (!n.data.config?.array_path?.trim() || !n.data.config?.output_key?.trim()) {
+        issues.push({
+          nodeId: n.id,
+          kind: "orphan",
+          message: `Loop "${n.data.name}" requires array_path and output_key`,
+        });
+      }
+    }
   }
   return issues;
 }