Automations add mirth style variable map

src/prerna/reactor/automation/AGENTS.md

@@ -66,6 +66,15 @@ references such as `${query_result[0].ID}` are resolved by
 `AutomationScopeResolver` while preserving the underlying Java value. The UI
 and durable history receive bounded presentation contracts, not raw tasks,
 engine objects, provider handles, Python variable names, or Insight internals.
+`data.transform` is a pure scope transform resolved directly by
+`AutomationDataTransformExecutor` - no Pixel call and no generated Python. It
+recursively resolves a configured field tree (nested objects/arrays, literals,
+and `${name.path}` references) into a brand-new JSON object built only from
+that tree; it never merges in any prior scope value. `data.declare-variable`
+builds its field tree the same way; besides its own `outputVar` entry, Java
+also spreads the declared names into scope as individually named variables -
+exactly like trigger globals - so later nodes can reference `${name}` directly
+instead of a dotted path into one output object.
 
 ## Node execution
 

src/prerna/reactor/automation/AutomationConstants.java

@@ -104,6 +104,8 @@ private AutomationConstants() {
 	public static final String OUTPUT_KIND_TABLE = "TABLE";
 	public static final String OUTPUT_VALUE = "OUTPUT_VALUE";
 	public static final String OUTPUT_PREVIEW = "OUTPUT_PREVIEW";
+	/** Bounded JSON of every run-scope variable as of this node's completion. */
+	public static final String SCOPE_SNAPSHOT = "SCOPE_SNAPSHOT";
 	public static final String ROOM_ID = "ROOM_ID";
 	public static final String WORKSPACE_ID = "WORKSPACE_ID";
 	public static final String MODEL_MESSAGE_ID = "MODEL_MESSAGE_ID";
@@ -212,6 +214,8 @@ private AutomationConstants() {
 	public static final String NODE_FUNCTION_EXECUTE = "function.execute";
 	public static final String NODE_APP_PIXEL = "app.pixel";
 	public static final String NODE_AGENT_RUN = "agent.run";
+	public static final String NODE_DATA_TRANSFORM = "data.transform";
+	public static final String NODE_DATA_DECLARE_VARIABLE = "data.declare-variable";
 	public static final String NODE_CONTROL_WAIT = "control.wait";
 	public static final String NODE_CONTROL_IF = "control.if";
 	public static final String NODE_CONTROL_JEV = "control.jev";
@@ -247,6 +251,7 @@ private AutomationConstants() {
 	public static final String CONFIG_GLOBALS = "globals";
 	public static final String CONFIG_DEFAULT_VALUE = "defaultValue";
 	public static final String CONFIG_SOURCE = "source";
+	public static final String CONFIG_FIELDS = "fields";
 	public static final String CONFIG_PATH = "path";
 	public static final String CONFIG_FORMAT = "format";
 	public static final String CONFIG_MISSING_VALUE = "missingValue";
@@ -303,6 +308,8 @@ private AutomationConstants() {
 	/** How the ERROR_MESSAGE of an agent node cut off by its waitTimeoutMs ends. */
 	public static final String AGENT_WAIT_TIMEOUT_ERROR = " did not complete before the wait timeout.";
 	public static final String CONFIG_APP_ID = "appId";
+	public static final String CONFIG_CSV_HAS_HEADER = "csvHasHeader";
+	public static final String DATA_FORMAT_CSV = "csv";
 
 	// -- Node execution defaults / bounds
 	// --------------------------------------------
@@ -445,6 +452,8 @@ private AutomationConstants() {
 	public static final String RESULT_SUMMARY = "summary";
 	/** Per-node variable descriptors used by Automation authoring clients. */
 	public static final String RESULT_SCOPE_VARIABLES = "scopeVariables";
+	/** Point-in-time run-scope snapshot for one node's run-history "Variables" view. */
+	public static final String RESULT_SCOPE_SNAPSHOT = "scopeSnapshot";
 
 	// -- Pixel execution defaults
 	// ----------------------------------------------------

src/prerna/reactor/automation/definition/AutomationDefinitionValidator.java

@@ -338,13 +338,82 @@ private static void validateNodeConfig(String nodeId, AutomationNodeType nodeTyp
 		case CONTROL_IF -> validateBranchConfig(nodeId, config);
 		case CONTROL_JEV -> validateJevBranchConfig(nodeId, config);
 		case CONTROL_LOOP -> validateLoopConfig(nodeId, config);
+		case DATA_TRANSFORM -> validateDataTransformFields(nodeId, config);
+		case DATA_DECLARE_VARIABLE -> validateDeclareVariableConfig(nodeId, config);
 		case STORAGE_LIST -> validateOptionalStringListOrPlaceholder(nodeId, config, "extensions");
 		case TRIGGER_START, DEVELOPER_PYTHON -> {
 			// These node types have no additional required configuration here.
 		}
 		}
 	}
 
+	/**
+	 * Requires at least one named field so the generated node builds a non-empty
+	 * JSON object. Each field's node is validated recursively since "object" and
+	 * "array" nodes can nest to any depth; "value" nodes are optional and resolve
+	 * to {@code None} when blank.
+	 */
+	private static void validateDataTransformFields(String nodeId, Map<String, Object> config) {
+		validateDataTransformFieldList(config.get("fields"), "Transform node '" + nodeId + "' config.fields");
+	}
+
+	/**
+	 * A {@code data.declare-variable} node builds a field tree exactly like
+	 * {@code data.transform}, but each top-level field is additionally spread into
+	 * scope as its own variable, so every top-level name must be a non-private
+	 * Python identifier, unique, and not a reserved runtime key.
+	 */
+	private static void validateDeclareVariableConfig(String nodeId, Map<String, Object> config) {
+		validateDataTransformFieldList(config.get("fields"), "Declare variable node '" + nodeId + "' config.fields");
+		Set<String> names = new HashSet<>();
+		for (Object fieldObj : (List<?>) config.get("fields")) {
+			Map<String, Object> field = requireMap(fieldObj, "Declare variable node '" + nodeId + "' config.fields");
+			String name = String.valueOf(field.get("name")).trim();
+			if (!name.matches("[A-Za-z][A-Za-z0-9_]*")) {
+				throw new IllegalArgumentException("Declare variable node '" + nodeId + "' variable '" + name
+						+ "' must be a non-private Python identifier.");
+			}
+			if (AutomationConstants.RESERVED_SCOPE_KEYS.contains(name)) {
+				throw new IllegalArgumentException("Declare variable node '" + nodeId + "' variable '" + name
+						+ "' is reserved for automation runtime metadata.");
+			}
+			if (!names.add(name)) {
+				throw new IllegalArgumentException(
+						"Declare variable node '" + nodeId + "' has duplicate variable: " + name + ".");
+			}
+		}
+	}
+
+	private static void validateDataTransformFieldList(Object fields, String path) {
+		if (!(fields instanceof List<?> list) || list.isEmpty()) {
+			throw new IllegalArgumentException(path + " must include at least one field.");
+		}
+		for (int index = 0; index < list.size(); index++) {
+			String fieldPath = path + "[" + index + "]";
+			Map<String, Object> field = requireMap(list.get(index), fieldPath);
+			requireNonblankString(field.get("name"), fieldPath + ".name");
+			validateDataTransformNode(field.get("node"), fieldPath + ".node");
+		}
+	}
+
+	private static void validateDataTransformNode(Object node, String path) {
+		Map<String, Object> map = requireMap(node, path);
+		Object kind = map.get("kind");
+		if ("object".equals(kind)) {
+			validateDataTransformFieldList(map.get("fields"), path + ".fields");
+		} else if ("array".equals(kind)) {
+			if (!(map.get("items") instanceof List<?> items)) {
+				throw new IllegalArgumentException(path + ".items must be an array.");
+			}
+			for (int index = 0; index < items.size(); index++) {
+				validateDataTransformNode(items.get(index), path + ".items[" + index + "]");
+			}
+		} else if (!"value".equals(kind) && !"boolean".equals(kind) && !"null".equals(kind)) {
+			throw new IllegalArgumentException(
+					path + ".kind must be one of value, boolean, null, array, or object.");
+		}
+	}
+
 	private static void validateDataExtractConfig(String nodeId, Map<String, Object> config) {
 		Object source = config.get(AutomationConstants.CONFIG_SOURCE);
 		if (source == null || source instanceof String string && string.isBlank()) {
@@ -353,10 +422,12 @@ private static void validateDataExtractConfig(String nodeId, Map<String, Object>
 		}
 		requireConfigString(nodeId, config, AutomationConstants.CONFIG_PATH);
 		String format = config.get(AutomationConstants.CONFIG_FORMAT) instanceof String value ? value : "auto";
-		if (!("auto".equals(format) || "json".equals(format) || "xml".equals(format))) {
+		if (!("auto".equals(format) || "json".equals(format) || "xml".equals(format)
+				|| AutomationConstants.DATA_FORMAT_CSV.equals(format))) {
 			throw new IllegalArgumentException(
-					"Node '" + nodeId + "' config.format must be auto, json, or xml.");
+					"Node '" + nodeId + "' config.format must be auto, json, xml, or csv.");
 		}
+		validateOptionalConfigBoolean(nodeId, config, AutomationConstants.CONFIG_CSV_HAS_HEADER);
 	}
 
 	private static boolean isRoutingNode(AutomationNodeType nodeType) {

src/prerna/reactor/automation/definition/AutomationNodeCatalog.java

@@ -153,13 +153,15 @@ private static List<AutomationNodeDefinition> createDefinitions() {
 				"Delete a file from connected storage.", true, false));
 
 		definitions.add(definition(AutomationNodeType.DATA_EXTRACT, "Extract value",
-				"Read one value from JSON, XML, or a file in this run's workspace.",
+				"Read one value from JSON, XML, CSV, or a file in this run's workspace.",
 				orderedConfig(AutomationConstants.CONFIG_SOURCE, "", AutomationConstants.CONFIG_PATH, "",
 						AutomationConstants.CONFIG_FORMAT, "auto", AutomationConstants.CONFIG_MISSING_VALUE, null,
-						AutomationConstants.CONFIG_NULL_VALUE, null),
+						AutomationConstants.CONFIG_NULL_VALUE, null, AutomationConstants.CONFIG_CSV_HAS_HEADER, true),
 				List.of(field(AutomationConstants.CONFIG_SOURCE, ConfigFieldType.TEXT, "Data or file path", true, ""),
 						field(AutomationConstants.CONFIG_PATH, ConfigFieldType.STRING, "Value path", true, ""),
 						field(AutomationConstants.CONFIG_FORMAT, ConfigFieldType.STRING, "Data format", true, "auto"),
+						field(AutomationConstants.CONFIG_CSV_HAS_HEADER, ConfigFieldType.BOOLEAN,
+								"First row is headers", false, true),
 						field(AutomationConstants.CONFIG_MISSING_VALUE, ConfigFieldType.JSON,
 								"If the path is missing", false, null),
 						field(AutomationConstants.CONFIG_NULL_VALUE, ConfigFieldType.JSON, "If the value is null", false,
@@ -190,6 +192,21 @@ private static List<AutomationNodeDefinition> createDefinitions() {
 								field("command", ConfigFieldType.TEXT, "Instruction", true, "")),
 						controlInputs(), controlAndResultOutputs()));
 
+		definitions.add(definition(AutomationNodeType.DATA_TRANSFORM, "Transform",
+				"Build a JSON object from field name/value pairs — no Python required.",
+				orderedConfig("fields", List.of()),
+				List.of(field("fields", ConfigFieldType.JSON, "Fields", true, List.of())),
+				List.of(outputField("result", OutputFieldType.OBJECT, "Result",
+						"JSON object built from the configured fields.", true)),
+				controlInputs(), controlAndResultOutputs()));
+		definitions.add(definition(AutomationNodeType.DATA_DECLARE_VARIABLE, "Declare variable",
+				"Declare one or more named variables so later steps can reference each one directly with ${name}.",
+				orderedConfig("fields", List.of()),
+				List.of(field("fields", ConfigFieldType.JSON, "Variables", true, List.of())),
+				List.of(outputField("result", OutputFieldType.OBJECT, "Result",
+						"JSON object of the declared variables.", true)),
+				controlInputs(), controlAndResultOutputs()));
+
 		definitions.add(definition(AutomationNodeType.CONTROL_WAIT, "Wait", "Pause the automation before continuing.",
 				Map.of("durationSeconds", 5),
 				List.of(boundedIntegerField("durationSeconds", "Wait for (seconds)", 5,

src/prerna/reactor/automation/definition/AutomationNodeType.java

@@ -81,6 +81,9 @@ public enum AutomationNodeType {
 	APP_PIXEL(AutomationConstants.NODE_APP_PIXEL, Category.APP, null, Permission.VIEW, true, true),
 	AGENT_RUN(AutomationConstants.NODE_AGENT_RUN, Category.AGENT, IEngine.CATALOG_TYPE.MODEL, Permission.VIEW, true,
 			true),
+	DATA_TRANSFORM(AutomationConstants.NODE_DATA_TRANSFORM, Category.DATA, null, Permission.NONE, true, true),
+	DATA_DECLARE_VARIABLE(AutomationConstants.NODE_DATA_DECLARE_VARIABLE, Category.DATA, null, Permission.NONE, true,
+			false),
 	CONTROL_WAIT(AutomationConstants.NODE_CONTROL_WAIT, Category.CONTROL, null, Permission.NONE, true, true),
 	CONTROL_IF(AutomationConstants.NODE_CONTROL_IF, Category.CONTROL, null, Permission.NONE, false, false),
 	CONTROL_JEV(AutomationConstants.NODE_CONTROL_JEV, Category.CONTROL, IEngine.CATALOG_TYPE.MODEL, Permission.VIEW,

src/prerna/reactor/automation/definition/AutomationScopeCatalog.java

@@ -81,20 +81,50 @@ public static Map<String, List<Map<String, Object>>> variablesByNode(
 				if (!possibleNodes.contains(sourceId)) {
 					continue;
 				}
+				String availability = guaranteedNodes.contains(sourceId) ? "guaranteed" : "conditional";
 				String outputVar = string(source.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR));
-				if (outputVar == null || outputVar.isBlank()) {
-					continue;
+				if (outputVar != null && !outputVar.isBlank()) {
+					variablesByName.put(outputVar,
+							variable(outputVar, "node", string(source.get(AutomationConstants.NODE_FIELD_LABEL)),
+									"Output from an earlier step.", sourceId, null, availability));
+				}
+				if (AutomationConstants.NODE_DATA_DECLARE_VARIABLE
+						.equals(source.get(AutomationConstants.NODE_FIELD_TYPE))) {
+					for (String name : declareVariableNames(source)) {
+						variablesByName.put(name,
+								variable(name, "node", string(source.get(AutomationConstants.NODE_FIELD_LABEL)),
+										"Variable declared by an earlier step.", sourceId, null, availability));
+					}
 				}
-				variablesByName.put(outputVar,
-						variable(outputVar, "node", string(source.get(AutomationConstants.NODE_FIELD_LABEL)),
-								"Output from an earlier step.", sourceId, null,
-								guaranteedNodes.contains(sourceId) ? "guaranteed" : "conditional"));
 			}
 			result.put(targetId, new ArrayList<>(variablesByName.values()));
 		}
 		return result;
 	}
 
+	/**
+	 * Reads the top-level variable names a {@code data.declare-variable} node
+	 * would save into scope, from its {@code data.transform}-style field tree.
+	 * Only top-level names are individually addressable, since Java spreads only
+	 * the resolved object's own keys, not nested paths.
+	 */
+	private static List<String> declareVariableNames(Map<String, Object> node) {
+		if (!(node.get(AutomationConstants.NODE_FIELD_CONFIG) instanceof Map<?, ?> config)
+				|| !(config.get("fields") instanceof List<?> fields)) {
+			return List.of();
+		}
+		List<String> names = new ArrayList<>();
+		for (Object fieldObj : fields) {
+			if (fieldObj instanceof Map<?, ?> field) {
+				String name = string(field.get("name"));
+				if (name != null && !name.isBlank()) {
+					names.add(name);
+				}
+			}
+		}
+		return names;
+	}
+
 	private static List<Map<String, Object>> sharedVariables(
 			AutomationDefinitionValidator.ValidatedDefinition definition) {
 		List<Map<String, Object>> variables = new ArrayList<>();