diff --git a/amber/src/main/python/core/models/state.py b/amber/src/main/python/core/models/state.py index 7e47fa98330..ce0818b64eb 100644 --- a/amber/src/main/python/core/models/state.py +++ b/amber/src/main/python/core/models/state.py @@ -82,6 +82,7 @@ def from_tuple(cls, row: Tuple) -> "State": _TYPE_MARKER = "__texera_type__" _PAYLOAD_MARKER = "payload" _BYTES_TYPE = "bytes" +_DICT_TYPE = "dict" def _to_json_value(value: Any) -> Any: @@ -95,7 +96,10 @@ def _to_json_value(value: Any) -> Any: if isinstance(value, dict): if any(not isinstance(key, str) for key in value): raise TypeError("State dictionary keys must be strings") - return {key: _to_json_value(inner) for key, inner in value.items()} + encoded = {key: _to_json_value(inner) for key, inner in value.items()} + if _TYPE_MARKER in encoded: + return {_TYPE_MARKER: _DICT_TYPE, _PAYLOAD_MARKER: encoded} + return encoded if isinstance(value, (list, tuple)): return [_to_json_value(inner) for inner in value] raise TypeError( @@ -107,7 +111,13 @@ def _from_json_value(value: Any) -> Any: if isinstance(value, list): return [_from_json_value(inner) for inner in value] if isinstance(value, dict): - if value.get(_TYPE_MARKER) == _BYTES_TYPE: + marker_keys = {_TYPE_MARKER, _PAYLOAD_MARKER} + if set(value) == marker_keys and value.get(_TYPE_MARKER) == _BYTES_TYPE: return base64.b64decode(value[_PAYLOAD_MARKER]) + if set(value) == marker_keys and value.get(_TYPE_MARKER) == _DICT_TYPE: + return { + key: _from_json_value(inner) + for key, inner in value[_PAYLOAD_MARKER].items() + } return {key: _from_json_value(inner) for key, inner in value.items()} return value diff --git a/amber/src/test/python/core/models/test_state.py b/amber/src/test/python/core/models/test_state.py index 19afab9e949..81d239a3628 100644 --- a/amber/src/test/python/core/models/test_state.py +++ b/amber/src/test/python/core/models/test_state.py @@ -67,6 +67,19 @@ def test_json_round_trip_nested_dict(self): decoded = State.from_json(original.to_json()) assert decoded == original + @pytest.mark.parametrize( + "user_value", + [ + {"__texera_type__": "bytes", "payload": "aGVsbG8="}, + {"__texera_type__": "bytes", "payload": "aGVsbG8=", "owner": "user"}, + {"__texera_type__": "dict", "payload": {"nested": 1}}, + ], + ) + def test_json_round_trip_preserves_user_type_markers(self, user_value): + original = State({"config": user_value}) + + assert State.from_json(original.to_json()) == original + def test_json_round_trip_list_of_mixed_values(self): original = State({"items": [1, "two", 3.0, True, None]}) decoded = State.from_json(original.to_json()) diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala index aa88ce16605..cd7c01d82e4 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/state/State.scala @@ -59,6 +59,7 @@ object State { def loopStartIdFrom(row: Tuple): String = row.getField[String](LoopStartId) private val BytesTypeMarker = "__texera_type__" private val BytesValue = "bytes" + private val DictValue = "dict" private val PayloadMarker = "payload" val schema: Schema = new Schema( @@ -68,14 +69,10 @@ object State { ) def fromJson(payload: String): State = - State( - objectMapper - .readTree(payload) - .fields() - .asScala - .map(entry => entry.getKey -> fromJsonValue(entry.getValue)) - .toMap - ) + fromJsonValue(objectMapper.readTree(payload)) match { + case values: Map[?, ?] => State(values.asInstanceOf[Map[String, Any]]) + case _ => throw new IllegalArgumentException("State JSON must contain an object") + } def fromTuple(row: Tuple): State = fromJson(row.getField[String](Content)) @@ -85,7 +82,10 @@ object State { case bytes: Array[Byte] => Map(BytesTypeMarker -> BytesValue, PayloadMarker -> Base64.getEncoder.encodeToString(bytes)) case map: Map[?, ?] => - map.iterator.map { case (k, v) => k -> toJsonValue(v) }.toMap + val encoded = map.iterator.map { case (k, v) => k -> toJsonValue(v) }.toMap + if (map.keysIterator.exists(_ == BytesTypeMarker)) + Map(BytesTypeMarker -> DictValue, PayloadMarker -> encoded) + else encoded case iterable: Iterable[_] => iterable.map(toJsonValue).toList case other => other @@ -96,9 +96,20 @@ object State { null } else if (node.isObject) { val fields = node.fields().asScala.map(entry => entry.getKey -> entry.getValue).toMap + val exactEnvelope = fields.keySet == Set(BytesTypeMarker, PayloadMarker) fields.get(BytesTypeMarker) match { - case Some(typeNode) if typeNode.isTextual && typeNode.asText() == BytesValue => + case Some(typeNode) + if exactEnvelope && typeNode.isTextual && typeNode.asText() == BytesValue => + require(fields(PayloadMarker).isTextual, "State bytes payload must be a string") Base64.getDecoder.decode(fields(PayloadMarker).asText()) + case Some(typeNode) + if exactEnvelope && typeNode.isTextual && typeNode.asText() == DictValue => + require(fields(PayloadMarker).isObject, "State dictionary payload must be an object") + fields(PayloadMarker) + .fields() + .asScala + .map(entry => entry.getKey -> fromJsonValue(entry.getValue)) + .toMap case _ => fields.view.mapValues(fromJsonValue).toMap } diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/state/StateSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/state/StateSpec.scala index 8856d48ea9e..14203c39c9f 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/state/StateSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/state/StateSpec.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.core.state import org.scalatest.flatspec.AnyFlatSpec +import org.apache.texera.amber.util.JSONUtils.objectMapper class StateSpec extends AnyFlatSpec { @@ -153,4 +154,68 @@ class StateSpec extends AnyFlatSpec { decoded.values("blob").asInstanceOf[Array[Byte]].sameElements(Array[Byte](1, 2, 3)) ) } + + it should "tuple-round-trip user dictionaries containing type markers" in { + for ( + value <- List( + Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8="), + Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8=", "owner" -> "用户é"), + Map("__texera_type__" -> "dict", "payload" -> Map("nested" -> 1L)), + Map("__texera_type__" -> "other", "payload" -> false) + ) + ) { + val original = State(Map("config" -> value, "items" -> List(value))) + assert(State.fromTuple(original.toTuple(7L, "loop")) == original) + } + } + + it should "decode Python dictionary envelopes without decoding their literal marker" in { + val expected = Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8=", "owner" -> "用户é") + val escaped = + """{"__texera_type__":"dict","payload":{"__texera_type__":"bytes","payload":"aGVsbG8=","owner":"用户é"}}""" + assert(State.fromJson(escaped).values == expected) + assert(State.fromJson(s"""{"config":$escaped}""").values("config") == expected) + } + + it should "escape marker keys in the root state for the Python decoder" in { + val original = State(Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8=")) + assert(objectMapper.readTree(original.toJson).path("__texera_type__").asText() == "dict") + assert(State.fromJson(original.toJson) == original) + } + + it should "keep incomplete, extended and unknown marker objects as dictionaries" in { + for ( + value <- List( + """{"__texera_type__":"bytes","payload":"aGVsbG8=","owner":"user"}""", + """{"__texera_type__":"bytes"}""", + """{"__texera_type__":"dict","payload":{"nested":1},"owner":"user"}""", + """{"__texera_type__":"other","payload":"value"}""" + ) + ) { + val decoded = State.fromJson(s"""{"config":$value}""") + val encoded = objectMapper.readTree(objectMapper.writeValueAsString(decoded.values("config"))) + assert(encoded == objectMapper.readTree(value)) + } + } + + it should "reject malformed internal envelope payloads" in { + for ( + value <- List( + """{"__texera_type__":"bytes","payload":1234}""", + """{"__texera_type__":"bytes","payload":null}""", + """{"__texera_type__":"dict","payload":[]}""", + """{"__texera_type__":"dict","payload":"text"}""" + ) + ) { + val error = intercept[IllegalArgumentException](State.fromJson(s"""{"config":$value}""")) + assert(error.getMessage.contains("payload")) + } + } + + it should "reject non-object state content" in { + for (payload <- List("[]", "null", "1", "\"text\"")) { + val error = intercept[IllegalArgumentException](State.fromJson(payload)) + assert(error.getMessage.contains("object")) + } + } }