Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions amber/src/main/python/core/models/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def from_tuple(cls, row: Tuple) -> "State":
_TYPE_MARKER = "__texera_type__"
_PAYLOAD_MARKER = "payload"
_BYTES_TYPE = "bytes"
_DICT_TYPE = "dict"


def _to_json_value(value: Any) -> Any:
Expand All @@ -95,7 +96,10 @@ def _to_json_value(value: Any) -> Any:
if isinstance(value, dict):
if any(not isinstance(key, str) for key in value):
raise TypeError("State dictionary keys must be strings")
return {key: _to_json_value(inner) for key, inner in value.items()}
encoded = {key: _to_json_value(inner) for key, inner in value.items()}
if _TYPE_MARKER in encoded:
return {_TYPE_MARKER: _DICT_TYPE, _PAYLOAD_MARKER: encoded}
return encoded
if isinstance(value, (list, tuple)):
return [_to_json_value(inner) for inner in value]
raise TypeError(
Expand All @@ -107,7 +111,13 @@ def _from_json_value(value: Any) -> Any:
if isinstance(value, list):
return [_from_json_value(inner) for inner in value]
if isinstance(value, dict):
if value.get(_TYPE_MARKER) == _BYTES_TYPE:
marker_keys = {_TYPE_MARKER, _PAYLOAD_MARKER}
if set(value) == marker_keys and value.get(_TYPE_MARKER) == _BYTES_TYPE:
return base64.b64decode(value[_PAYLOAD_MARKER])
if set(value) == marker_keys and value.get(_TYPE_MARKER) == _DICT_TYPE:
return {
key: _from_json_value(inner)
for key, inner in value[_PAYLOAD_MARKER].items()
}
Comment thread
carloea2 marked this conversation as resolved.
Resolved
return {key: _from_json_value(inner) for key, inner in value.items()}
return value
13 changes: 13 additions & 0 deletions amber/src/test/python/core/models/test_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,19 @@ def test_json_round_trip_nested_dict(self):
decoded = State.from_json(original.to_json())
assert decoded == original

@pytest.mark.parametrize(
"user_value",
[
{"__texera_type__": "bytes", "payload": "aGVsbG8="},
{"__texera_type__": "bytes", "payload": "aGVsbG8=", "owner": "user"},
{"__texera_type__": "dict", "payload": {"nested": 1}},
],
)
def test_json_round_trip_preserves_user_type_markers(self, user_value):
original = State({"config": user_value})

assert State.from_json(original.to_json()) == original

def test_json_round_trip_list_of_mixed_values(self):
original = State({"items": [1, "two", 3.0, True, None]})
decoded = State.from_json(original.to_json())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ object State {
def loopStartIdFrom(row: Tuple): String = row.getField[String](LoopStartId)
private val BytesTypeMarker = "__texera_type__"
private val BytesValue = "bytes"
private val DictValue = "dict"
private val PayloadMarker = "payload"

val schema: Schema = new Schema(
Expand All @@ -68,14 +69,10 @@ object State {
)

def fromJson(payload: String): State =
State(
objectMapper
.readTree(payload)
.fields()
.asScala
.map(entry => entry.getKey -> fromJsonValue(entry.getValue))
.toMap
)
fromJsonValue(objectMapper.readTree(payload)) match {
case values: Map[?, ?] => State(values.asInstanceOf[Map[String, Any]])
case _ => throw new IllegalArgumentException("State JSON must contain an object")
}

def fromTuple(row: Tuple): State = fromJson(row.getField[String](Content))

Expand All @@ -85,7 +82,10 @@ object State {
case bytes: Array[Byte] =>
Map(BytesTypeMarker -> BytesValue, PayloadMarker -> Base64.getEncoder.encodeToString(bytes))
case map: Map[?, ?] =>
map.iterator.map { case (k, v) => k -> toJsonValue(v) }.toMap
val encoded = map.iterator.map { case (k, v) => k -> toJsonValue(v) }.toMap
if (map.keysIterator.exists(_ == BytesTypeMarker))
Map(BytesTypeMarker -> DictValue, PayloadMarker -> encoded)
else encoded
case iterable: Iterable[_] =>
iterable.map(toJsonValue).toList
case other => other
Expand All @@ -96,9 +96,20 @@ object State {
null
} else if (node.isObject) {
val fields = node.fields().asScala.map(entry => entry.getKey -> entry.getValue).toMap
val exactEnvelope = fields.keySet == Set(BytesTypeMarker, PayloadMarker)
fields.get(BytesTypeMarker) match {
case Some(typeNode) if typeNode.isTextual && typeNode.asText() == BytesValue =>
case Some(typeNode)
if exactEnvelope && typeNode.isTextual && typeNode.asText() == BytesValue =>
require(fields(PayloadMarker).isTextual, "State bytes payload must be a string")
Base64.getDecoder.decode(fields(PayloadMarker).asText())
case Some(typeNode)
if exactEnvelope && typeNode.isTextual && typeNode.asText() == DictValue =>
require(fields(PayloadMarker).isObject, "State dictionary payload must be an object")
fields(PayloadMarker)
.fields()
.asScala
.map(entry => entry.getKey -> fromJsonValue(entry.getValue))
.toMap
case _ =>
fields.view.mapValues(fromJsonValue).toMap
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.texera.amber.core.state

import org.scalatest.flatspec.AnyFlatSpec
import org.apache.texera.amber.util.JSONUtils.objectMapper

class StateSpec extends AnyFlatSpec {

Expand Down Expand Up @@ -153,4 +154,68 @@ class StateSpec extends AnyFlatSpec {
decoded.values("blob").asInstanceOf[Array[Byte]].sameElements(Array[Byte](1, 2, 3))
)
}

it should "tuple-round-trip user dictionaries containing type markers" in {
for (
value <- List(
Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8="),
Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8=", "owner" -> "用户é"),
Map("__texera_type__" -> "dict", "payload" -> Map("nested" -> 1L)),
Map("__texera_type__" -> "other", "payload" -> false)
)
) {
val original = State(Map("config" -> value, "items" -> List(value)))
assert(State.fromTuple(original.toTuple(7L, "loop")) == original)
}
}

it should "decode Python dictionary envelopes without decoding their literal marker" in {
val expected = Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8=", "owner" -> "用户é")
val escaped =
"""{"__texera_type__":"dict","payload":{"__texera_type__":"bytes","payload":"aGVsbG8=","owner":"用户é"}}"""
assert(State.fromJson(escaped).values == expected)
assert(State.fromJson(s"""{"config":$escaped}""").values("config") == expected)
}

it should "escape marker keys in the root state for the Python decoder" in {
val original = State(Map("__texera_type__" -> "bytes", "payload" -> "aGVsbG8="))
assert(objectMapper.readTree(original.toJson).path("__texera_type__").asText() == "dict")
assert(State.fromJson(original.toJson) == original)
}

it should "keep incomplete, extended and unknown marker objects as dictionaries" in {
for (
value <- List(
"""{"__texera_type__":"bytes","payload":"aGVsbG8=","owner":"user"}""",
"""{"__texera_type__":"bytes"}""",
"""{"__texera_type__":"dict","payload":{"nested":1},"owner":"user"}""",
"""{"__texera_type__":"other","payload":"value"}"""
)
) {
val decoded = State.fromJson(s"""{"config":$value}""")
val encoded = objectMapper.readTree(objectMapper.writeValueAsString(decoded.values("config")))
assert(encoded == objectMapper.readTree(value))
}
}

it should "reject malformed internal envelope payloads" in {
for (
value <- List(
"""{"__texera_type__":"bytes","payload":1234}""",
"""{"__texera_type__":"bytes","payload":null}""",
"""{"__texera_type__":"dict","payload":[]}""",
"""{"__texera_type__":"dict","payload":"text"}"""
)
) {
val error = intercept[IllegalArgumentException](State.fromJson(s"""{"config":$value}"""))
assert(error.getMessage.contains("payload"))
}
}

it should "reject non-object state content" in {
for (payload <- List("[]", "null", "1", "\"text\"")) {
val error = intercept[IllegalArgumentException](State.fromJson(payload))
assert(error.getMessage.contains("object"))
}
}
}
Loading