From d7007e7733b925b4d836c15570b91b6b957518a9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 00:25:09 -0800 Subject: [PATCH 1/5] test(logical_types): failing tests for refusing a logical type nested in a dict (ITL-1150) Co-Authored-By: Claude Opus 5.5 --- .../test_nested_logical_type_refusal.py | 371 ++++++++++++++++++ .../test_set_of_primitives.py | 35 +- 2 files changed, 385 insertions(+), 21 deletions(-) create mode 100644 tests/test_logical_types/test_nested_logical_type_refusal.py diff --git a/tests/test_logical_types/test_nested_logical_type_refusal.py b/tests/test_logical_types/test_nested_logical_type_refusal.py new file mode 100644 index 000000000..fcd82f695 --- /dev/null +++ b/tests/test_logical_types/test_nested_logical_type_refusal.py @@ -0,0 +1,371 @@ +"""A logical type or native container where storage cannot keep it is refused (ITL-1150). + +Arrow cannot hold an extension type inside a struct field, and a Python ``dict`` is stored +as a list of key/value structs (a tuple as a struct). A logical type or a native container +(``set[int]``, ``list[set[int]]``) as a dict key or value would not read back as declared: a +set would read as a list. Such a declared type is refused when it is declared (a pod's +parameter or output, a source's declared or inferred schema, a user-class field), with an +error that names the column or ``Class.field``, the type and ITL-1150. + +The positions that do keep the declared type stay accepted: a set, a list of sets and any +logical type as a column, in a list, and as a field of a dataclass, an ``orcapod.dataclass`` +or a pydantic model, read back as declared from the node, from ``as_source()`` and from a +job loaded in a fresh process. +""" + +from __future__ import annotations + +import dataclasses +import json +import re +import subprocess +import sys +import textwrap +import uuid +from pathlib import Path +from typing import Any + +import pydantic +import pytest + +import orcapod as op +from orcapod.databases import InMemoryArrowDatabase +from orcapod.operators import MapData +from orcapod.sources import DictSource + +# A declared type whose value would not read back as declared -> one value of it. +REFUSED: dict[str, tuple[Any, Any]] = { + "dict_value_set_int": (dict[str, set[int]], {"a": {2, 1}}), + "dict_value_set_str": (dict[str, set[str]], {"a": {"y", "x"}}), + "dict_value_list_of_sets": (dict[str, list[set[int]]], {"a": [{2, 1}]}), + "dict_value_optional_set": (dict[str, set[int] | None], {"a": {1}}), + "dict_key_set": (dict[set[int], int], {}), + "list_of_dict_of_sets": (list[dict[str, set[int]]], [{"a": {1}}]), + "dict_value_path": (dict[str, Path], {"a": Path("/a")}), + "dict_value_set_of_paths": (dict[str, set[Path]], {"a": {Path("/a")}}), +} + + +@dataclasses.dataclass +class HoldsDictOfSets: + x: dict[str, set[int]] + + +class ModelHoldsDictOfSets(pydantic.BaseModel): + x: dict[str, set[int]] + + +@dataclasses.dataclass +class Fields: + """An exact class whose fields hold logical types and native containers.""" + + s: set[int] + t: set[str] + p: Path + u: uuid.UUID + ls: list[set[int]] + lp: list[Path] + + +@op.dataclass +class StructuralFields: + s: set[int] + p: Path + ls: list[set[int]] + + +class ModelFields(pydantic.BaseModel): + s: set[int] + p: Path + ls: list[set[int]] + + +def _producer(declared: Any, value: Any) -> Any: + """A function pod with one input ``i`` that returns ``value`` as ``out``.""" + + def produce(i: int) -> Any: + return value + + produce.__annotations__["return"] = declared + return op.function_pod(output_keys="out")(produce) + + +def _consumer(declared: Any) -> Any: + """A function pod whose parameter ``got`` is declared ``declared``.""" + + def consume(got: Any) -> int: + return 1 + + consume.__annotations__["got"] = declared + return op.function_pod(output_keys="n")(consume) + + +def _refusal(column: str, declared: Any) -> str: + """The pattern a refusal of ``declared`` at ``column`` matches.""" + return rf"(?s){column}.*{_escaped(declared)}.*ITL-1150" + + +def _escaped(declared: Any) -> str: + return re.escape(str(declared)) + + +@pytest.mark.parametrize("case", list(REFUSED)) +class TestRefused: + """Every declaration of a type that storage cannot keep is refused when declared.""" + + def test_a_pod_output(self, case: str) -> None: + declared, value = REFUSED[case] + with pytest.raises(ValueError, match=_refusal("'out'", declared)): + _producer(declared, value) + + def test_a_pod_parameter(self, case: str) -> None: + declared, _ = REFUSED[case] + with pytest.raises(ValueError, match=_refusal("'got'", declared)): + _consumer(declared) + + def test_a_declared_source_schema(self, case: str) -> None: + declared, value = REFUSED[case] + with pytest.raises(ValueError, match=_refusal("'v'", declared)): + DictSource( + [{"k": 0, "v": value}], + tag_columns=["k"], + data_schema={"k": int, "v": declared}, + source_id="one_row", + ) + + +def test_a_source_schema_inferred_from_rows_is_refused() -> None: + """Rows holding a dict of sets infer ``dict[str, set[int]]``, which is refused.""" + with pytest.raises(ValueError, match=_refusal("'v'", dict[str, set[int]])): + DictSource([{"k": 0, "v": {"a": {1, 2}}}], tag_columns=["k"], source_id="x") + + +@pytest.mark.parametrize("cls", [HoldsDictOfSets, ModelHoldsDictOfSets]) +def test_a_user_class_field_is_refused_naming_the_field(cls: type) -> None: + """A class field typed ``dict[str, set[int]]`` is refused, naming ``Class.field``.""" + with pytest.raises( + ValueError, match=rf"(?s){cls.__qualname__}\.x.*{_escaped(dict[str, set[int]])}" + ) as info: + _producer(cls, cls(x={"a": {1}})) + assert "ITL-1150" in str(info.value) + + +def test_the_refusal_says_how_to_keep_the_value() -> None: + """The message points to a dataclass field, or the plain type, as the way out.""" + with pytest.raises(ValueError) as info: + _producer(dict[str, set[int]], {"a": {1}}) + message = str(info.value) + assert "dataclass" in message + assert "dict[str, list[int]]" in message + + +# --------------------------------------------------------------------------- +# Accepted: the positions that keep the declared type +# --------------------------------------------------------------------------- + +ACCEPTED: dict[str, tuple[Any, Any]] = { + "set_int": (set[int], {3, 1, 2}), + "list_of_sets": (list[set[int]], [{2, 1}, {3}]), + "list_of_paths": (list[Path], [Path("/a"), Path("/b")]), + "dict_of_lists": (dict[str, list[int]], {"a": [2, 1]}), + "dict_of_ints": (dict[str, int], {"a": 1}), + "list_of_dicts": (list[dict[str, int]], [{"a": 1}]), + "dataclass_fields": ( + Fields, + Fields( + s={2, 1}, + t={"b", "a"}, + p=Path("/a"), + u=uuid.UUID(int=5), + ls=[{2, 1}], + lp=[Path("/b")], + ), + ), + "structural_fields": ( + StructuralFields, + StructuralFields(s={2, 1}, p=Path("/a"), ls=[{3}]), + ), + "pydantic_fields": (ModelFields, ModelFields(s={2, 1}, p=Path("/a"), ls=[{3}])), +} + + +def _skeleton(value: Any) -> Any: + """The value with every container and field typed, so a set never equals a list.""" + if isinstance(value, set): + return ("set", sorted((_skeleton(v) for v in value), key=repr)) + if isinstance(value, list): + return ("list", [_skeleton(v) for v in value]) + if isinstance(value, dict): + return ("dict", sorted((k, _skeleton(v)) for k, v in value.items())) + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return ( + type(value).__qualname__, + [ + (f.name, _skeleton(getattr(value, f.name))) + for f in dataclasses.fields(value) + ], + ) + if isinstance(value, pydantic.BaseModel): + return ( + type(value).__qualname__, + [ + (name, _skeleton(getattr(value, name))) + for name in type(value).model_fields + ], + ) + return (type(value).__name__, repr(value)) + + +@pytest.mark.parametrize("case", list(ACCEPTED)) +def test_an_accepted_type_reads_back_as_declared(case: str) -> None: + """The node, a downstream pod and ``as_source()`` read the value as it was returned.""" + declared, value = ACCEPTED[case] + received: list[Any] = [] + + def consume(got: Any) -> int: + received.append(got) + return 1 + + consume.__annotations__["got"] = declared + source = DictSource([{"k": 0, "i": 0}], tag_columns=["k"], source_id="one_row") + job = op.PipelineJob(name="accepted", database=InMemoryArrowDatabase()) + with job: + produced = _producer(declared, value).pod(source, label="produce") + renamed = MapData({"out": "got"})(produced, label="rename") + op.function_pod(output_keys="n")(consume).pod(renamed, label="consume") + job.run() + + expected = _skeleton(value) + node = job.nodes["produce"] + stored = node.as_source() + for stream in (node, stored): + (_, data), *rest = list(stream.iter_data()) + assert not rest + assert _skeleton(data.as_dict()["out"]) == expected + assert [_skeleton(v) for v in received] == [expected] + assert stored.output_schema()[1]["out"] == declared + + +_CLASSES = textwrap.dedent( + """\ + import dataclasses + import uuid + from pathlib import Path + + import pydantic + + import orcapod + + + @dataclasses.dataclass + class Fields: + s: set[int] + p: Path + u: uuid.UUID + ls: list[set[int]] + + + @orcapod.dataclass + class StructuralFields: + s: set[int] + p: Path + ls: list[set[int]] + + + class ModelFields(pydantic.BaseModel): + s: set[int] + p: Path + ls: list[set[int]] + + + VALUES = { + "Fields": lambda: Fields(s={2, 1}, p=Path("/a"), u=uuid.UUID(int=5), ls=[{2, 1}]), + "StructuralFields": lambda: StructuralFields(s={2, 1}, p=Path("/a"), ls=[{3}]), + "ModelFields": lambda: ModelFields(s={2, 1}, p=Path("/a"), ls=[{3}]), + } + + + def skeleton(value): + if isinstance(value, set): + return ["set", sorted((skeleton(v) for v in value), key=repr)] + if isinstance(value, list): + return ["list", [skeleton(v) for v in value]] + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return [ + type(value).__qualname__, + [[f.name, skeleton(getattr(value, f.name))] for f in dataclasses.fields(value)], + ] + if isinstance(value, pydantic.BaseModel): + return [ + type(value).__qualname__, + [[n, skeleton(getattr(value, n))] for n in type(value).model_fields], + ] + return [type(value).__name__, repr(value)] + """ +) + +_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + mode, work, class_name = sys.argv[2], sys.argv[3], sys.argv[4] + + import classes + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.sources import DictSource + + cls = getattr(classes, class_name) + + + @function_pod(output_keys="out") + def produce(i: int) -> cls: + return classes.VALUES[class_name]() + + + if mode == "save": + source = DictSource([{"k": 0, "i": 0}], tag_columns=["k"], source_id="one_row") + job = PipelineJob(name="fields", database=DeltaTableDatabase(work + "/store")) + with job: + produce.pod(source, label="produce") + job.run() + job.save(work + "/job.json") + else: + job = PipelineJob.load(work + "/job.json", database=DeltaTableDatabase(work + "/store")) + _, data = next(iter(job.nodes["produce"].as_source().iter_data())) + print(json.dumps({ + "value": classes.skeleton(data.as_dict()["out"]), + "row_hash": data.content_hash().to_string(), + })) + """ +) + + +def _run(classes_dir: Path, mode: str, work: Path, class_name: str) -> dict[str, Any]: + result = subprocess.run( + [sys.executable, "-c", _SCRIPT, str(classes_dir), mode, str(work), class_name], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-3000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@pytest.mark.parametrize("class_name", ["Fields", "StructuralFields", "ModelFields"]) +def test_user_class_fields_read_back_as_declared_in_a_fresh_process( + class_name: str, tmp_path: Path +) -> None: + """A loaded job reads class fields holding sets and paths as declared, same row hash.""" + classes_dir = tmp_path / "classes" + classes_dir.mkdir() + (classes_dir / "classes.py").write_text(_CLASSES) + saved = _run(classes_dir, "save", tmp_path, class_name) + loaded = _run(classes_dir, "load", tmp_path, class_name) + assert loaded == saved + assert saved["value"][0] == class_name + assert any(field[1][0] == "set" for field in saved["value"][1]) diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index 5c69ec877..ea4a2bba7 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -245,12 +245,12 @@ def test_nested_and_optional_sets_reach_a_downstream_pod_as_sets( assert _out_values(job.nodes["produce"].as_source()) == [value] -def test_a_set_inside_a_dict_value_is_still_stored() -> None: - """``dict[str, set[int]]`` keeps working (plain storage inside the dict value).""" - job = _run_job( - InMemoryArrowDatabase(), dict[str, set[int]], {"a": {2, 1}}, use_async=False - ) - assert len(job.nodes["produce"].as_table()) == 1 +def test_a_set_inside_a_dict_value_is_refused() -> None: + """``dict[str, set[int]]`` is refused: a set there would read back as a list (ITL-1150).""" + with pytest.raises(ValueError, match="ITL-1150"): + _run_job( + InMemoryArrowDatabase(), dict[str, set[int]], {"a": {2, 1}}, use_async=False + ) def test_frozenset_stays_refused() -> None: @@ -702,22 +702,15 @@ def test_two_integer_widths_of_a_list_of_sets_live_in_one_process( } -def test_a_list_of_sets_inside_a_dict_value_keeps_plain_storage() -> None: - """``dict[str, list[set[int]]]`` is stored as it was before sets were containers.""" - source = DictSource( - [{"k": 0, "v": {"a": [{2, 1}]}}], - tag_columns=["k"], - data_schema={"k": int, "v": dict[str, list[set[int]]]}, - source_id="one_row", - ) - assert source.as_table().schema.field("v").type == pa.large_list( - pa.struct( - [ - pa.field("key", pa.large_string()), - pa.field("value", pa.large_list(pa.large_list(pa.int64()))), - ] +def test_a_list_of_sets_inside_a_dict_value_is_refused() -> None: + """``dict[str, list[set[int]]]`` is refused as a source schema (ITL-1150).""" + with pytest.raises(ValueError, match="ITL-1150"): + DictSource( + [{"k": 0, "v": {"a": [{2, 1}]}}], + tag_columns=["k"], + data_schema={"k": int, "v": dict[str, list[set[int]]]}, + source_id="one_row", ) - ) def test_the_factory_reads_a_set_element_as_its_arrow_type_does() -> None: From 5571561c771e360be7f175539f49c4bfb7ff66ae Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 00:25:46 -0800 Subject: [PATCH 2/5] docs: a logical type nested in a dict is refused at definition (ITL-1150) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 6 ++++-- CLAUDE.md | 6 ++++-- changelog.d/ITL-1150.md | 23 +++++++++++++++++++++++ changelog.d/ITL-611.md | 3 --- docs/concepts/function-pods.md | 10 ++++++++++ docs/concepts/logical-type-identifiers.md | 11 ++++++++--- 6 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 changelog.d/ITL-1150.md diff --git a/.zed/rules b/.zed/rules index 649ad9ee3..d0ecac354 100644 --- a/.zed/rules +++ b/.zed/rules @@ -296,8 +296,10 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension both as `set[int]`, as for lists), so it is registered by identifier only, never bound to a Python type (`binds_python_type`), and `set[T]` maps to Arrow through `T`. `list[T]` of a primitive stays a plain - `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps - its plain storage (Arrow has no extension type in a struct field). A container's elements + `large_list`: never wrap it. A native container, like any logical type, inside a dict key + or value or a tuple field is refused at definition (Arrow has no extension type in a struct + field; developer, November 15, 2026, ITL-1150): one guard, `nested_storage.require_storable_in_struct_field`, + never a silent strip to storage. A container's elements are value-hashed only when its innermost element is a logical type. A saved schema rebuilds an unregistered native container from its identifier (`serialization.parse_arrow_type_string`). `frozenset[T]` is refused (ITL-1140). diff --git a/CLAUDE.md b/CLAUDE.md index dcf04224d..b856b2ccc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -307,8 +307,10 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension both as `set[int]`, as for lists), so it is registered by identifier only, never bound to a Python type (`binds_python_type`), and `set[T]` maps to Arrow through `T`. `list[T]` of a primitive stays a plain - `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps - its plain storage (Arrow has no extension type in a struct field). A container's elements + `large_list`: never wrap it. A native container, like any logical type, inside a dict key + or value or a tuple field is refused at definition (Arrow has no extension type in a struct + field; developer, November 15, 2026, ITL-1150): one guard, `nested_storage.require_storable_in_struct_field`, + never a silent strip to storage. A container's elements are value-hashed only when its innermost element is a logical type. A saved schema rebuilds an unregistered native container from its identifier (`serialization.parse_arrow_type_string`). `frozenset[T]` is refused (ITL-1140). diff --git a/changelog.d/ITL-1150.md b/changelog.d/ITL-1150.md new file mode 100644 index 000000000..90096c8c9 --- /dev/null +++ b/changelog.d/ITL-1150.md @@ -0,0 +1,23 @@ +### Breaking changes and migration + +#### A logical type or native container as a dict key or value is refused where it is declared (ITL-1150) + +A dictionary is stored as a list of key/value structs, and Arrow cannot hold an extension type +inside a struct field. A set of a primitive there was stored silently as a plain list: +`dict[str, set[int]]` read back from `as_source()`, in a downstream pod and in a job loaded in +another process as `dict[str, list[int]]`, and a source declared with it took the schema +`dict[str, list[int]]`. Every other logical type in that position (`Path`, `UUID`, a user +class, `numpy.ndarray`, `list[Path]`, `set[Path]`) was already refused. + +Now any logical type or native container as a dict key or value, at any depth, is refused +when it is declared: a function pod's parameter or output, a source's declared or inferred +schema, and a field of a dataclass, an `orcapod.dataclass` or a pydantic model. Refused, for +example: `dict[str, set[int]]`, `dict[str, set[str]]`, `dict[str, list[set[int]]]`, +`dict[set[int], int]`, `list[dict[str, set[int]]]`, a dataclass field typed +`dict[str, set[int]]`. The error, `NestedTypeNotStorableError` (a `ValueError`), names the +column or `Class.field`, the declared type and ITL-1150. + +Migration: hold the set in a dataclass field (a field of a dataclass, an `orcapod.dataclass` +or a pydantic model keeps `set[int]`, `Path`, `UUID` and the other logical types on every +read path), or declare the plain type (`dict[str, list[int]]`). No stored data, stored path or +hash changes: only declarations that used to be accepted are refused. diff --git a/changelog.d/ITL-611.md b/changelog.d/ITL-611.md index 882c1066f..1779e7990 100644 --- a/changelog.d/ITL-611.md +++ b/changelog.d/ITL-611.md @@ -45,8 +45,5 @@ form are combined. Delete the stored tables of such nodes and run again. ### Known limitations -- Inside a dict value or a tuple field (`dict[str, set[int]]`, `dict[str, list[set[int]]]`), a - set of a primitive keeps its plain `large_list` storage, because Arrow cannot hold an extension type inside a struct - field; read back without the declared type, such a value is a list. (ITL-611) - `frozenset[T]` is not supported: declare `set[T]`. Support is to be evaluated in ITL-1140. (ITL-611) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 3ffa69841..84dd37030 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -250,6 +250,16 @@ before standalone iteration, table materialization, or sync/async storage can di Errors name the function, output, declared type and returned type; nested locations show structure without including returned dictionary keys or converter messages. +A declared type that storage cannot keep is refused when the pod is defined, before any value +is returned. A dictionary is stored as a list of key/value structs, and Arrow cannot hold a +logical type inside a struct field, so a logical type or native container (`Path`, `UUID`, a +user class, `numpy.ndarray`, `set[int]`, `list[set[int]]`, `list[Path]`) as a dictionary key +or value, at any depth (`dict[str, set[int]]`, `list[dict[str, set[int]]]`, a dataclass field +typed `dict[str, set[int]]`), raises `NestedTypeNotStorableError` naming the parameter or +output (ITL-1150). The same types are faithful as a column, inside a list, and as a field of a +dataclass, an `orcapod.dataclass` or a pydantic model: hold such a value in a dataclass +field, or declare the plain type (`dict[str, list[int]]`). + Outputs are validated against the declared storage schema immediately, and accepted numeric exceptions are normalized so downstream pods, hashes and stored rows see the same value. Eager validation never replaces a compatible user-class object; stored reads keep their diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index eeba60163..6cf9989f0 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -163,9 +163,14 @@ print(labels_type.storage_type) - The elements are stored in the canonical set order (see the hashing reference, section 1e), so equal sets store, and hash, identically. A set row never hashes like the list row with the same elements, because the container's identifier is part of the row hash. -- **Known limit:** inside a dict value or a tuple field (`dict[str, set[int]]`, - `dict[str, list[set[int]]]`), a set of a primitive keeps its plain `large_list` storage, because Arrow cannot hold an extension - type inside a struct field; read back without the declared type, such a value is a list. +- A native container, like any logical type, cannot be a dict key or value or a tuple field + yet (`dict[str, set[int]]`, `dict[str, list[set[int]]]`, `dict[set[int], int]`, + `list[dict[str, set[int]]]`): Arrow cannot hold an extension type inside a struct field, so + the value would read back as a list. Such a declared type is refused where it is declared + (a pod's parameter or output, a source's schema, a user-class field), with a + `NestedTypeNotStorableError` that names the column or `Class.field` (ITL-1150). Hold the set + in a dataclass field, where it keeps its type, or declare the plain type + (`dict[str, list[int]]`). - `frozenset[T]` is not supported: declare `set[T]`. Support is to be evaluated in ITL-1140. A container type you implement yourself is not a native container. It is an ordinary logical From 14e93e277231388cd8eaf6e394724ad85d0632c4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 00:33:38 -0800 Subject: [PATCH 3/5] fix(semantic_types)!: refuse a logical type nested in a dict key or value (ITL-1150) A native container (set[int], set[str], list[set[int]]) as a dict key or value was silently stored as its plain storage and read back as a list from as_source(), a downstream pod and a fresh-process load. One guard, nested_storage.require_storable_in_struct_field, now refuses any Arrow extension type at a struct-field position (dict key and value, tuple field) on both converter paths, replacing the ET1 raises and deleting _plain_set_storage. Callers name the column or Class.field; with_context now keeps the class of either nested refusal. Co-Authored-By: Claude Opus 5.5 --- .../logical_types/user_class_logical_type.py | 7 +- src/orcapod/semantic_types/nested_none.py | 16 ++- src/orcapod/semantic_types/nested_storage.py | 47 ++++++++ .../semantic_types/universal_converter.py | 110 +++++------------- 4 files changed, 93 insertions(+), 87 deletions(-) create mode 100644 src/orcapod/semantic_types/nested_storage.py diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index d6337969a..fe505eb4c 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -31,6 +31,7 @@ require_restorable_none, with_context, ) +from orcapod.semantic_types.nested_storage import NestedTypeNotStorableError from orcapod.logical_types.registry import ( make_arrow_extension_type, make_polars_extension_type, @@ -314,6 +315,8 @@ def _storage( RecursiveTypeError: If the class refers back to itself, naming the path. NestedNoneNotRestorableError: If a field admits a None a read cannot restore (ITL-1144), naming the field. + NestedTypeNotStorableError: If a field's type holds a logical type as a + dict key or value (ITL-1150), naming the field. """ hints = uci.type_hints(python_type) arrow_fields = [] @@ -331,9 +334,9 @@ def _storage( UserClassLogicalType, ), ) - except NestedNoneNotRestorableError as exc: + except (NestedNoneNotRestorableError, NestedTypeNotStorableError) as exc: raise with_context( - f"{python_type.__qualname__}.{field_name}", exc + f"{python_type.__qualname__}.{field_name} ({annotation})", exc ) from exc except TypeError as exc: if isinstance(exc, RecursiveTypeError): diff --git a/src/orcapod/semantic_types/nested_none.py b/src/orcapod/semantic_types/nested_none.py index 8e337bfcb..5484f645d 100644 --- a/src/orcapod/semantic_types/nested_none.py +++ b/src/orcapod/semantic_types/nested_none.py @@ -13,7 +13,7 @@ import types import typing from collections.abc import Callable -from typing import TYPE_CHECKING, Any, get_args, get_origin +from typing import TYPE_CHECKING, Any, TypeVar, get_args, get_origin if TYPE_CHECKING: import pyarrow as pa @@ -110,10 +110,14 @@ def unrestorable_none_error(arrow_type: pa.DataType) -> NestedNoneNotRestorableE ) -def with_context( - context: str, error: NestedNoneNotRestorableError -) -> NestedNoneNotRestorableError: - """The same refusal with the column or field it concerns put first. +_Refusal = TypeVar("_Refusal", bound=ValueError) + + +def with_context(context: str, error: _Refusal) -> _Refusal: + """The same refusal, of the same class, with the column or field it concerns put first. + + Used for every refusal of a declared nested type: a None a read cannot restore + (ITL-1144) and a logical type storage cannot keep (ITL-1150). Args: context: What the caller knows the refusal concerns, e.g. ``"Column 'v'"``. @@ -122,4 +126,4 @@ def with_context( Returns: A new error whose message starts with ``context``. """ - return NestedNoneNotRestorableError(f"{context}: {error}") + return type(error)(f"{context}: {error}") diff --git a/src/orcapod/semantic_types/nested_storage.py b/src/orcapod/semantic_types/nested_storage.py new file mode 100644 index 000000000..37ee98ba6 --- /dev/null +++ b/src/orcapod/semantic_types/nested_storage.py @@ -0,0 +1,47 @@ +"""Which nested positions storage can keep a logical type at (ITL-1150). + +A Python ``dict`` is stored as a list of key/value structs and a tuple as a struct (or a +fixed-size list), and Arrow cannot hold an extension type inside a struct field or a list +value field. A logical type or a native container (``set[int]``, ``list[set[int]]``) as a +dict key or value or a tuple field would therefore be stored as its plain storage and read +back as something else than declared (a set as a list). Until such positions are stored +faithfully, a declared type with one is refused where it is declared, so no value is stored +lossily. A field of a user class is not such a position: the class decodes each field by its +declared type. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pyarrow as pa + +_FOLLOW_UP = "ITL-1150" + + +class NestedTypeNotStorableError(ValueError): + """A logical type is declared where storage cannot keep it (ITL-1150).""" + + +def require_storable_in_struct_field(arrow_type: pa.DataType) -> None: + """Refuse a logical type or native container as a dict key or value or a tuple field. + + Args: + arrow_type: The Arrow type the declared key, value or field type registers as. + + Raises: + NestedTypeNotStorableError: If ``arrow_type`` is an extension type (a logical type + or a native container such as ``set[int]``). + """ + import pyarrow as pa + + if isinstance(arrow_type, pa.ExtensionType): + raise NestedTypeNotStorableError( + f"a value of Arrow extension type {arrow_type.extension_name!r} cannot be a " + f"dict key or value or a tuple field yet: Arrow cannot hold an extension type " + f"inside a struct field, so the value would not read back as declared (a set " + f"would read as a list). Hold it in a dataclass field, where it keeps its type, " + f"or declare the plain type (e.g. dict[str, list[int]] for " + f"dict[str, set[int]]). Follow-up: {_FOLLOW_UP}." + ) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 338f25153..a7a0bc16a 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -35,6 +35,10 @@ unrestorable_none_error, with_context, ) +from orcapod.semantic_types.nested_storage import ( + NestedTypeNotStorableError, + require_storable_in_struct_field, +) from orcapod.semantic_types.type_inference import infer_python_schema_from_pylist_data from orcapod.types import DataType, Schema, SchemaLike from orcapod.utils import arrow_utils @@ -253,6 +257,8 @@ def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: NestedNoneNotRestorableError: If an annotation declares a None at a nested position that a read cannot restore (ITL-1144), naming the column. + NestedTypeNotStorableError: If an annotation declares a logical type as + a dict key or value or a tuple field (ITL-1150), naming the column. """ if self._logical_type_registry is None: return @@ -260,7 +266,10 @@ def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: for name, annotation in schema.items(): try: self._register_annotation(annotation) - except NestedNoneNotRestorableError as exc: + except ( + NestedNoneNotRestorableError, + NestedTypeNotStorableError, + ) as exc: raise with_context(f"Column {name!r} ({annotation})", exc) from exc def _register_annotation(self, annotation: Any) -> None: @@ -421,34 +430,20 @@ def _register_python_class_impl( raise ValueError(_FROZENSET_UNSUPPORTED.format(annotation=annotation)) # dict[K, V] → pa.large_list(struct{key: K, value: V}). - # Raise if K or V resolves to an extension type: the key/value land inside struct - # fields, which also forbids extension types (ET1 in design_issues/). + # Refuse K or V that resolves to an extension type (a logical type or a native + # container): the key/value land inside struct fields, which cannot hold one + # (ITL-1150). if origin is dict: if len(args) < 2: raise ValueError( "Unparameterized 'dict' is not supported. Use 'dict[K, V]' with concrete " "key and value types (e.g. dict[str, int])." ) - # A set of a plain type keeps its plain storage inside a dict: Arrow - # cannot hold its extension type in a struct field (ITL-611). The guard - # sees the stored value type, the type a read of the value decides on. - key_arrow = self._plain_set_storage(self.register_python_class(args[0])) - val_arrow = self._plain_set_storage(self.register_python_class(args[1])) + key_arrow = self.register_python_class(args[0]) + val_arrow = self.register_python_class(args[1]) + require_storable_in_struct_field(key_arrow) + require_storable_in_struct_field(val_arrow) require_restorable_none(args[1], val_arrow, self._is_user_class_extension) - if isinstance(key_arrow, pa.ExtensionType): - raise ValueError( - f"'dict[{args[0]}, ...]' is not yet supported: the key type maps to Arrow " - f"extension type {key_arrow.extension_name!r}, which cannot be preserved " - f"inside a struct field due to an Arrow limitation (ET1 in design_issues/). " - f"Native dict-of-logical-type support is tracked in PLT-1732." - ) - if isinstance(val_arrow, pa.ExtensionType): - raise ValueError( - f"'dict[..., {args[1]}]' is not yet supported: the value type maps to Arrow " - f"extension type {val_arrow.extension_name!r}, which cannot be preserved " - f"inside a struct field due to an Arrow limitation (ET1 in design_issues/). " - f"Native dict-of-logical-type support is tracked in PLT-1732." - ) return pa.large_list( pa.struct([pa.field("key", key_arrow), pa.field("value", val_arrow)]) ) @@ -571,38 +566,6 @@ def _make_or_get_arrow_element_set_type( existing = lt return existing.get_arrow_extension_type() - def _plain_set_storage(self, arrow_type: "pa.DataType") -> "pa.DataType": - """Return a set of a plain element as its storage; any other type as it is. - - Arrow cannot hold an extension type inside a struct field, so a set of a - plain element type keeps the plain ``large_list`` storage it had before it - was a native container where it is a dict value or a tuple field (ITL-611). - So does a native container whose innermost element is such a set - (``list[set[int]]``): its storage holds no extension type either. - - Args: - arrow_type: An Arrow type the converter produced. - - Returns: - The storage type of a set of a plain element, or of a native container - whose innermost element is one; otherwise ``arrow_type``. - """ - from orcapod.logical_types.list_logical_type_factory import ListLogicalType - - if not isinstance(arrow_type, pa.ExtensionType) or ( - self._logical_type_registry is None - ): - return arrow_type - lt = self._logical_type_registry.get_by_arrow_extension_name( - arrow_type.extension_name - ) - innermost = lt - while isinstance(innermost, ListLogicalType): - if innermost.element_logical_type is None: - return arrow_type.storage_type - innermost = innermost.element_logical_type - return arrow_type - def _find_factory_for_class( self, python_type: type, @@ -1048,7 +1011,12 @@ def python_schema_to_arrow_schema(self, python_schema: SchemaLike) -> pa.Schema: """ fields = [] for field_name, python_type in python_schema.items(): - arrow_type = self.python_type_to_arrow_type(python_type) + try: + arrow_type = self.python_type_to_arrow_type(python_type) + except (NestedNoneNotRestorableError, NestedTypeNotStorableError) as exc: + raise with_context( + f"Column {field_name!r} ({python_type})", exc + ) from exc nullable = _is_optional_type(python_type) fields.append(pa.field(field_name, arrow_type, nullable=nullable)) return pa.schema(fields) @@ -1475,31 +1443,15 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: if len(set(args)) == 1: # Homogeneous tuple → fixed-size list - element_type = self._plain_set_storage( - self.python_type_to_arrow_type(args[0]) - ) - if hasattr(element_type, "extension_name"): - raise ValueError( - f"'tuple[{args[0]}, ...]' is not supported: the element type maps to " - f"Arrow extension type {element_type.extension_name!r}, which cannot be " - f"preserved inside a list or struct field due to an Arrow limitation " - f"(ET1 in design_issues/)." - ) + element_type = self.python_type_to_arrow_type(args[0]) + require_storable_in_struct_field(element_type) return pa.list_(element_type, len(args)) else: # Heterogeneous tuple → struct with indexed fields fields = [] for i, arg_type in enumerate(args): - field_type = self._plain_set_storage( - self.python_type_to_arrow_type(arg_type) - ) - if hasattr(field_type, "extension_name"): - raise ValueError( - f"'tuple[..., {arg_type}, ...]' is not supported: element type " - f"maps to Arrow extension type {field_type.extension_name!r}, which " - f"cannot be preserved inside a struct field due to an Arrow limitation " - f"(ET1 in design_issues/)." - ) + field_type = self.python_type_to_arrow_type(arg_type) + require_storable_in_struct_field(field_type) fields.append((f"f{i}", field_type)) return pa.struct(fields) @@ -1509,10 +1461,10 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: raise ValueError( f"dict type must have exactly two type arguments, got: {args}" ) - key_type = self._plain_set_storage(self.python_type_to_arrow_type(args[0])) - value_type = self._plain_set_storage( - self.python_type_to_arrow_type(args[1]) - ) + key_type = self.python_type_to_arrow_type(args[0]) + value_type = self.python_type_to_arrow_type(args[1]) + require_storable_in_struct_field(key_type) + require_storable_in_struct_field(value_type) require_restorable_none(args[1], value_type, self._is_user_class_extension) key_value_struct = pa.struct([("key", key_type), ("value", value_type)]) return pa.large_list(key_value_struct) From 6ba37ca10100c18f93d8d6d9f21aec04e5c0473c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 00:37:52 -0800 Subject: [PATCH 4/5] test(semantic_types): a set as a dict value is refused before the None guard (ITL-1150) Co-Authored-By: Claude Opus 5.5 --- .../test_nested_none_guard.py | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index d55dce070..77df3e423 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -385,17 +385,14 @@ def test_the_type_is_refused(self, annotation: Any) -> None: class TestSetOfAPrimitive: """The guard against a set of a primitive, now a native container (ITL-611). - Inside a dict value a set of a primitive keeps its plain ``large_list`` storage; the - guard decides on that stored type, and its decision is the one it made when the set - was plain storage everywhere: a set read back where a None was is never a None. + A set as a dict value is refused before this guard looks at it: storage cannot keep + a native container inside a struct field (ITL-1150). """ @pytest.mark.parametrize( "annotation", [ - dict[str, set[int] | None], dict[str, list[set[int] | None]], - dict[str, set[Path] | None], list[set[int] | None], set[uuid.UUID | None], ], @@ -405,25 +402,23 @@ def test_a_none_in_place_of_a_set_is_refused(self, annotation: Any) -> None: with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): _source(None, annotation) - def test_a_dict_of_sets_of_a_primitive_is_accepted(self) -> None: - value = {"a": {2, 1}, "b": set()} - read, written_hash = _read_v(_source(value, dict[str, set[int]])) - # Plain storage inside a dict value: the elements come back as stored (ITL-611). - assert {k: sorted(v) for k, v in read.items()} == {"a": [1, 2], "b": []} - _, read_hash = _read_v(_source(value, dict[str, set[int]])) - assert read_hash == written_hash - - def test_a_none_element_inside_a_set_in_a_dict_is_accepted(self) -> None: - value = {"a": {None, 1}} - read, written_hash = _read_v(_source(value, dict[str, set[int | None]])) - assert read == {"a": [None, 1]} - _, read_hash = _read_v(_source(value, dict[str, set[int | None]])) - assert read_hash == written_hash - - def test_a_dict_of_sets_of_a_logical_type_stays_refused(self) -> None: - """No None involved: the set's extension type cannot sit in a struct field.""" - with pytest.raises(ValueError, match="ET1") as info: - _source({"a": {P}}, dict[str, set[Path]]) + @pytest.mark.parametrize( + ("annotation", "value"), + [ + (dict[str, set[int]], {"a": {2, 1}, "b": set()}), + (dict[str, set[int | None]], {"a": {None, 1}}), + (dict[str, set[int] | None], {"a": None}), + (dict[str, set[Path] | None], {"a": None}), + (dict[str, set[Path]], {"a": {P}}), + ], + ids=str, + ) + def test_a_set_as_a_dict_value_is_refused_before_the_none_guard( + self, annotation: Any, value: Any + ) -> None: + """The set's extension type cannot sit in a struct field (ITL-1150).""" + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1150") as info: + _source(value, annotation) assert "ITL-1144" not in str(info.value) @pytest.mark.parametrize( From 1991a2c4c0abc5e511d11c19a00e815149e504d5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 01:35:28 -0800 Subject: [PATCH 5/5] docs(semantic_types): state the tuple-field refusal accurately (ITL-1150) Co-Authored-By: Claude Sonnet 5.5 --- changelog.d/ITL-1150.md | 9 ++++++++- src/orcapod/semantic_types/universal_converter.py | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/changelog.d/ITL-1150.md b/changelog.d/ITL-1150.md index 90096c8c9..84ef1be8b 100644 --- a/changelog.d/ITL-1150.md +++ b/changelog.d/ITL-1150.md @@ -14,7 +14,14 @@ when it is declared: a function pod's parameter or output, a source's declared o schema, and a field of a dataclass, an `orcapod.dataclass` or a pydantic model. Refused, for example: `dict[str, set[int]]`, `dict[str, set[str]]`, `dict[str, list[set[int]]]`, `dict[set[int], int]`, `list[dict[str, set[int]]]`, a dataclass field typed -`dict[str, set[int]]`. The error, `NestedTypeNotStorableError` (a `ValueError`), names the +`dict[str, set[int]]`. + +A tuple annotation was never accepted by a function pod or a source (registration rejects it +as an unsupported annotation, and still does). The only path that took a set inside a tuple +was a direct call of the type converter (`python_schema_to_arrow_schema({"x": tuple[int, set[int]]})`), +which stored the set as a plain list and now refuses it too. + +The error, `NestedTypeNotStorableError` (a `ValueError`), names the column or `Class.field`, the declared type and ITL-1150. Migration: hold the set in a dataclass field (a field of a dataclass, an `orcapod.dataclass` diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index a7a0bc16a..eb1c03139 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -258,7 +258,12 @@ def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: nested position that a read cannot restore (ITL-1144), naming the column. NestedTypeNotStorableError: If an annotation declares a logical type as - a dict key or value or a tuple field (ITL-1150), naming the column. + a dict key or value (ITL-1150), naming the column. A tuple + annotation is not covered here: registration does not support + tuples and raises a plain ``ValueError`` (``Unsupported + annotation``). The tuple-field refusal applies where a tuple is + converted to Arrow, in ``python_type_to_arrow_type`` and + ``python_schema_to_arrow_schema``. """ if self._logical_type_registry is None: return