From f8e06cc1c95ff95b142bd5562162a61a7acd6939 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:35:42 -0800 Subject: [PATCH 01/12] docs(plans): ITL-1144 measurement of None inside container columns Co-Authored-By: Claude Opus 5.5 --- ...11-15-itl-1144-none-in-containers-guard.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md diff --git a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md new file mode 100644 index 000000000..06ba9c954 --- /dev/null +++ b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md @@ -0,0 +1,77 @@ +# ITL-1144: guard against a None inside a container column that a read cannot restore + +Status: measurement done (main fe433eab, PR #393 head 8b75ccea); proposal sent to the +coordinator; implementation waits for the reply. + +## Measurement + +Scripts: the scratchpad `measure/` directory of this session (`cases.py`, `measure.py`, +`probe2.py`, `probe3.py`, `probe4.py`); results summarized here. + +Paths: A = function output in a job, read back by `node.as_source()`; B = `DictSource` +built from Python values, read by `iter_data()`; C = the saved job loaded in a fresh +process, `as_source()`; D = a downstream function in the job annotated with the type. +"Row hash" = `Data.content_hash()` of a one-row `DictSource` holding the value read, +compared with the same for the value written. + +Stored Arrow always holds the null correctly (write side faithful), and the in-memory +row hash equals the stored row hash in every case. The loss is only on the read to Python. + +| element | container | main (A, B, C, D agree) | PR #393 head | +|---|---|---|---| +| int, str, bytes, datetime | `list[T \| None]` | OK | OK | +| same | `set[T \| None]` | None kept; set reads back as list (ITL-611), hash equal | OK | +| same | `list[list[T \| None]]` | OK | OK | +| same | `list[set[T] \| None]` | LOST `[None,{1}]` -> `[[],[1]]`, hash differs | LOST `[set(),{1}]`, hash differs | +| same | `set[T] \| None` (top-level None) | OK | OK | +| same | `dict[str, T \| None]` | OK | OK | +| same | dataclass field `x: T \| None` | OK | OK | +| same | dataclass field `xs: list[T \| None]` | OK | OK | +| UUID, Path | `list[T \| None]`, `set[T \| None]`, `list[list[T \| None]]`, field `x: T \| None`, field `xs: list[T \| None]` | CRASH `TypeError` on read | same | +| UUID, Path | `list[set[T] \| None]` | LOST `[set(), {u}]`, hash differs | same | +| UUID, Path | `set[T] \| None` | OK | OK | +| UUID, Path, user classes | `dict[str, T \| None]` | refused at definition already (ET1, PLT-1732) | same | +| dataclass, orcapod.dataclass | `list[T \| None]`, `list[list[T \| None]]`, `set[T] \| None`, field `x`, field `xs` | OK | OK | +| dataclass, orcapod.dataclass | `set[T \| None]`, `list[set[T] \| None]` | n/a (instances unhashable) | n/a | + +Extra probes (path B): + +- LOST (None reads as `[]`, `{}` or `set()`, hash differs): `list[list[int] | None]`, + `list[dict[str, int] | None]`, `dict[str, list[int] | None]`, `dict[str, set[int] | None]`, + `dict[str, dict[str, int] | None]`, `list[list[UUID] | None]`, `list[list[Pt] | None]`, + `list[list[SPt] | None]`, dataclass fields `x: list[int] | None`, `set[int] | None`, + `dict[str, int] | None`, `list[UUID] | None`. +- CRASH on read: `list[UPath | None]`, `list[np.ndarray | None]`, `list[torch.Tensor | None]`, + `list[pd.DataFrame | None]`, `list[pl.DataFrame | None]`, dataclass field + `x: np.ndarray | None`, pydantic field `x: UUID | None`. +- OK: `list[Pt | None]`, `list[PydanticModel | None]`, dataclass field + `x: PydanticModel | None`, a top-level None of any type. +- Not reached by a declaration: `DictSource` declared `list[list[int]]` given `[None, [1]]` + accepts it and reads `[[], [1]]`; `ArrowTableSource` of `large_list>` + holding `[None, [1]]` types as `list[list[int]] | None` (ITL-1136) and reads `[[], [1]]`. + A function output refuses a None in a non-optional position (`OutputTypeError`). +- An in-memory pod -> pod chain (no job, no storage, no source built from the values) + keeps the None for every refused type. + +Rule found: a None at a NESTED position (list or set element, dict value, user-class +field) is restored only when the position's type is a primitive (int, float, str, bool, +bytes, datetime, date, Literal) or a user class (dataclass, orcapod.dataclass, pydantic). +A container (list, set, dict, and on PR #393 the set logical type) reads as empty, silently, +with a different hash; any other logical type (UUID, Path, UPath, numpy, torch, pandas, +polars) crashes. A top-level None is always restored. + +## Proposal (sent to the coordinator) + +- (a) Refuse at type level in `register_python_class` (every declared type: function + inputs and outputs, `DictSource` data schema, user-class fields): `X | None` at a nested + position where X is a container or a logical type other than a user class. The caller + that knows the column or field adds its name. +- (c) Read backstop in the nested decoders (`UniversalTypeConverter.storage_to_python`, + and the list element, dict value and struct field loops of + `_create_arrow_to_python_converter`): a null at a container or non-user-class logical + position raises instead of returning an empty value or a `TypeError`. +- Not (b): writes are faithful. + +## Plan + +(after the coordinator's reply) From a8815e5731abd4ed14c7f0850a79dfc54c879867 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:38:32 -0800 Subject: [PATCH 02/12] docs(plans): ITL-1144 operator-built lists measured Co-Authored-By: Claude Opus 5.5 --- ...11-15-itl-1144-none-in-containers-guard.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md index 06ba9c954..d30a6067e 100644 --- a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md +++ b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md @@ -53,6 +53,28 @@ Extra probes (path B): - An in-memory pod -> pod chain (no job, no storage, no source built from the values) keeps the None for every refused type. +Operator-built lists (coordinator follow-up; `ops.py`): source tags `g`, `k` (`k` null in +one row for Batch), one nullable data column per job; GroupBy(by=g), Batch(2), MergeJoin. +Read via the plain stream, the operator job node (`iter_data`), its LOG cache +`as_source()`, a downstream echo function (what it receives, its `as_source()`), and the +echo node of the job loaded in a fresh process. + +| operator | column | predicted | every path | +|---|---|---|---| +| GroupBy, Batch | `m: int \| None` | `list[int \| None]` | OK `[None, 2]` (loaded job too); Batch list tag `k` `[None, 8]` OK | +| GroupBy, Batch | `l: list[int] \| None`, `t: set[int] \| None` | `list[list[int] \| None]` | LOST `[[], [1]]`; the echo function receives and STORES `[[], [1]]` | +| GroupBy, Batch | `p: Path \| None`, `u: UUID \| None` | `list[Path \| None]`, `list[UUID \| None]` | CRASH `TypeError` on read; job.run() counts the downstream row as failed | +| MergeJoin | any colliding column with a None (even int) | `list[T \| None]` | CRASH in MergeJoin itself: `'<' not supported between NoneType` (separate bug) | + +Out-of-scope findings met on the way: +- MergeJoin cannot merge a colliding column holding a None (sort of None), any type. +- A function after GroupBy/Batch whose output key equals an input data column name: its + `as_source()` raises `ArrowNotImplementedError: Unsupported cast from + large_list to large_utf8` (source-info column list vs scalar). +- A job with a `list[Path]` output loaded in a fresh process before anything registers + `list[Path]`: "Cannot restore the output schema ... list[orcapod.path]" and + `as_source()` returns `['/tmp/a']` (str, not Path): a wrong value, no None involved. + Rule found: a None at a NESTED position (list or set element, dict value, user-class field) is restored only when the position's type is a primitive (int, float, str, bool, bytes, datetime, date, Literal) or a user class (dataclass, orcapod.dataclass, pydantic). From a2475ede1ced3a9b3d8d25f3e2d5d2c061ac43a5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:43:45 -0800 Subject: [PATCH 03/12] docs(plans): ITL-1144 guard plan Co-Authored-By: Claude Opus 5.5 --- ...11-15-itl-1144-none-in-containers-guard.md | 94 ++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md index d30a6067e..8386c851e 100644 --- a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md +++ b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md @@ -94,6 +94,98 @@ polars) crashes. A top-level None is always restored. position raises instead of returning an empty value or a `TypeError`. - Not (b): writes are faithful. +## Coordinator decision (November 15, 2026) + +Approve (a) + (c), not (b). (a) refuses exactly the LOST/CRASH nested positions; every OK +cell stays accepted (`list[int | None]`, GroupBy/Batch over a nullable int column: run PR +#346's `tests/test_core/alignment` against this branch and report). State and test what +happens to GroupBy/Batch over a nullable list, set, Path or UUID column. The error names +the type, the column or `Class.field` (added by the caller), ITL-1144 as the follow-up, and +what to do. Changelog under "Breaking changes and migration", saying plainly that an +in-memory pod-to-pod chain that worked is refused too. MergeJoin None crash: give a 5-line +repro, not ours. No hash, golden or stored-table change. Draft PR, "Refs ITL-1144". + ## Plan -(after the coordinator's reply) +> **For agentic workers:** executed inline (sensei:executing-plans), TDD. + +**Goal:** no Python value read from a stored or Arrow column is a wrong value because of a +None at a nested position: refuse the declaration (a), and raise at read (c). + +**Architecture:** one new module, `src/orcapod/semantic_types/nested_none.py`, holds the +rule (`none_is_restorable(arrow_type, lookup)`), the error +(`NestedNoneNotRestorableError(ValueError)`) and the refusal +(`require_restorable_none(annotation, arrow_type, lookup)`). It is used by +(a) `UniversalTypeConverter._register_python_class_impl` (list and set element, dict value) +and `UserClassLogicalType` field registration; (c) the Arrow-to-Python converter's nested +decoders (`storage_to_python`, list element, dict value, struct field). Callers that know +the column (`ensure_types_registered_for_schemas`, `arrow_table_to_python_dicts`) or the +field (`UserClassLogicalType`) re-raise with its name. + +The rule, on the Arrow type of the nested position: an extension type is restorable only +when its logical type is a `UserClassLogicalType` (dataclass, `orcapod.dataclass`, +pydantic); a list, large list, fixed-size list, struct or map is not; anything else +(primitive, timestamp, date, decimal) is. + +### Task 1: failing tests (public interface) + +File: `tests/test_semantic_types/test_nested_none_guard.py`. +- Refused at definition, parametrized: `DictSource(data_schema={"v": T})` and a function pod + output `-> T` and input `v: T` raise `ValueError` matching `'v'`, `ITL-1144`, for T in + `list[set[int] | None]`, `list[list[int] | None]`, `list[dict[str, int] | None]`, + `dict[str, list[int] | None]`, `list[UUID | None]`, `set[UUID | None]`, + `list[Path | None]`, `list[list[UUID] | None]`, `list[np.ndarray | None]`; a dataclass + with field `x: list[int] | None`, `x: UUID | None`, `x: Path | None`, used as a column, + raises naming `.x`. +- Accepted and round-tripping through `DictSource` (read value == written, row hash of the + read value == row hash of the written value): `list[int | None]`, `list[str | None]`, + `list[list[int | None]]`, `dict[str, int | None]`, `set[int] | None` (None), + `list[UUID] | None` (None), `UUID | None` (None), `list[Pt | None]`, + `list[SPt | None]` (orcapod.dataclass), dataclass fields `x: int | None`, + `xs: list[int | None]`, `p: Pt | None`. +- Raised at read (c): `ArrowTableSource` of `large_list>` holding + `[None, [1]]`; `DictSource` declared `list[list[int]]` given `[None, [1]]`; GroupBy and + Batch over a nullable `list[int]`, `Path`, `UUID` column: reading the rows raises + `ValueError` naming the column and ITL-1144. GroupBy and Batch over a nullable int column + read `[None, 2]` (accepted). +- Run: `uv run pytest tests/test_semantic_types/test_nested_none_guard.py` -> refused and + read cases FAIL, accepted cases PASS. + +### Task 2: the rule module and (a) + +- `nested_none.py` as above. `lookup` is the converter's + `get_logical_type_by_arrow_extension_name`. +- In `_register_python_class_impl`: after `inner = self.register_python_class(args[0])` in + the list and set branches, and after `val_arrow` in the dict branch, call + `require_restorable_none(args[i], inner, self.get_logical_type_by_arrow_extension_name)`. +- `ensure_types_registered_for_schemas` iterates `schema.items()` and re-raises + `NestedNoneNotRestorableError(f"Column {name!r} ({annotation}): {exc}")`. +- `UserClassLogicalType` field loop: call the check on each field annotation and its arrow + type before stripping the extension, re-raise `f"{cls.__qualname__}.{field}: {exc}"`. + +### Task 3: (c) read backstop + +- `UniversalTypeConverter.storage_to_python`: a None value with a non-restorable Arrow type + raises `NestedNoneNotRestorableError` (it serves ListLogicalType elements and user-class + fields; a top-level null never reaches it: `arrow_table_to_python_dicts` and the hashing + visitor handle it first). +- `_create_arrow_to_python_converter`: list element, dict value and struct field decoders + wrap the element converter so a None at a non-restorable position raises. +- `arrow_table_to_python_dicts` re-raises with the column name. + +### Task 4: docs, changelog, design issue + +- `docs/concepts/function-pods.md` "Value types": a paragraph on refused nested optionals + (which, why, what to do, ITL-1144). Doc samples run, so a sample shows the refusal. +- `changelog.d/ITL-1144.md`: Breaking changes and migration. +- `design_issues/ITL-1144-NN1.md` (`## src/orcapod/semantic_types/`), status in progress + (guard in; the full fix open). + +### Task 5: verification + +- Targeted: the new test file, `tests/test_semantic_types`, `tests/test_user_types`, + `tests/test_logical_types`, `tests/test_hashing` (golden unchanged), `tests/test_docs`. +- PR #346's alignment suite against this branch (checked out into a scratch copy of the + tests, not committed). +- Full `uv run pytest tests/`, ruff, basedpyright. Merge origin/main, push, draft PR + "Refs ITL-1144". From 5e6c6519e1436b3c6131de7ff4f86d76ed2195ed Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:44:25 -0800 Subject: [PATCH 04/12] test(semantic_types): a nested None a read cannot restore is refused (ITL-1144, failing) Co-Authored-By: Claude Opus 5.5 --- .../test_nested_none_guard.py | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 tests/test_semantic_types/test_nested_none_guard.py diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py new file mode 100644 index 000000000..9b3b94152 --- /dev/null +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -0,0 +1,221 @@ +"""A None at a nested position that a read cannot restore is refused (ITL-1144). + +A None inside a container (a list or set element, a dict value, a user-class field) is +stored faithfully, but reading the stored column back to Python restored it only for a +primitive or a user class: a container read as empty and any other logical type crashed. +Until the read restores them (ITL-1144 follow-up), such declarations are refused when the +type is declared, and a stored column holding such a None raises when it is read. +""" + +import dataclasses +import uuid +from pathlib import Path +from typing import Any + +import numpy as np +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.operators import Batch, GroupBy +from orcapod.sources import ArrowTableSource, DictSource + +U = uuid.UUID(int=1) +P = Path("/tmp/itl-1144") + + +@dataclasses.dataclass +class Point: + x: int + + +@op.dataclass +class StructuralPoint: + x: int + + +@dataclasses.dataclass +class OptionalList: + x: list[int] | None + + +@dataclasses.dataclass +class OptionalUUID: + x: uuid.UUID | None + + +@dataclasses.dataclass +class OptionalPath: + x: Path | None + + +@dataclasses.dataclass +class OptionalScalars: + x: int | None + xs: list[int | None] + p: Point | None + + +REFUSED_TYPES = [ + list[set[int] | None], + list[list[int] | None], + list[dict[str, int] | None], + dict[str, list[int] | None], + list[uuid.UUID | None], + set[uuid.UUID | None], + list[Path | None], + list[list[uuid.UUID] | None], + list[np.ndarray | None], +] + +REFUSED_CLASSES = [OptionalList, OptionalUUID, OptionalPath] + + +def _source(value: Any, annotation: Any) -> DictSource: + """A one-row source holding ``value`` in column ``v`` declared as ``annotation``.""" + return DictSource( + [{"id": 1, "v": value}], + tag_columns=["id"], + data_schema={"id": int, "v": annotation}, + source_id="v", + ) + + +def _read_v(source: Any) -> tuple[Any, str]: + """The value of column ``v`` read back, and its row's data hash.""" + [(_, data)] = list(source.iter_data()) + return data.as_dict()["v"], data.content_hash().to_string() + + +def _rows(stream: Any) -> list[tuple[dict[str, Any], dict[str, Any]]]: + """Every row of ``stream`` as Python values.""" + return [(tag.as_dict(), data.as_dict()) for tag, data in stream.iter_data()] + + +class TestRefusedAtDefinition: + """A declared type with a None the read cannot restore is refused, naming the column.""" + + @pytest.mark.parametrize("annotation", REFUSED_TYPES, ids=str) + def test_a_source_schema_is_refused(self, annotation: Any) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + _source(None, annotation) + + @pytest.mark.parametrize("annotation", REFUSED_TYPES, ids=str) + def test_a_function_output_is_refused(self, annotation: Any) -> None: + def produce(n: int) -> Any: + return None + + produce.__annotations__["return"] = annotation + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + op.function_pod(output_keys="v")(produce) + + @pytest.mark.parametrize("annotation", REFUSED_TYPES, ids=str) + def test_a_function_input_is_refused(self, annotation: Any) -> None: + def consume(v: Any) -> int: + return 0 + + consume.__annotations__["v"] = annotation + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + op.function_pod(output_keys="n")(consume) + + @pytest.mark.parametrize("cls", REFUSED_CLASSES, ids=lambda c: c.__name__) + def test_a_class_field_is_refused_naming_the_field(self, cls: type) -> None: + with pytest.raises( + ValueError, match=rf"(?s)Column 'v'.*{cls.__qualname__}\.x.*ITL-1144" + ): + _source(cls(None), cls) + + def test_the_message_says_what_to_do(self) -> None: + with pytest.raises(ValueError, match=r"list\[int\] \| None") as info: + _source([None], list[list[int] | None]) + assert "without '| None'" in str(info.value) + + +ACCEPTED = [ + (list[int | None], [None, 1]), + (list[str | None], [None, "a"]), + (list[list[int | None]], [[None, 1]]), + (dict[str, int | None], {"j": 1, "k": None}), + (set[int] | None, None), + (list[uuid.UUID] | None, None), + (uuid.UUID | None, None), + (list[Point | None], [None, Point(1)]), + (list[StructuralPoint | None], [None, StructuralPoint(1)]), + (OptionalScalars, OptionalScalars(None, [None, 1], None)), +] + + +class TestAccepted: + """A None that a read restores stays accepted: same value, same row hash.""" + + @pytest.mark.parametrize(("annotation", "value"), ACCEPTED, ids=str) + def test_round_trips(self, annotation: Any, value: Any) -> None: + read, written_hash = _read_v(_source(value, annotation)) + assert read == value + _, read_hash = _read_v(_source(read, annotation)) + assert read_hash == written_hash + + +class TestRaisedAtRead: + """A stored None the read cannot restore raises instead of a wrong value.""" + + def test_an_arrow_table_with_a_null_inner_list(self) -> None: + table = pa.table( + { + "id": pa.array([1], pa.int64()), + "v": pa.array([[None, [1]]], pa.large_list(pa.large_list(pa.int64()))), + } + ) + source = ArrowTableSource(table, tag_columns=["id"], source_id="v") + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + _read_v(source) + + def test_a_none_given_for_a_non_optional_inner_list(self) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + _read_v(_source([None, [1]], list[list[int]])) + + +def _members(annotation: Any, first: Any, second: Any) -> DictSource: + """Two rows of group ``g=1``: column ``m`` holds ``first`` then ``second``.""" + return DictSource( + [{"g": 1, "k": 1, "m": first}, {"g": 1, "k": 2, "m": second}], + tag_columns=["g", "k"], + data_schema={"g": int, "k": int, "m": annotation}, + source_id="m", + ) + + +OPERATORS = { + "group_by": lambda s: GroupBy(by=["g"])(s), + "batch": lambda s: Batch(2)(s), +} + + +class TestReducingOperators: + """GroupBy and Batch list a nullable column's values: a None stays where restorable.""" + + @pytest.mark.parametrize("operator", OPERATORS) + def test_a_nullable_int_column_round_trips(self, operator: str) -> None: + out = OPERATORS[operator](_members(int | None, None, 2)) + [(_, data)] = _rows(out) + assert data["m"] == [None, 2] + + @pytest.mark.parametrize("operator", OPERATORS) + @pytest.mark.parametrize( + ("annotation", "second"), + [(list[int] | None, [1]), (set[int] | None, {1}), (Path | None, P), (uuid.UUID | None, U)], + ids=["list", "set", "Path", "UUID"], + ) + def test_a_nullable_container_or_logical_column_raises_at_read( + self, operator: str, annotation: Any, second: Any + ) -> None: + out = OPERATORS[operator](_members(annotation, None, second)) + with pytest.raises(ValueError, match=r"(?s)Column 'm'.*ITL-1144"): + _rows(out) + + def test_a_function_declared_for_such_a_list_is_refused(self) -> None: + def consume(m: list[list[int] | None]) -> int: + return 0 + + with pytest.raises(ValueError, match=r"(?s)Column 'm'.*ITL-1144"): + op.function_pod(output_keys="n")(consume) From c6f96e89165decbf4a07ef6d74bd84457e73da84 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:46:14 -0800 Subject: [PATCH 05/12] fix(semantic_types): refuse a nested None a read cannot restore (ITL-1144) A None inside a container (list or set element, dict value, user-class field) is stored faithfully, but the read restored it only for a primitive or a user class: a container read as empty and any other logical type crashed. Declaring such a position is refused, and a stored null at such a position raises at read, naming the column. Co-Authored-By: Claude Opus 5.5 --- .../logical_types/user_class_logical_type.py | 17 +++ src/orcapod/semantic_types/nested_none.py | 127 ++++++++++++++++++ .../semantic_types/universal_converter.py | 105 ++++++++++++--- .../test_nested_none_guard.py | 4 - 4 files changed, 227 insertions(+), 26 deletions(-) create mode 100644 src/orcapod/semantic_types/nested_none.py diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 92ad15ca7..7cea6b0d5 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -26,6 +26,11 @@ import orcapod.logical_types.user_class_identity as uci from orcapod.logical_types.base_logical_type import BaseLogicalType from orcapod.logical_types.structural_class import class_for_record +from orcapod.semantic_types.nested_none import ( + NestedNoneNotRestorableError, + require_restorable_none, + with_context, +) from orcapod.logical_types.registry import ( make_arrow_extension_type, make_polars_extension_type, @@ -307,6 +312,8 @@ def _storage( Raises: ValueError: If the type hints cannot be resolved. RecursiveTypeError: If the class refers back to itself, naming the path. + NestedNoneNotRestorableError: If a field admits a None a read cannot + restore (ITL-1144), naming the field. """ hints = uci.type_hints(python_type) arrow_fields = [] @@ -315,6 +322,16 @@ def _storage( annotation = hints.get(field_name, Any) try: arrow_type = converter.register_python_class(annotation) + # A field is a nested position: its None must be readable (ITL-1144). + require_restorable_none( + annotation, + arrow_type, + converter.get_logical_type_by_arrow_extension_name, + ) + except NestedNoneNotRestorableError as exc: + raise with_context( + f"{python_type.__qualname__}.{field_name}", exc + ) from exc except TypeError as exc: if isinstance(exc, RecursiveTypeError): raise RecursiveTypeError( diff --git a/src/orcapod/semantic_types/nested_none.py b/src/orcapod/semantic_types/nested_none.py new file mode 100644 index 000000000..5637b7a08 --- /dev/null +++ b/src/orcapod/semantic_types/nested_none.py @@ -0,0 +1,127 @@ +"""Which None at a nested position a read restores (ITL-1144). + +A None inside a container column (a list or set element, a dict value, a user-class +field) is stored faithfully as an Arrow null. Reading the column back to Python restores +it only where the position's type is a primitive or a user class: a container read as an +empty container and any other logical type failed. Until the read restores every such +None (the ITL-1144 follow-up), a declared type with such a position is refused, and a +stored null at such a position raises when it is read, so no wrong value gets out. +""" + +from __future__ import annotations + +import types +import typing +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, get_args, get_origin + +if TYPE_CHECKING: + import pyarrow as pa + + from orcapod.logical_types.protocols import LogicalTypeProtocol + +LogicalTypeLookup = Callable[[str], "LogicalTypeProtocol | None"] +"""Find the registered logical type of an Arrow extension name.""" + +_FOLLOW_UP = "ITL-1144" + + +class NestedNoneNotRestorableError(ValueError): + """A None at a nested position cannot be read back faithfully (ITL-1144).""" + + +def none_is_restorable(arrow_type: pa.DataType, lookup: LogicalTypeLookup) -> bool: + """Whether a read restores a null at a nested position of ``arrow_type`` as None. + + A user class (dataclass, ``orcapod.dataclass``, pydantic model) and every scalar + type are restored; a list, struct or map reads as an empty value, and any other + logical type fails. + + Args: + arrow_type: The Arrow type of the nested position. + lookup: Finds the logical type registered for an extension name. + + Returns: + True when a null there is read back as None. + """ + import pyarrow as pa + + from orcapod.logical_types.user_class_logical_type import UserClassLogicalType + + if isinstance(arrow_type, pa.ExtensionType): + return isinstance(lookup(arrow_type.extension_name), UserClassLogicalType) + return not ( + pa.types.is_list(arrow_type) + or pa.types.is_large_list(arrow_type) + or pa.types.is_fixed_size_list(arrow_type) + or pa.types.is_list_view(arrow_type) + or pa.types.is_large_list_view(arrow_type) + or pa.types.is_struct(arrow_type) + or pa.types.is_map(arrow_type) + ) + + +def _is_optional(annotation: Any) -> bool: + """Whether ``annotation`` is a union that includes None.""" + origin = get_origin(annotation) + return (origin is typing.Union or origin is types.UnionType) and type( + None + ) in get_args(annotation) + + +def require_restorable_none( + annotation: Any, arrow_type: pa.DataType, lookup: LogicalTypeLookup +) -> None: + """Refuse a nested position declared ``X | None`` whose None a read cannot restore. + + Args: + annotation: The declared type of the nested position (a list or set element, a + dict value, a user-class field). + arrow_type: The Arrow type ``annotation`` registers as. + lookup: Finds the logical type registered for an extension name. + + Raises: + NestedNoneNotRestorableError: If ``annotation`` admits None and a stored None + there would not be read back as None. + """ + if _is_optional(annotation) and not none_is_restorable(arrow_type, lookup): + raise NestedNoneNotRestorableError( + f"{annotation} is not supported inside a container or as a class field yet: " + f"a None there is stored but not read back faithfully (it would read as an " + f"empty value or fail). Declare it without '| None', or hold the value in a " + f"dataclass and make that optional (e.g. list[MyData | None]). Follow-up: " + f"{_FOLLOW_UP}." + ) + + +def unrestorable_none_error(arrow_type: pa.DataType) -> NestedNoneNotRestorableError: + """The error for a stored null read at a nested position it cannot be restored at. + + Args: + arrow_type: The Arrow type of the nested position holding the null. + + Returns: + The error to raise. + """ + return NestedNoneNotRestorableError( + f"a stored None inside a container, at a position of Arrow type {arrow_type}, " + f"cannot be read back faithfully yet (it would read as an empty value or fail). " + f"Remove the None before the value is put in a container (e.g. DropNulls on " + f"the column before GroupBy or Batch), or hold the value in a dataclass. " + f"Follow-up: {_FOLLOW_UP}." + ) + + +def with_context( + context: str, error: NestedNoneNotRestorableError +) -> NestedNoneNotRestorableError: + """The same refusal with the column or field it concerns put first. + + Args: + context: What the caller knows the refusal concerns, e.g. ``"Column 'v'"``. + error: The refusal raised below the caller. + + Returns: + A new error whose message starts with ``context``. + """ + return NestedNoneNotRestorableError(f"{context}: {error}") diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 5c752b21e..c5847d1b7 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -28,6 +28,13 @@ sort_dict_entries, sort_set_elements, ) +from orcapod.semantic_types.nested_none import ( + NestedNoneNotRestorableError, + none_is_restorable, + require_restorable_none, + unrestorable_none_error, + with_context, +) from orcapod.semantic_types.type_inference import infer_python_schema_from_pylist_data from orcapod.types import DataType, Schema, SchemaLike from orcapod.utils import arrow_utils @@ -238,23 +245,33 @@ def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: Raises: TypeError: If a leaf class has no registered ``LogicalType`` and no registered factory covers it. + NestedNoneNotRestorableError: If an annotation declares a None at a + nested position that a read cannot restore (ITL-1144), naming the + column. """ if self._logical_type_registry is None: return for schema in schemas: - for annotation in schema.values(): - origin = get_origin(annotation) - if origin is typing.Union or origin is types.UnionType: - # Union types (e.g. str | Path) are valid in function input - # schemas — they express that the pod accepts either concrete - # type. Register each non-None branch so its LogicalType is - # available when a stream is bound; the union itself has no - # Arrow representation. - for branch in get_args(annotation): - if branch is not type(None): - self.register_python_class(branch) - else: - self.register_python_class(annotation) + for name, annotation in schema.items(): + try: + self._register_annotation(annotation) + except NestedNoneNotRestorableError as exc: + raise with_context(f"Column {name!r} ({annotation})", exc) from exc + + def _register_annotation(self, annotation: Any) -> None: + """Register one schema annotation; a union registers each non-None branch.""" + origin = get_origin(annotation) + if origin is typing.Union or origin is types.UnionType: + # Union types (e.g. str | Path) are valid in function input + # schemas — they express that the pod accepts either concrete + # type. Register each non-None branch so its LogicalType is + # available when a stream is bound; the union itself has no + # Arrow representation. + for branch in get_args(annotation): + if branch is not type(None): + self.register_python_class(branch) + else: + self.register_python_class(annotation) def register_python_class(self, annotation: Any) -> "pa.DataType": """Register a Python type annotation and return its Arrow type. @@ -279,6 +296,9 @@ def register_python_class(self, annotation: Any) -> "pa.DataType": TypeError: If a concrete class has no registered ``LogicalType`` and no factory covers it, or if a circular dependency is detected. ValueError: If a complex (non-Optional) union is encountered. + NestedNoneNotRestorableError: If a list or set element or a dict value is + declared ``X | None`` and a stored None there cannot be read back + (ITL-1144). """ in_progress = _register_in_progress.get() if in_progress is None: @@ -361,6 +381,9 @@ def _register_python_class_impl( "element type (e.g. list[int], list[str])." ) inner = self.register_python_class(args[0]) + require_restorable_none( + args[0], inner, self.get_logical_type_by_arrow_extension_name + ) if self._logical_type_registry is not None and hasattr( inner, "extension_name" ): @@ -379,6 +402,9 @@ def _register_python_class_impl( "element type (e.g. set[int], set[str])." ) inner = self.register_python_class(args[0]) + require_restorable_none( + args[0], inner, self.get_logical_type_by_arrow_extension_name + ) if self._logical_type_registry is not None and hasattr( inner, "extension_name" ): @@ -400,6 +426,9 @@ def _register_python_class_impl( ) key_arrow = self.register_python_class(args[0]) val_arrow = self.register_python_class(args[1]) + require_restorable_none( + args[1], val_arrow, self.get_logical_type_by_arrow_extension_name + ) if isinstance(key_arrow, pa.ExtensionType): raise ValueError( f"'dict[{args[0]}, ...]' is not yet supported: the key type maps to Arrow " @@ -819,8 +848,18 @@ def storage_to_python(self, storage_value: Any, annotation: Any) -> Any: Returns: A Python object of the type described by ``annotation``. + + Raises: + NestedNoneNotRestorableError: If ``storage_value`` is None and a None of + ``annotation`` cannot be read back faithfully (ITL-1144). This method + decodes nested positions (list elements, class fields); a column's own + null never reaches it. """ arrow_type = self.python_type_to_arrow_type(annotation) + if storage_value is None and not none_is_restorable( + arrow_type, self.get_logical_type_by_arrow_extension_name + ): + raise unrestorable_none_error(arrow_type) converter_fn = self.get_arrow_to_python_converter(arrow_type) return converter_fn(storage_value) @@ -1211,7 +1250,10 @@ def arrow_table_to_python_dicts( python_dict = {} for field_name, value in row.items(): if value is not None: - python_dict[field_name] = converters[field_name](value) + try: + python_dict[field_name] = converters[field_name](value) + except NestedNoneNotRestorableError as exc: + raise with_context(f"Column {field_name!r}", exc) from exc else: python_dict[field_name] = None python_dicts.append(python_dict) @@ -1822,9 +1864,7 @@ def _create_arrow_to_python_converter( value_field = next(f for f in element_type if f.name == "value") key_converter = self.get_arrow_to_python_converter(key_field.type) - value_converter = self.get_arrow_to_python_converter( - value_field.type - ) + value_converter = self._nested_value_converter(value_field.type) return lambda value: ( { @@ -1837,7 +1877,7 @@ def _create_arrow_to_python_converter( ) # Regular list - element_converter = self.get_arrow_to_python_converter(element_type) + element_converter = self._nested_value_converter(element_type) if pa.types.is_fixed_size_list(arrow_type): # Fixed-size list → tuple @@ -1857,7 +1897,7 @@ def _create_arrow_to_python_converter( n = len(get_args(python_type)) # prepare list of converters converters = [ - self.get_arrow_to_python_converter(arrow_type.field(f"f{i}").type) + self._nested_value_converter(arrow_type.field(f"f{i}").type) for i in range(n) ] # this is a heterogeneous tuple @@ -1868,9 +1908,7 @@ def _create_arrow_to_python_converter( # Create converters for each field field_converters = {} for field in arrow_type: - field_converters[field.name] = self.get_arrow_to_python_converter( - field.type - ) + field_converters[field.name] = self._nested_value_converter(field.type) return lambda value: ( { @@ -1892,6 +1930,29 @@ def _create_arrow_to_python_converter( # Default passthrough return lambda value: value + def _nested_value_converter(self, arrow_type: pa.DataType) -> Callable[[Any], Any]: + """The converter of a nested position (list element, dict value, struct field). + + A null at a position whose None a read cannot restore raises instead of reading + as an empty value or failing inside the element's converter (ITL-1144). + + Args: + arrow_type: The Arrow type of the nested position. + + Returns: + A function converting one stored value of ``arrow_type`` to Python. + """ + convert = self.get_arrow_to_python_converter(arrow_type) + if none_is_restorable(arrow_type, self.get_logical_type_by_arrow_extension_name): + return convert + + def convert_or_refuse(value: Any) -> Any: + if value is None: + raise unrestorable_none_error(arrow_type) + return convert(value) + + return convert_or_refuse + def is_dynamic_typeddict(self, python_type: type) -> bool: """Check if a type is one of our dynamically created TypedDicts.""" return python_type in self._typeddict_to_struct_signature diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index 9b3b94152..b067cd44f 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -170,10 +170,6 @@ def test_an_arrow_table_with_a_null_inner_list(self) -> None: with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): _read_v(source) - def test_a_none_given_for_a_non_optional_inner_list(self) -> None: - with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): - _read_v(_source([None, [1]], list[list[int]])) - def _members(annotation: Any, first: Any, second: Any) -> DictSource: """Two rows of group ``g=1``: column ``m`` holds ``first`` then ``second``.""" From c5187a2d1955a4638992ac87b0e88cff780d4f4b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:51:21 -0800 Subject: [PATCH 06/12] fix(semantic_types): docs, changelog and design issue for the nested None guard (ITL-1144) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1144.md | 4 ++ design_issues/ITL-1144-NN1.md | 30 ++++++++++++ docs/concepts/function-pods.md | 46 +++++++++++++++++++ .../logical_types/user_class_logical_type.py | 5 +- src/orcapod/semantic_types/nested_none.py | 22 ++++----- .../semantic_types/universal_converter.py | 32 ++++++++----- .../operators/test_async_sync_equivalence.py | 5 +- .../test_nested_none_guard.py | 9 +++- 8 files changed, 126 insertions(+), 27 deletions(-) create mode 100644 changelog.d/ITL-1144.md create mode 100644 design_issues/ITL-1144-NN1.md diff --git a/changelog.d/ITL-1144.md b/changelog.d/ITL-1144.md new file mode 100644 index 000000000..6187ee2d7 --- /dev/null +++ b/changelog.d/ITL-1144.md @@ -0,0 +1,4 @@ +### Breaking changes and migration + +- A `None` inside a container is refused where a read could not give it back (ITL-1144). A list or set element, a dict value or a class field declared `X | None`, where `X` is a container (`list`, `set`, `dict`) or a logical type other than your own class (`Path`, `UPath`, `UUID`, `numpy.ndarray`, `torch.Tensor`, pandas and Polars values), now raises `ValueError` when the type is declared, as a function's parameter or return value, a source's `data_schema`, or a field of a dataclass or pydantic model. Examples: `list[set[int] | None]`, `list[list[int] | None]`, `dict[str, list[int] | None]`, `list[Path | None]`, `set[UUID | None]`, a field `x: UUID | None` or `x: list[int] | None`. The error names the column (or `Class.field`), the type and ITL-1144. Such a `None` was stored correctly, but read back as an empty container (`[]`, `set()`, `{}`), with a different row hash, or the read failed with a `TypeError`: through `as_source()`, a `DictSource`, a loaded job and a downstream function. An in-memory pod-to-pod chain that never stored or re-read the values did keep the `None`; it is refused too. Declare the inner type without `| None`, or hold the value in a dataclass and make that optional (`list[MyData | None]`). Still accepted, unchanged: a `None` inside a container where the inner type is a scalar (`list[int | None]`, `dict[str, float | None]`, a field `x: int | None`) or your own class (`list[MyData | None]`), and a `None` at the top level of any column (`list[Path] | None`). No hash, stored table or golden value changes. +- Reading a stored column that holds such a `None` raises `ValueError` naming the column, instead of returning a wrong value or a `TypeError` (ITL-1144). This covers an Arrow table given to `ArrowTableSource` with a null inner list or struct, and `GroupBy` or `Batch` over a nullable list, set, `Path` or `UUID` column, whose output rows read as Python values now raise. `GroupBy` and `Batch` over a nullable scalar column (`list[int | None]`) are unchanged. Drop the nulls first (`DropNulls` on that column) or keep the value in a dataclass. diff --git a/design_issues/ITL-1144-NN1.md b/design_issues/ITL-1144-NN1.md new file mode 100644 index 000000000..7607bbe6c --- /dev/null +++ b/design_issues/ITL-1144-NN1.md @@ -0,0 +1,30 @@ +## `src/orcapod/semantic_types/universal_converter.py` + +### NN1 — A None inside a container is stored but not read back for a container or a logical type +**Status:** in progress +**Severity:** critical +**Issue:** ITL-1144 + +A `None` inside a container column (a list or set element, a dict value, a user-class field) +is stored faithfully as an Arrow null, and the in-memory row hash equals the stored row hash. +Reading the stored column back to Python restored it only where the position's type is a +scalar or a user class. A container read as an empty one (`list[set[int] | None]` holding +`[None, {1}]` read `[[], [1]]`; a dataclass field `x: list[int] | None` read `[]`), with a +different row hash; any other logical type failed (`list[UUID | None]`, +`list[Path | None]`, `list[np.ndarray | None]`: `TypeError`). Every read path agreed: +`as_source()`, `DictSource`, a loaded job, a downstream function, and `GroupBy` or `Batch` +over a nullable list, set, `Path` or `UUID` column, where a downstream function received +and stored the wrong value. An in-memory pod-to-pod chain kept the `None`. The decoders +map a null to an empty value (`if value else []`) or hand it to a logical type that does not +accept it. + +**Fix (guard, v0.3.0):** `semantic_types/nested_none.py` states which nested None a read +restores. `register_python_class` and the user-class field registration refuse a declared +`X | None` at a nested position whose None cannot be restored +(`NestedNoneNotRestorableError`, naming the column or `Class.field`), and the nested +decoders raise on a stored null there instead of returning a wrong value (Arrow-sourced +columns, whose item nullability the Python type does not show, ITL-1136). Still open: read +these None values back faithfully (a read-side change; storage is already right, so no +stored value or hash moves) and lift the guard. Land ITL-1136's fix only after it: reading +item nullability into the Python type would make every default-nullable nested list read +from Arrow a refused type. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index ae34d30f2..c2460596b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -1116,6 +1116,52 @@ for tag, data in count_rows.pod(readings).iter_data(): # s2 3 ``` +### A `None` inside a container + +A `None` inside a list, a set, a dict value or a class field is accepted where its type is +a scalar (`int`, `float`, `str`, `bool`, `bytes`, `datetime`, ...) or a class of your own +(a dataclass, an `orcapod.dataclass` or a pydantic model): `list[int | None]`, +`dict[str, float | None]`, `list[MyData | None]` and a field `x: int | None` store and read +back exactly. + +Where the type inside is a container (`list`, `set`, `dict`) or one of the logical types +above (`Path`, `UUID`, `numpy.ndarray`, ...), a stored `None` is not yet read back +faithfully: it would read as an empty container or fail. Such a declaration is refused when +the type is declared: `list[set[int] | None]`, `list[Path | None]`, `dict[str, list[int] | +None]` or a dataclass field `x: UUID | None`, as a function's parameter or return value, a +source's schema or a class field. The error names the column (or the class field) and the +type. Declare the inner type without `| None`, or hold the value in a dataclass and make +that optional (`list[MyData | None]`). A `None` at the top level of a column +(`list[Path] | None`, `UUID | None`) is accepted as before. Lifting this restriction is +ITL-1144. + +```python +try: + DictSource( + data=[{"sample": "s1", "labels": [{1, 2}, None]}], + tag_columns=["sample"], + data_schema={"sample": str, "labels": list[set[int] | None]}, + ) +except ValueError as error: + print(str(error).split(": ")[0]) +# Column 'labels' (list[set[int] | None]) + +labels = DictSource( + data=[{"sample": "s1", "counts": [3, None, 1]}], + tag_columns=["sample"], + data_schema={"sample": str, "counts": list[int | None]}, +) +for tag, data in labels.iter_data(): + print(data["counts"]) +# [3, None, 1] +``` + +A column that already holds such a `None` (an Arrow table given to `ArrowTableSource`, or +`GroupBy` or `Batch` over a column of a nullable list, set, `Path` or `UUID`) raises a +`ValueError` naming the column when its values are read as Python values, instead of +returning a wrong value. Drop the nulls +first (`DropNulls` on that column) or keep the value in a dataclass. + ## Multiple input streams If you pass multiple streams to a function pod, they are automatically joined (using diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 7cea6b0d5..d6337969a 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -326,7 +326,10 @@ def _storage( require_restorable_none( annotation, arrow_type, - converter.get_logical_type_by_arrow_extension_name, + lambda name: isinstance( + converter.get_logical_type_by_arrow_extension_name(name), + UserClassLogicalType, + ), ) except NestedNoneNotRestorableError as exc: raise with_context( diff --git a/src/orcapod/semantic_types/nested_none.py b/src/orcapod/semantic_types/nested_none.py index 5637b7a08..92d202448 100644 --- a/src/orcapod/semantic_types/nested_none.py +++ b/src/orcapod/semantic_types/nested_none.py @@ -18,10 +18,10 @@ if TYPE_CHECKING: import pyarrow as pa - from orcapod.logical_types.protocols import LogicalTypeProtocol - -LogicalTypeLookup = Callable[[str], "LogicalTypeProtocol | None"] -"""Find the registered logical type of an Arrow extension name.""" +IsUserClass = Callable[[str], bool] +"""Whether an Arrow extension name is the logical type of a user class (a dataclass, +an ``orcapod.dataclass`` or a pydantic model). The caller supplies it, so this module +does not import the user-class logical types, which import it.""" _FOLLOW_UP = "ITL-1144" @@ -30,7 +30,7 @@ class NestedNoneNotRestorableError(ValueError): """A None at a nested position cannot be read back faithfully (ITL-1144).""" -def none_is_restorable(arrow_type: pa.DataType, lookup: LogicalTypeLookup) -> bool: +def none_is_restorable(arrow_type: pa.DataType, is_user_class: IsUserClass) -> bool: """Whether a read restores a null at a nested position of ``arrow_type`` as None. A user class (dataclass, ``orcapod.dataclass``, pydantic model) and every scalar @@ -39,17 +39,15 @@ def none_is_restorable(arrow_type: pa.DataType, lookup: LogicalTypeLookup) -> bo Args: arrow_type: The Arrow type of the nested position. - lookup: Finds the logical type registered for an extension name. + is_user_class: Whether an extension name is a user class's logical type. Returns: True when a null there is read back as None. """ import pyarrow as pa - from orcapod.logical_types.user_class_logical_type import UserClassLogicalType - if isinstance(arrow_type, pa.ExtensionType): - return isinstance(lookup(arrow_type.extension_name), UserClassLogicalType) + return is_user_class(arrow_type.extension_name) return not ( pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type) @@ -70,7 +68,7 @@ def _is_optional(annotation: Any) -> bool: def require_restorable_none( - annotation: Any, arrow_type: pa.DataType, lookup: LogicalTypeLookup + annotation: Any, arrow_type: pa.DataType, is_user_class: IsUserClass ) -> None: """Refuse a nested position declared ``X | None`` whose None a read cannot restore. @@ -78,13 +76,13 @@ def require_restorable_none( annotation: The declared type of the nested position (a list or set element, a dict value, a user-class field). arrow_type: The Arrow type ``annotation`` registers as. - lookup: Finds the logical type registered for an extension name. + is_user_class: Whether an extension name is a user class's logical type. Raises: NestedNoneNotRestorableError: If ``annotation`` admits None and a stored None there would not be read back as None. """ - if _is_optional(annotation) and not none_is_restorable(arrow_type, lookup): + if _is_optional(annotation) and not none_is_restorable(arrow_type, is_user_class): raise NestedNoneNotRestorableError( f"{annotation} is not supported inside a container or as a class field yet: " f"a None there is stored but not read back faithfully (it would read as an " diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index c5847d1b7..a6ac32259 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -381,9 +381,7 @@ def _register_python_class_impl( "element type (e.g. list[int], list[str])." ) inner = self.register_python_class(args[0]) - require_restorable_none( - args[0], inner, self.get_logical_type_by_arrow_extension_name - ) + require_restorable_none(args[0], inner, self._is_user_class_extension) if self._logical_type_registry is not None and hasattr( inner, "extension_name" ): @@ -402,9 +400,7 @@ def _register_python_class_impl( "element type (e.g. set[int], set[str])." ) inner = self.register_python_class(args[0]) - require_restorable_none( - args[0], inner, self.get_logical_type_by_arrow_extension_name - ) + require_restorable_none(args[0], inner, self._is_user_class_extension) if self._logical_type_registry is not None and hasattr( inner, "extension_name" ): @@ -426,9 +422,7 @@ def _register_python_class_impl( ) key_arrow = self.register_python_class(args[0]) val_arrow = self.register_python_class(args[1]) - require_restorable_none( - args[1], val_arrow, self.get_logical_type_by_arrow_extension_name - ) + require_restorable_none(args[1], val_arrow, self._is_user_class_extension) if isinstance(key_arrow, pa.ExtensionType): raise ValueError( f"'dict[{args[0]}, ...]' is not yet supported: the key type maps to Arrow " @@ -857,7 +851,7 @@ def storage_to_python(self, storage_value: Any, annotation: Any) -> Any: """ arrow_type = self.python_type_to_arrow_type(annotation) if storage_value is None and not none_is_restorable( - arrow_type, self.get_logical_type_by_arrow_extension_name + arrow_type, self._is_user_class_extension ): raise unrestorable_none_error(arrow_type) converter_fn = self.get_arrow_to_python_converter(arrow_type) @@ -1930,6 +1924,22 @@ def _create_arrow_to_python_converter( # Default passthrough return lambda value: value + def _is_user_class_extension(self, extension_name: str) -> bool: + """Whether *extension_name* is the logical type of a user class (ITL-1144). + + Args: + extension_name: An Arrow extension name. + + Returns: + True for a registered dataclass, ``orcapod.dataclass`` or pydantic type. + """ + from orcapod.logical_types.user_class_logical_type import UserClassLogicalType + + return isinstance( + self.get_logical_type_by_arrow_extension_name(extension_name), + UserClassLogicalType, + ) + def _nested_value_converter(self, arrow_type: pa.DataType) -> Callable[[Any], Any]: """The converter of a nested position (list element, dict value, struct field). @@ -1943,7 +1953,7 @@ def _nested_value_converter(self, arrow_type: pa.DataType) -> Callable[[Any], An A function converting one stored value of ``arrow_type`` to Python. """ convert = self.get_arrow_to_python_converter(arrow_type) - if none_is_restorable(arrow_type, self.get_logical_type_by_arrow_extension_name): + if none_is_restorable(arrow_type, self._is_user_class_extension): return convert def convert_or_refuse(value: Any) -> Any: diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 33fdf63e4..d94b76545 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -241,7 +241,10 @@ def async_rows(op: StaticOutputOperatorPod, *sources: StreamProtocol) -> list[Ro def _row_key(tag: TagProtocol) -> str: - return repr(sorted(tag.as_dict().items())) + # The tag's Arrow values, not ``as_dict()``: a null nested in a container tag + # (``[[1], None]``) cannot be read back to Python yet and raises (ITL-1144). + [row] = tag.as_table().to_pylist() + return repr(sorted(row.items())) def _row_schema(tag: TagProtocol, data: DataProtocol) -> pa.Schema: diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index b067cd44f..8e3f60bbe 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -65,7 +65,7 @@ class OptionalScalars: set[uuid.UUID | None], list[Path | None], list[list[uuid.UUID] | None], - list[np.ndarray | None], + list[np.ndarray | None], # pyright: ignore[reportMissingTypeArgument] # the bare class is how a pod declares it ] REFUSED_CLASSES = [OptionalList, OptionalUUID, OptionalPath] @@ -199,7 +199,12 @@ def test_a_nullable_int_column_round_trips(self, operator: str) -> None: @pytest.mark.parametrize("operator", OPERATORS) @pytest.mark.parametrize( ("annotation", "second"), - [(list[int] | None, [1]), (set[int] | None, {1}), (Path | None, P), (uuid.UUID | None, U)], + [ + (list[int] | None, [1]), + (set[int] | None, {1}), + (Path | None, P), + (uuid.UUID | None, U), + ], ids=["list", "set", "Path", "UUID"], ) def test_a_nullable_container_or_logical_column_raises_at_read( From 52e0634372faab15bd91f1194afb2343dc50e1d9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 17:05:02 -0800 Subject: [PATCH 07/12] docs(plans): ITL-1144 plan status Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-15-itl-1144-none-in-containers-guard.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md index 8386c851e..635146bb8 100644 --- a/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md +++ b/superpowers/plans/2026-11-15-itl-1144-none-in-containers-guard.md @@ -1,7 +1,7 @@ # ITL-1144: guard against a None inside a container column that a read cannot restore -Status: measurement done (main fe433eab, PR #393 head 8b75ccea); proposal sent to the -coordinator; implementation waits for the reply. +Status: implemented (guard: (a) + (c)); measurement on main fe433eab and PR #393 head 8b75ccea; +approved by the coordinator November 15, 2026. Draft PR open; full fix still open (ITL-1144). ## Measurement From fe5ba58d1c95a7dec74e691b79eee608f30795b9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 17:50:53 -0800 Subject: [PATCH 08/12] fix(semantic_types): refuse a None a container type does not admit; keep map nulls readable (ITL-1144) CodeRabbit round 1 on PR #394: the list, set and dict write converters stored a None given where the declared type does not admit one as an empty container. They now raise UnsupportedValueError, which the caller prefixes with the column name. A null Arrow map element is read back as None again: its decoder passes a null through. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1144.md | 1 + design_issues/ITL-1144-NN1.md | 5 +- docs/concepts/function-pods.md | 5 ++ src/orcapod/semantic_types/nested_none.py | 8 +- .../semantic_types/universal_converter.py | 45 ++++++++--- .../test_nested_none_guard.py | 79 ++++++++++++++++++- 6 files changed, 126 insertions(+), 17 deletions(-) diff --git a/changelog.d/ITL-1144.md b/changelog.d/ITL-1144.md index 6187ee2d7..f5fbe2926 100644 --- a/changelog.d/ITL-1144.md +++ b/changelog.d/ITL-1144.md @@ -2,3 +2,4 @@ - A `None` inside a container is refused where a read could not give it back (ITL-1144). A list or set element, a dict value or a class field declared `X | None`, where `X` is a container (`list`, `set`, `dict`) or a logical type other than your own class (`Path`, `UPath`, `UUID`, `numpy.ndarray`, `torch.Tensor`, pandas and Polars values), now raises `ValueError` when the type is declared, as a function's parameter or return value, a source's `data_schema`, or a field of a dataclass or pydantic model. Examples: `list[set[int] | None]`, `list[list[int] | None]`, `dict[str, list[int] | None]`, `list[Path | None]`, `set[UUID | None]`, a field `x: UUID | None` or `x: list[int] | None`. The error names the column (or `Class.field`), the type and ITL-1144. Such a `None` was stored correctly, but read back as an empty container (`[]`, `set()`, `{}`), with a different row hash, or the read failed with a `TypeError`: through `as_source()`, a `DictSource`, a loaded job and a downstream function. An in-memory pod-to-pod chain that never stored or re-read the values did keep the `None`; it is refused too. Declare the inner type without `| None`, or hold the value in a dataclass and make that optional (`list[MyData | None]`). Still accepted, unchanged: a `None` inside a container where the inner type is a scalar (`list[int | None]`, `dict[str, float | None]`, a field `x: int | None`) or your own class (`list[MyData | None]`), and a `None` at the top level of any column (`list[Path] | None`). No hash, stored table or golden value changes. - Reading a stored column that holds such a `None` raises `ValueError` naming the column, instead of returning a wrong value or a `TypeError` (ITL-1144). This covers an Arrow table given to `ArrowTableSource` with a null inner list or struct, and `GroupBy` or `Batch` over a nullable list, set, `Path` or `UUID` column, whose output rows read as Python values now raise. `GroupBy` and `Batch` over a nullable scalar column (`list[int | None]`) are unchanged. Drop the nulls first (`DropNulls` on that column) or keep the value in a dataclass. +- A source given a `None` where its declared type does not allow one now refuses instead of storing an empty container (ITL-1144). A `None` for a list, set or dict whose declared type is not `| None`, inside a column's value (`[None, [1]]` for `list[list[int]]`, `{"k": None}` for `dict[str, list[int]]`, a dataclass field `xs: list[int]` given `None`) or as the column's own value, was stored as `[]`, `set()` or `{}`. `DictSource`, `ListSource` and any other source built from Python values now raise `UnsupportedValueError` (a `ValueError`) naming the column and the declared type. A function output already refused such a value (`OutputTypeError`). A `None` at a position declared `| None` is unchanged, and no hash, stored table or golden value of a valid value changes. diff --git a/design_issues/ITL-1144-NN1.md b/design_issues/ITL-1144-NN1.md index 7607bbe6c..bcd5aa340 100644 --- a/design_issues/ITL-1144-NN1.md +++ b/design_issues/ITL-1144-NN1.md @@ -23,7 +23,10 @@ restores. `register_python_class` and the user-class field registration refuse a `X | None` at a nested position whose None cannot be restored (`NestedNoneNotRestorableError`, naming the column or `Class.field`), and the nested decoders raise on a stored null there instead of returning a wrong value (Arrow-sourced -columns, whose item nullability the Python type does not show, ITL-1136). Still open: read +columns, whose item nullability the Python type does not show, ITL-1136). A None given for a list, +set or dict whose declared type does not admit one, which the write converters stored as an +empty container, is refused at write (`UnsupportedValueError`, naming the column); an Arrow +map, whose decoder passes a null through, stays readable. Still open: read these None values back faithfully (a read-side change; storage is already right, so no stored value or hash moves) and lift the guard. Land ITL-1136's fix only after it: reading item nullability into the Python type would make every default-nullable nested list read diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index c2460596b..5b7c09c31 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -1162,6 +1162,11 @@ A column that already holds such a `None` (an Arrow table given to `ArrowTableSo returning a wrong value. Drop the nulls first (`DropNulls` on that column) or keep the value in a dataclass. +A `None` given where the declared type does not allow one (`[None, [1]]` for +`list[list[int]]`) is refused when a source is built from it, naming the column: it used to +be stored as an empty list. A function returning one is refused as described in +[Return values must match the declared type](#return-values-must-match-the-declared-type). + ## Multiple input streams If you pass multiple streams to a function pod, they are automatically joined (using diff --git a/src/orcapod/semantic_types/nested_none.py b/src/orcapod/semantic_types/nested_none.py index 92d202448..8e337bfcb 100644 --- a/src/orcapod/semantic_types/nested_none.py +++ b/src/orcapod/semantic_types/nested_none.py @@ -33,9 +33,10 @@ class NestedNoneNotRestorableError(ValueError): def none_is_restorable(arrow_type: pa.DataType, is_user_class: IsUserClass) -> bool: """Whether a read restores a null at a nested position of ``arrow_type`` as None. - A user class (dataclass, ``orcapod.dataclass``, pydantic model) and every scalar - type are restored; a list, struct or map reads as an empty value, and any other - logical type fails. + A user class (dataclass, ``orcapod.dataclass``, pydantic model), every scalar type + and an Arrow map (its decoder passes a null through; a Python ``dict`` is stored as + a list of key/value structs, never as a map) are restored; a list or struct reads + as an empty value, and any other logical type fails. Args: arrow_type: The Arrow type of the nested position. @@ -55,7 +56,6 @@ def none_is_restorable(arrow_type: pa.DataType, is_user_class: IsUserClass) -> b or pa.types.is_list_view(arrow_type) or pa.types.is_large_list_view(arrow_type) or pa.types.is_struct(arrow_type) - or pa.types.is_map(arrow_type) ) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index a6ac32259..ad87b9c99 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -1735,7 +1735,7 @@ def _convert_datetime(dt: datetime) -> datetime: def convert_list(value: Any) -> list[Any]: """Convert list storage, preserving set order independence on updates.""" if value is None: - return [] + raise _none_not_admitted(python_type) elements = [element_converter(item) for item in value] # An Arrow schema remembers a plain set only as a list. Python set # inputs still need canonical storage order; actual lists keep theirs. @@ -1748,28 +1748,32 @@ def convert_list(value: Any) -> list[Any]: elif origin is dict: key_converter = self.get_python_to_arrow_converter(args[0]) value_converter = self.get_python_to_arrow_converter(args[1]) + # A dict is order independent: entries are stored in canonical key order # so equal dicts store and hash identically (ITL-905). - return lambda value: ( - sort_dict_entries( + def convert_dict(value: Any) -> list[dict[str, Any]]: + if value is None: + raise _none_not_admitted(python_type) + return sort_dict_entries( [ {"key": key_converter(k), "value": value_converter(v)} for k, v in value.items() ] ) - if value is not None - else [] - ) + + return convert_dict elif origin is set: # A set has no order: elements are stored sorted, never in iteration # order, which depends on PYTHONHASHSEED for str and bytes (ITL-905). element_converter = self.get_python_to_arrow_converter(args[0]) - return lambda value: ( - sort_set_elements([element_converter(item) for item in value]) - if value is not None - else [] - ) + + def convert_set(value: Any) -> list[Any]: + if value is None: + raise _none_not_admitted(python_type) + return sort_set_elements([element_converter(item) for item in value]) + + return convert_set elif origin is tuple: if len(set(args)) == 1: @@ -1995,6 +1999,25 @@ def get_cache_stats(self) -> dict[str, int]: # Public API functions +def _none_not_admitted(python_type: DataType) -> UnsupportedValueError: + """The error for a None given where a container type without None is declared. + + The container converters stored such a None as an empty container, a silent value + change (ITL-1144). The caller adds the column name. + + Args: + python_type: The declared container type the None was given for. + + Returns: + The error to raise. + """ + return UnsupportedValueError( + f"a None is given where {python_type} is declared, which does not admit None " + f"(it would be stored as an empty container). Give an empty container, or " + f"declare the column's own type '| None' if a missing value is meant." + ) + + def python_type_to_arrow_type( python_type: type, data_context: DataContext | str | None = None ) -> pa.DataType: diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index 8e3f60bbe..a1bea3441 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -18,7 +18,9 @@ import orcapod as op from orcapod.operators import Batch, GroupBy -from orcapod.sources import ArrowTableSource, DictSource +from orcapod.core.streams import ArrowTableStream +from orcapod.errors import OutputTypeError +from orcapod.sources import ArrowTableSource, DataFrameSource, DictSource, ListSource U = uuid.UUID(int=1) P = Path("/tmp/itl-1144") @@ -220,3 +222,78 @@ def consume(m: list[list[int] | None]) -> int: with pytest.raises(ValueError, match=r"(?s)Column 'm'.*ITL-1144"): op.function_pod(output_keys="n")(consume) + + +@dataclasses.dataclass +class ListField: + xs: list[int] + + +class TestNoneWhereTheTypeDoesNotAllowOne: + """A None given at a non-optional container position is refused at write. + + It was stored as an empty container (``[]``, ``set()``, ``{}``): a silent value + change (ITL-1144, CodeRabbit on PR #394). + """ + + @pytest.mark.parametrize( + ("annotation", "value"), + [ + (list[list[int]], [None, [1]]), + (list[set[int]], [None, {1}]), + (list[dict[str, int]], [None]), + (dict[str, list[int]], {"k": None}), + ], + ids=str, + ) + def test_a_dict_source_refuses(self, annotation: Any, value: Any) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*does not admit None"): + _source(value, annotation) + + def test_a_class_field_refuses(self) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*does not admit None"): + _source(ListField(None), ListField) # pyright: ignore[reportArgumentType] # the wrong value is the case under test + + def test_a_list_source_refuses(self) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*does not admit None"): + ListSource( + name="v", + data=[[None, [1]]], + data_schema={"element_index": int, "v": list[list[int]]}, + ) + + def test_a_data_frame_source_raises_at_read(self) -> None: + # Polars keeps the null inner list; reading it to Python raises. + source = DataFrameSource({"id": [1], "v": [[None, [1]]]}, tag_columns=["id"]) + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + _read_v(source) + + def test_a_function_output_refuses(self) -> None: + def produce(n: int) -> list[list[int]]: + return [None, [1]] # pyright: ignore[reportReturnType] # the wrong value is the case under test + + pod = op.function_pod(output_keys="v")(produce).pod + stream = pod(DictSource([{"id": 1, "n": 1}], tag_columns=["id"], source_id="n")) + with pytest.raises(OutputTypeError): + list(stream.iter_data()) + + def test_a_none_at_an_optional_position_is_unchanged(self) -> None: + read, _ = _read_v(_source([None, 1], list[int | None])) + assert read == [None, 1] + + +def test_a_null_map_element_still_reads_as_none() -> None: + """A null Arrow map element is read back as None, as before (CodeRabbit, PR #394). + + A source refuses a map column (the Arrow hasher cannot hash it); a raw + ``ArrowTableStream`` holds one as given, and its map decoder passes a null through. + """ + map_type = pa.map_(pa.large_string(), pa.int64()) + table = pa.table( + { + "id": pa.array([1], pa.int64()), + "v": pa.array([[None, [("a", 1)]]], pa.large_list(map_type)), + } + ) + [(_, data)] = list(ArrowTableStream(table, tag_columns=["id"]).iter_data()) + assert data.as_dict()["v"][0] is None From 1edd5d3f95f41ee426db11babcd0157fa0d3a4f8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 17:53:07 -0800 Subject: [PATCH 09/12] test(semantic_types): type the map read-back value (ITL-1144) Co-Authored-By: Claude Opus 5.5 --- tests/test_semantic_types/test_nested_none_guard.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index a1bea3441..462c8da6c 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -296,4 +296,5 @@ def test_a_null_map_element_still_reads_as_none() -> None: } ) [(_, data)] = list(ArrowTableStream(table, tag_columns=["id"]).iter_data()) - assert data.as_dict()["v"][0] is None + value: Any = data.as_dict()["v"] + assert value[0] is None From 64f882eb271dc7ef8011e32fc89d9cd643a09562 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:34:42 -0800 Subject: [PATCH 10/12] fix(semantic_types): refuse a None for a non-optional logical type; raise on a null dict entry (ITL-1144) CodeRabbit round 2 on PR #394. A None given where a logical type is declared without '| None' reached the type's encoder: a Path was stored as the string 'None', a list[UUID] as []. It now raises UnsupportedValueError, which the caller prefixes with the column name. A null key/value entry of a stored dict raises at read instead of being dropped. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1144.md | 4 +- .../semantic_types/universal_converter.py | 48 ++++++++++----- .../test_nested_none_guard.py | 59 +++++++++++++++++++ 3 files changed, 93 insertions(+), 18 deletions(-) diff --git a/changelog.d/ITL-1144.md b/changelog.d/ITL-1144.md index f5fbe2926..cd79950a6 100644 --- a/changelog.d/ITL-1144.md +++ b/changelog.d/ITL-1144.md @@ -1,5 +1,5 @@ ### Breaking changes and migration - A `None` inside a container is refused where a read could not give it back (ITL-1144). A list or set element, a dict value or a class field declared `X | None`, where `X` is a container (`list`, `set`, `dict`) or a logical type other than your own class (`Path`, `UPath`, `UUID`, `numpy.ndarray`, `torch.Tensor`, pandas and Polars values), now raises `ValueError` when the type is declared, as a function's parameter or return value, a source's `data_schema`, or a field of a dataclass or pydantic model. Examples: `list[set[int] | None]`, `list[list[int] | None]`, `dict[str, list[int] | None]`, `list[Path | None]`, `set[UUID | None]`, a field `x: UUID | None` or `x: list[int] | None`. The error names the column (or `Class.field`), the type and ITL-1144. Such a `None` was stored correctly, but read back as an empty container (`[]`, `set()`, `{}`), with a different row hash, or the read failed with a `TypeError`: through `as_source()`, a `DictSource`, a loaded job and a downstream function. An in-memory pod-to-pod chain that never stored or re-read the values did keep the `None`; it is refused too. Declare the inner type without `| None`, or hold the value in a dataclass and make that optional (`list[MyData | None]`). Still accepted, unchanged: a `None` inside a container where the inner type is a scalar (`list[int | None]`, `dict[str, float | None]`, a field `x: int | None`) or your own class (`list[MyData | None]`), and a `None` at the top level of any column (`list[Path] | None`). No hash, stored table or golden value changes. -- Reading a stored column that holds such a `None` raises `ValueError` naming the column, instead of returning a wrong value or a `TypeError` (ITL-1144). This covers an Arrow table given to `ArrowTableSource` with a null inner list or struct, and `GroupBy` or `Batch` over a nullable list, set, `Path` or `UUID` column, whose output rows read as Python values now raise. `GroupBy` and `Batch` over a nullable scalar column (`list[int | None]`) are unchanged. Drop the nulls first (`DropNulls` on that column) or keep the value in a dataclass. -- A source given a `None` where its declared type does not allow one now refuses instead of storing an empty container (ITL-1144). A `None` for a list, set or dict whose declared type is not `| None`, inside a column's value (`[None, [1]]` for `list[list[int]]`, `{"k": None}` for `dict[str, list[int]]`, a dataclass field `xs: list[int]` given `None`) or as the column's own value, was stored as `[]`, `set()` or `{}`. `DictSource`, `ListSource` and any other source built from Python values now raise `UnsupportedValueError` (a `ValueError`) naming the column and the declared type. A function output already refused such a value (`OutputTypeError`). A `None` at a position declared `| None` is unchanged, and no hash, stored table or golden value of a valid value changes. +- Reading a stored column that holds such a `None` raises `ValueError` naming the column, instead of returning a wrong value or a `TypeError` (ITL-1144). A null key/value entry of a stored dict raises too, instead of being dropped. This covers an Arrow table given to `ArrowTableSource` with a null inner list or struct, and `GroupBy` or `Batch` over a nullable list, set, `Path` or `UUID` column, whose output rows read as Python values now raise. `GroupBy` and `Batch` over a nullable scalar column (`list[int | None]`) are unchanged. Drop the nulls first (`DropNulls` on that column) or keep the value in a dataclass. +- A source given a `None` where its declared type does not allow one now refuses instead of storing an empty container or a wrong value (ITL-1144). A `None` for a list, set or dict whose declared type is not `| None`, inside a column's value (`[None, [1]]` for `list[list[int]]`, `{"k": None}` for `dict[str, list[int]]`, a dataclass field `xs: list[int]` given `None`) or as the column's own value, was stored as `[]`, `set()` or `{}`. A `None` for a logical type declared without `| None` reached the type's encoder: a `Path` was stored as the string `'None'` and read back as `Path('None')`, a `list[UUID]` as `[]`, and a `UUID` failed with an `AttributeError`; a user class raised `TypeError` and now raises the same `UnsupportedValueError`. `DictSource`, `ListSource` and any other source built from Python values now raise `UnsupportedValueError` (a `ValueError`) naming the column and the declared type. A function output already refused such a value (`OutputTypeError`). A `None` at a position declared `| None` is unchanged, and no hash, stored table or golden value of a valid value changes. diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index ad87b9c99..e6c2e49bb 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -1683,7 +1683,16 @@ def _create_python_to_arrow_converter( if lt is not None: _lt = lt _self = self - return lambda value: _lt.python_to_storage(value, _self) + + def convert_logical(value: Any) -> Any: + # A None at a position whose type does not admit one never reaches + # the logical type's encoder, which would store a wrong value (a + # Path stored the string 'None', a list[UUID] an empty list). + if value is None: + raise _none_not_admitted(python_type) + return _lt.python_to_storage(value, _self) + + return convert_logical # Get the Arrow type for this Python type # TODO: check if this step is necessary @@ -1864,15 +1873,20 @@ def _create_arrow_to_python_converter( key_converter = self.get_arrow_to_python_converter(key_field.type) value_converter = self._nested_value_converter(value_field.type) - return lambda value: ( - { - key_converter(item["key"]): value_converter(item["value"]) - for item in value - if item is not None - } - if value - else {} - ) + def convert_entries(value: Any) -> dict[Any, Any]: + if not value: + return {} + converted = {} + for item in value: + # A null entry is not a key/value pair: never drop it. + if item is None: + raise unrestorable_none_error(element_type) + converted[key_converter(item["key"])] = value_converter( + item["value"] + ) + return converted + + return convert_entries # Regular list element_converter = self._nested_value_converter(element_type) @@ -2000,21 +2014,23 @@ def get_cache_stats(self) -> dict[str, int]: # Public API functions def _none_not_admitted(python_type: DataType) -> UnsupportedValueError: - """The error for a None given where a container type without None is declared. + """The error for a None given where a type that does not admit None is declared. - The container converters stored such a None as an empty container, a silent value - change (ITL-1144). The caller adds the column name. + The container converters stored such a None as an empty container and a logical + type's encoder as a wrong value (a ``Path`` as the string ``'None'``): a silent + value change (ITL-1144). The caller adds the column name. Args: - python_type: The declared container type the None was given for. + python_type: The declared type the None was given for. Returns: The error to raise. """ return UnsupportedValueError( f"a None is given where {python_type} is declared, which does not admit None " - f"(it would be stored as an empty container). Give an empty container, or " - f"declare the column's own type '| None' if a missing value is meant." + f"(it would be stored as an empty container or a wrong value). Give a value of " + f"that type, or declare the column's own type '| None' if a missing value is " + f"meant." ) diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index 462c8da6c..da30822bb 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -298,3 +298,62 @@ def test_a_null_map_element_still_reads_as_none() -> None: [(_, data)] = list(ArrowTableStream(table, tag_columns=["id"]).iter_data()) value: Any = data.as_dict()["v"] assert value[0] is None + + +class TestNoneForANonOptionalLogicalType: + """A None given where a logical type is declared without ``| None`` is refused. + + The logical type's own encoder took it: a ``Path`` stored the string ``'None'`` + and read back ``Path('None')``; a ``list[UUID]`` stored ``[]`` (CodeRabbit round 2, + PR #394). + """ + + def test_a_path_given_none_is_not_stored_as_the_string_none(self) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*does not admit None"): + _source(None, Path) + + @pytest.mark.parametrize( + ("annotation", "value"), + [ + (list[Path], [None]), + (list[uuid.UUID], None), + (set[uuid.UUID], None), + (list[list[uuid.UUID]], [None]), + (uuid.UUID, None), + ], + ids=str, + ) + def test_a_dict_source_refuses(self, annotation: Any, value: Any) -> None: + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*does not admit None"): + _source(value, annotation) + + @pytest.mark.parametrize( + ("annotation", "value"), + [ + (Path | None, None), + (list[uuid.UUID] | None, None), + (list[Path] | None, None), + ], + ids=str, + ) + def test_an_optional_column_still_takes_none( + self, annotation: Any, value: Any + ) -> None: + read, _ = _read_v(_source(value, annotation)) + assert read is None + + +def test_a_null_dict_entry_raises_at_read() -> None: + """A null key/value entry of a stored dict is not dropped (CodeRabbit round 2).""" + entries = pa.large_list( + pa.struct([("key", pa.large_string()), ("value", pa.int64())]) + ) + table = pa.table( + { + "id": pa.array([1], pa.int64()), + "v": pa.array([[None, {"key": "a", "value": 1}]], entries), + } + ) + source = ArrowTableSource(table, tag_columns=["id"], source_id="v") + with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): + _read_v(source) From 8d89cb5108eb8e837acf14e823938b5b59284e78 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 20:05:58 -0800 Subject: [PATCH 11/12] docs(function_pods): DropNulls removes null rows, not a None nested in a list (ITL-1144) CodeRabbit on PR #394. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index ea3c7356c..128a16f9b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -1166,8 +1166,11 @@ for tag, data in labels.iter_data(): A column that already holds such a `None` (an Arrow table given to `ArrowTableSource`, or `GroupBy` or `Batch` over a column of a nullable list, set, `Path` or `UUID`) raises a `ValueError` naming the column when its values are read as Python values, instead of -returning a wrong value. Drop the nulls -first (`DropNulls` on that column) or keep the value in a dataclass. +returning a wrong value. Before `GroupBy` or `Batch`, `DropNulls` on that column removes +the rows whose value is `None`, so no `None` reaches the lists they build. `DropNulls` reads +only whether a column's own value is null, so it cannot remove a `None` already nested inside +a list: remove it from an Arrow table before giving the table to `ArrowTableSource`. Keeping +the value in a dataclass (`list[MyData | None]`) works on both paths. A `None` given where the declared type does not allow one (`[None, [1]]` for `list[list[int]]`) is refused when a source is built from it, naming the column: it used to From c383102d22e6f70aab136ddc2c2a44c8cfeb3735 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 20:54:56 -0800 Subject: [PATCH 12/12] fix(semantic_types): refuse a nested None in a registry-free converter too (ITL-1144) Co-Authored-By: Claude Sonnet 5.5 --- .../semantic_types/universal_converter.py | 7 ++++++ .../test_nested_none_guard.py | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index e6c2e49bb..ddc717f6f 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -1343,6 +1343,9 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: f"list type must have exactly one type argument, got: {args}" ) element_type = self.python_type_to_arrow_type(args[0]) + require_restorable_none( + args[0], element_type, self._is_user_class_extension + ) if self._logical_type_registry is not None and hasattr( element_type, "extension_name" ): @@ -1392,6 +1395,7 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: ) key_type = self.python_type_to_arrow_type(args[0]) value_type = self.python_type_to_arrow_type(args[1]) + require_restorable_none(args[1], value_type, self._is_user_class_extension) key_value_struct = pa.struct([("key", key_type), ("value", value_type)]) return pa.large_list(key_value_struct) @@ -1434,6 +1438,9 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: f"set type must have exactly one type argument, got: {args}" ) element_type = self.python_type_to_arrow_type(args[0]) + require_restorable_none( + args[0], element_type, self._is_user_class_extension + ) if self._logical_type_registry is not None and hasattr( element_type, "extension_name" ): diff --git a/tests/test_semantic_types/test_nested_none_guard.py b/tests/test_semantic_types/test_nested_none_guard.py index da30822bb..ec107331d 100644 --- a/tests/test_semantic_types/test_nested_none_guard.py +++ b/tests/test_semantic_types/test_nested_none_guard.py @@ -357,3 +357,26 @@ def test_a_null_dict_entry_raises_at_read() -> None: source = ArrowTableSource(table, tag_columns=["id"], source_id="v") with pytest.raises(ValueError, match=r"(?s)Column 'v'.*ITL-1144"): _read_v(source) + + +class TestConverterWithoutRegistry: + """A converter with no logical type registry refuses the same declarations.""" + + # Only element types a registry-free converter can derive; a logical type there + # is already unsupported without a registry. + @pytest.mark.parametrize( + "annotation", + [ + list[list[int] | None], + set[list[int] | None], + list[dict[str, int] | None], + dict[str, list[int] | None], + ], + ids=str, + ) + def test_the_type_is_refused(self, annotation: Any) -> None: + from orcapod.semantic_types.universal_converter import UniversalTypeConverter + + converter = UniversalTypeConverter() + with pytest.raises(ValueError, match="ITL-1144"): + converter.python_type_to_arrow_type(annotation)