From 92e0f53145d8c01077e3a11f93155b97bc5988d4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:25:40 -0800 Subject: [PATCH 01/21] docs(specs): ITL-611 design for set of primitives (proposal) Co-Authored-By: Claude Opus 5.5 --- ...-11-14-itl-611-set-of-primitives-design.md | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md diff --git a/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md b/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md new file mode 100644 index 000000000..4d59440e2 --- /dev/null +++ b/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md @@ -0,0 +1,178 @@ +# ITL-611: `set[primitive]` as a native container — design + +Status: proposal, awaiting coordinator / developer decision (2026-11-14). +Measured on `main` 810d1b03 with a probe (job on Delta and in-memory, sync and async, +`as_source()`, a fresh-process `PipelineJob.load`), then with a throwaway prototype of +the proposal (not committed; kept as a patch outside the repo). + +## (a) Measured on main + +`set[T]` returned by a function pod, read back: + +| `T` | stored Arrow type | identifier | read with schema: node / downstream pod `set[T]` | `as_source()` (same process) | loaded job, fresh process | +|---|---|---|---|---|---| +| `int` | `large_list` | none (plain) | set / **list** | **list**, schema says `set[int]` | schema `list[int]`, values list | +| `str` | `large_list` | none | set / **list** | **list** | `list[str]` | +| `float` | `large_list` | none | set / **list** | **list** | `list[float]` | +| `bool` | `large_list` | none | set / **list** | **list** | `list[bool]` | +| `bytes` | `large_list` | none | set / **list** | **list** | `list[bytes]` | +| `datetime` | `large_list` | none | set / **list** | **list** | `list[datetime]` | +| `date` | `large_list` | none | set / **list** | **list** | `list[date]` | +| `Path` | `extension` | `{"category": "set", "element_ext_name": "orcapod.path", "element_ext_metadata": "{\"version\":1}"}` | set / set | set | **schema cannot be restored** (warning); values `list[str]` | +| `UUID` | `extension` | same shape | set / set | set | **cannot restore**; values `list[bytes]`; **row hash differs** from in-process | +| user dataclass | `extension]>` | same shape | set / set | set | **cannot restore**; values `list[dict]` | +| `tuple[int, str]` | (pod output refused: `Unsupported annotation: tuple[int, str]`) | | | | | +| `list[set[int]]` | `large_list>` | none | list of sets / **list of lists** | list of lists | list of lists | +| `set[int] \| None` | `large_list` | none | set / **list** | **list** | `list[int] \| None` | +| `frozenset[int]` | refused: `Unsupported annotation: frozenset[int]` | | | | | +| `list[int]` (control) | `large_list` | none | list / list | list | list | + +Hashes on main: + +- **Row hash of `{1, 2, 3, 10}` as `set[int]` equals the row hash of `[1, 2, 3, 10]` as + `list[int]`** (`arrow_v1:0000028fcaa6…`): the set identity is in no hash. +- Type hash of `set[int]` (semantic, `GenericAliasHandler`) differs from `list[int]`, so + schema and pipeline hashes already distinguish them. +- Semantic value hash of a Python set (`hash_object`) is order-independent (canonical JSON + text order, ITL-942); `set[datetime]`/`set[date]`/`set[Path]` values have no + `hash_object` handler (unrelated to storage). +- `set[UUID]` in a fresh process: the stored view loses the extension and the row hash + changes (`…48f810…` in process, `…f5a773…` fresh). If the type is registered before + `PipelineJob.load`, the fresh read is a set and the hash matches: the gap is only that + `serialization.parse_arrow_type_string` resolves an extension name already registered + in this process and never rebuilds a native container from its identifier. + +Storage order on main: plain `set[T]` is ordered by `canonical_order.sort_set_elements` +(None first; numeric; str by code point; bytes unsigned bytewise; date/datetime by +instant; NaN, incomparable or duplicate storage values raise `CanonicalOrderError`). +`set[logical]` is ordered by `ListLogicalType.python_to_storage` (natural sort, `repr` +fallback; ITL-922 covers its NaN gap). Measured: `{3,1,2,10}` → `[1,2,3,10]`; +`{"b","a","10","2"}` → `["10","2","a","b"]`; `{b"y",b"x",b"\xff"}` → `[b"x",b"y",b"\xff"]`; +`{True, 2}` as `set[int]` → `[1, 2]`; `{1.0, nan}` → the row fails. + +## (b) Proposal + +`set[T]` where `T` has no logical type is stored as the same native container as +`set[logical]`: a `ListLogicalType` of container kind `set`, storage +`large_list`, no `version` key. Its identifier is the container kind plus the +element's identifier; for a primitive element the element's identifier is its +large-normalized Arrow type text (`str(normalize_to_large_types(element_type))`): + +| Python | extension name | extension metadata (exact bytes) | +|---|---|---| +| `set[int]` | `set[int64]` | `{"category": "set", "element_arrow_type": "int64"}` | +| `set[str]` | `set[large_string]` | `{"category": "set", "element_arrow_type": "large_string"}` | +| `set[bytes]` | `set[large_binary]` | `{"category": "set", "element_arrow_type": "large_binary"}` | +| `set[float]` | `set[double]` | `{"category": "set", "element_arrow_type": "double"}` | +| `set[datetime]` | `set[timestamp[us, tz=UTC]]` | `{"category": "set", "element_arrow_type": "timestamp[us, tz=UTC]"}` | +| `list[set[int]]` | `list[set[int64]]` | `{"category": "list", "element_ext_name": "set[int64]", "element_ext_metadata": "{\"category\": \"set\", \"element_arrow_type\": \"int64\"}"}` | + +The key `element_arrow_type` (not `element_ext_name`) marks a primitive element, so a +logical type can never be confused with an Arrow type name. JSON is written as the +existing container metadata is (`json.dumps`, insertion order). + +- `list[primitive]` is unchanged (plain `large_list`); a regression test pins it. +- `set[T | None]` gets the same identifier as `set[T]`, as `set[Path | None]` does today + (element nullability is not in a native container's identifier; see ITL-1136, which this + does not change). +- Element order: unchanged, `sort_set_elements` on the converted storage values, so the + stored bytes of every value are as before and only the type identity moves. +- Reading: `storage_to_python` converts each element with the converter and returns a + `set`. `ListLogicalTypeFactory.reconstruct_from_arrow` rebuilds the type from + `element_arrow_type` (checking it equals the storage's element type). +- Hashing: the `SemanticHashingVisitor` value-hashes a native container only when its + innermost element is a logical type; a container of a primitive element is hashed as + stored, with its identifier in the field metadata (starfix `include_metadata`). + Without this, `bytes` (which has a semantic value handler) would be value-hashed per + element in process and hashed as stored in a fresh read: measured mismatch in the + prototype. + +## (c) What moves + +Moves (measured with the prototype): + +- Stored Arrow type of every `set[primitive]` column (top level, `Optional`, element of a + `list`), sync and async, Delta and in-memory. +- Row (data) hash of every row holding such a column: all nine measured cases moved. So: + function-node input hashes of rows that take a set input (cache miss), the default + `source_id` and source content hash of an in-memory source with such a column. + +Unchanged (measured): + +- Type, schema and pipeline hashes (the Python type `set[int]` is unchanged and hashed as a + generic alias): node pipeline hashes identical on main and prototype. +- Semantic value hash of a Python set; anything about `list[primitive]` and + `set[logical]`. +- User-class identifiers whose fields are `set[...]` (fields stay plain storage inside the + struct; identity record is built from Python types). +- **Golden: all 1282 entries pass unchanged with the prototype.** No golden entry pins a + `set[primitive]` column. Related entries, none move: `semantic: value.set.ints`, + `value.frozenset.ints`, `value.set.strs`, `value.set.mixed_str_equal` (Python-value + hashes); `semantic: type.set_uuid`, `schema: single_field.set_uuid.*`, + `datagram: set_uuid.*`, `arrow: extension_no_values.*.set_of_uuid` (logical element); + `pipeline: column_selection.select_data_spelling.set` (a Python set of column names). + Proposal: add golden entries (additions only) for `set[int]`, `set[str]`, `set[bytes]` + arrow type, schema hash and one datagram row each. + +Data context: CLAUDE.md says a change to how native containers are stored or hashed is a +data context change. `std:v1` is not released (`released_in: null`), so the change lands +inside `std:v1` with no new manifest and no key change. + +## (d) A v0.3 development store holding `set[int]` as a plain list + +Measured (main writes a Delta store, prototype code runs over it): + +- Same input again: cache hit; the node serves the stored plain list. Values are `list`, + declared type `set[int]`; downstream pod gets a `list`. No error. +- One new input: the run raises `ArrowInvalid: Schema ... was different` when the stored + plain rows and the new extension rows are concatenated; the new row was already appended + (Delta's append check compares storage only), so the table now mixes both forms. +- No `LogicalTypeIdentifierError`: the stored column has no extension metadata, so the + logical type layout rule has nothing to refuse. + +Options: + +1. **Leave stores; document.** Only stores from unreleased v0.3 development builds hold + this layout (v0.3 does not read 0.2.0 tables at all). Changelog "Breaking changes and + migration": clear such nodes. Cost: none; leaves the silent `list` and the crash. +2. **Refuse explicitly.** Where a stored column is plain storage but the node declares a + native container (read of stored results, Delta/connector append check), raise naming + the column, the stored type and the expected identifier, telling the user to clear + the node. Cost: two or three code paths, more tests. +3. Read the plain list as the set: forbidden by "never wrap a stored column by name alone". + +Recommendation: 1, unless the developer wants 2. + +## (e) `as_source()` / `DerivedSource` reading a set as a list + +Fixed by the proposal in process (measured). In a fresh process / loaded job the change +would otherwise regress `set[int]` from "reads as `list[int]`" to "cannot restore" (as +`set[Path]` already fails). Proposal: in scope, make `parse_arrow_type_string` rebuild an +unregistered native container from its identifier through the registered factory +(`register_logical_type_from_arrow_metadata`). That also fixes the same pre-existing gap +for `set[Path]`, `set[UUID]`, `list[Path]`. + +## (f) `frozenset[T]` + +1. Leave refused (today: `ValueError: Unsupported annotation: frozenset[int]`). Recommended. +2. Own native container kind `frozenset` (`frozenset[int64]`), reads back a `frozenset`. +3. Store as `set`: a pod annotated `frozenset[int]` would receive a `set`; rejected by type + strictness. + +## (g) Order + +See (a): unchanged, `sort_set_elements` per element type; NaN, incomparable and duplicate +storage values raise `CanonicalOrderError` (the row fails under the error policy). A +Python set cannot hold both `1` and `True` or both `0.0` and `-0.0`; in `set[int]` a +`bool` is converted to `int` before ordering. Two equal sets therefore store the same +list, and the row hash (starfix over the stored list plus the identifier) is equal. + +## Out-of-scope findings + +- `dict[K, set[primitive]]`: works today (plain). Under the proposal the value would be an + extension inside a struct, which Arrow cannot hold; `dict[K, set[Path]]` already raises. + Proposal: keep the plain storage inside a dict value (identity lost there, as today; + PLT-1732 covers dict of logical types) rather than start raising. +- `tuple[int, str]` (heterogeneous tuple) as a pod output is refused by + `register_python_class` though `python_type_to_arrow_type` supports it. +- Loaded-job reads of native containers of logical types (above, (e)). From da7d976c32acdc0fe864d6f7d940e1a0842f4ba7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:28:41 -0800 Subject: [PATCH 02/21] test(logical_types): failing tests for a set of primitives (ITL-611) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-611-set-of-primitives.md | 119 +++++++ .../test_set_of_primitives.py | 301 ++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-611-set-of-primitives.md create mode 100644 tests/test_logical_types/test_set_of_primitives.py diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md new file mode 100644 index 000000000..7ecc960e1 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -0,0 +1,119 @@ +# ITL-611 set of primitives — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `set[T]` of a primitive element type is stored as the native set container (an +Arrow extension type), so the set identity survives every read; `list[T]` is unchanged. + +**Architecture:** `ListLogicalType` (kind `set`) gains a primitive-element form whose +element identifier is the large-normalized Arrow type text (metadata key +`element_arrow_type`). The converter returns it for `set[T]` when `T` has no logical type, +except inside a dict value (plain storage kept). The hash visitor value-hashes a native +container only when its innermost element is a logical type. The saved-schema parser +rebuilds an unregistered native container from its identifier through the registered +factory, so a loaded job in a fresh process restores it. + +**Tech Stack:** Python, PyArrow extension types, Delta Lake, pytest (`uv run`). + +Spec: `superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md`. + +**Decision state (2026-11-14):** coordinator approved the recommendations +(dev stores: document only; loaded-job restore in scope; frozenset stays refused; golden +entries added; dict values keep plain storage). D1 (identifier format) and the row-hash +movement are waiting on the developer. **No library code until the coordinator's go.** + +--- + +## File structure + +- Modify `src/orcapod/logical_types/list_logical_type_factory.py`: primitive-element form + of `ListLogicalType`; `reconstruct_from_arrow` handles `element_arrow_type`. +- Modify `src/orcapod/semantic_types/universal_converter.py`: both `set` branches + (`_register_python_class_impl`, `_convert_python_to_arrow`) return the container for a + primitive element; dict branch keeps plain storage for a primitive set value. +- Modify `src/orcapod/hashing/visitors.py`: `_hashes_extension` / `visit_extension` + value-hash only a logical innermost element. +- Modify `src/orcapod/pipeline/serialization.py` (`parse_arrow_type_string`): rebuild an + unregistered native container from its identifier. +- Test `tests/test_logical_types/test_set_of_primitives.py` (new): public-interface tests. +- Golden: `tests/test_hashing/golden/cases.py`: add `set_int`, `set_str`, `set_bytes` to + `_semantic_types()` (new type and schema entries) and a datagram case; regenerate and + check that the diff only adds lines. +- Docs: `docs/concepts/logical-type-identifiers.md` (native containers), + `docs/reference/hashing.md` §1e, CLAUDE.md + `.zed/rules` (container rule), + `changelog.d/ITL-611.md`, `design_issues/ITL-611-.md`. + +## Task 1: Failing tests (allowed now) + +**Files:** Create `tests/test_logical_types/test_set_of_primitives.py`. + +- [ ] Step 1: write tests (public interface only): + - job round trip, parametrized over `int, str, float, bool, bytes, datetime, date` × + `InMemoryArrowDatabase`, `DeltaTableDatabase` × `run` / `run_async`: node rows are sets, + a downstream pod annotated `set[T]` receives a `set`, `as_source()` rows are sets and its + schema is `set[T]`. + - stored Arrow type of `set[int]`, `set[str]`, `set[bytes]`: extension name and exact + metadata bytes from the spec table; storage `large_list`. + - `list[int]`, `list[str]`, `list[bytes]` pin: plain `large_list`, read as `list`. + - row hash of a `set[int]` row differs from the `list[int]` row with the same elements; + two equal sets built in different orders hash equal. + - `list[set[int]]` and `set[int] | None` round trip through a downstream pod. + - fresh process (subprocess): a saved job over Delta, loaded with `PipelineJob.load`, + `as_source()` reads `set[int]` / `set[str]` / `set[bytes]` / `set[Path]` as sets with + schema `set[T]`, and the row hash equals the in-process one. + - `dict[str, set[int]]` still stores and reads (plain storage inside the dict value). + - `frozenset[int]` stays refused. +- [ ] Step 2: run `uv run pytest tests/test_logical_types/test_set_of_primitives.py -q`; + expect the set tests to FAIL on main and the pins (`list`, `dict`, `frozenset`) to pass. +- [ ] Step 3: commit `test(logical_types): failing tests for set of primitives (ITL-611)`. + +## Task 2: Primitive-element `ListLogicalType` (after go) + +- [ ] `ListLogicalType` accepts a primitive element: `ListLogicalType.of_arrow_element( + element_python_type, element_arrow_type)` (set only). Name `set[]`, metadata + `json.dumps({"category": "set", "element_arrow_type": text})`, storage + `large_list(element)`, `also_reads` = own metadata. `python_to_storage`: converter's + element converter, then `sort_set_elements` (canonical, unchanged storage bytes). + `storage_to_python`: converter's Arrow→Python element converter, `set(...)`. +- [ ] `reconstruct_from_arrow`: `element_arrow_type` present → element type is the + normalized storage value type; its text must equal the metadata (else `ValueError`); + Python element type from `converter.arrow_type_to_python_type`. +- [ ] Converter: `set` branches call a `_make_or_get_primitive_set_type`; idempotent by + extension name (so `set[int | None]` reuses `set[int64]`). Dict branch: a value that is a + primitive set container uses its storage type. +- [ ] Run Task 1 tests; commit `feat(logical_types): set of primitives is a native container`. + +## Task 3: Hash visitor rule + +- [ ] `_hashes_extension` and `visit_extension`: unwrap to the innermost element; value-hash + only when that element type has a logical type registered (not merely a semantic handler). +- [ ] Run `uv run pytest tests/test_hashing -q` (golden unchanged) and the set tests; + commit `fix(hashing): a container of a primitive element is hashed as stored`. + +## Task 4: Loaded-job restore of native containers + +- [ ] `parse_arrow_type_string`: unregistered extension name → if metadata is a native + container (`category` in `list`/`set`), rebuild through + `type_converter.register_logical_type_from_arrow_metadata(name, metadata, storage)` + where storage is derived from the element (Arrow text, or the element logical type's + storage after recursively resolving it); then `require_readable`. +- [ ] Run fresh-process tests and `tests/test_pipeline -q`; commit + `fix(pipeline): a loaded job restores native containers from their identifier`. + +## Task 5: Golden additions, docs, changelog, design issue + +- [ ] Add golden cases for `set[int]`, `set[str]`, `set[bytes]`; regenerate; `git diff` + shows added lines only. +- [ ] Docs, CLAUDE.md, `.zed/rules`, `changelog.d/ITL-611.md` (Changed + Breaking: + dev stores holding a plain `set[...]` column must be cleared), `design_issues/ITL-611-SP1.md`. +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, basedpyright (exit code), + full `uv run pytest tests/`. + +## Task 6: PR + +- [ ] Merge `origin/main`, rerun suite if it moved; push; draft PR (`Fixes ITL-611`); + `orca linear attach`; `orca worktree set --worktree current --pr `; CodeRabbit rounds. + +## Progress notes + +- 2026-11-14: design sent; prototype patch measured (kept outside the repo); Task 1 next. diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py new file mode 100644 index 000000000..bfcbf78b1 --- /dev/null +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -0,0 +1,301 @@ +"""A ``set[T]`` of a primitive element type keeps its set identity (ITL-611). + +``set[T]`` is stored as the native set container, an Arrow extension type named +``set[]``, whatever ``T`` is; ``list[T]`` of a primitive stays a plain +``large_list``. So a set reads back as a set everywhere: from the node, in a downstream +pod, from ``as_source()`` and from a job loaded in a fresh process; and a set row never +hashes like the list row with the same elements. +""" + +from __future__ import annotations + +import datetime as dt +import json +import pathlib +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.sources import DictSource + +UTC = dt.timezone.utc + +# element type -> (one set value, the large-normalized Arrow type text of the element) +PRIMITIVE_SETS: dict[str, tuple[Any, Any, str]] = { + "int": (int, {3, 1, 2, 10}, "int64"), + "str": (str, {"b", "a", "10", "2"}, "large_string"), + "float": (float, {1.5, -2.5, 10.0}, "double"), + "bool": (bool, {True, False}, "bool"), + "bytes": (bytes, {b"y", b"x", b"\xff"}, "large_binary"), + "datetime": ( + dt.datetime, + {dt.datetime(2026, 1, 2, tzinfo=UTC), dt.datetime(2025, 1, 2, tzinfo=UTC)}, + "timestamp[us, tz=UTC]", + ), + "date": (dt.date, {dt.date(2026, 1, 2), dt.date(2025, 1, 2)}, "date32[day]"), +} + + +def _producer(declared: Any, value: Any) -> Any: + """Return a function pod with one input ``i`` that returns ``value`` as ``out``.""" + + def produce(i: int) -> Any: + return value + + produce.__annotations__["return"] = declared + return op.function_pod(output_keys="out")(produce) + + +def _consumer(declared: Any) -> Any: + """Return a function pod that reports the Python type of its ``out`` input.""" + + def consume(out: Any) -> str: + if isinstance(out, list): + return "list[" + ",".join(type(x).__name__ for x in out) + "]" + return type(out).__name__ + + consume.__annotations__["out"] = declared + return op.function_pod(output_keys="got")(consume) + + +def _source() -> DictSource: + return DictSource([{"k": 0, "i": 0}], tag_columns=["k"], source_id="one_row") + + +def _run_job(database: Any, declared: Any, value: Any, *, use_async: bool) -> Any: + """Run a job ``produce -> consume`` and return it.""" + job = op.PipelineJob(name="sets", database=database) + with job: + produced = _producer(declared, value).pod(_source(), label="produce") + _consumer(declared).pod(produced, label="consume") + if use_async: + job.run_async() + else: + job.run() + return job + + +def _out_values(stream: Any) -> list[Any]: + return [data.as_dict()["out"] for _, data in stream.iter_data()] + + +def _database(kind: str, tmp_path: Path) -> Any: + if kind == "delta": + return DeltaTableDatabase(tmp_path / "store") + return InMemoryArrowDatabase() + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("database_kind", ["memory", "delta"]) +@pytest.mark.parametrize("element", list(PRIMITIVE_SETS)) +class TestJobRoundTrip: + """A set of a primitive reads back as a set from every reader in the process.""" + + def test_node_downstream_pod_and_as_source_read_a_set( + self, element: str, database_kind: str, use_async: bool, tmp_path: Path + ) -> None: + element_type, value, _ = PRIMITIVE_SETS[element] + declared = set[element_type] + job = _run_job( + _database(database_kind, tmp_path), declared, value, use_async=use_async + ) + node = job.nodes["produce"] + assert _out_values(node) == [value] + assert all(isinstance(v, set) for v in _out_values(node)) + + got = job.nodes["consume"].as_table().column("got").to_pylist() + assert got == ["set"] + + stored = node.as_source() + assert stored.output_schema()[1]["out"] == declared + assert _out_values(stored) == [value] + assert all(isinstance(v, set) for v in _out_values(stored)) + + +@pytest.mark.parametrize("element", list(PRIMITIVE_SETS)) +def test_stored_arrow_type_is_the_set_container(element: str) -> None: + """The stored column is ``extension]>`` over ``large_list``.""" + element_type, value, element_text = PRIMITIVE_SETS[element] + job = _run_job(InMemoryArrowDatabase(), set[element_type], value, use_async=False) + arrow_type = job.nodes["produce"].as_table().schema.field("out").type + assert isinstance(arrow_type, pa.ExtensionType) + assert arrow_type.extension_name == f"set[{element_text}]" + assert json.loads(arrow_type.__arrow_ext_serialize__()) == { + "category": "set", + "element_arrow_type": element_text, + } + assert pa.types.is_large_list(arrow_type.storage_type) + + +@pytest.mark.parametrize( + ("declared", "value", "name", "metadata"), + [ + ( + set[int], + {2, 1}, + "set[int64]", + b'{"category": "set", "element_arrow_type": "int64"}', + ), + ( + set[str], + {"x"}, + "set[large_string]", + b'{"category": "set", "element_arrow_type": "large_string"}', + ), + ( + set[bytes], + {b"x"}, + "set[large_binary]", + b'{"category": "set", "element_arrow_type": "large_binary"}', + ), + ], + ids=["int", "str", "bytes"], +) +def test_identifier_bytes( + declared: Any, value: Any, name: str, metadata: bytes +) -> None: + """The identifier of a set of a primitive: exact name and metadata, no version key.""" + source = DictSource( + [{"k": 0, "s": value}], + tag_columns=["k"], + data_schema={"k": int, "s": declared}, + ) + arrow_type = source.as_table().schema.field("s").type + assert arrow_type.extension_name == name + assert arrow_type.__arrow_ext_serialize__() == metadata + + +@pytest.mark.parametrize( + ("declared", "value", "arrow_type"), + [ + (list[int], [3, 1, 2], pa.large_list(pa.int64())), + (list[str], ["b", "a"], pa.large_list(pa.large_string())), + (list[bytes], [b"y", b"x"], pa.large_list(pa.large_binary())), + ], + ids=["int", "str", "bytes"], +) +def test_list_of_a_primitive_is_unchanged( + declared: Any, value: Any, arrow_type: pa.DataType +) -> None: + """``list[T]`` of a primitive stays a plain ``large_list`` and reads back as a list.""" + job = _run_job(InMemoryArrowDatabase(), declared, value, use_async=False) + node = job.nodes["produce"] + assert node.as_table().schema.field("out").type == arrow_type + assert _out_values(node) == [value] + assert _out_values(node.as_source()) == [value] + assert job.nodes["consume"].as_table().column("got").to_pylist() == [ + "list[" + ",".join(type(x).__name__ for x in value) + "]" + ] + + +def _row_hash(declared: Any, value: Any) -> Any: + source = DictSource( + [{"k": 0, "s": value}], + tag_columns=["k"], + data_schema={"k": int, "s": declared}, + source_id="row_hash", + ) + _, data = next(iter(source.iter_data())) + return data.content_hash() + + +def test_a_set_row_does_not_hash_like_the_list_row() -> None: + """``{1, 2, 3}`` as ``set[int]`` and ``[1, 2, 3]`` as ``list[int]`` are two rows.""" + assert _row_hash(set[int], {3, 1, 2}) != _row_hash(list[int], [1, 2, 3]) + + +def test_equal_sets_hash_equal() -> None: + """The stored order is canonical, so equal sets give one row hash.""" + assert _row_hash(set[str], {"a", "b", "c"}) == _row_hash(set[str], {"c", "a", "b"}) + + +@pytest.mark.parametrize( + ("declared", "value", "expected"), + [ + (list[set[int]], [{3, 1}, {2}], "list[set,set]"), + (set[int] | None, {3, 1, 2}, "set"), + ], + ids=["list_of_sets", "optional_set"], +) +def test_nested_and_optional_sets_reach_a_downstream_pod_as_sets( + declared: Any, value: Any, expected: str +) -> None: + """A set inside a list, and an optional set, reach a downstream pod as sets.""" + job = _run_job(InMemoryArrowDatabase(), declared, value, use_async=False) + assert job.nodes["consume"].as_table().column("got").to_pylist() == [expected] + assert _out_values(job.nodes["produce"].as_source()) == [value] + + +def test_a_set_inside_a_dict_value_is_still_stored() -> None: + """``dict[str, set[int]]`` keeps working (plain storage inside the dict value).""" + job = _run_job( + InMemoryArrowDatabase(), dict[str, set[int]], {"a": {2, 1}}, use_async=False + ) + assert len(job.nodes["produce"].as_table()) == 1 + + +def test_frozenset_stays_refused() -> None: + """``frozenset[T]`` is not a supported annotation.""" + with pytest.raises(ValueError, match="frozenset"): + _run_job( + InMemoryArrowDatabase(), frozenset[int], frozenset({1}), use_async=False + ) + + +_FRESH_READ = textwrap.dedent( + """ + import json, sys + from orcapod import PipelineJob + from orcapod.databases import DeltaTableDatabase + + job = PipelineJob.load(sys.argv[1], database=DeltaTableDatabase(sys.argv[2])) + stored = job.nodes["produce"].as_source() + _, data = next(iter(stored.iter_data())) + value = data.as_dict()["out"] + print(json.dumps({ + "schema": str(stored.output_schema()[1]["out"]), + "python_type": type(value).__name__, + "elements": sorted(repr(v) for v in value), + "row_hash": data.content_hash().to_string(), + })) + """ +) + +FRESH_CASES: dict[str, tuple[Any, Any]] = { + "int": (set[int], {3, 1, 2}), + "str": (set[str], {"b", "a"}), + "bytes": (set[bytes], {b"y", b"x"}), + "path": (set[pathlib.Path], {pathlib.Path("/a/b"), pathlib.Path("/a/c")}), +} + + +@pytest.mark.parametrize("case", list(FRESH_CASES)) +def test_a_loaded_job_in_a_fresh_process_reads_a_set(case: str, tmp_path: Path) -> None: + """A job loaded in another process restores the set type, values and row hash.""" + declared, value = FRESH_CASES[case] + store = tmp_path / "store" + job = _run_job(DeltaTableDatabase(store), declared, value, use_async=False) + saved = tmp_path / "job.json" + job.save(saved) + _, data = next(iter(job.nodes["produce"].as_source().iter_data())) + in_process_hash = data.content_hash().to_string() + + result = subprocess.run( + [sys.executable, "-c", _FRESH_READ, str(saved), str(store)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + read = json.loads(result.stdout.strip().splitlines()[-1]) + assert read["schema"] == str(declared) + assert read["python_type"] == "set" + assert read["elements"] == sorted(repr(v) for v in value) + assert read["row_hash"] == in_process_hash From 3e4c3596041880c6707756795cfb5745201c649d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:37:42 -0800 Subject: [PATCH 03/21] feat(logical_types): a set of a primitive is the native set container (ITL-611) set[T] with no logical type for T is stored as ListLogicalType of kind set over an ArrowElement: name set[], metadata {"category": "set", "element_arrow_type": ...}, no version key, elements in the canonical set order. list[T] of a primitive stays a plain large_list. A set inside a dict value or tuple field keeps its plain storage. The hash visitor value-hashes a container's elements only when the innermost element is a logical type. frozenset[T] is refused with a message naming ITL-1140. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/hashing/visitors.py | 58 +++-- .../list_logical_type_factory.py | 206 ++++++++++++++++-- .../semantic_types/universal_converter.py | 112 +++++++++- .../test_set_of_primitives.py | 3 +- 4 files changed, 321 insertions(+), 58 deletions(-) diff --git a/src/orcapod/hashing/visitors.py b/src/orcapod/hashing/visitors.py index 3ac720279..33b34b1aa 100644 --- a/src/orcapod/hashing/visitors.py +++ b/src/orcapod/hashing/visitors.py @@ -300,7 +300,8 @@ def _hashes_extension(self, extension_type: "pa.ExtensionType") -> bool: The same type-level decision ``visit_extension`` takes for a non-null value: a registered value hasher for the Python type, or for the innermost - element of a list- or set-backed extension type. + element of a list- or set-backed extension type when that element is a + logical type. """ python_type = self._type_converter.arrow_type_to_python_type(extension_type) if typing.get_origin(python_type) in (list, set) and pa.types.is_large_list( @@ -308,19 +309,38 @@ def _hashes_extension(self, extension_type: "pa.ExtensionType") -> bool: ): args = typing.get_args(python_type) if args: - inner = args[0] - while typing.get_origin(inner) in (list, set): - inner_args = typing.get_args(inner) - if not inner_args: - break - inner = inner_args[0] - return isinstance( - inner, type - ) and self._python_hasher.type_handler_registry.has_handler(inner) + return self._hashes_container_elements(args[0]) if python_type is typing.Any or not isinstance(python_type, type): return False return self._python_hasher.type_handler_registry.has_handler(python_type) + def _hashes_container_elements(self, element_python_type: Any) -> bool: + """Return True if a native container's elements are hashed by value. + + Unwraps list and set nesting to the innermost element type. Its values are + hashed by value only when it is a logical type with a registered value + hasher. A container of a plain Arrow element (``set[bytes]``, ITL-611) is + hashed as stored, like a plain ``list[bytes]`` column, even where the + semantic hasher has a handler for the element's Python type. + + Args: + element_python_type: The Python type of the container's elements. + + Returns: + Whether the elements are replaced by their value hashes. + """ + inner = element_python_type + while typing.get_origin(inner) in (list, set): + inner_args = typing.get_args(inner) + if not inner_args: + break + inner = inner_args[0] + return ( + isinstance(inner, type) + and self._type_converter.get_logical_type(inner) is not None + and self._python_hasher.type_handler_registry.has_handler(inner) + ) + def visit_extension( self, extension_type: "pa.ExtensionType", @@ -370,19 +390,11 @@ def visit_extension( elem_python_type = args[0] # Type-driven hashability: unwrap list/set nesting to the innermost - # non-container type and check whether it has a semantic handler. - # Decision is made once per column (not per row) so empty and null - # inner lists are handled correctly without crashing. - inner = elem_python_type - while typing.get_origin(inner) in (list, set): - inner_args = typing.get_args(inner) - if not inner_args: - break - inner = inner_args[0] - hashable = isinstance( - inner, type - ) and self._python_hasher.type_handler_registry.has_handler(inner) - if not hashable: + # non-container type and check whether it is a logical type with a + # semantic handler. Decision is made once per column (not per row) + # so empty and null inner lists are handled correctly without + # crashing. + if not self._hashes_container_elements(elem_python_type): # Innermost element type has no semantic handler — whole-column # passthrough, identical to main-branch behaviour. return extension_type, storage_value diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 2d229cfaa..50853df15 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -1,5 +1,6 @@ -"""ListLogicalType: wraps list[T]/set[T] (where T is an Arrow extension type) as a -top-level Arrow extension type with storage ``large_list()``. +"""ListLogicalType: wraps list[T]/set[T] as a top-level Arrow extension type with +storage ``large_list()``, where ``T`` is a logical type (``list`` and ``set``) +or, for ``set`` only, a plain Arrow type (ITL-611). This solves the ET1 constraint (Arrow cannot embed extension types inside list value positions): by promoting the extension metadata to the outermost field level, the @@ -12,6 +13,17 @@ - Metadata JSON: ``{"category": "list"|"set", "element_ext_name": "...", "element_ext_metadata": null_or_string}`` - ``python_type`` returns the full generic alias ``list[uuid.UUID]`` (not bare ``list``) +A set of a primitive (developer, November 14, 2026, ITL-611). ``set[T]`` where ``T`` has +no logical type (``set[int]``, ``set[str]``, ``set[bytes]``) is the same native set +container: without it a set is stored as a plain ``large_list`` and is read back, and +hashed, as the list with the same elements. Its element is an ``ArrowElement``, whose +identifier is its Arrow type normalized to the large variants, as text +(``str(normalize_to_large_types(t))``): the extension name is ``set[int64]`` and the +metadata ``{"category": "set", "element_arrow_type": "int64"}``. The key +``element_arrow_type`` (never ``element_ext_name``) marks a plain Arrow element, so it +can never be mistaken for a logical type's name. ``list[T]`` of a primitive is not a +native container: it stays a plain ``large_list``, because a list is a list in Arrow too. + Native containers (developer, November 11, 2026, ITL-935). ``list[T]`` and ``set[T]`` of a logical type ``T`` are the *native containers*, and the only ones. They are logical types only because Arrow cannot hold an extension type inside a list value @@ -36,6 +48,7 @@ import json import logging +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from orcapod.logical_types.base_logical_type import BaseLogicalType @@ -60,27 +73,51 @@ LIST_CATEGORY = "list" SET_CATEGORY = "set" +ELEMENT_ARROW_TYPE_KEY = "element_arrow_type" +"""Metadata key naming a set's plain Arrow element type (ITL-611).""" + + +@dataclass(frozen=True) +class ArrowElement: + """The element of a native set that is not a logical type: a plain Arrow type. + + Attributes: + python_type: The element's Python type (``int`` for ``set[int]``). + arrow_type: The element's Arrow type; ``ListLogicalType`` normalizes it to + its large variants. + """ + + python_type: Any + arrow_type: "pa.DataType" class ListLogicalType(BaseLogicalType): - """Logical type for ``list[T]`` or ``set[T]`` where T maps to an Arrow extension type. + """Logical type for the native containers ``list[T]`` and ``set[T]``. Wraps a homogeneous list or set of elements whose Python type corresponds to a registered Arrow extension type (e.g. ``list[uuid.UUID]`` where ``uuid.UUID`` - maps to ``orcapod.uuid``). The resulting Arrow extension type has: + maps to ``orcapod.uuid``), or a set of a plain Arrow element type + (``set[int]``, an ``ArrowElement``). The resulting Arrow extension type has: - - Extension name: ``list[]`` or ``set[]`` + - Extension name: ``list[]`` or ``set[]``; + for a plain element ``set[]`` - Storage type: ``pa.large_list()`` — ET1 safe (no nested extension) - - Metadata: JSON with ``category``, ``element_ext_name``, and ``element_ext_metadata`` + - Metadata: JSON with ``category``, ``element_ext_name``, and ``element_ext_metadata``; + for a plain element ``category`` and ``element_arrow_type`` Args: - element_logical_type: The logical type of individual list elements (e.g. ``LogicalUUID()`` - for ``uuid.UUID`` elements). The element Python type and Arrow extension type are - derived from this logical type. + element_logical_type: The element: the logical type of individual elements + (e.g. ``LogicalUUID()`` for ``uuid.UUID`` elements), from which the element + Python type and Arrow extension type are derived, or an ``ArrowElement`` + for a set of a plain Arrow type. is_set: If ``True``, uses ``set[T]`` semantics (``storage_to_python`` returns a ``set``; ``python_to_storage`` sorts storage values for determinism). Defaults to ``False`` (``list[T]`` semantics). + Raises: + ValueError: If the element is an ``ArrowElement`` and ``is_set`` is ``False``: a + list of a plain Arrow type is a plain ``large_list``, not a logical type. + Example: >>> from orcapod.logical_types.builtin_logical_types import LogicalUUID >>> lt = ListLogicalType(LogicalUUID(), is_set=False) @@ -92,16 +129,27 @@ class ListLogicalType(BaseLogicalType): def __init__( self, - element_logical_type: "LogicalTypeProtocol", + element_logical_type: "LogicalTypeProtocol | ArrowElement", *, is_set: bool = False, ) -> None: - self._element_logical_type = element_logical_type - self._element_python_type = element_logical_type.python_type self._is_set = is_set self._arrow_ext: pa.ExtensionType | None = None self._polars_ext: pl.BaseExtension | None = None + if isinstance(element_logical_type, ArrowElement): + if not is_set: + raise ValueError( + f"A list of the plain Arrow type {element_logical_type.arrow_type} " + "is a plain large_list, not a list logical type; only a set of a " + "plain Arrow type is a native container." + ) + self._init_arrow_element(element_logical_type) + return + + self._element_logical_type: LogicalTypeProtocol | None = element_logical_type + self._element_python_type = element_logical_type.python_type + # Derive Arrow extension details from the element logical type. element_ext_type = element_logical_type.get_arrow_extension_type() element_ext_name = element_ext_type.extension_name @@ -150,6 +198,30 @@ def _container_metadata(element_metadata: bytes) -> bytes: # Compose the extension name from the category and element extension name. self._logical_type_name = f"{category}[{element_ext_name}]" + def _init_arrow_element(self, element: ArrowElement) -> None: + """Set up a set of a plain Arrow element type (ITL-611). + + The element's identifier is its Arrow type normalized to the large variants, + as text. No ``"version"`` key, as for every native container. + + Args: + element: The element's Python and Arrow types. + """ + from orcapod.utils.arrow_utils import normalize_to_large_types + + element_arrow_type = normalize_to_large_types( + element.arrow_type, keep_null=True + ) + element_text = str(element_arrow_type) + self._element_logical_type = None + self._element_python_type = element.python_type + self._metadata_bytes = json.dumps( + {"category": SET_CATEGORY, ELEMENT_ARROW_TYPE_KEY: element_text} + ).encode("utf-8") + self._also_reads = (self._metadata_bytes,) + self._storage_type = pa.large_list(element_arrow_type) + self._logical_type_name = f"{SET_CATEGORY}[{element_text}]" + @property def logical_type_name(self) -> str: """Unique orcapod identifier for this logical type. @@ -160,11 +232,12 @@ def logical_type_name(self) -> str: return self._logical_type_name @property - def element_logical_type(self) -> LogicalTypeProtocol: + def element_logical_type(self) -> LogicalTypeProtocol | None: """The logical type of the list's elements. Returns: - The element logical type this list type was built from. + The element logical type this list type was built from, or ``None`` for a + set of a plain Arrow element type. """ return self._element_logical_type @@ -247,9 +320,10 @@ def python_to_storage( the element Python type. If ``value`` is ``None``, returns an empty list. For ``set[T]`` (``is_set=True``), the resulting list is sorted to guarantee - deterministic storage order regardless of Python set iteration order. Sorting - is attempted naturally; if elements are not comparable (``TypeError``), falls - back to ``repr``-based ordering. + deterministic storage order regardless of Python set iteration order. For a + logical element sorting is attempted naturally; if elements are not comparable + (``TypeError``), falls back to ``repr``-based ordering. A plain Arrow element is + put in the canonical order of ``canonical_order.sort_set_elements``. Args: value: A Python list (or set) of elements of type ``element_python_type``, @@ -259,6 +333,10 @@ def python_to_storage( Returns: A plain Python list of storage values suitable for Arrow large_list storage. For set semantics the list is sorted. + + Raises: + CanonicalOrderError: For a set of a plain Arrow type holding a NaN, elements + that cannot be ordered, or two elements with one storage value. """ if value is None: return [] @@ -270,6 +348,13 @@ def python_to_storage( for item in value ] if self._is_set: + if self._element_logical_type is None: + # A set of a plain Arrow type keeps the canonical order a plain set + # had before it was a native container (ITL-905), so the stored + # elements are unchanged (ITL-611). + from orcapod.semantic_types.canonical_order import sort_set_elements + + return sort_set_elements(elements) try: elements = sorted(elements) except TypeError: @@ -341,8 +426,9 @@ def create_for_python_type( ``ListLogicalType``. Args: - python_type: A ``list[T]`` or ``set[T]`` generic alias where ``T`` - maps to a registered ``LogicalType``. + python_type: A ``list[T]`` generic alias where ``T`` maps to a registered + ``LogicalType``, or a ``set[T]`` of any element type the converter + supports. converter: The active converter for recursive element-type resolution. Returns: @@ -350,7 +436,8 @@ def create_for_python_type( Raises: ValueError: If ``python_type`` is not ``list[T]`` or ``set[T]``, if - it lacks a type argument, or if ``T`` has no registered ``LogicalType``. + it lacks a type argument, or if ``T`` of a ``list[T]`` has no + registered ``LogicalType``. """ import typing @@ -373,6 +460,15 @@ def create_for_python_type( # Directly look up (and register if needed) the LogicalType for the element. element_lt = converter.get_logical_type_for_python_type(element_annotation) + if element_lt is None and is_set: + # A set of a plain Arrow type is a native container too (ITL-611). + return ListLogicalType( + ArrowElement( + element_annotation, + converter.register_python_class(element_annotation), + ), + is_set=True, + ) if element_lt is None: raise ValueError( f"ListLogicalTypeFactory.create_for_python_type: element type " @@ -396,19 +492,25 @@ def reconstruct_from_arrow( ensuring the element logical type is registered before constructing the outer ``ListLogicalType``. Handles arbitrary nesting depth via recursion. + A set of a plain Arrow element (metadata key ``"element_arrow_type"``) is + rebuilt from the storage's element type, which must be the type the metadata + names (ITL-611). + Args: arrow_extension_name: Extension name (e.g. ``"list[orcapod.uuid]"``). storage_type: Outer storage type (``large_list()``). - metadata: Parsed metadata dict; must contain ``"category"`` and - ``"element_ext_name"``; ``"element_ext_metadata"`` may be ``None``. + metadata: Parsed metadata dict; must contain ``"category"`` and either + ``"element_ext_name"`` (``"element_ext_metadata"`` may be ``None``) or, + for a set, ``"element_arrow_type"``. converter: Active converter for recursive element registration. Returns: A ``ListLogicalType`` ready for registration. Raises: - ValueError: If ``storage_type`` is not a list type, or required metadata - keys are missing. + ValueError: If ``storage_type`` is not a list type, required metadata + keys are missing, or the storage's element type is not the + ``"element_arrow_type"`` the metadata names. """ if not (pa.types.is_large_list(storage_type) or pa.types.is_list(storage_type)): raise ValueError( @@ -416,6 +518,16 @@ def reconstruct_from_arrow( f"type for {arrow_extension_name!r}, got {storage_type!r}." ) + element_arrow_text = metadata.get(ELEMENT_ARROW_TYPE_KEY) + if element_arrow_text is not None: + return self._reconstruct_arrow_element_set( + arrow_extension_name, + storage_type.value_type, + metadata, + element_arrow_text, + converter, + ) + element_ext_name = metadata.get("element_ext_name") if not element_ext_name: raise ValueError( @@ -453,3 +565,49 @@ def reconstruct_from_arrow( is_set, ) return ListLogicalType(element_logical_type, is_set=is_set) + + def _reconstruct_arrow_element_set( + self, + arrow_extension_name: str, + element_storage_type: "pa.DataType", + metadata: dict, + element_arrow_text: str, + converter: "TypeConverterProtocol", + ) -> ListLogicalType: + """Rebuild a set of a plain Arrow element type from stored metadata (ITL-611). + + Args: + arrow_extension_name: Extension name (e.g. ``"set[int64]"``). + element_storage_type: The value type of the stored list. + metadata: Parsed metadata dict. + element_arrow_text: The ``"element_arrow_type"`` value of ``metadata``. + converter: Active converter, for the element's Python type. + + Returns: + The ``ListLogicalType`` of the set. + + Raises: + ValueError: If the category is not ``set``, or the stored element type, + normalized to its large variants, is not the type the metadata names. + """ + from orcapod.utils.arrow_utils import normalize_to_large_types + + if metadata.get("category") != SET_CATEGORY: + raise ValueError( + f"ListLogicalTypeFactory.reconstruct_from_arrow: {arrow_extension_name!r} " + f"names a plain Arrow element type, which only a set may have; " + f"metadata={metadata!r}." + ) + element_arrow_type = normalize_to_large_types( + element_storage_type, keep_null=True + ) + if str(element_arrow_type) != element_arrow_text: + raise ValueError( + f"ListLogicalTypeFactory.reconstruct_from_arrow: {arrow_extension_name!r} " + f"names the element type {element_arrow_text!r}, but its storage holds " + f"{element_arrow_type}." + ) + element_python_type = converter.arrow_type_to_python_type(element_arrow_type) + return ListLogicalType( + ArrowElement(element_python_type, element_arrow_type), is_set=True + ) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 5c752b21e..665028af1 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -56,6 +56,11 @@ # Using a ContextVar (rather than an instance attribute) keeps it thread-safe, # coroutine-safe, and explicitly scoped to the active call chain without # polluting the converter instance with temporary state. +_FROZENSET_UNSUPPORTED = ( + "{annotation!r} is not supported: declare set[T] instead. Support for frozenset " + "is to be evaluated in ITL-1140." +) + _register_in_progress: contextvars.ContextVar[set[type] | None] = ( contextvars.ContextVar("_register_in_progress", default=None) ) @@ -371,7 +376,8 @@ def _register_python_class_impl( return self._make_or_get_list_logical_type(element_lt, is_set=False) return pa.large_list(inner) - # set[T] → pa.large_list(T). Same restriction as list[T] unless T has a LogicalType. + # set[T] → the native set container, a ListLogicalType extension: over T's + # logical type when T has one, otherwise over T's plain Arrow type (ITL-611). if origin is set: if not args: raise ValueError( @@ -387,7 +393,10 @@ def _register_python_class_impl( ) if element_lt is not None: return self._make_or_get_list_logical_type(element_lt, is_set=True) - return pa.large_list(inner) + return self._make_or_get_arrow_element_set_type(args[0], inner) + + if origin is frozenset: + raise ValueError(_FROZENSET_UNSUPPORTED.format(annotation=annotation)) # dict[K, V] → pa.large_list(struct{key: K, value: V}). # Raise if K or V resolves to an extension type: the key/value land inside struct @@ -398,8 +407,10 @@ def _register_python_class_impl( "Unparameterized 'dict' is not supported. Use 'dict[K, V]' with concrete " "key and value types (e.g. dict[str, int])." ) - key_arrow = self.register_python_class(args[0]) - val_arrow = self.register_python_class(args[1]) + # A set of a plain type keeps its plain storage inside a dict: Arrow + # cannot hold its extension type in a struct field (ITL-611). + key_arrow = self._plain_set_storage(self.register_python_class(args[0])) + val_arrow = self._plain_set_storage(self.register_python_class(args[1])) if isinstance(key_arrow, pa.ExtensionType): raise ValueError( f"'dict[{args[0]}, ...]' is not yet supported: the key type maps to Arrow " @@ -491,6 +502,78 @@ def _make_or_get_list_logical_type( self._logical_type_registry.register_logical_type(lt) return lt.get_arrow_extension_type() + def _make_or_get_arrow_element_set_type( + self, element_annotation: Any, element_arrow_type: "pa.DataType" + ) -> "pa.DataType": + """Return the native set container of a plain Arrow element type (ITL-611). + + Created and registered on first use, looked up by extension name after that, + like ``_make_or_get_list_logical_type``. The element's Python type is its + annotation without ``None``: ``set[int | None]`` and ``set[int]`` are one + container (``set[int64]``), as ``set[Path | None]`` and ``set[Path]`` are, so + the Python type a column reads as never depends on which was declared first. + + Args: + element_annotation: The annotation of the set's elements. + element_arrow_type: The Arrow type the element annotation maps to; an + extension type here (a type with no registered logical type) is not + wrapped. + + Returns: + The ``pa.ExtensionType`` of the set, or ``pa.large_list(element)`` when the + converter has no registry or the element is an extension type. + """ + from orcapod.logical_types.list_logical_type_factory import ( + ArrowElement, + ListLogicalType, + ) + + if self._logical_type_registry is None or isinstance( + element_arrow_type, pa.BaseExtensionType + ): + return pa.large_list(element_arrow_type) + origin = get_origin(element_annotation) + if origin is typing.Union or origin is types.UnionType: + non_none = [a for a in get_args(element_annotation) if a is not type(None)] + if len(non_none) == 1: + element_annotation = non_none[0] + lt = ListLogicalType( + ArrowElement(element_annotation, element_arrow_type), is_set=True + ) + existing = self._logical_type_registry.get_by_arrow_extension_name( + lt.logical_type_name + ) + if existing is None: + self._logical_type_registry.register_logical_type(lt) + existing = lt + return existing.get_arrow_extension_type() + + def _plain_set_storage(self, arrow_type: "pa.DataType") -> "pa.DataType": + """Return a set of a plain element as its storage; any other type as it is. + + Arrow cannot hold an extension type inside a struct field, so a set of a + plain element type keeps the plain ``large_list`` storage it had before it + was a native container where it is a dict value or a tuple field (ITL-611). + + Args: + arrow_type: An Arrow type the converter produced. + + Returns: + The storage type of a set of a plain element; otherwise ``arrow_type``. + """ + from orcapod.logical_types.list_logical_type_factory import ListLogicalType + + if not isinstance(arrow_type, pa.ExtensionType) or ( + self._logical_type_registry is None + ): + return arrow_type + lt = self._logical_type_registry.get_by_arrow_extension_name( + arrow_type.extension_name + ) + if isinstance(lt, ListLogicalType) and lt.element_logical_type is None: + return arrow_type.storage_type + return arrow_type + def _find_factory_for_class( self, python_type: type, @@ -1324,7 +1407,9 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: if len(set(args)) == 1: # Homogeneous tuple → fixed-size list - element_type = self.python_type_to_arrow_type(args[0]) + element_type = self._plain_set_storage( + self.python_type_to_arrow_type(args[0]) + ) if hasattr(element_type, "extension_name"): raise ValueError( f"'tuple[{args[0]}, ...]' is not supported: the element type maps to " @@ -1337,7 +1422,9 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: # Heterogeneous tuple → struct with indexed fields fields = [] for i, arg_type in enumerate(args): - field_type = self.python_type_to_arrow_type(arg_type) + field_type = self._plain_set_storage( + self.python_type_to_arrow_type(arg_type) + ) if hasattr(field_type, "extension_name"): raise ValueError( f"'tuple[..., {arg_type}, ...]' is not supported: element type " @@ -1354,8 +1441,10 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: raise ValueError( f"dict type must have exactly two type arguments, got: {args}" ) - key_type = self.python_type_to_arrow_type(args[0]) - value_type = self.python_type_to_arrow_type(args[1]) + key_type = self._plain_set_storage(self.python_type_to_arrow_type(args[0])) + value_type = self._plain_set_storage( + self.python_type_to_arrow_type(args[1]) + ) key_value_struct = pa.struct([("key", key_type), ("value", value_type)]) return pa.large_list(key_value_struct) @@ -1391,7 +1480,7 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: ) return self.python_type_to_arrow_type(next(iter(value_types))) - # Handle set types → lists + # Handle set types → the native set container (ITL-611) elif origin is set: if len(args) != 1: raise ValueError( @@ -1406,7 +1495,10 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: ) if element_lt is not None: return self._make_or_get_list_logical_type(element_lt, is_set=True) - return pa.large_list(element_type) + return self._make_or_get_arrow_element_set_type(args[0], element_type) + + elif origin is frozenset: + raise ValueError(_FROZENSET_UNSUPPORTED.format(annotation=python_type)) else: raise ValueError(f"Unsupported generic type: {origin}") diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index bfcbf78b1..6a13dbaf8 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -23,6 +23,7 @@ import orcapod as op from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator from orcapod.sources import DictSource UTC = dt.timezone.utc @@ -76,7 +77,7 @@ def _run_job(database: Any, declared: Any, value: Any, *, use_async: bool) -> An produced = _producer(declared, value).pod(_source(), label="produce") _consumer(declared).pod(produced, label="consume") if use_async: - job.run_async() + job.run(orchestrator=AsyncPipelineOrchestrator()) else: job.run() return job From eebc06322cb40fe83c4045d1cb627a60f3812a4c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:50:59 -0800 Subject: [PATCH 04/21] fix(pipeline): a loaded job restores native containers from their identifier (ITL-611) A saved schema names an extension type by its identifier only. A native container (list[T], set[T]) is fully described by it, so parse_arrow_type_string now rebuilds an unregistered one through the registered factory, as a read of stored data does. A job loaded in a fresh process restores set[int] and set[Path] columns as sets, with the in-process row hash. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/serialization.py | 75 +++++++++++++++++++ .../test_set_of_primitives.py | 3 +- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index a7146b066..3f3920e82 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -963,6 +963,11 @@ def parse_arrow_type_string(type_str: str, type_converter: Any | None = None) -> type_converter = resolve_context(None).type_converter logical_type = type_converter.get_logical_type_by_arrow_extension_name(name) + if logical_type is None: + # A native container (list[T], set[T]) is fully described by its + # identifier: rebuild it, as a read of stored data does (ITL-611). + _register_native_container(name, metadata, type_converter) + logical_type = type_converter.get_logical_type_by_arrow_extension_name(name) if logical_type is None: raise ValueError(f"Unregistered extension type: {name!r}") require_readable(logical_type, metadata) @@ -1015,6 +1020,76 @@ def parse_arrow_type_string(type_str: str, type_converter: Any | None = None) -> raise ValueError(f"Unknown nested Arrow type: {type_str!r}") +def _native_container_storage(metadata: bytes, type_converter: Any) -> Any | None: + """Return the storage type a native container's identifier describes. + + A native container's metadata names its element: a plain Arrow type + (``element_arrow_type``, a set of a primitive) or a logical type + (``element_ext_name`` and ``element_ext_metadata``). The storage is a + ``large_list`` of the element's storage. A logical element must be registered + in this process, or be a native container itself. + + Args: + metadata: The container's extension metadata. + type_converter: The converter that resolves a logical element. + + Returns: + The storage type, or ``None`` if ``metadata`` is not a native container's or + names a logical element that is not registered. + """ + import pyarrow as pa + + from orcapod.logical_types.list_logical_type_factory import ( + ELEMENT_ARROW_TYPE_KEY, + LIST_CATEGORY, + SET_CATEGORY, + ) + + try: + description = _json.loads(metadata.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + return None + if not isinstance(description, dict) or description.get("category") not in ( + LIST_CATEGORY, + SET_CATEGORY, + ): + return None + element_text = description.get(ELEMENT_ARROW_TYPE_KEY) + if isinstance(element_text, str): + return pa.large_list(parse_arrow_type_string(element_text, type_converter)) + element_name = description.get("element_ext_name") + if not isinstance(element_name, str): + return None + element_metadata = (description.get("element_ext_metadata") or "").encode("utf-8") + element = type_converter.get_logical_type_by_arrow_extension_name(element_name) + if element is not None: + return pa.large_list(element.get_arrow_extension_type().storage_type) + element_storage = _native_container_storage(element_metadata, type_converter) + if element_storage is None: + return None + return pa.large_list(element_storage) + + +def _register_native_container(name: str, metadata: bytes, type_converter: Any) -> None: + """Register the native container a saved identifier names, if it is one. + + Saved schemas hold an extension type as its identifier only. A native container + (``list[T]`` or ``set[T]``) is fully described by it, so it is rebuilt through + the registered factory, as reading stored data rebuilds it; any other logical + type must already be registered in this process. + + Args: + name: The extension name. + metadata: The extension metadata. + type_converter: The converter to register the container with. + """ + storage = _native_container_storage(metadata, type_converter) + if storage is not None: + type_converter.register_logical_type_from_arrow_metadata( + name, metadata, storage + ) + + def _parse_arrow_field(text: str, type_converter: Any | None) -> Any: """Restore a nested field's type and nullability from schema text.""" import pyarrow as pa diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index 6a13dbaf8..5d55a56c9 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -274,6 +274,7 @@ def test_frozenset_stays_refused() -> None: "str": (set[str], {"b", "a"}), "bytes": (set[bytes], {b"y", b"x"}), "path": (set[pathlib.Path], {pathlib.Path("/a/b"), pathlib.Path("/a/c")}), + "list_of_sets": (list[set[int]], [{3, 1}, {2}]), } @@ -297,6 +298,6 @@ def test_a_loaded_job_in_a_fresh_process_reads_a_set(case: str, tmp_path: Path) assert result.returncode == 0, result.stderr[-2000:] read = json.loads(result.stdout.strip().splitlines()[-1]) assert read["schema"] == str(declared) - assert read["python_type"] == "set" + assert read["python_type"] == type(value).__name__ assert read["elements"] == sorted(repr(v) for v in value) assert read["row_hash"] == in_process_hash From 4f1ba4279cc905ae608e0034a0ef39b350c0913d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:51:43 -0800 Subject: [PATCH 05/21] test(golden): add set[int], set[str], set[bytes] entries (ITL-611) Additions only: no existing golden value changes. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/cases.py | 8 +++++ tests/test_hashing/golden/data/datagram.json | 36 ++++++++++++++++++++ tests/test_hashing/golden/data/schema.json | 6 ++++ tests/test_hashing/golden/data/semantic.json | 3 ++ 4 files changed, 53 insertions(+) diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index a2e444c86..132923b0b 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -512,6 +512,10 @@ def _semantic_types() -> dict[str, Any]: "dataclass_subclass": GoldenChild, "dataclass_with_structural": GoldenHolder, "list_dataclass": list[GoldenPoint], + # Added by ITL-611: a set of a primitive is the native set container. + "set_int": set[int], + "set_str": set[str], + "set_bytes": set[bytes], } @@ -987,6 +991,10 @@ def _datagram_rows() -> dict[str, list[dict[str, Any]]]: {"k": "r1", "c": GoldenHolder(GoldenInner(1), "a")}, {"k": "r2", "c": GoldenHolder(GoldenInner(2), "b")}, ], + # Added by ITL-611: a set of a primitive is the native set container. + "set_int": [{"k": "r1", "c": {10, 2, 1}}, {"k": "r2", "c": {-1}}], + "set_str": [{"k": "r1", "c": {"b", "a", "10"}}, {"k": "r2", "c": {"x"}}], + "set_bytes": [{"k": "r1", "c": {b"y", b"\x00"}}, {"k": "r2", "c": {b"x"}}], } diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/datagram.json index 807c9afc1..1a73227d6 100644 --- a/tests/test_hashing/golden/data/datagram.json +++ b/tests/test_hashing/golden/data/datagram.json @@ -298,6 +298,42 @@ "_tag_source_id::a7cdef49c8d02a3910137a97c6deac6bc87493cadc102bfce49cb195725fae7a", "_tag_record_id::a7cdef49c8d02a3910137a97c6deac6bc87493cadc102bfce49cb195725fae7a" ], + "set_bytes.content_hash": "semantic_v1:ee042c0dfb3d2bcaba484e0d01a9129ce07cb79a8984fd0465b202ef949a6500", + "set_bytes.pipeline_hash": "semantic_v1:e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951", + "set_bytes.row[{\"k\":\"r1\"}]._tag_record_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "518dfec57f2c51998591fe459d19d3df", + "set_bytes.row[{\"k\":\"r1\"}]._tag_source_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "golden_set_bytes", + "set_bytes.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002ebef00311f86221db68f0ba0d3ac6e5f27da0895a8e8c0aa4bd38607958f86d1", + "set_bytes.row[{\"k\":\"r2\"}]._tag_record_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "7a4a4d5d3c5c54c5a27390332d0ea93c", + "set_bytes.row[{\"k\":\"r2\"}]._tag_source_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "golden_set_bytes", + "set_bytes.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002ef16814e8ce9ed31ebed7787fbcac1a2a38a85299e719d3d65fb2aafe40eea35", + "set_bytes.system_tag_columns": [ + "_tag_source_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951", + "_tag_record_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951" + ], + "set_int.content_hash": "semantic_v1:f7e30f37e72860f035c440f04cf6b646fa22321bc07e15b5c8efeb49f6df2795", + "set_int.pipeline_hash": "semantic_v1:07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a", + "set_int.row[{\"k\":\"r1\"}]._tag_record_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "12fcd3ffe39f510b9d47f7ea2c0fcfa2", + "set_int.row[{\"k\":\"r1\"}]._tag_source_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "golden_set_int", + "set_int.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000026d5018d7edf30da5500878c8ae6f97b0cda14f6ac6cf07c640998091382250a6", + "set_int.row[{\"k\":\"r2\"}]._tag_record_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "9f913c859c6a56478883a077d10413de", + "set_int.row[{\"k\":\"r2\"}]._tag_source_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "golden_set_int", + "set_int.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:00000255f2f160238ee464d3292a9a0ea1ae5c4797473a662f4dff02586fab4f26b2c6", + "set_int.system_tag_columns": [ + "_tag_source_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a", + "_tag_record_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a" + ], + "set_str.content_hash": "semantic_v1:e98a3b5bbc53d63e4625873e18000e9a0483ecc607e2bce8b169c5f8d4f4f726", + "set_str.pipeline_hash": "semantic_v1:a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93", + "set_str.row[{\"k\":\"r1\"}]._tag_record_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "1db3719420995abab9ef75a223c5baea", + "set_str.row[{\"k\":\"r1\"}]._tag_source_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "golden_set_str", + "set_str.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000020420d58c35f4830d949a6e844cb0be834c39370b05309ca944555e78e761be2c", + "set_str.row[{\"k\":\"r2\"}]._tag_record_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "107803948e885973a6746072bde16429", + "set_str.row[{\"k\":\"r2\"}]._tag_source_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "golden_set_str", + "set_str.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002e887edd4898d61012724d36ccf15cbd49156e5b3b9e9798784f3b5918164cf40", + "set_str.system_tag_columns": [ + "_tag_source_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93", + "_tag_record_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93" + ], "set_uuid.content_hash": "semantic_v1:d355e7723650062a578bd830a61a66ae19f7acf01d4b7b8cd36581ad662df144", "set_uuid.pipeline_hash": "semantic_v1:5b561315f1796dce0f0cd9a4a98df81a781ca0c0a2c71cefd349418aeaebce07", "set_uuid.row[{\"k\":\"r1\"}]._tag_record_id::5b561315f1796dce0f0cd9a4a98df81a781ca0c0a2c71cefd349418aeaebce07": "2d3990a5b84f5cf0a5c846c448a25267", diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/schema.json index a2038c006..b48a97a86 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/schema.json @@ -95,6 +95,12 @@ "single_field.pydantic.schema_hash": "semantic_v1:7e65c46de4b5daceba5a06df5f24c8153e09507d42cf943c20ac4b398fe48cd1", "single_field.pydantic_optional.arrow_type": "extension[struct]", "single_field.pydantic_optional.schema_hash": "semantic_v1:0c522a5f2a7d2753c5ebce3177674afaab4f3ebe161eede4a6b326223955020e", + "single_field.set_bytes.arrow_type": "extension[large_list]", + "single_field.set_bytes.schema_hash": "semantic_v1:ebea8d27a27ad902d3240ab1c1c997917dbd9867bb355c032d978f9eb65f0b63", + "single_field.set_int.arrow_type": "extension[large_list]", + "single_field.set_int.schema_hash": "semantic_v1:4e218be4db0a70bcf577f41d3dafee4b736c49fed180382e1b9cf0276ac19a89", + "single_field.set_str.arrow_type": "extension[large_list]", + "single_field.set_str.schema_hash": "semantic_v1:8d35a0a6003d99fbe64eabfb07f23ae36b662807f76ed02388b16cb4e6442070", "single_field.set_uuid.arrow_type": "extension[large_list]", "single_field.set_uuid.schema_hash": "semantic_v1:6ba2432e14f1026834a8b86ada09574f9da55a25d2cb237cc75fefd5761d1734", "single_field.str.arrow_type": "large_string", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index 107b9986d..66e2d16e6 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -39,6 +39,9 @@ "type.path": "semantic_v1:7f0da67eaf9d048c9d1791fbe223333cfcc8018c66878647c8b7ae8571c72bd3", "type.pydantic": "semantic_v1:58c6aad8ca32918a34da1dddb62c72392079b9326b021aa3a4c57de4ae0a4ea8", "type.pydantic_optional": "semantic_v1:37902c428bb3aff581d9dbdba4c74abc1aa359ebb062a055c9df3957a90f0d63", + "type.set_bytes": "semantic_v1:b2e5f021326d97334f537a250636b04b0158bbf2c31a3ba0bd0424cd4878c447", + "type.set_int": "semantic_v1:c72f9a5a8a09bf0a6d4992b86c9bbfa18a7cbd3b254af423ba2eabb42aa8d068", + "type.set_str": "semantic_v1:5f6179ffc21288249fc67bbb1c5cfb7329d39151014d973f2e4e00a7b916dfec", "type.set_uuid": "semantic_v1:33d4c1747923a88b8ff5b7eec15d697fdcd11dce11ea0c8071ce76032f00e9ef", "type.str": "semantic_v1:309d607de6633b959ae5118171f079b2830f4b1340064fac065652e305b038ad", "type.structural": "semantic_v1:e050877645fb628a5ae83ad6c88398a91f2437a599a65054127056b221077564", From 2b4916f87b5dbb8b64a2751d0b45f9f115e40864 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:53:40 -0800 Subject: [PATCH 06/21] docs: a set of a primitive is a native container (ITL-611) Concepts and hashing reference, CLAUDE.md and .zed/rules, changelog fragment and design issue NC1. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 9 +++++ CLAUDE.md | 9 +++++ changelog.d/ITL-611.md | 44 +++++++++++++++++++++++ design_issues/ITL-611-NC1.md | 35 ++++++++++++++++++ docs/concepts/logical-type-identifiers.md | 37 ++++++++++++++++++- docs/reference/hashing.md | 15 +++++++- 6 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 changelog.d/ITL-611.md create mode 100644 design_issues/ITL-611-NC1.md diff --git a/.zed/rules b/.zed/rules index 59e335f4f..67be3d954 100644 --- a/.zed/rules +++ b/.zed/rules @@ -285,6 +285,15 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension identifier is the container kind plus the element's identifier; the type hash treats them like any generic alias (origin and arguments). A change to how they are stored or hashed is a data context (global hash version) change, not a logical type change. +- `set[T]` of a primitive is a native container too (developer, November 14, 2026, ITL-611): + `ListLogicalType` over an `ArrowElement`, extension name `set[]` + (`set[int64]`), metadata `{"category": "set", "element_arrow_type": "int64"}`, elements in + the canonical set order (`sort_set_elements`). `list[T]` of a primitive stays a plain + `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps + its plain storage (Arrow has no extension type in a struct field). A container's elements + are value-hashed only when its innermost element is a logical type. A saved schema rebuilds + an unregistered native container from its identifier (`serialization.parse_arrow_type_string`). + `frozenset[T]` is refused (ITL-1140). - On read, stored metadata must be one of the type's `readable_extension_metadata()`; otherwise raise `LogicalTypeIdentifierError`. Never wrap a stored column by name alone and never fall back to the storage type. List an older value in `also_reads` only when the diff --git a/CLAUDE.md b/CLAUDE.md index 4567f8ec4..ff9f100a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -296,6 +296,15 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension identifier is the container kind plus the element's identifier; the type hash treats them like any generic alias (origin and arguments). A change to how they are stored or hashed is a data context (global hash version) change, not a logical type change. +- `set[T]` of a primitive is a native container too (developer, November 14, 2026, ITL-611): + `ListLogicalType` over an `ArrowElement`, extension name `set[]` + (`set[int64]`), metadata `{"category": "set", "element_arrow_type": "int64"}`, elements in + the canonical set order (`sort_set_elements`). `list[T]` of a primitive stays a plain + `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps + its plain storage (Arrow has no extension type in a struct field). A container's elements + are value-hashed only when its innermost element is a logical type. A saved schema rebuilds + an unregistered native container from its identifier (`serialization.parse_arrow_type_string`). + `frozenset[T]` is refused (ITL-1140). - On read, stored metadata must be one of the type's `readable_extension_metadata()`; otherwise raise `LogicalTypeIdentifierError`. Never wrap a stored column by name alone and never fall back to the storage type. List an older value in `also_reads` only when the diff --git a/changelog.d/ITL-611.md b/changelog.d/ITL-611.md new file mode 100644 index 000000000..91a35538b --- /dev/null +++ b/changelog.d/ITL-611.md @@ -0,0 +1,44 @@ +### Breaking changes and migration + +#### A set of a primitive is stored as the native set container; its rows hash differently (ITL-611) + +A column declared `set[T]`, where `T` has no logical type (`set[int]`, `set[str]`, +`set[bytes]`, `set[datetime]`), is now stored under the Arrow extension type +`set[]` (for example `set[int64]`, metadata +`{"category": "set", "element_arrow_type": "int64"}`) instead of a plain `large_list`. This +moves: + +- the stored Arrow type of such a column (at the top level, as `set[T] | None`, and as the + element of a list: `list[set[int]]` is stored as `list[set[int64]]`); +- the row hash of every row that holds such a column, and with it the input hash of a function + pod that takes such a column (its results are computed again), and the default `source_id` + and content hash of an in-memory source with such a column. + +Type, schema and pipeline hashes do not move, nor does anything about `list[T]` or about a set +of a logical type. No released version wrote such a row: only stores written by development +builds of v0.3 hold a set of a primitive as a plain list. Such a store is not refused: a node +serves its stored values as lists, and a run that adds a row raises when the stored and the new +form are combined. Delete the stored tables of such nodes and run again. + +### Fixed + +- **A `set[T]` of a primitive reads back as a set everywhere.** A downstream pod annotated + `set[int]` received a `list`, even inside the job that computed it; `node.as_source()` read + a `list` under a schema that said `set[int]`; and a job loaded in another process said + `list[int]`. The row `{1, 2, 3}` as `set[int]` also hashed exactly like `[1, 2, 3]` as + `list[int]`. A set now keeps its type on every path: the node, a downstream pod, + `as_source()`, `DerivedSource`, and a job loaded in a fresh process, sync and async, on + Delta and in memory. `list[T]` of a primitive is unchanged (a plain `large_list`). (ITL-611) +- **A job loaded in a fresh process restores `list[T]` and `set[T]` columns of a logical type** + (`set[Path]`, `set[UUID]`, `list[Path]`): their saved type could not be restored unless the + type had been used in that process first, and the stored view read the raw storage, with a + row hash that differed from the one in the process that wrote it. A saved schema now rebuilds + such a container from its identifier. (ITL-611) + +### Known limitations + +- Inside a dict value or a tuple field (`dict[str, set[int]]`), a set of a primitive keeps its + plain `large_list` storage, because Arrow cannot hold an extension type inside a struct + field; read back without the declared type, such a value is a list. (ITL-611) +- `frozenset[T]` is not supported: declare `set[T]`. Support is to be evaluated in ITL-1140. + (ITL-611) diff --git a/design_issues/ITL-611-NC1.md b/design_issues/ITL-611-NC1.md new file mode 100644 index 000000000..f30647712 --- /dev/null +++ b/design_issues/ITL-611-NC1.md @@ -0,0 +1,35 @@ +## `src/orcapod/logical_types/` + +### NC1 — A `set[T]` of a primitive was stored, read back and hashed as a list +**Status:** resolved +**Severity:** high +**Issue:** ITL-611 + +`set[T]` where `T` has no logical type (`set[int]`, `set[str]`, `set[bytes]`, +`set[datetime]`) was stored as a plain `large_list`, the same Arrow type as `list[T]`. The +set identity was in no stored table and in no row hash: a downstream pod annotated +`set[int]` received a `list` even inside the job that computed it, `node.as_source()` read a +`list` under a schema that said `set[int]`, a job loaded in another process said `list[int]`, +and the row `{1, 2, 3}` as `set[int]` hashed exactly like `[1, 2, 3]` as `list[int]`. A +`set[T]` of a logical type (`set[Path]`) was already the native set container, but a job +loaded in a fresh process could not restore it (`parse_arrow_type_string` resolved only +extension names already registered in the process) and read its raw storage, with a row hash +that differed from the in-process one. + +**Fix:** the developer approved (November 14, 2026) storing a set of a primitive as the +native set container: `ListLogicalType` over an `ArrowElement`, extension name +`set[]` (`set[int64]`), metadata +`{"category": "set", "element_arrow_type": "int64"}`, no version key, elements in the +canonical set order of `canonical_order.sort_set_elements` (unchanged, so the stored values +are as before and only the type identity moves). `list[T]` of a primitive stays a plain +`large_list`. A container's elements are value-hashed only when its innermost element is a +logical type (a `set[bytes]` is hashed as stored, as a `list[bytes]` column is). A saved +schema rebuilds an unregistered native container from its identifier through the registered +factory, so a loaded job restores `set[int]` and `set[Path]` alike. The row hash of every row +holding a set of a primitive moves (documented; no golden value moved, entries for +`set[int]`, `set[str]` and `set[bytes]` were added). Known limits: inside a dict value or a +tuple field a set of a primitive keeps its plain storage (Arrow has no extension type in a +struct field); `frozenset[T]` is refused, its support to be evaluated in ITL-1140; a store +written by a development build of v0.3 that holds a set of a primitive as a plain list is not +refused on read (it serves lists, and a run that adds a row to it raises when the two forms +are concatenated): clear such nodes. diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index a6db58a16..22179aac4 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -114,7 +114,8 @@ containers*. They are logical types only because Arrow cannot hold an extension a list; conceptually they are an ordinary list or set of `T`. By design: - **A native container has no version of its own.** Its metadata is JSON with `category` - (`list` or `set`) and the element's extension name and metadata, and nothing else. Its + (`list` or `set`) and the element's extension name and metadata (for a set of a primitive, + the element's Arrow type; see below), and nothing else. Its identifier is the container kind plus the element's identifier, so it changes exactly when the element's identifier changes. Because the element's metadata is written into that JSON as text, a logical type used inside `list[T]` or `set[T]` must have extension @@ -127,6 +128,40 @@ a list; conceptually they are an ordinary list or set of `T`. By design: hashing**, recorded by the data context (the global hash version), never a change of a logical type. +**A set of a primitive is a native container too** (ITL-611). `set[T]` where `T` has no +logical type (`set[int]`, `set[str]`, `set[bytes]`, `set[datetime]`) is stored under the +wrapper `set[]`, so a set reads back as a set everywhere: from the +node, in a downstream pod, from `as_source()` and from a job loaded in another process. +The element's identifier is its Arrow type in the large variants, as text, under the +metadata key `element_arrow_type` (never `element_ext_name`, which names a logical type): + +```python +from orcapod.sources import DictSource + +source = DictSource( + data=[{"k": "r", "labels": {"b", "a"}}], + tag_columns=["k"], + data_schema={"k": str, "labels": set[str]}, +) +labels_type = source.as_table().schema.field("labels").type +print(labels_type.extension_name) +# set[large_string] +print(labels_type.__arrow_ext_serialize__().decode()) +# {"category": "set", "element_arrow_type": "large_string"} +``` + +- `list[T]` of a primitive is not a native container: it stays a plain `large_list`, because + a list is a list in Arrow too. +- `set[T | None]` has the identifier of `set[T]`, as `set[Path | None]` has that of + `set[Path]`: the element's nullability is not part of a native container's identifier. +- The elements are stored in the canonical set order (see the hashing reference, section 1e), + so equal sets store, and hash, identically. A set row never hashes like the list row with + the same elements, because the container's identifier is part of the row hash. +- **Known limit:** inside a dict value or a tuple field (`dict[str, set[int]]`), a set of a + primitive keeps its plain `large_list` storage, because Arrow cannot hold an extension + type inside a struct field; read back without the declared type, such a value is a list. +- `frozenset[T]` is not supported: declare `set[T]`. Support is to be evaluated in ITL-1140. + A container type you implement yourself is not a native container. It is an ordinary logical type with its own identifier, which you declare and change like that of any other type. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index c5f112b78..5e1c4cd0a 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -582,7 +582,9 @@ counts column order. A Polars frame and the equal pandas frame never share a has A `dict` and a `set` have no order, so equal values must hash equally whatever order they were built in. Arrow stores both as lists: `dict[K, V]` as `large_list>` -and a plain `set[T]` as `large_list`. Orcapod puts them in canonical order at the point +and a `set[T]` of a primitive as the native set container `set[]` over +`large_list` (ITL-611; see [Logical Type Identifiers](../concepts/logical-type-identifiers.md)). +Orcapod puts them in canonical order at the point where a Python value becomes Arrow, so the stored list, and therefore the row hash, does not depend on insertion order (dicts) or on `PYTHONHASHSEED` (sets of `str` or `bytes`). @@ -661,6 +663,17 @@ of affected inputs, and changing storage order would move row hashes and what a `set[T]` of a logical type (`set[Path]`, `set[UUID]`) is ordered by `ListLogicalType`, not by this rule. +**The set's identity is part of the row hash** (ITL-611). A set of a primitive is stored +under its native container type, and the Arrow hasher hashes an extension column's +identifier (extension name and metadata) with its stored values. So `{1, 2, 3}` as +`set[int]` and `[1, 2, 3]` as `list[int]` are two different rows, although both store the +list `[1, 2, 3]`. The elements of a set of a primitive are hashed as stored, never replaced +by their value hashes: only a container whose innermost element is a logical type is (a +`set[bytes]` hashes like its stored `large_list` plus the identifier, although +the semantic hasher has a handler for `bytes`). Before ITL-611 a set of a primitive was a +plain `large_list` and hashed exactly like the list row; such rows were written only by +development builds of v0.3. + --- ## 2. Source Provenance & System Tags From abe9d7a2b0dd1b99470df0911370211bfd5f3d19 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:57:58 -0800 Subject: [PATCH 07/21] refactor: type the native-container helpers (ITL-611) Co-Authored-By: Claude Opus 5.5 --- .../list_logical_type_factory.py | 2 +- src/orcapod/pipeline/serialization.py | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 50853df15..30c946ead 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -570,7 +570,7 @@ def _reconstruct_arrow_element_set( self, arrow_extension_name: str, element_storage_type: "pa.DataType", - metadata: dict, + metadata: dict[str, Any], element_arrow_text: str, converter: "TypeConverterProtocol", ) -> ListLogicalType: diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 3f3920e82..df58160b8 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -7,7 +7,7 @@ import logging from collections.abc import Collection from enum import Enum -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from orcapod.types import Schema from orcapod.logical_types.identifier import ( @@ -1046,21 +1046,26 @@ def _native_container_storage(metadata: bytes, type_converter: Any) -> Any | Non ) try: - description = _json.loads(metadata.decode("utf-8")) + description: object = _json.loads(metadata.decode("utf-8")) except (UnicodeDecodeError, ValueError): return None - if not isinstance(description, dict) or description.get("category") not in ( - LIST_CATEGORY, - SET_CATEGORY, - ): + if not isinstance(description, dict): return None - element_text = description.get(ELEMENT_ARROW_TYPE_KEY) + fields = cast("dict[str, object]", description) + if fields.get("category") not in (LIST_CATEGORY, SET_CATEGORY): + return None + element_text = fields.get(ELEMENT_ARROW_TYPE_KEY) if isinstance(element_text, str): return pa.large_list(parse_arrow_type_string(element_text, type_converter)) - element_name = description.get("element_ext_name") + element_name = fields.get("element_ext_name") if not isinstance(element_name, str): return None - element_metadata = (description.get("element_ext_metadata") or "").encode("utf-8") + element_metadata_text = fields.get("element_ext_metadata") + element_metadata = ( + element_metadata_text.encode("utf-8") + if isinstance(element_metadata_text, str) + else b"" + ) element = type_converter.get_logical_type_by_arrow_extension_name(element_name) if element is not None: return pa.large_list(element.get_arrow_extension_type().storage_type) From e7709d2c2e11f4aabab88f54503cf89cb648d031 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 15:09:18 -0800 Subject: [PATCH 08/21] docs(plans): ITL-611 follow-up dispatch steps --- .../plans/2026-11-14-itl-611-set-of-primitives.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index 7ecc960e1..ded9528b9 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -117,3 +117,15 @@ movement are waiting on the developer. **No library code until the coordinator's ## Progress notes - 2026-11-14: design sent; prototype patch measured (kept outside the repo); Task 1 next. + +## Follow-up dispatch (2026-11-14, task_ac04b84f8f2a) + +- [ ] Merge `origin/main` (834ee639, PR #391 ITL-1137: `TypeObjectHandler` and + `canonical_annotation_str` register a type before hashing it). Check its interaction with + the hash-visitor rule (Task 3) and native-container registration (Task 2). +- [ ] Targeted tests (3.12, one process): tests/test_logical_types, tests/test_hashing (golden, + label rule), tests/test_semantic_types, tests/test_pipeline loaded-job tests, ITL-611 tests. +- [ ] Golden diff against main adds lines only. +- [ ] Push; CI green on 3.11/3.12/3.13. PR stays DRAFT (never `gh pr ready`). +- [ ] After the developer marks ready: CodeRabbit rounds (plan via coordinator `ask`). +- [ ] Reproduce: `hash_object` of a set of datetimes; is it reachable in a row hash via #393? From 595d3363c4de58ef81e2acb380a76ce30c86781f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 15:13:24 -0800 Subject: [PATCH 09/21] docs(plans): ITL-611 follow-up notes --- .../plans/2026-11-14-itl-611-set-of-primitives.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index ded9528b9..3f77590ae 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -129,3 +129,15 @@ movement are waiting on the developer. **No library code until the coordinator's - [ ] Push; CI green on 3.11/3.12/3.13. PR stays DRAFT (never `gh pr ready`). - [ ] After the developer marks ready: CodeRabbit rounds (plan via coordinator `ask`). - [ ] Reproduce: `hash_object` of a set of datetimes; is it reachable in a row hash via #393? + +### Follow-up notes + +- Merged origin/main 834ee639 cleanly (merge commit 7c1d48f0). #391 changes only + `TypeObjectHandler`/`canonical_annotation_str` (lookups of plain `type` objects) and + `SchemaHandler` order; the visitor's container rule reads the element type from a + registered Arrow extension, so a lookup is enough there. Targeted suites: 3885 passed, + 31 skipped, 1 xfailed (3.12). Golden diff vs main: 53 insertions, 0 deletions. +- `hash_object` has no handler for `datetime` (bare or in a set): pre-existing, untouched. + Not reachable in a row hash: `set[datetime]` elements are not a logical type, so the + column is hashed as stored (verified: a pod returning an aware `set[datetime]` gives a row + hash). Naive datetimes are refused as pod outputs for scalar/list/set alike (existing rule). From 3ec7738913ec766e98af4f7a0a1d132974470749 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:20:57 -0800 Subject: [PATCH 10/21] docs(plans): ITL-611 CodeRabbit round 1 plan --- .../plans/2026-11-14-itl-611-set-of-primitives.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index 3f77590ae..bce2adb2e 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -141,3 +141,14 @@ movement are waiting on the developer. **No library code until the coordinator's Not reachable in a row hash: `set[datetime]` elements are not a logical type, so the column is hashed as stored (verified: a pod returning an aware `set[datetime]` gives a row hash). Naive datetimes are refused as pod outputs for scalar/list/set alike (existing rule). + +### CodeRabbit round 1 (review 5908025110 on b22cb6ff), plan pending coordinator + +- 4576307086 (Literal first wins) — ACCEPT. Reproduced: after a `set[Literal[1, 2]]` source, + a `set[int]` source's schema is `set[Literal[1, 2]]` and its pipeline hash differs from the + same source built alone. Fix in `_make_or_get_arrow_element_set_type`: the element Python + type is `arrow_type_to_python_type(element Arrow type)` (what fresh-process reconstruction + uses), never the first annotation. Test first (public: two DictSources, both orders). +- 4576307078 (None in `list[set[int] | None]`) — DECLINE as pre-existing: main reads the same + column as `[[], [1], []]`; `list[set[UUID] | None]` reads `[set(), ...]` on main; + `list[UUID | None]` crashes on read on main. Report out of scope. From 8b75ccea12079fa79b890feb73572a81406c0ade Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:24:07 -0800 Subject: [PATCH 11/21] fix(semantic_types): a set's element Python type comes from its Arrow type (ITL-611) A set of a plain element kept the first annotation registered under its extension name, so after set[Literal[1, 2]] a set[int] source reported set[Literal[1, 2]] and its pipeline hash depended on what the process declared first. The element Python type is now the Python type of the element Arrow type, as fresh-process reconstruction reads it. Co-Authored-By: Claude Opus 5.5 --- .../semantic_types/universal_converter.py | 27 +++++----- .../2026-11-14-itl-611-set-of-primitives.md | 2 +- .../test_set_of_primitives.py | 53 +++++++++++++++++++ 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 665028af1..09959b4c7 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -393,7 +393,7 @@ def _register_python_class_impl( ) if element_lt is not None: return self._make_or_get_list_logical_type(element_lt, is_set=True) - return self._make_or_get_arrow_element_set_type(args[0], inner) + return self._make_or_get_arrow_element_set_type(inner) if origin is frozenset: raise ValueError(_FROZENSET_UNSUPPORTED.format(annotation=annotation)) @@ -503,18 +503,19 @@ def _make_or_get_list_logical_type( return lt.get_arrow_extension_type() def _make_or_get_arrow_element_set_type( - self, element_annotation: Any, element_arrow_type: "pa.DataType" + self, element_arrow_type: "pa.DataType" ) -> "pa.DataType": """Return the native set container of a plain Arrow element type (ITL-611). Created and registered on first use, looked up by extension name after that, - like ``_make_or_get_list_logical_type``. The element's Python type is its - annotation without ``None``: ``set[int | None]`` and ``set[int]`` are one - container (``set[int64]``), as ``set[Path | None]`` and ``set[Path]`` are, so - the Python type a column reads as never depends on which was declared first. + like ``_make_or_get_list_logical_type``. The element's Python type is the + Python type of its Arrow type, never the annotation that declared it: + ``set[int]``, ``set[int | None]`` and ``set[Literal[1, 2]]`` are one container + (``set[int64]``) that reads as ``set[int]``, as a job loaded in a fresh process + reads it, so neither a column's Python type nor a schema hash depends on which + annotation was declared first. Args: - element_annotation: The annotation of the set's elements. element_arrow_type: The Arrow type the element annotation maps to; an extension type here (a type with no registered logical type) is not wrapped. @@ -532,13 +533,11 @@ def _make_or_get_arrow_element_set_type( element_arrow_type, pa.BaseExtensionType ): return pa.large_list(element_arrow_type) - origin = get_origin(element_annotation) - if origin is typing.Union or origin is types.UnionType: - non_none = [a for a in get_args(element_annotation) if a is not type(None)] - if len(non_none) == 1: - element_annotation = non_none[0] lt = ListLogicalType( - ArrowElement(element_annotation, element_arrow_type), is_set=True + ArrowElement( + self.arrow_type_to_python_type(element_arrow_type), element_arrow_type + ), + is_set=True, ) existing = self._logical_type_registry.get_by_arrow_extension_name( lt.logical_type_name @@ -1495,7 +1494,7 @@ def _convert_python_to_arrow(self, python_type: DataType) -> pa.DataType: ) if element_lt is not None: return self._make_or_get_list_logical_type(element_lt, is_set=True) - return self._make_or_get_arrow_element_set_type(args[0], element_type) + return self._make_or_get_arrow_element_set_type(element_type) elif origin is frozenset: raise ValueError(_FROZENSET_UNSUPPORTED.format(annotation=python_type)) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index bce2adb2e..14d707863 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -142,7 +142,7 @@ movement are waiting on the developer. **No library code until the coordinator's column is hashed as stored (verified: a pod returning an aware `set[datetime]` gives a row hash). Naive datetimes are refused as pod outputs for scalar/list/set alike (existing rule). -### CodeRabbit round 1 (review 5908025110 on b22cb6ff), plan pending coordinator +### CodeRabbit round 1 (review 5908025110 on b22cb6ff), plan approved by coordinator, fix done - 4576307086 (Literal first wins) — ACCEPT. Reproduced: after a `set[Literal[1, 2]]` source, a `set[int]` source's schema is `set[Literal[1, 2]]` and its pipeline hash differs from the diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index 5d55a56c9..b601f3b81 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -301,3 +301,56 @@ def test_a_loaded_job_in_a_fresh_process_reads_a_set(case: str, tmp_path: Path) assert read["python_type"] == type(value).__name__ assert read["elements"] == sorted(repr(v) for v in value) assert read["row_hash"] == in_process_hash + + +_REGISTRATION_ORDER = textwrap.dedent( + """ + import json, sys + from typing import Literal + from orcapod.sources import DictSource + + declared = {"int": set[int], "literal": set[Literal[1, 2]]} + report = {} + for name in sys.argv[1:]: + source = DictSource( + [{"k": 0, "v": {1}}], + tag_columns=["k"], + data_schema={"k": int, "v": declared[name]}, + source_id="one_row", + ) + report[name] = { + "schema": str(source.output_schema()[1]["v"]), + "pipeline_hash": source.pipeline_hash().to_string(), + } + print(json.dumps(report)) + """ +) + + +def _declare_in_fresh_process(*names: str) -> dict[str, dict[str, str]]: + """Build one source per name, in order, in a fresh process; report each.""" + result = subprocess.run( + [sys.executable, "-c", _REGISTRATION_ORDER, *names], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_a_set_type_does_not_depend_on_what_was_declared_first() -> None: + """``set[int]`` reads as ``set[int]`` even after ``set[Literal[1, 2]]``. + + Both annotations map to the one container ``set[int64]``; its Python type is the + element Arrow type's Python type, so neither the schema nor the pipeline hash of a + source depends on what the process declared before it. + """ + alone = _declare_in_fresh_process("int")["int"] + literal_first = _declare_in_fresh_process("literal", "int") + int_first = _declare_in_fresh_process("int", "literal") + + assert alone["schema"] == "set[int]" + for report in (literal_first, int_first): + assert report["int"] == alone + assert report["literal"] == alone From 7a4797f835d25b7c661b0efe2b37dba173c6b418 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:38:29 -0800 Subject: [PATCH 12/21] docs(plans): ITL-611 CodeRabbit round 2 finding --- .../plans/2026-11-14-itl-611-set-of-primitives.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index 14d707863..590573207 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -152,3 +152,15 @@ movement are waiting on the developer. **No library code until the coordinator's - 4576307078 (None in `list[set[int] | None]`) — DECLINE as pre-existing: main reads the same column as `[[], [1], []]`; `list[set[UUID] | None]` reads `[set(), ...]` on main; `list[UUID | None]` crashes on read on main. Report out of scope. + +### CodeRabbit round 2 (review 5908180501 on 8b75ccea): thread 4576412950, widths collide + +- Reproduced. HEAD: set[np.int8] then set[int] -> set[int] gets the int8 container and a + set[int] source raises "python_type 'set' is already bound to 'set[int8]'"; reverse order: + set[np.int8] raises. b22cb6ff: same failure via a job loaded in a fresh process (reconstruct + reads int8 as int and binds set[int]). main: plain lists, no collision. +- Root cause: the registry binds each Python type to one logical type; many element Arrow + types read as one Python type (int8/int64, float/double, timestamp units/tz, date32/64). +- Proposed (A): plain-element set containers are registered by identifier only, never bound + to a Python type; set[T] -> Arrow goes through T's Arrow type, as list[T] does. Pending + coordinator decision. From f74a939164aaf1dbf64883f3b29377982a0c81cc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:45:02 -0800 Subject: [PATCH 13/21] fix(logical_types): a set of a plain element is found by its identifier only (ITL-611) Many element Arrow types read as one Python type (int8 and int64 as int), but the registry bound each Python type to one logical type. So after set[np.int8] a set[int] source took the int8 container or raised, and the reverse order raised too; a job loaded in a fresh process with a set[np.int8] node made every later set[int] declaration raise. A set of a plain element now opts out of the Python-type binding (binds_python_type = False): set[T] maps to Arrow through T's own Arrow type, as list[T] does, and reads as set[]. No identifier, stored value or golden value changes. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 +- CLAUDE.md | 5 +- changelog.d/ITL-611.md | 5 +- .../list_logical_type_factory.py | 12 ++ src/orcapod/logical_types/registry.py | 19 ++- .../2026-11-14-itl-611-set-of-primitives.md | 2 +- .../test_set_of_primitives.py | 118 ++++++++++++++++++ 7 files changed, 157 insertions(+), 9 deletions(-) diff --git a/.zed/rules b/.zed/rules index bb7ec1d78..3d321e737 100644 --- a/.zed/rules +++ b/.zed/rules @@ -288,7 +288,10 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension - `set[T]` of a primitive is a native container too (developer, November 14, 2026, ITL-611): `ListLogicalType` over an `ArrowElement`, extension name `set[]` (`set[int64]`), metadata `{"category": "set", "element_arrow_type": "int64"}`, elements in - the canonical set order (`sort_set_elements`). `list[T]` of a primitive stays a plain + the canonical set order (`sort_set_elements`). It reads as `set[]` (`set[int8]` and `set[int64]` both as `set[int]`, as for lists), so it is + registered by identifier only, never bound to a Python type (`binds_python_type`), and + `set[T]` maps to Arrow through `T`. `list[T]` of a primitive stays a plain `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps its plain storage (Arrow has no extension type in a struct field). A container's elements are value-hashed only when its innermost element is a logical type. A saved schema rebuilds diff --git a/CLAUDE.md b/CLAUDE.md index f4db8e412..28d90e497 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -299,7 +299,10 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension - `set[T]` of a primitive is a native container too (developer, November 14, 2026, ITL-611): `ListLogicalType` over an `ArrowElement`, extension name `set[]` (`set[int64]`), metadata `{"category": "set", "element_arrow_type": "int64"}`, elements in - the canonical set order (`sort_set_elements`). `list[T]` of a primitive stays a plain + the canonical set order (`sort_set_elements`). It reads as `set[]` (`set[int8]` and `set[int64]` both as `set[int]`, as for lists), so it is + registered by identifier only, never bound to a Python type (`binds_python_type`), and + `set[T]` maps to Arrow through `T`. `list[T]` of a primitive stays a plain `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps its plain storage (Arrow has no extension type in a struct field). A container's elements are value-hashed only when its innermost element is a logical type. A saved schema rebuilds diff --git a/changelog.d/ITL-611.md b/changelog.d/ITL-611.md index 91a35538b..443955092 100644 --- a/changelog.d/ITL-611.md +++ b/changelog.d/ITL-611.md @@ -28,7 +28,10 @@ form are combined. Delete the stored tables of such nodes and run again. `list[int]`. The row `{1, 2, 3}` as `set[int]` also hashed exactly like `[1, 2, 3]` as `list[int]`. A set now keeps its type on every path: the node, a downstream pod, `as_source()`, `DerivedSource`, and a job loaded in a fresh process, sync and async, on - Delta and in memory. `list[T]` of a primitive is unchanged (a plain `large_list`). (ITL-611) + Delta and in memory. Its Python type is that of the element's Arrow type, as for a list: + `set[np.int8]` and `set[int]` both read as `set[int]`, each column keeps its own width, and + both can be declared in one process. `list[T]` of a primitive is unchanged (a plain + `large_list`). (ITL-611) - **A job loaded in a fresh process restores `list[T]` and `set[T]` columns of a logical type** (`set[Path]`, `set[UUID]`, `list[Path]`): their saved type could not be restored unless the type had been used in that process first, and the stored view read the raw storage, with a diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 30c946ead..b9f028da8 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -114,6 +114,13 @@ class ListLogicalType(BaseLogicalType): ``set``; ``python_to_storage`` sorts storage values for determinism). Defaults to ``False`` (``list[T]`` semantics). + Attributes: + binds_python_type: Whether the registry binds ``python_type`` to this logical + type. ``False`` for a set of a plain Arrow element: ``set[int]`` is the + Python type of ``set[int64]`` and of ``set[int8]`` alike, so such a set is + found by its identifier only, and ``set[T]`` maps to Arrow through ``T``'s + own Arrow type, as ``list[T]`` does (ITL-611). + Raises: ValueError: If the element is an ``ArrowElement`` and ``is_set`` is ``False``: a list of a plain Arrow type is a plain ``large_list``, not a logical type. @@ -127,6 +134,8 @@ class ListLogicalType(BaseLogicalType): list[uuid.UUID] """ + binds_python_type: bool = True + def __init__( self, element_logical_type: "LogicalTypeProtocol | ArrowElement", @@ -213,6 +222,9 @@ def _init_arrow_element(self, element: ArrowElement) -> None: element.arrow_type, keep_null=True ) element_text = str(element_arrow_type) + # Many element Arrow types read as one Python type (int8 and int64 as int), + # so a set of a plain element is found by its identifier only (ITL-611). + self.binds_python_type = False self._element_logical_type = None self._element_python_type = element.python_type self._metadata_bytes = json.dumps( diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index b8a1ddc4e..2b1a692ae 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -331,7 +331,9 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: Raises: ValueError: If any of the three keys (``logical_type_name``, Arrow extension name, ``python_type``) is already bound to a - *different* ``LogicalTypeProtocol`` in this registry, or if the + *different* ``LogicalTypeProtocol`` in this registry (the + ``python_type`` only for a logical type that binds it, see + ``binds_python_type`` on ``ListLogicalType``), or if the Arrow extension name contains ``;`` (the separator between name and metadata in the identifier text that is hashed). """ @@ -344,10 +346,16 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: ) py_type = logical_type.python_type logical_name = logical_type.logical_type_name + # A logical type whose Python type is shared by other logical types (a set + # of a plain Arrow element: ``set[int]`` reads as ``set[int64]`` and as + # ``set[int8]``) is found by its identifier only, never by its Python type. + binds_python_type = getattr(logical_type, "binds_python_type", True) existing_by_logical = self._by_logical_name.get(logical_name) existing_by_arrow = self._by_arrow_name.get(arrow_ext_name) - existing_by_python = self._by_python_type.get(py_type) + existing_by_python = ( + self._by_python_type.get(py_type) if binds_python_type else None + ) if ( existing_by_python is None @@ -379,11 +387,11 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: f"'{existing.logical_type_name}'." ) - # Idempotent check: all three keys already bound to this same instance. + # Idempotent check: every key already bound to this same instance. if ( existing_by_logical is logical_type and existing_by_arrow is logical_type - and existing_by_python is logical_type + and (existing_by_python is logical_type or not binds_python_type) ): return @@ -410,7 +418,8 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: # Store three-way binding. self._by_logical_name[logical_name] = logical_type self._by_arrow_name[arrow_ext_name] = logical_type - self._by_python_type[py_type] = logical_type + if binds_python_type: + self._by_python_type[py_type] = logical_type def _rebind( self, previous: LogicalTypeProtocol, logical_type: LogicalTypeProtocol diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index 590573207..d9de94418 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -163,4 +163,4 @@ movement are waiting on the developer. **No library code until the coordinator's types read as one Python type (int8/int64, float/double, timestamp units/tz, date32/64). - Proposed (A): plain-element set containers are registered by identifier only, never bound to a Python type; set[T] -> Arrow goes through T's Arrow type, as list[T] does. Pending - coordinator decision. + coordinator decision. APPROVED (A) and implemented. diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index b601f3b81..ad5499d7e 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -354,3 +354,121 @@ def test_a_set_type_does_not_depend_on_what_was_declared_first() -> None: for report in (literal_first, int_first): assert report["int"] == alone assert report["literal"] == alone + + +_TWO_WIDTHS = textwrap.dedent( + """ + import json, sys + import numpy as np + from orcapod.sources import DictSource + + declared = {"int8": (set[np.int8], {1, 2}), "int64": (set[int], {1, 300})} + report = {} + for name in sys.argv[1:]: + annotation, value = declared[name] + source = DictSource( + [{"k": 0, "v": value}], + tag_columns=["k"], + data_schema={"k": int, "v": annotation}, + source_id="one_row", + ) + _, data = next(iter(source.iter_data())) + read = data.as_dict()["v"] + report[name] = { + "arrow_type": str(source.as_table().schema.field("v").type), + "schema": str(source.output_schema()[1]["v"]), + "read": sorted(int(v) for v in read), + "read_type": type(read).__name__, + } + print(json.dumps(report)) + """ +) + + +@pytest.mark.parametrize("order", [("int8", "int64"), ("int64", "int8")]) +def test_two_integer_widths_of_a_set_live_in_one_process( + order: tuple[str, str], +) -> None: + """``set[np.int8]`` and ``set[int]`` are two containers in one process, in any order. + + Both read as ``set[int]``, as ``list[np.int8]`` and ``list[int]`` both read as + ``list[int]``; each column keeps its own element width. + """ + result = subprocess.run( + [sys.executable, "-c", _TWO_WIDTHS, *order], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report["int8"] == { + "arrow_type": "extension>", + "schema": "set[int]", + "read": [1, 2], + "read_type": "set", + } + assert report["int64"] == { + "arrow_type": "extension>", + "schema": "set[int]", + "read": [1, 300], + "read_type": "set", + } + + +_LOAD_THEN_DECLARE = textwrap.dedent( + """ + import json, sys + from orcapod import PipelineJob + from orcapod.databases import DeltaTableDatabase + from orcapod.sources import DictSource + + job = PipelineJob.load(sys.argv[1], database=DeltaTableDatabase(sys.argv[2])) + stored = job.nodes["produce"].as_source() + _, data = next(iter(stored.iter_data())) + loaded = data.as_dict()["out"] + source = DictSource( + [{"k": 0, "v": {1, 300}}], + tag_columns=["k"], + data_schema={"k": int, "v": set[int]}, + source_id="one_row", + ) + _, data = next(iter(source.iter_data())) + print(json.dumps({ + "loaded": sorted(int(v) for v in loaded), + "declared_arrow_type": str(source.as_table().schema.field("v").type), + "declared": sorted(data.as_dict()["v"]), + })) + """ +) + + +def test_a_loaded_set_of_one_width_leaves_other_widths_declarable( + tmp_path: Path, +) -> None: + """A job loaded with a ``set[np.int8]`` node does not take over ``set[int]``.""" + import numpy as np + + store = tmp_path / "store" + job = _run_job( + DeltaTableDatabase(store), + set[np.int8], + {np.int8(1), np.int8(2)}, + use_async=False, + ) + saved = tmp_path / "job.json" + job.save(saved) + + result = subprocess.run( + [sys.executable, "-c", _LOAD_THEN_DECLARE, str(saved), str(store)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report == { + "loaded": [1, 2], + "declared_arrow_type": "extension>", + "declared": [1, 300], + } From 077262df5fbddc8143954285e28a507d0f38d8c1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:01:51 -0800 Subject: [PATCH 14/21] docs(plans): ITL-611 set metadata element_kind arrow Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-611-set-of-primitives.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index d9de94418..9d6e2dc92 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -164,3 +164,28 @@ movement are waiting on the developer. **No library code until the coordinator's - Proposed (A): plain-element set containers are registered by identifier only, never bound to a Python type; set[T] -> Arrow goes through T's Arrow type, as list[T] does. Pending coordinator decision. APPROVED (A) and implemented. + +### Metadata format change (developer decision, November 15, 2026), task_261a2b5ec576 + +Decision: "make the logical extension name contain the element as is, but remove redundancy +for arrow type by simply using element_kind arrow". A set of a primitive keeps its name +(`set[int64]`); metadata becomes exactly `{"category": "set", "element_kind": "arrow"}`. +Containers of logical elements unchanged. + +- [ ] Tests first (test_set_of_primitives.py): identifier/metadata for set[int], set[str], + set[bytes], set[datetime]; list[set[int]] element_ext_metadata is the new inner metadata; + fresh-process read rebuilds set[int]/set[str] from storage (existing fresh test covers it, + with the new metadata); a stored column `set[...]` with `element_kind: arrow` over a + non-list storage, and over a list whose element is not the one the name names, is refused + with LogicalTypeIdentifierError (fresh process, so the name is not already registered). +- [ ] Writer `_init_arrow_element`: metadata `{"category": "set", "element_kind": "arrow"}`; + constant `ELEMENT_KIND_KEY = "element_kind"`, `ARROW_ELEMENT_KIND = "arrow"`. +- [ ] Reader `reconstruct_from_arrow`: `element_kind == "arrow"` -> element from storage + (`large_list`, list accepted as before and normalized), rebuilt name must equal the + stored name; otherwise LogicalTypeIdentifierError. Other element_kind values refused. +- [ ] Saved-schema rebuild (`serialization._native_container_storage`): element text comes + from the extension name `set[]` when element_kind is arrow (needs the name). +- [ ] Docs: logical-type-identifiers.md, CLAUDE.md + .zed/rules, changelog fragment, + design_issues/ITL-611-NC1.md, spec table, PR description. +- [ ] Golden: regenerate only; diff vs origin/main must remain additions only. +- [ ] Targeted suites, push, CI, one signed `@coderabbitai review`. From 24cc3d9f15e39ea0d10f406a07cb1b4a2e2ef7ab Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:05:07 -0800 Subject: [PATCH 15/21] fix(logical_types): a set of a primitive writes element_kind arrow, element read from storage (ITL-611) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/logical_types/identifier.py | 8 +- .../list_logical_type_factory.py | 113 +++++++---- src/orcapod/pipeline/serialization.py | 26 ++- .../test_set_of_primitives.py | 190 +++++++++++++++--- 4 files changed, 259 insertions(+), 78 deletions(-) diff --git a/src/orcapod/logical_types/identifier.py b/src/orcapod/logical_types/identifier.py index 523cfa03c..840066ff0 100644 --- a/src/orcapod/logical_types/identifier.py +++ b/src/orcapod/logical_types/identifier.py @@ -102,9 +102,13 @@ def for_column(self, column: str) -> "LogicalTypeIdentifierError": column: The name of the column the identifier was stored with. Returns: - A new ``LogicalTypeIdentifierError`` with ``column`` set. + A new ``LogicalTypeIdentifierError`` with ``column`` set and this + error's notes. """ - return LogicalTypeIdentifierError(self.stored, self.readable, column) + error = LogicalTypeIdentifierError(self.stored, self.readable, column) + for note in getattr(self, "__notes__", ()): + error.add_note(note) + return error def version_metadata(version: int) -> bytes: diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index b9f028da8..f1fabd453 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -18,9 +18,12 @@ container: without it a set is stored as a plain ``large_list`` and is read back, and hashed, as the list with the same elements. Its element is an ``ArrowElement``, whose identifier is its Arrow type normalized to the large variants, as text -(``str(normalize_to_large_types(t))``): the extension name is ``set[int64]`` and the -metadata ``{"category": "set", "element_arrow_type": "int64"}``. The key -``element_arrow_type`` (never ``element_ext_name``) marks a plain Arrow element, so it +(``str(normalize_to_large_types(t))``): the extension name is ``set[int64]``, which +holds the element as it is and is the registry key of the exact type, and the metadata +is ``{"category": "set", "element_kind": "arrow"}`` (developer, November 15, 2026). The +metadata does not repeat the element type: a reader takes it from the storage, +``large_list``, and refuses a storage whose element is not the one the name +names. ``element_kind`` (never ``element_ext_name``) marks a plain Arrow element, so it can never be mistaken for a logical type's name. ``list[T]`` of a primitive is not a native container: it stays a plain ``large_list``, because a list is a list in Arrow too. @@ -73,8 +76,10 @@ LIST_CATEGORY = "list" SET_CATEGORY = "set" -ELEMENT_ARROW_TYPE_KEY = "element_arrow_type" -"""Metadata key naming a set's plain Arrow element type (ITL-611).""" +ELEMENT_KIND_KEY = "element_kind" +"""Metadata key saying what a set's element is; only ``ARROW_ELEMENT_KIND`` (ITL-611).""" +ARROW_ELEMENT_KIND = "arrow" +"""The ``element_kind`` of a set of a plain Arrow element: the storage's element type.""" @dataclass(frozen=True) @@ -103,7 +108,7 @@ class ListLogicalType(BaseLogicalType): for a plain element ``set[]`` - Storage type: ``pa.large_list()`` — ET1 safe (no nested extension) - Metadata: JSON with ``category``, ``element_ext_name``, and ``element_ext_metadata``; - for a plain element ``category`` and ``element_arrow_type`` + for a plain element ``category`` and ``element_kind`` (``"arrow"``) Args: element_logical_type: The element: the logical type of individual elements @@ -211,7 +216,9 @@ def _init_arrow_element(self, element: ArrowElement) -> None: """Set up a set of a plain Arrow element type (ITL-611). The element's identifier is its Arrow type normalized to the large variants, - as text. No ``"version"`` key, as for every native container. + as text, and enters the extension name only; the metadata says just that the + element is a plain Arrow type, which a reader takes from the storage. No + ``"version"`` key, as for every native container. Args: element: The element's Python and Arrow types. @@ -228,7 +235,7 @@ def _init_arrow_element(self, element: ArrowElement) -> None: self._element_logical_type = None self._element_python_type = element.python_type self._metadata_bytes = json.dumps( - {"category": SET_CATEGORY, ELEMENT_ARROW_TYPE_KEY: element_text} + {"category": SET_CATEGORY, ELEMENT_KIND_KEY: ARROW_ELEMENT_KIND} ).encode("utf-8") self._also_reads = (self._metadata_bytes,) self._storage_type = pa.large_list(element_arrow_type) @@ -504,42 +511,39 @@ def reconstruct_from_arrow( ensuring the element logical type is registered before constructing the outer ``ListLogicalType``. Handles arbitrary nesting depth via recursion. - A set of a plain Arrow element (metadata key ``"element_arrow_type"``) is - rebuilt from the storage's element type, which must be the type the metadata - names (ITL-611). + A set of a plain Arrow element (``"element_kind": "arrow"``) is rebuilt from + the storage's element type; the rebuilt extension name must be the stored one + (ITL-611). Args: arrow_extension_name: Extension name (e.g. ``"list[orcapod.uuid]"``). storage_type: Outer storage type (``large_list()``). metadata: Parsed metadata dict; must contain ``"category"`` and either ``"element_ext_name"`` (``"element_ext_metadata"`` may be ``None``) or, - for a set, ``"element_arrow_type"``. + for a set, ``"element_kind"``. converter: Active converter for recursive element registration. Returns: A ``ListLogicalType`` ready for registration. Raises: - ValueError: If ``storage_type`` is not a list type, required metadata - keys are missing, or the storage's element type is not the - ``"element_arrow_type"`` the metadata names. + LogicalTypeIdentifierError: If the metadata names a plain Arrow element + and the storage is not a list of the element the extension name + names, or ``"element_kind"`` is not ``"arrow"``. + ValueError: If ``storage_type`` is not a list type, or required metadata + keys are missing. """ + if ELEMENT_KIND_KEY in metadata: + return self._reconstruct_arrow_element_set( + arrow_extension_name, storage_type, metadata, converter + ) + if not (pa.types.is_large_list(storage_type) or pa.types.is_list(storage_type)): raise ValueError( f"ListLogicalTypeFactory.reconstruct_from_arrow: expected a list storage " f"type for {arrow_extension_name!r}, got {storage_type!r}." ) - element_arrow_text = metadata.get(ELEMENT_ARROW_TYPE_KEY) - if element_arrow_text is not None: - return self._reconstruct_arrow_element_set( - arrow_extension_name, - storage_type.value_type, - metadata, - element_arrow_text, - converter, - ) - element_ext_name = metadata.get("element_ext_name") if not element_ext_name: raise ValueError( @@ -581,45 +585,66 @@ def reconstruct_from_arrow( def _reconstruct_arrow_element_set( self, arrow_extension_name: str, - element_storage_type: "pa.DataType", + storage_type: "pa.DataType", metadata: dict[str, Any], - element_arrow_text: str, converter: "TypeConverterProtocol", ) -> ListLogicalType: """Rebuild a set of a plain Arrow element type from stored metadata (ITL-611). + The element is the storage's element type: the metadata does not repeat it. + The set built over it must have the stored extension name, so a storage whose + element is not the one the name names is refused. + Args: arrow_extension_name: Extension name (e.g. ``"set[int64]"``). - element_storage_type: The value type of the stored list. + storage_type: The stored storage type (``large_list``). metadata: Parsed metadata dict. - element_arrow_text: The ``"element_arrow_type"`` value of ``metadata``. converter: Active converter, for the element's Python type. Returns: The ``ListLogicalType`` of the set. Raises: - ValueError: If the category is not ``set``, or the stored element type, - normalized to its large variants, is not the type the metadata names. + LogicalTypeIdentifierError: If the metadata is not a set's with + ``"element_kind": "arrow"``, the storage is not a list, or the set + built over the storage's element is not named ``arrow_extension_name``. """ + from orcapod.logical_types.identifier import ( + LogicalTypeIdentifier, + LogicalTypeIdentifierError, + ) from orcapod.utils.arrow_utils import normalize_to_large_types - if metadata.get("category") != SET_CATEGORY: - raise ValueError( - f"ListLogicalTypeFactory.reconstruct_from_arrow: {arrow_extension_name!r} " - f"names a plain Arrow element type, which only a set may have; " - f"metadata={metadata!r}." + def refused(reason: str) -> LogicalTypeIdentifierError: + error = LogicalTypeIdentifierError( + LogicalTypeIdentifier( + arrow_extension_name, json.dumps(metadata).encode("utf-8") + ), + (), + ) + error.add_note(reason) + return error + + if ( + metadata.get("category") != SET_CATEGORY + or metadata.get(ELEMENT_KIND_KEY) != ARROW_ELEMENT_KIND + ): + raise refused( + f'Only a set has a plain element, with "{ELEMENT_KIND_KEY}": ' + f'"{ARROW_ELEMENT_KIND}".' ) + if not (pa.types.is_large_list(storage_type) or pa.types.is_list(storage_type)): + raise refused(f"Its storage {storage_type} is not a list.") element_arrow_type = normalize_to_large_types( - element_storage_type, keep_null=True + storage_type.value_type, keep_null=True ) - if str(element_arrow_type) != element_arrow_text: - raise ValueError( - f"ListLogicalTypeFactory.reconstruct_from_arrow: {arrow_extension_name!r} " - f"names the element type {element_arrow_text!r}, but its storage holds " - f"{element_arrow_type}." - ) element_python_type = converter.arrow_type_to_python_type(element_arrow_type) - return ListLogicalType( + logical_type = ListLogicalType( ArrowElement(element_python_type, element_arrow_type), is_set=True ) + if logical_type.logical_type_name != arrow_extension_name: + raise refused( + f"Its storage {storage_type} holds the elements of " + f"{logical_type.logical_type_name!r}." + ) + return logical_type diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index df58160b8..2ad9843d9 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -1020,16 +1020,20 @@ def parse_arrow_type_string(type_str: str, type_converter: Any | None = None) -> raise ValueError(f"Unknown nested Arrow type: {type_str!r}") -def _native_container_storage(metadata: bytes, type_converter: Any) -> Any | None: +def _native_container_storage( + name: str, metadata: bytes, type_converter: Any +) -> Any | None: """Return the storage type a native container's identifier describes. - A native container's metadata names its element: a plain Arrow type - (``element_arrow_type``, a set of a primitive) or a logical type + A native container's element is a plain Arrow type (a set of a primitive, + ``"element_kind": "arrow"``: the element type is the text inside the extension + name, ``set[]``) or a logical type its metadata names (``element_ext_name`` and ``element_ext_metadata``). The storage is a ``large_list`` of the element's storage. A logical element must be registered in this process, or be a native container itself. Args: + name: The container's extension name. metadata: The container's extension metadata. type_converter: The converter that resolves a logical element. @@ -1040,7 +1044,8 @@ def _native_container_storage(metadata: bytes, type_converter: Any) -> Any | Non import pyarrow as pa from orcapod.logical_types.list_logical_type_factory import ( - ELEMENT_ARROW_TYPE_KEY, + ARROW_ELEMENT_KIND, + ELEMENT_KIND_KEY, LIST_CATEGORY, SET_CATEGORY, ) @@ -1054,8 +1059,11 @@ def _native_container_storage(metadata: bytes, type_converter: Any) -> Any | Non fields = cast("dict[str, object]", description) if fields.get("category") not in (LIST_CATEGORY, SET_CATEGORY): return None - element_text = fields.get(ELEMENT_ARROW_TYPE_KEY) - if isinstance(element_text, str): + if fields.get(ELEMENT_KIND_KEY) == ARROW_ELEMENT_KIND: + prefix = f"{SET_CATEGORY}[" + if not (name.startswith(prefix) and name.endswith("]")): + return None + element_text = name[len(prefix) : -1] return pa.large_list(parse_arrow_type_string(element_text, type_converter)) element_name = fields.get("element_ext_name") if not isinstance(element_name, str): @@ -1069,7 +1077,9 @@ def _native_container_storage(metadata: bytes, type_converter: Any) -> Any | Non element = type_converter.get_logical_type_by_arrow_extension_name(element_name) if element is not None: return pa.large_list(element.get_arrow_extension_type().storage_type) - element_storage = _native_container_storage(element_metadata, type_converter) + element_storage = _native_container_storage( + element_name, element_metadata, type_converter + ) if element_storage is None: return None return pa.large_list(element_storage) @@ -1088,7 +1098,7 @@ def _register_native_container(name: str, metadata: bytes, type_converter: Any) metadata: The extension metadata. type_converter: The converter to register the container with. """ - storage = _native_container_storage(metadata, type_converter) + storage = _native_container_storage(name, metadata, type_converter) if storage is not None: type_converter.register_logical_type_from_arrow_metadata( name, metadata, storage diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index ad5499d7e..e3b180390 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -130,39 +130,33 @@ def test_stored_arrow_type_is_the_set_container(element: str) -> None: assert arrow_type.extension_name == f"set[{element_text}]" assert json.loads(arrow_type.__arrow_ext_serialize__()) == { "category": "set", - "element_arrow_type": element_text, + "element_kind": "arrow", } assert pa.types.is_large_list(arrow_type.storage_type) +SET_METADATA = b'{"category": "set", "element_kind": "arrow"}' +"""The extension metadata of every set of a primitive: the element is the storage's.""" + + @pytest.mark.parametrize( - ("declared", "value", "name", "metadata"), + ("declared", "value", "name"), [ + (set[int], {2, 1}, "set[int64]"), + (set[str], {"x"}, "set[large_string]"), + (set[bytes], {b"x"}, "set[large_binary]"), ( - set[int], - {2, 1}, - "set[int64]", - b'{"category": "set", "element_arrow_type": "int64"}', - ), - ( - set[str], - {"x"}, - "set[large_string]", - b'{"category": "set", "element_arrow_type": "large_string"}', - ), - ( - set[bytes], - {b"x"}, - "set[large_binary]", - b'{"category": "set", "element_arrow_type": "large_binary"}', + set[dt.datetime], + {dt.datetime(2026, 1, 2, tzinfo=UTC)}, + "set[timestamp[us, tz=UTC]]", ), ], - ids=["int", "str", "bytes"], + ids=["int", "str", "bytes", "datetime"], ) -def test_identifier_bytes( - declared: Any, value: Any, name: str, metadata: bytes -) -> None: - """The identifier of a set of a primitive: exact name and metadata, no version key.""" +def test_identifier_bytes(declared: Any, value: Any, name: str) -> None: + """The identifier of a set of a primitive: the name holds the element Arrow type, + the metadata only says the element is a plain Arrow type; no version key. + """ source = DictSource( [{"k": 0, "s": value}], tag_columns=["k"], @@ -170,7 +164,24 @@ def test_identifier_bytes( ) arrow_type = source.as_table().schema.field("s").type assert arrow_type.extension_name == name - assert arrow_type.__arrow_ext_serialize__() == metadata + assert arrow_type.__arrow_ext_serialize__() == SET_METADATA + assert pa.types.is_large_list(arrow_type.storage_type) + + +def test_a_list_of_sets_embeds_the_inner_set_identifier() -> None: + """``list[set[int]]`` names the inner set by its name and its new metadata.""" + source = DictSource( + [{"k": 0, "s": [{2, 1}]}], + tag_columns=["k"], + data_schema={"k": int, "s": list[set[int]]}, + ) + arrow_type = source.as_table().schema.field("s").type + assert arrow_type.extension_name == "list[set[int64]]" + assert json.loads(arrow_type.__arrow_ext_serialize__()) == { + "category": "list", + "element_ext_name": "set[int64]", + "element_ext_metadata": SET_METADATA.decode(), + } @pytest.mark.parametrize( @@ -472,3 +483,134 @@ def test_a_loaded_set_of_one_width_leaves_other_widths_declarable( "declared_arrow_type": "extension>", "declared": [1, 300], } + + +_READ_STORED_COLUMN = textwrap.dedent( + """ + import json, sys + import pyarrow as pa + from orcapod.contexts import get_default_context + from orcapod.logical_types.identifier import LogicalTypeIdentifierError + from orcapod.sources import ArrowTableSource + + name, metadata, storage = sys.argv[1], sys.argv[2].encode(), sys.argv[3] + columns = { + "large_list": pa.array([[1, 2, 3]], pa.large_list(pa.int64())), + "large_list": pa.array([[1, 2]], pa.large_list(pa.int8())), + "large_list": pa.array( + [["a", "b"]], pa.large_list(pa.large_string()) + ), + "large_string": pa.array(["a"], pa.large_string()), + "int64": pa.array([1], pa.int64()), + } + column = columns[storage] + field_metadata = { + b"ARROW:extension:name": name.encode(), + b"ARROW:extension:metadata": metadata, + } + table = pa.table( + [pa.array([0], pa.int64()), column], + schema=pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("c", column.type, nullable=False, metadata=field_metadata), + ] + ), + ) + try: + table = get_default_context().type_converter.load_logical_types(table) + except LogicalTypeIdentifierError as exc: + print(json.dumps({ + "refused": type(exc).__name__, + "message": str(exc), + "notes": getattr(exc, "__notes__", []), + })) + sys.exit(0) + source = ArrowTableSource(table, tag_columns=["k"], source_id="stored") + _, data = next(iter(source.iter_data())) + value = data.as_dict()["c"] + print(json.dumps({ + "arrow_type": str(table.schema.field("c").type), + "metadata": table.schema.field("c").type.__arrow_ext_serialize__().decode(), + "schema": str(source.output_schema()[1]["c"]), + "python_type": type(value).__name__, + "elements": sorted(value), + })) + """ +) + + +def _read_stored_column_in_fresh_process( + name: str, metadata: bytes, storage: str +) -> dict[str, Any]: + """Read a stored column (field-metadata channel) in a fresh process; report it.""" + result = subprocess.run( + [sys.executable, "-c", _READ_STORED_COLUMN, name, metadata.decode(), storage], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@pytest.mark.parametrize( + ("name", "storage", "schema", "elements"), + [ + ("set[int64]", "large_list", "set[int]", [1, 2, 3]), + ("set[large_string]", "large_list", "set[str]", ["a", "b"]), + ], + ids=["int", "str"], +) +def test_a_fresh_process_rebuilds_a_set_from_its_storage( + name: str, storage: str, schema: str, elements: list[Any] +) -> None: + """A stored set column is rebuilt from its name, its metadata and its storage alone.""" + report = _read_stored_column_in_fresh_process(name, SET_METADATA, storage) + assert report["metadata"] == SET_METADATA.decode() + assert report["arrow_type"].startswith(f"extension<{name}<") + assert report["schema"] == schema + assert report["python_type"] == "set" + assert report["elements"] == elements + + +@pytest.mark.parametrize( + ("name", "storage", "reason"), + [ + ("set[int64]", "large_string", "is not a list"), + ("set[int64]", "int64", "is not a list"), + ("set[int64]", "large_list", "holds the elements of 'set[int8]'"), + ], + ids=["string_storage", "int_storage", "other_element_width"], +) +def test_a_set_whose_storage_does_not_hold_its_element_is_refused( + name: str, storage: str, reason: str +) -> None: + """``element_kind: arrow`` over a storage that is not a list of the named element is + refused: the element comes from the storage, so the two must agree. + """ + report = _read_stored_column_in_fresh_process(name, SET_METADATA, storage) + assert report.get("refused") == "LogicalTypeIdentifierError", report + assert "'c'" in report["message"] + assert any(reason in note for note in report["notes"]), report + + +def test_a_set_with_the_unreleased_element_arrow_type_metadata_is_refused() -> None: + """Only ``{"category": "set", "element_kind": "arrow"}`` is read for a set of a + primitive; the form this pull request first wrote was never released. + """ + result = subprocess.run( + [ + sys.executable, + "-c", + _READ_STORED_COLUMN, + "set[int64]", + '{"category": "set", "element_arrow_type": "int64"}', + "large_list", + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0 + assert "ValueError" in result.stderr From e7bf4e095151096f004b4908ea57a50144f0831f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:05:50 -0800 Subject: [PATCH 16/21] test(golden): ITL-611 set data hashes under the element_kind metadata Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/data/datagram.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/datagram.json index 1a73227d6..027eb1111 100644 --- a/tests/test_hashing/golden/data/datagram.json +++ b/tests/test_hashing/golden/data/datagram.json @@ -302,10 +302,10 @@ "set_bytes.pipeline_hash": "semantic_v1:e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951", "set_bytes.row[{\"k\":\"r1\"}]._tag_record_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "518dfec57f2c51998591fe459d19d3df", "set_bytes.row[{\"k\":\"r1\"}]._tag_source_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "golden_set_bytes", - "set_bytes.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002ebef00311f86221db68f0ba0d3ac6e5f27da0895a8e8c0aa4bd38607958f86d1", + "set_bytes.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:00000209b1234d474584dcaba0bf1697326d8a7d17af08e0663849d8e9edd61159dc7c", "set_bytes.row[{\"k\":\"r2\"}]._tag_record_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "7a4a4d5d3c5c54c5a27390332d0ea93c", "set_bytes.row[{\"k\":\"r2\"}]._tag_source_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951": "golden_set_bytes", - "set_bytes.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002ef16814e8ce9ed31ebed7787fbcac1a2a38a85299e719d3d65fb2aafe40eea35", + "set_bytes.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002b82f8de0c4d706e1adfab09b0a0721217f99c45992a902dab3dcd87595e89b4e", "set_bytes.system_tag_columns": [ "_tag_source_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951", "_tag_record_id::e61a21b4303d9aaaad4bc7acbbcbc232919fae2c9c84f8c61dd166f75607d951" @@ -314,10 +314,10 @@ "set_int.pipeline_hash": "semantic_v1:07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a", "set_int.row[{\"k\":\"r1\"}]._tag_record_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "12fcd3ffe39f510b9d47f7ea2c0fcfa2", "set_int.row[{\"k\":\"r1\"}]._tag_source_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "golden_set_int", - "set_int.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000026d5018d7edf30da5500878c8ae6f97b0cda14f6ac6cf07c640998091382250a6", + "set_int.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002c66e08689c6546e8e2e4c75480f12aeb8fa8f03117cb4bbca20239054da1226d", "set_int.row[{\"k\":\"r2\"}]._tag_record_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "9f913c859c6a56478883a077d10413de", "set_int.row[{\"k\":\"r2\"}]._tag_source_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a": "golden_set_int", - "set_int.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:00000255f2f160238ee464d3292a9a0ea1ae5c4797473a662f4dff02586fab4f26b2c6", + "set_int.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:00000250eadd1390c914bb77edc5286000569c65c11d01eeb038f5ca657646cb405265", "set_int.system_tag_columns": [ "_tag_source_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a", "_tag_record_id::07fc17e5d283de983dbfb456c28dee31905f70b9714038c5f6977ea4f816467a" @@ -326,10 +326,10 @@ "set_str.pipeline_hash": "semantic_v1:a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93", "set_str.row[{\"k\":\"r1\"}]._tag_record_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "1db3719420995abab9ef75a223c5baea", "set_str.row[{\"k\":\"r1\"}]._tag_source_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "golden_set_str", - "set_str.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000020420d58c35f4830d949a6e844cb0be834c39370b05309ca944555e78e761be2c", + "set_str.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002d3577efbb77c8cd29859c39d1ec4540028c928669c3c17817682a657580a915c", "set_str.row[{\"k\":\"r2\"}]._tag_record_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "107803948e885973a6746072bde16429", "set_str.row[{\"k\":\"r2\"}]._tag_source_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93": "golden_set_str", - "set_str.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002e887edd4898d61012724d36ccf15cbd49156e5b3b9e9798784f3b5918164cf40", + "set_str.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002ca9566cd0df49261b343372e28e82696239c316519fb1b93883f06ba78696b65", "set_str.system_tag_columns": [ "_tag_source_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93", "_tag_record_id::a8da17a17f8cea1f96f383d788cb81caf5424d810c9b5faa28b190502cfe7b93" From 928482fc17f4f219ece71a71b931a40638e1b385 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:07:01 -0800 Subject: [PATCH 17/21] docs(logical_types): set of a primitive metadata is element_kind arrow (ITL-611) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 10 ++++--- CLAUDE.md | 10 ++++--- changelog.d/ITL-611.md | 4 +-- design_issues/ITL-611-NC1.md | 5 +++- docs/concepts/logical-type-identifiers.md | 12 ++++++--- ...-11-14-itl-611-set-of-primitives-design.md | 26 +++++++++++-------- 6 files changed, 44 insertions(+), 23 deletions(-) diff --git a/.zed/rules b/.zed/rules index 3d321e737..efde5960b 100644 --- a/.zed/rules +++ b/.zed/rules @@ -287,9 +287,13 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension a data context (global hash version) change, not a logical type change. - `set[T]` of a primitive is a native container too (developer, November 14, 2026, ITL-611): `ListLogicalType` over an `ArrowElement`, extension name `set[]` - (`set[int64]`), metadata `{"category": "set", "element_arrow_type": "int64"}`, elements in - the canonical set order (`sort_set_elements`). It reads as `set[]` (`set[int8]` and `set[int64]` both as `set[int]`, as for lists), so it is + (`set[int64]`), metadata `{"category": "set", "element_kind": "arrow"}` (developer, + November 15, 2026: the name holds the element as it is and is the registry key; the + metadata never repeats it), elements in the canonical set order (`sort_set_elements`). A + reader takes the element from the storage (`large_list`) and refuses + (`LogicalTypeIdentifierError`) a storage that is not a list of the element the name names. + It reads as `set[]` (`set[int8]` and `set[int64]` + both as `set[int]`, as for lists), so it is registered by identifier only, never bound to a Python type (`binds_python_type`), and `set[T]` maps to Arrow through `T`. `list[T]` of a primitive stays a plain `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps diff --git a/CLAUDE.md b/CLAUDE.md index 28d90e497..2e94446fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -298,9 +298,13 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension a data context (global hash version) change, not a logical type change. - `set[T]` of a primitive is a native container too (developer, November 14, 2026, ITL-611): `ListLogicalType` over an `ArrowElement`, extension name `set[]` - (`set[int64]`), metadata `{"category": "set", "element_arrow_type": "int64"}`, elements in - the canonical set order (`sort_set_elements`). It reads as `set[]` (`set[int8]` and `set[int64]` both as `set[int]`, as for lists), so it is + (`set[int64]`), metadata `{"category": "set", "element_kind": "arrow"}` (developer, + November 15, 2026: the name holds the element as it is and is the registry key; the + metadata never repeats it), elements in the canonical set order (`sort_set_elements`). A + reader takes the element from the storage (`large_list`) and refuses + (`LogicalTypeIdentifierError`) a storage that is not a list of the element the name names. + It reads as `set[]` (`set[int8]` and `set[int64]` + both as `set[int]`, as for lists), so it is registered by identifier only, never bound to a Python type (`binds_python_type`), and `set[T]` maps to Arrow through `T`. `list[T]` of a primitive stays a plain `large_list`: never wrap it. Inside a dict value or tuple field a set of a primitive keeps diff --git a/changelog.d/ITL-611.md b/changelog.d/ITL-611.md index 443955092..c99dac459 100644 --- a/changelog.d/ITL-611.md +++ b/changelog.d/ITL-611.md @@ -5,8 +5,8 @@ A column declared `set[T]`, where `T` has no logical type (`set[int]`, `set[str]`, `set[bytes]`, `set[datetime]`), is now stored under the Arrow extension type `set[]` (for example `set[int64]`, metadata -`{"category": "set", "element_arrow_type": "int64"}`) instead of a plain `large_list`. This -moves: +`{"category": "set", "element_kind": "arrow"}`: the element type is in the name and is read +from the storage, `large_list`) instead of a plain `large_list`. This moves: - the stored Arrow type of such a column (at the top level, as `set[T] | None`, and as the element of a list: `list[set[int]]` is stored as `list[set[int64]]`); diff --git a/design_issues/ITL-611-NC1.md b/design_issues/ITL-611-NC1.md index f30647712..87ab16a35 100644 --- a/design_issues/ITL-611-NC1.md +++ b/design_issues/ITL-611-NC1.md @@ -19,7 +19,10 @@ that differed from the in-process one. **Fix:** the developer approved (November 14, 2026) storing a set of a primitive as the native set container: `ListLogicalType` over an `ArrowElement`, extension name `set[]` (`set[int64]`), metadata -`{"category": "set", "element_arrow_type": "int64"}`, no version key, elements in the +`{"category": "set", "element_kind": "arrow"}` (developer, November 15, 2026: the name +holds the element as it is and is the registry key; the metadata does not repeat it, and a +reader takes the element from the storage, refusing a storage that is not a list of the +element the name names), no version key, elements in the canonical set order of `canonical_order.sort_set_elements` (unchanged, so the stored values are as before and only the type identity moves). `list[T]` of a primitive stays a plain `large_list`. A container's elements are value-hashed only when its innermost element is a diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index 22179aac4..09d629ed1 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -132,8 +132,12 @@ a list; conceptually they are an ordinary list or set of `T`. By design: logical type (`set[int]`, `set[str]`, `set[bytes]`, `set[datetime]`) is stored under the wrapper `set[]`, so a set reads back as a set everywhere: from the node, in a downstream pod, from `as_source()` and from a job loaded in another process. -The element's identifier is its Arrow type in the large variants, as text, under the -metadata key `element_arrow_type` (never `element_ext_name`, which names a logical type): +The element's identifier is its Arrow type in the large variants, as text, inside the +extension name, which is the key the type is registered under. The metadata does not repeat +it: `"element_kind": "arrow"` (never `element_ext_name`, which names a logical type) says +that the element is a plain Arrow type, which a reader takes from the storage +(`large_list`). A stored column whose storage is not a list of the element its +name names is refused with `LogicalTypeIdentifierError`. ```python from orcapod.sources import DictSource @@ -147,7 +151,9 @@ labels_type = source.as_table().schema.field("labels").type print(labels_type.extension_name) # set[large_string] print(labels_type.__arrow_ext_serialize__().decode()) -# {"category": "set", "element_arrow_type": "large_string"} +# {"category": "set", "element_kind": "arrow"} +print(labels_type.storage_type) +# large_list ``` - `list[T]` of a primitive is not a native container: it stays a plain `large_list`, because diff --git a/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md b/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md index 4d59440e2..ee5b498d1 100644 --- a/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md +++ b/superpowers/specs/2026-11-14-itl-611-set-of-primitives-design.md @@ -60,15 +60,18 @@ large-normalized Arrow type text (`str(normalize_to_large_types(element_type))`) | Python | extension name | extension metadata (exact bytes) | |---|---|---| -| `set[int]` | `set[int64]` | `{"category": "set", "element_arrow_type": "int64"}` | -| `set[str]` | `set[large_string]` | `{"category": "set", "element_arrow_type": "large_string"}` | -| `set[bytes]` | `set[large_binary]` | `{"category": "set", "element_arrow_type": "large_binary"}` | -| `set[float]` | `set[double]` | `{"category": "set", "element_arrow_type": "double"}` | -| `set[datetime]` | `set[timestamp[us, tz=UTC]]` | `{"category": "set", "element_arrow_type": "timestamp[us, tz=UTC]"}` | -| `list[set[int]]` | `list[set[int64]]` | `{"category": "list", "element_ext_name": "set[int64]", "element_ext_metadata": "{\"category\": \"set\", \"element_arrow_type\": \"int64\"}"}` | - -The key `element_arrow_type` (not `element_ext_name`) marks a primitive element, so a -logical type can never be confused with an Arrow type name. JSON is written as the +| `set[int]` | `set[int64]` | `{"category": "set", "element_kind": "arrow"}` | +| `set[str]` | `set[large_string]` | `{"category": "set", "element_kind": "arrow"}` | +| `set[bytes]` | `set[large_binary]` | `{"category": "set", "element_kind": "arrow"}` | +| `set[float]` | `set[double]` | `{"category": "set", "element_kind": "arrow"}` | +| `set[datetime]` | `set[timestamp[us, tz=UTC]]` | `{"category": "set", "element_kind": "arrow"}` | +| `list[set[int]]` | `list[set[int64]]` | `{"category": "list", "element_ext_name": "set[int64]", "element_ext_metadata": "{\"category\": \"set\", \"element_kind\": \"arrow\"}"}` | + +`element_kind: arrow` (not `element_ext_name`) marks a primitive element, so a +logical type can never be confused with an Arrow type name. Developer decision, November 15, +2026: the element type is in the name only (the registry key, unique per element type); the +metadata does not repeat it, and a reader takes the element from the storage. (The first +form, `element_arrow_type`, was never released.) JSON is written as the existing container metadata is (`json.dumps`, insertion order). - `list[primitive]` is unchanged (plain `large_list`); a regression test pins it. @@ -78,8 +81,9 @@ existing container metadata is (`json.dumps`, insertion order). - Element order: unchanged, `sort_set_elements` on the converted storage values, so the stored bytes of every value are as before and only the type identity moves. - Reading: `storage_to_python` converts each element with the converter and returns a - `set`. `ListLogicalTypeFactory.reconstruct_from_arrow` rebuilds the type from - `element_arrow_type` (checking it equals the storage's element type). + `set`. `ListLogicalTypeFactory.reconstruct_from_arrow` rebuilds the type from the + storage's element type and refuses (`LogicalTypeIdentifierError`) a storage that is not a + list or whose set would have another name than the stored one. - Hashing: the `SemanticHashingVisitor` value-hashes a native container only when its innermost element is a logical type; a container of a primitive element is hashed as stored, with its identifier in the field metadata (starfix `include_metadata`). From c4f488788c16ec2f05cae420efafeafeb20ecc18 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:23:41 -0800 Subject: [PATCH 18/21] fix(semantic_types): a registered logical type refuses a stored column of another storage (ITL-611) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-611.md | 5 +++ .../semantic_types/universal_converter.py | 31 ++++++++++++++--- .../2026-11-14-itl-611-set-of-primitives.md | 11 +++++++ .../test_set_of_primitives.py | 33 +++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/changelog.d/ITL-611.md b/changelog.d/ITL-611.md index c99dac459..ab1ed9f88 100644 --- a/changelog.d/ITL-611.md +++ b/changelog.d/ITL-611.md @@ -38,6 +38,11 @@ form are combined. Delete the stored tables of such nodes and run again. row hash that differed from the one in the process that wrote it. A saved schema now rebuilds such a container from its identifier. (ITL-611) +- **A stored column whose storage is not its logical type's storage raises + `LogicalTypeIdentifierError`**, naming the column and both storage types, whether or not the + type was registered in the process before (it raised a `TypeError` from Arrow when it was). + (ITL-611) + ### Known limitations - Inside a dict value or a tuple field (`dict[str, set[int]]`), a set of a primitive keeps its diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 09959b4c7..a6f38aecf 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -800,13 +800,20 @@ def register_logical_type_from_arrow_metadata( Raises: LogicalTypeIdentifierError: If the extension name is known (or was reconstructed by a factory) and ``extension_metadata`` is not one - of the values the logical type declares readable. + of the values the logical type declares readable, or the name is + known and ``storage_type`` (in its large variants) is not the + registered type's storage. ValueError: If metadata is missing, malformed, lacks ``"category"``, or no factory is registered for the category. """ import json as _json - from orcapod.logical_types.identifier import require_readable + from orcapod.logical_types.identifier import ( + LogicalTypeIdentifier, + LogicalTypeIdentifierError, + require_readable, + ) + from orcapod.utils.arrow_utils import normalize_to_large_types if self._logical_type_registry is None: raise ValueError( @@ -819,9 +826,25 @@ def register_logical_type_from_arrow_metadata( arrow_extension_name ) if lt is not None: - # The name is known: the stored metadata must be one the type reads. + # The name is known: the stored metadata must be one the type reads, + # and the stored storage the type's own; never wrap by name alone. require_readable(lt, extension_metadata or b"") - return lt.get_arrow_extension_type() + extension_type = lt.get_arrow_extension_type() + if normalize_to_large_types( + storage_type, keep_null=True + ) != normalize_to_large_types(extension_type.storage_type, keep_null=True): + error = LogicalTypeIdentifierError( + LogicalTypeIdentifier( + arrow_extension_name, extension_metadata or b"" + ), + (), + ) + error.add_note( + f"Its storage {storage_type} is not the storage of " + f"{arrow_extension_name!r}, {extension_type.storage_type}." + ) + raise error + return extension_type # Missing metadata — cannot auto-register if not extension_metadata: diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index 9d6e2dc92..a247252d9 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -189,3 +189,14 @@ Containers of logical elements unchanged. design_issues/ITL-611-NC1.md, spec table, PR description. - [ ] Golden: regenerate only; diff vs origin/main must remain additions only. - [ ] Targeted suites, push, CI, one signed `@coderabbitai review`. +- Done: writer/reader/serialization, docs, golden (6 set_* data hashes, vs main still 53+/0-), + pushed 928482fc, CI green. + +### CodeRabbit round 3 (review 5908556654 on 24cc3d9f), plan approved by coordinator + +- 4576679626 (registered name: storage not checked) — ACCEPT, generic: the registry-hit + branch of `register_logical_type_from_arrow_metadata` compares the stored storage with the + registered type's (large-normalized) and raises `LogicalTypeIdentifierError` with a note. + Was refused before too, by Arrow's `TypeError`. Hashing, logical types, semantic types, + pipeline and database suites pass with it (8322 passed). +- ast-grep notes (jsonify x2, subprocess x2): DECLINE, false positives; one signed comment. diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index e3b180390..9c4d15ac0 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -614,3 +614,36 @@ def test_a_set_with_the_unreleased_element_arrow_type_metadata_is_refused() -> N ) assert result.returncode != 0 assert "ValueError" in result.stderr + + +@pytest.mark.parametrize( + "storage", + [pa.array(["a"], pa.large_string()), pa.array([[1]], pa.large_list(pa.int8()))], + ids=["string_storage", "other_element_width"], +) +def test_a_registered_set_whose_storage_does_not_hold_its_element_is_refused( + storage: "pa.Array[Any]", +) -> None: + """With ``set[int64]`` already registered, a stored ``set[int64]`` column over another + storage is refused as it is in a fresh process, with ``LogicalTypeIdentifierError``. + """ + from orcapod.contexts import get_default_context + from orcapod.logical_types.identifier import LogicalTypeIdentifierError + from orcapod.semantic_types.universal_converter import UniversalTypeConverter + + _row_hash(set[int], {1}) # registers set[int64] in this process + field_metadata = { + b"ARROW:extension:name": b"set[int64]", + b"ARROW:extension:metadata": SET_METADATA, + } + table = pa.table( + [storage], + schema=pa.schema([pa.field("c", storage.type, metadata=field_metadata)]), + ) + converter = get_default_context().type_converter + assert isinstance(converter, UniversalTypeConverter) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + converter.load_logical_types(table) + assert excinfo.value.column == "c" + notes = getattr(excinfo.value, "__notes__", []) + assert any(str(storage.type) in note for note in notes), notes From f548f5cd78d66eb71091ec09c54ce277dcb71314 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 18:51:25 -0800 Subject: [PATCH 19/21] fix(logical_types): a list of sets of a plain element is found by its identifier and lowered in a dict value (ITL-611) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-611.md | 4 +- docs/concepts/logical-type-identifiers.md | 4 +- .../list_logical_type_factory.py | 9 ++- .../semantic_types/universal_converter.py | 12 ++- .../2026-11-14-itl-611-set-of-primitives.md | 9 +++ .../test_set_of_primitives.py | 80 +++++++++++++++++++ 6 files changed, 110 insertions(+), 8 deletions(-) diff --git a/changelog.d/ITL-611.md b/changelog.d/ITL-611.md index ab1ed9f88..882c1066f 100644 --- a/changelog.d/ITL-611.md +++ b/changelog.d/ITL-611.md @@ -45,8 +45,8 @@ form are combined. Delete the stored tables of such nodes and run again. ### Known limitations -- Inside a dict value or a tuple field (`dict[str, set[int]]`), a set of a primitive keeps its - plain `large_list` storage, because Arrow cannot hold an extension type inside a struct +- Inside a dict value or a tuple field (`dict[str, set[int]]`, `dict[str, list[set[int]]]`), a + set of a primitive keeps its plain `large_list` storage, because Arrow cannot hold an extension type inside a struct field; read back without the declared type, such a value is a list. (ITL-611) - `frozenset[T]` is not supported: declare `set[T]`. Support is to be evaluated in ITL-1140. (ITL-611) diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index 09d629ed1..eeba60163 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -163,8 +163,8 @@ print(labels_type.storage_type) - The elements are stored in the canonical set order (see the hashing reference, section 1e), so equal sets store, and hash, identically. A set row never hashes like the list row with the same elements, because the container's identifier is part of the row hash. -- **Known limit:** inside a dict value or a tuple field (`dict[str, set[int]]`), a set of a - primitive keeps its plain `large_list` storage, because Arrow cannot hold an extension +- **Known limit:** inside a dict value or a tuple field (`dict[str, set[int]]`, + `dict[str, list[set[int]]]`), a set of a primitive keeps its plain `large_list` storage, because Arrow cannot hold an extension type inside a struct field; read back without the declared type, such a value is a list. - `frozenset[T]` is not supported: declare `set[T]`. Support is to be evaluated in ITL-1140. diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index f1fabd453..66523bbc1 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -124,7 +124,8 @@ class ListLogicalType(BaseLogicalType): type. ``False`` for a set of a plain Arrow element: ``set[int]`` is the Python type of ``set[int64]`` and of ``set[int8]`` alike, so such a set is found by its identifier only, and ``set[T]`` maps to Arrow through ``T``'s - own Arrow type, as ``list[T]`` does (ITL-611). + own Arrow type, as ``list[T]`` does (ITL-611). A container takes its + element's value, so ``list[set[int]]`` is found by its identifier only too. Raises: ValueError: If the element is an ``ArrowElement`` and ``is_set`` is ``False``: a @@ -163,6 +164,12 @@ def __init__( self._element_logical_type: LogicalTypeProtocol | None = element_logical_type self._element_python_type = element_logical_type.python_type + # A container of an element found by its identifier only (a set of a plain + # element) is found by its identifier only too: ``list[set[int8]]`` and + # ``list[set[int64]]`` both read as ``list[set[int]]`` (ITL-611). + self.binds_python_type = getattr( + element_logical_type, "binds_python_type", True + ) # Derive Arrow extension details from the element logical type. element_ext_type = element_logical_type.get_arrow_extension_type() diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index a6f38aecf..4a847a29f 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -553,12 +553,15 @@ def _plain_set_storage(self, arrow_type: "pa.DataType") -> "pa.DataType": Arrow cannot hold an extension type inside a struct field, so a set of a plain element type keeps the plain ``large_list`` storage it had before it was a native container where it is a dict value or a tuple field (ITL-611). + So does a native container whose innermost element is such a set + (``list[set[int]]``): its storage holds no extension type either. Args: arrow_type: An Arrow type the converter produced. Returns: - The storage type of a set of a plain element; otherwise ``arrow_type``. + The storage type of a set of a plain element, or of a native container + whose innermost element is one; otherwise ``arrow_type``. """ from orcapod.logical_types.list_logical_type_factory import ListLogicalType @@ -569,8 +572,11 @@ def _plain_set_storage(self, arrow_type: "pa.DataType") -> "pa.DataType": lt = self._logical_type_registry.get_by_arrow_extension_name( arrow_type.extension_name ) - if isinstance(lt, ListLogicalType) and lt.element_logical_type is None: - return arrow_type.storage_type + innermost = lt + while isinstance(innermost, ListLogicalType): + if innermost.element_logical_type is None: + return arrow_type.storage_type + innermost = innermost.element_logical_type return arrow_type def _find_factory_for_class( diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index a247252d9..b9307d81f 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -200,3 +200,12 @@ Containers of logical elements unchanged. Was refused before too, by Arrow's `TypeError`. Hashing, logical types, semantic types, pipeline and database suites pass with it (8322 passed). - ast-grep notes (jsonify x2, subprocess x2): DECLINE, false positives; one signed comment. + +### CodeRabbit CLI (pre-push gate) on e3baffa8: 2 findings, both reproduced, fixed + +- minor list_logical_type_factory.py:234 — list[set[np.int8]] then list[set[int]] raised + (python_type 'list' already bound); main: plain lists, fine. Fix: a container takes its + element's `binds_python_type`. Test: both orders, fresh process. +- major universal_converter.py:572 — dict[str, list[set[int]]] raised (extension in struct); + main: plain storage. Fix: `_plain_set_storage` lowers a container whose innermost element is + a plain set. Tuple part declined: tuple[int, list[int]] is unsupported on main too. diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index 9c4d15ac0..5435efdda 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -647,3 +647,83 @@ def test_a_registered_set_whose_storage_does_not_hold_its_element_is_refused( assert excinfo.value.column == "c" notes = getattr(excinfo.value, "__notes__", []) assert any(str(storage.type) in note for note in notes), notes + + +_TWO_WIDTHS_IN_A_LIST = textwrap.dedent( + """ + import json, sys + import numpy as np + from orcapod.sources import DictSource + + declared = { + "int8": (list[set[np.int8]], [{1, 2}]), + "int64": (list[set[int]], [{1, 300}]), + } + report = {} + for name in sys.argv[1:]: + annotation, value = declared[name] + source = DictSource( + [{"k": 0, "v": value}], + tag_columns=["k"], + data_schema={"k": int, "v": annotation}, + source_id="one_row", + ) + _, data = next(iter(source.iter_data())) + read = data.as_dict()["v"] + report[name] = { + "arrow_type": str(source.as_table().schema.field("v").type), + "schema": str(source.output_schema()[1]["v"]), + "read": [sorted(int(v) for v in s) for s in read], + "read_type": [type(s).__name__ for s in read], + } + print(json.dumps(report)) + """ +) + + +@pytest.mark.parametrize("order", [("int8", "int64"), ("int64", "int8")]) +def test_two_integer_widths_of_a_list_of_sets_live_in_one_process( + order: tuple[str, str], +) -> None: + """``list[set[np.int8]]`` and ``list[set[int]]`` are two containers in one process. + + A list of a set of a plain element is found by its identifier only, as the set is. + """ + result = subprocess.run( + [sys.executable, "-c", _TWO_WIDTHS_IN_A_LIST, *order], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report["int8"] == { + "arrow_type": "extension>", + "schema": "list[set[int]]", + "read": [[1, 2]], + "read_type": ["set"], + } + assert report["int64"] == { + "arrow_type": "extension>", + "schema": "list[set[int]]", + "read": [[1, 300]], + "read_type": ["set"], + } + + +def test_a_list_of_sets_inside_a_dict_value_keeps_plain_storage() -> None: + """``dict[str, list[set[int]]]`` is stored as it was before sets were containers.""" + source = DictSource( + [{"k": 0, "v": {"a": [{2, 1}]}}], + tag_columns=["k"], + data_schema={"k": int, "v": dict[str, list[set[int]]]}, + source_id="one_row", + ) + assert source.as_table().schema.field("v").type == pa.large_list( + pa.struct( + [ + pa.field("key", pa.large_string()), + pa.field("value", pa.large_list(pa.large_list(pa.int64()))), + ] + ) + ) From 106648bc1b484dde3b58d62801b764828e833878 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 19:36:19 -0800 Subject: [PATCH 20/21] fix(logical_types): the list factory reads a set element as its Arrow type does (ITL-611) Co-Authored-By: Claude Opus 5.5 --- .../list_logical_type_factory.py | 9 ++++++--- .../2026-11-14-itl-611-set-of-primitives.md | 8 ++++++++ .../test_set_of_primitives.py | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 66523bbc1..5854b0e02 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -487,11 +487,14 @@ def create_for_python_type( # Directly look up (and register if needed) the LogicalType for the element. element_lt = converter.get_logical_type_for_python_type(element_annotation) if element_lt is None and is_set: - # A set of a plain Arrow type is a native container too (ITL-611). + # A set of a plain Arrow type is a native container too (ITL-611). Its + # element Python type is that of the element Arrow type, as on the read + # path, never the annotation (``Literal[1, 2]`` reads as ``int``). + element_arrow_type = converter.register_python_class(element_annotation) return ListLogicalType( ArrowElement( - element_annotation, - converter.register_python_class(element_annotation), + converter.arrow_type_to_python_type(element_arrow_type), + element_arrow_type, ), is_set=True, ) diff --git a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md index b9307d81f..56f427893 100644 --- a/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md +++ b/superpowers/plans/2026-11-14-itl-611-set-of-primitives.md @@ -209,3 +209,11 @@ Containers of logical elements unchanged. - major universal_converter.py:572 — dict[str, list[set[int]]] raised (extension in struct); main: plain storage. Fix: `_plain_set_storage` lowers a container whose innermost element is a plain set. Tuple part declined: tuple[int, list[int]] is unsupported on main too. + +### Merged main 118b0379 (typecheck hotfix #396). CodeRabbit CLI on the merge (pre-push gate) + +- Run 1: 0 findings, but 11 files `review_failed` (nothing reviewed); retried once. +- Run 2: 1 minor finding, list_logical_type_factory.py:493: the factory's write path for a + set of a plain element kept the annotation (`Literal[1, 2]`) as the element Python type. + Not reachable from the public converter path (generic sets take the converter's own set + branch), but fixed for consistency with the read path; test calls the factory directly. diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index 5435efdda..af68fddd0 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -727,3 +727,22 @@ def test_a_list_of_sets_inside_a_dict_value_keeps_plain_storage() -> None: ] ) ) + + +def test_the_factory_reads_a_set_element_as_its_arrow_type_does() -> None: + """``ListLogicalTypeFactory.create_for_python_type`` gives a set of a plain element + the Python type of its element Arrow type, as reconstruction from Arrow does. + """ + from typing import Literal + + from orcapod.contexts import get_default_context + from orcapod.logical_types.list_logical_type_factory import ListLogicalTypeFactory + + converter = get_default_context().type_converter + # The factory's write path is what this test is about, so it is called directly. + created = ListLogicalTypeFactory().create_for_python_type( + set[Literal[1, 2]], + converter, # pyright: ignore[reportArgumentType] + ) + assert created.logical_type_name == "set[int64]" + assert created.python_type == set[int] From 76679d74e3e5a50117996f0a6127d24e5733cdaa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 20:26:26 -0800 Subject: [PATCH 21/21] fix(logical_types): a set whose metadata names no element is refused as an identifier error (ITL-611) Co-Authored-By: Claude Sonnet 5.5 --- .../list_logical_type_factory.py | 9 ++++++++- .../test_set_of_primitives.py | 19 +++++-------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 5854b0e02..4a453ca15 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -539,7 +539,8 @@ def reconstruct_from_arrow( Raises: LogicalTypeIdentifierError: If the metadata names a plain Arrow element and the storage is not a list of the element the extension name - names, or ``"element_kind"`` is not ``"arrow"``. + names, ``"element_kind"`` is not ``"arrow"``, or a set's metadata + names no element at all. ValueError: If ``storage_type`` is not a list type, or required metadata keys are missing. """ @@ -555,6 +556,12 @@ def reconstruct_from_arrow( ) element_ext_name = metadata.get("element_ext_name") + if not element_ext_name and metadata.get("category") == SET_CATEGORY: + # A set's metadata that names no element is not a form this version + # reads (a set of a primitive writes ``element_kind``). + return self._reconstruct_arrow_element_set( + arrow_extension_name, storage_type, metadata, converter + ) if not element_ext_name: raise ValueError( f"ListLogicalTypeFactory.reconstruct_from_arrow: missing 'element_ext_name' " diff --git a/tests/test_logical_types/test_set_of_primitives.py b/tests/test_logical_types/test_set_of_primitives.py index af68fddd0..5c69ec877 100644 --- a/tests/test_logical_types/test_set_of_primitives.py +++ b/tests/test_logical_types/test_set_of_primitives.py @@ -599,21 +599,12 @@ def test_a_set_with_the_unreleased_element_arrow_type_metadata_is_refused() -> N """Only ``{"category": "set", "element_kind": "arrow"}`` is read for a set of a primitive; the form this pull request first wrote was never released. """ - result = subprocess.run( - [ - sys.executable, - "-c", - _READ_STORED_COLUMN, - "set[int64]", - '{"category": "set", "element_arrow_type": "int64"}', - "large_list", - ], - capture_output=True, - text=True, - check=False, + report = _read_stored_column_in_fresh_process( + "set[int64]", + b'{"category": "set", "element_arrow_type": "int64"}', + "large_list", ) - assert result.returncode != 0 - assert "ValueError" in result.stderr + assert report.get("refused") == "LogicalTypeIdentifierError", report @pytest.mark.parametrize(