From 9f76ebbba1549676e63be90890d6c2c76e795e26 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:27:28 -0800 Subject: [PATCH 1/8] fix(converter): check the logical type identifier when mapping an Arrow type to a Python type (ITL-975) Co-Authored-By: Claude Fable 5.2 --- .../semantic_types/universal_converter.py | 17 +- .../2026-11-12-itl-975-hash-step-follow-up.md | 155 ++++++++++++++++++ .../test_logical_type_identifiers.py | 40 +++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index b26072f83..620cbd35f 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -987,10 +987,20 @@ def arrow_schema_to_python_schema(self, arrow_schema: pa.Schema) -> Schema: Round-trip guarantee: - ``int`` → ``nullable=False`` → ``int`` - ``int | None`` → ``nullable=True`` → ``int | None`` + + Raises: + LogicalTypeIdentifierError: If a column (or a member of one) has the + extension name of a registered logical type with metadata that type + does not read; the error names the column. """ + from orcapod.logical_types.identifier import LogicalTypeIdentifierError + fields = {} for field in arrow_schema: - python_type = self.arrow_type_to_python_type(field.type) + try: + python_type = self.arrow_type_to_python_type(field.type) + except LogicalTypeIdentifierError as exc: + raise exc.for_column(field.name) from exc if field.nullable and python_type is not Any: python_type = python_type | None fields[field.name] = python_type @@ -1351,6 +1361,11 @@ def _convert_arrow_to_python(self, arrow_type: pa.DataType) -> type | Any: arrow_type.extension_name ) if lt is not None: + # The registered class is named only for a column whose values that + # class reads: never by extension name alone. + from orcapod.logical_types.identifier import require_readable + + require_readable(lt, arrow_type.__arrow_ext_serialize__()) return lt.python_type # Handle basic types diff --git a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md new file mode 100644 index 000000000..29fd9fe86 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md @@ -0,0 +1,155 @@ +# ITL-975 Hash Step Follow-up Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the seven small findings left after PR #324's review (ITL-975), each with a +regression test that fails first, without moving any golden value, stored path or stored schema. + +**Architecture:** Three two-to-five line source fixes (a readable-metadata check in the +Arrow-to-Python type mapping, a flag reset in `FunctionJobNode.set_ephemeral_store`, a data +context check at the top of `CachedDataFunction.call` / `async_call`), three test-only +hardenings, and one sentence in an existing changelog fragment. + +**Tech Stack:** Python, pyarrow, pytest, `uv run`. + +**Source of the fixes:** the evaluation report's revision plan +(`.../scratchpad/itl967_base_findings/report.md`; its items 24, 5, 22, 23, 3, 19, 20 are +items 1-7 of the issue). + +**Out of scope (do not touch):** per-row async checks, operator `forward`, cache-hit context +(ITL-951); read-only operator node (ITL-904). + +--- + +### Task 1 (issue item 1): Arrow type to Python type checks the identifier + +**Files:** +- Modify: `src/orcapod/semantic_types/universal_converter.py` (`_convert_arrow_to_python`, about line 1345) +- Test: `tests/test_logical_types/test_logical_type_identifiers.py` + +- [ ] Failing test, after `test_values_of_an_unreadable_identifier_are_not_decoded`: + +```python +def test_python_type_of_an_unreadable_identifier_is_refused() -> None: + """Mapping an Arrow type to a Python type checks the identifier, also when nested.""" + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + table = _typed_thing_column(b'{"v":1}') + column_type = table.schema.field("c").type + + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(column_type) + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_schema_to_python_schema(table.schema) + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(pa.struct([pa.field("t", column_type)])) + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(pa.large_list(column_type)) # if Arrow allows it + + +def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> None: + lenient = _converter(_fresh_context(_thing_logical_type(b'{"v":2}', also_reads=(b'{"v":1}',)))) + column_type = _typed_thing_column(b'{"v":1}').schema.field("c").type + assert lenient.arrow_type_to_python_type(column_type) is Thing +``` + +- [ ] Run: `uv run pytest tests/test_logical_types/test_logical_type_identifiers.py -k python_type_of -q` — first test FAILS (returns the class). +- [ ] Fix: before `return lt.python_type`: + +```python + if lt is not None: + # The schema names the registered class only for a column whose + # values that class reads (never by extension name alone). + from orcapod.logical_types.identifier import require_readable + + require_readable(lt, arrow_type.__arrow_ext_serialize__()) + return lt.python_type +``` + + Check whether a result cache keyed by Arrow type sits in front of this method; if so the + check must run before the cache is consulted or the cache must not hold a refused type. +- [ ] Run the file; PASS. Commit `fix(converter): ...`. + +### Task 2 (item 2): `set_ephemeral_store` resets the stale-table flags + +**Files:** +- Modify: `src/orcapod/core/nodes/function_node.py` (`FunctionJobNode.set_ephemeral_store`, about line 1268) +- Test: `tests/test_pipeline/test_stored_data_context.py`, class `TestSideEffectFunctionNodes` + +- [ ] Failing test `test_a_late_attached_ephemeral_store_is_checked_again`: build the `notify` + node on a fresh store; `get_all_records()` (the node checks its clean stores); build an + ephemeral store holding the result rows (`rdb_v2`) of a written store relabelled `OTHER` + with other keys; attach it through the public `set_ephemeral_store` and + `NodeConfig(is_result_ephemeral=True)` (through `PipelineJob.set_ephemeral_store` / + `apply_node_config` if the paths allow, else on the node); `run()` must raise + `SideEffectLogDataContextError` and `delivered == []`. +- [ ] Run; FAILS (node runs, delivers `[2, 4]`). +- [ ] Fix, at the top of `FunctionJobNode.set_ephemeral_store`: + +```python + # Another store has its own tables of other generations to account for. + self._stale_tables_checked = False + self._current_tables_checked = False + self._unlisted_store_reported = False +``` + +- [ ] Run class; PASS. Commit. + +### Task 3 (item 3): data context check before the cache bypass + +**Files:** +- Modify: `src/orcapod/core/data_function.py` (`CachedDataFunction.call`, `async_call`) +- Test: `tests/test_pipeline/test_stored_data_context.py`, class `TestInputOfAnotherDataContext` + +- [ ] Failing test, parametrized over sync/async and over + `{}`, `{"skip_cache_lookup": True}`, `{"skip_cache_lookup": True, "skip_cache_insert": True}`: + `CachedDataFunction(double.pod.data_function, result_database=store)`; call with + `Data({"x": 1}, data_context=other_context)`; expect `InputValidationError` naming both + contexts, `calls == []`, no table written. +- [ ] Run; the two bypass variants FAIL (function called). +- [ ] Fix: first statement of both methods: + +```python + # Checked before anything else: with ``skip_cache_lookup`` the lookup, which + # also checks it, does not run, and the function must not be called. + require_same_data_context(data.data_context_key, self.data_context_key) +``` + + (import from `orcapod.contexts.stored`; check for an import cycle.) +- [ ] Run; PASS. Commit. + +### Task 4 (item 4): env var in `test_list_table_paths.py` + +- [ ] Show failure: `FSSPEC_S3_ENDPOINT_URL=https://example.test uv run pytest tests/test_databases/test_list_table_paths.py -k http_endpoint -q` FAILS. +- [ ] Add `monkeypatch.delenv("FSSPEC_S3_ENDPOINT_URL", raising=False)` in + `test_an_http_endpoint_from_the_environment_follows_the_same_rule` (and check the sibling + tests of the class under the same variable; clear it wherever it changes the outcome). +- [ ] Same command PASSES. + +### Task 5 (item 5): `changelog.d/ITL-944.md` Migration paragraph + +- [ ] "side-effect nodes fail on such a store" gains "unless the store cannot list its + tables (then the node warns and runs)". `uv run python scripts/assemble_changelog.py --check`. + +### Task 6 (item 6): label literals in `test_serialization_helpers.py` + +- [ ] Replace the eight `"semantic_v1:..."` literals with strings built from + `SEMANTIC_HASH_LABEL` (`orcapod.hashing.semantic_hashing.semantic_hasher`), e.g. + `f"{SEMANTIC_HASH_LABEL}:abc123"`. Run the file. + +### Task 7 (item 7): key-list length in `test_wrong_hash_findings.py` + +- [ ] `sync_keys = stored_keys(...)`, `async_keys = stored_keys(...)`; + `assert len(sync_keys) == len(async_keys) == 2` then equality. Run the test. + +### Task 8: changelog fragment, verification, PR + +- [ ] `changelog.d/ITL-975.md`: `### Fixed` with two bullets (items 1 and 3; item 2 too). +- [ ] `uv run ruff format .`, `uv run ruff check .`, `uv run --with basedpyright basedpyright` (exit 0), + `uv run pytest tests/` on 3.12, golden regeneration with empty diff, + `scripts/assemble_changelog.py --check`, `scripts/design_issues.py --check`. +- [ ] Merge origin/main if moved, push, PR against main (`Fixes ITL-975`), link in Orca, + CodeRabbit rounds, `worker_done` with the reviewed sha. + +## Notes for a successor + +(progress notes are appended here) diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index 24ce53934..daf903ac8 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -555,6 +555,46 @@ def test_values_of_an_unreadable_identifier_are_not_decoded() -> None: strict.arrow_table_to_python_dicts(table) +def test_python_type_of_an_unreadable_identifier_is_refused() -> None: + """Mapping an Arrow type to a Python type checks the identifier (ITL-975). + + The registered class is never named by extension name alone: a schema must not + promise ``Thing`` for a column whose values will be refused. + """ + current = _thing_logical_type(b'{"v":2}') + strict = _converter(_fresh_context(current)) + table = _typed_thing_column(b'{"v":1}') + column_type = table.schema.field("c").type + # The current identifier maps to the class; having mapped it changes nothing below. + assert strict.arrow_type_to_python_type(current.get_arrow_extension_type()) is Thing + + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(column_type) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + strict.arrow_schema_to_python_schema(table.schema) + assert "'c'" in str(excinfo.value) + # A member of a struct column is checked like a top-level column. + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(pa.struct([pa.field("t", column_type)])) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + strict.arrow_schema_to_python_schema( + pa.schema([pa.field("s", pa.struct([pa.field("t", column_type)]))]) + ) + assert "'s'" in str(excinfo.value) + + +def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> None: + """An older identifier the type declares readable maps to the registered class.""" + lenient = _converter( + _fresh_context(_thing_logical_type(b'{"v":2}', also_reads=(b'{"v":1}',))) + ) + table = _typed_thing_column(b'{"v":1}') + + assert lenient.arrow_type_to_python_type(table.schema.field("c").type) is Thing + schema = lenient.arrow_schema_to_python_schema(table.schema) + assert schema["c"] == Thing | None + + def test_list_wrapper_written_before_identifiers_is_read() -> None: """A list column whose metadata records no element metadata is read as the current list type. From d8ca95d41983e818b5a5d8141befaa5798492cf3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:28:14 -0800 Subject: [PATCH 2/8] fix(data_function): check the input's data context before the cache bypass (ITL-975) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/data_function.py | 19 +++++++++- .../test_pipeline/test_stored_data_context.py | 36 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index 13a97817f..7a1c6ee94 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -15,6 +15,7 @@ from orcapod.config import OrcapodConfig from orcapod.contexts import DataContext +from orcapod.contexts.stored import require_same_data_context from orcapod.core.base import TraceableBase from orcapod.core.datagrams import Datagram, Data from orcapod.hashing.hash_utils import ( @@ -1070,7 +1071,15 @@ def call( the underlying function. They are intentionally excluded from the cache key — such arguments are deterministic per row and pod, so the same input data always produces the same output. + + Raises: + InputValidationError: If ``data`` is of another data context than this + function's; the wrapped function has not been called then, whatever + ``skip_cache_lookup`` and ``skip_cache_insert`` are. """ + # Checked before anything else: with ``skip_cache_lookup`` the lookup, which + # checks it too, does not run, and the function must not be called. + require_same_data_context(data.data_context_key, self.data_context_key) output_data = None if not skip_cache_lookup: module_logger.info("Checking for cache...") @@ -1105,7 +1114,15 @@ async def async_call( skip_cache_insert: bool = False, **kwargs: Any, ) -> DataProtocol | None: - """Async counterpart of ``call`` with cache check and recording.""" + """Async counterpart of ``call`` with cache check and recording. + + Raises: + InputValidationError: If ``data`` is of another data context than this + function's; the wrapped function has not been called then. + """ + # Checked before anything else: with ``skip_cache_lookup`` the lookup, which + # checks it too, does not run, and the function must not be called. + require_same_data_context(data.data_context_key, self.data_context_key) output_data = None if not skip_cache_lookup: module_logger.info("Checking for cache...") diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 15e233c4c..5a1795ea4 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -1329,6 +1329,42 @@ def test_cached_function_refuses_an_output_of_another_data_context( _assert_names_both_contexts(info.value) assert reopened(store).list_sources() == [] + @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) + @pytest.mark.parametrize( + "flags", + [ + {}, + {"skip_cache_lookup": True}, + {"skip_cache_lookup": True, "skip_cache_insert": True}, + ], + ids=["default", "no_lookup", "no_lookup_no_insert"], + ) + def test_cached_function_refuses_an_input_of_another_data_context( + self, + store: Store, + other_context: contexts.DataContext, + flags: dict[str, bool], + asynchronous: bool, + ): + """The input's data context is checked before the function is called (ITL-975). + + Also when the cache lookup, which checks it too, is skipped. + """ + import asyncio + + cached = CachedDataFunction(double.pod.data_function, result_database=store) + data = Data({"x": 1}, data_context=other_context) + + with pytest.raises(InputValidationError) as info: + if asynchronous: + _ = asyncio.run(cached.async_call(data, **flags)) + else: + _ = cached.call(data, **flags) + + _assert_names_both_contexts(info.value) + assert calls == [] + assert reopened(store).list_sources() == [] + def test_cached_source_keeps_its_data_context_on_cached_rows(self, store: Store): """Rows read back from the cache carry the source's data context, not the default.""" table = pa.table({"k": ["a", "b"], "x": [1, 2]}) From b67b0cccd483e75d530eb15b18416d776bca49b9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:29:05 -0800 Subject: [PATCH 3/8] fix(function_node): check a newly attached ephemeral store for stale tables (ITL-975) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/nodes/function_node.py | 5 +++ .../test_pipeline/test_stored_data_context.py | 35 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 826123f68..b33f458e9 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1283,6 +1283,11 @@ def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: store: The ``ArrowDatabaseProtocol`` to use for ephemeral result storage, or ``None`` to detach and revert to persistent-only writes. """ + # Another store has its own tables of other generations to account for, + # as in ``attach_databases``. + self._stale_tables_checked = False + self._current_tables_checked = False + self._unlisted_store_reported = False self.ephemeral_result_store = store if store is not None and self._function_pod is not None: self._ephemeral_cached_pod = CachedFunctionPod( diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 5a1795ea4..f46a5ad79 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -48,7 +48,7 @@ from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.side_effects import SideEffectPod, sink_pod from orcapod.sources import ArrowTableSource, DictSource -from orcapod.types import CacheMode +from orcapod.types import CacheMode, NodeConfig Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator Store = InMemoryArrowDatabase | DeltaTableDatabase @@ -1134,6 +1134,39 @@ def test_a_newly_attached_store_is_checked_again( assert delivered == [] + @pytest.mark.parametrize("read_first", [False, True], ids=["unread", "read_first"]) + def test_a_late_attached_ephemeral_store_is_checked_again( + self, new_store: Callable[[], Store], read_first: bool + ): + """An ephemeral store attached after the node checked its stores is checked too + (ITL-975): completions of ``OTHER`` in it raise and nothing is delivered. + """ + relabelled = reopened(_notify_store(new_store, copy_as_other_context)) + ephemeral = new_store() + for path in relabelled.list_sources(): + if path[-1] == "rdb_v2": + rows = relabelled.get_all_records(path, record_id_column=RID) + assert rows is not None + # An ephemeral store holds the result table at the path relative to + # the job's result database. + assert path[:2] == (JOB_NAME, "_result") + ephemeral.add_records(path[2:], rows, record_id_column=RID) + ephemeral.flush() + assert [path[-1] for path in reopened(ephemeral).list_sources()] == ["rdb_v2"] + job = _notify_job(new_store()) + if read_first: + # The node checks its (clean) stores when it first reads them. + assert job.nodes["notify"].get_all_records() is None + + job.set_ephemeral_store(ephemeral) + job.apply_node_config(NodeConfig(is_result_ephemeral=True)) + with pytest.raises(SideEffectLogDataContextError) as info: + _ = job.run() + + assert delivered == [] + assert info.value.found_context == OTHER + assert [path[-1] for path in info.value.record_paths] == ["rdb_v2"] + def test_plain_function_node_in_the_same_store_is_only_reported( self, new_store: Callable[[], Store] ): From c264bae4c34c2df255082b45d2a95839a67e3441 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:29:50 -0800 Subject: [PATCH 4/8] test: harden three tests and qualify the ITL-944 migration sentence; changelog fragment (ITL-975) Co-Authored-By: Claude Fable 5.2 --- changelog.d/ITL-944.md | 3 ++- changelog.d/ITL-975.md | 11 +++++++++++ tests/test_databases/test_list_table_paths.py | 4 ++++ tests/test_hashing/test_wrong_hash_findings.py | 6 +++++- .../test_pipeline/test_serialization_helpers.py | 17 +++++++++-------- 5 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 changelog.d/ITL-975.md diff --git a/changelog.d/ITL-944.md b/changelog.d/ITL-944.md index 2b22afdc2..8b801eec1 100644 --- a/changelog.d/ITL-944.md +++ b/changelog.d/ITL-944.md @@ -124,7 +124,8 @@ read its tables as results, and computes every input again into its own tables. **Migration:** none is needed to run the function, operator and source nodes of a v0.2 store under v0.3, at the price of computing everything again; side-effect nodes fail - on such a store, and on a fresh store they perform every side effect again. A tool that converts old-layout tables is planned (ITL-913, ITL-914); until + on such a store, unless the store cannot list its tables (then the node warns and + runs), and on a fresh store they perform every side effect again. A tool that converts old-layout tables is planned (ITL-913, ITL-914); until then they are kept, untouched, for it. ### Added diff --git a/changelog.d/ITL-975.md b/changelog.d/ITL-975.md new file mode 100644 index 000000000..7d3245b44 --- /dev/null +++ b/changelog.d/ITL-975.md @@ -0,0 +1,11 @@ +### Fixed + +- `arrow_type_to_python_type` and `arrow_schema_to_python_schema` no longer name a logical + type's Python class for a column (or a member of a struct column) whose extension metadata + that type does not read: they raise `LogicalTypeIdentifierError`, as reading the values + already did (ITL-975). +- `CachedDataFunction.call` and `async_call` refuse an input of another data context before + the function is called, also with `skip_cache_lookup=True` (ITL-975). +- A side-effecting function node checks an ephemeral result store attached with + `set_ephemeral_store` for completions of another data context, also when the store is + attached after the node first read its stores (ITL-975). diff --git a/tests/test_databases/test_list_table_paths.py b/tests/test_databases/test_list_table_paths.py index b1d081d90..95654bde9 100644 --- a/tests/test_databases/test_list_table_paths.py +++ b/tests/test_databases/test_list_table_paths.py @@ -208,6 +208,10 @@ def filesystem(protocol: str, **options: object) -> DirFileSystem: return DirFileSystem(path=str(tmp_path), fs=LocalFileSystem()) monkeypatch.setattr(fsspec, "filesystem", filesystem) + # The endpoint is also read from the environment: these tests must not depend + # on what the machine running them has set (ITL-975). + monkeypatch.delenv("FSSPEC_S3_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) _fill(DeltaTableDatabase(tmp_path / "bucket" / "root")) return calls diff --git a/tests/test_hashing/test_wrong_hash_findings.py b/tests/test_hashing/test_wrong_hash_findings.py index 37a8589c1..093562627 100644 --- a/tests/test_hashing/test_wrong_hash_findings.py +++ b/tests/test_hashing/test_wrong_hash_findings.py @@ -705,7 +705,11 @@ async def run_async() -> tuple[Any, Any]: return db, node async_db, async_node = asyncio.run(run_async()) - assert stored_keys(sync_db, sync_node) == stored_keys(async_db, async_node) + sync_keys = stored_keys(sync_db, sync_node) + async_keys = stored_keys(async_db, async_node) + # One key per input row on each path: two equal empty lists must not pass. + assert len(sync_keys) == len(async_keys) == 2 + assert sync_keys == async_keys # --------------------------------------------------------------------------- diff --git a/tests/test_pipeline/test_serialization_helpers.py b/tests/test_pipeline/test_serialization_helpers.py index ce232e4e3..874ed8f46 100644 --- a/tests/test_pipeline/test_serialization_helpers.py +++ b/tests/test_pipeline/test_serialization_helpers.py @@ -20,6 +20,7 @@ serialize_schema, ) from orcapod.databases.in_memory_databases import InMemoryArrowDatabase +from orcapod.hashing.semantic_hashing.semantic_hasher import SEMANTIC_HASH_LABEL from orcapod.core.operators import Join, Batch from orcapod.types import Schema @@ -695,15 +696,15 @@ def test_source_proxy_from_node_descriptor_fields(): source_config = {"source_type": "dict", "source_id": "my_src"} node_descriptor = { - "content_hash": "semantic_v1:abc123", - "pipeline_hash": "semantic_v1:def456", + "content_hash": f"{SEMANTIC_HASH_LABEL}:abc123", + "pipeline_hash": f"{SEMANTIC_HASH_LABEL}:def456", "output_schema": { "tag": {"x": "int64"}, "data": {"result": "int64"}, }, } proxy = _source_proxy_from_config(source_config, node_descriptor=node_descriptor) - assert proxy.content_hash().to_string() == "semantic_v1:abc123" + assert proxy.content_hash().to_string() == f"{SEMANTIC_HASH_LABEL}:abc123" def test_source_proxy_from_config_backward_compat(): @@ -713,13 +714,13 @@ def test_source_proxy_from_config_backward_compat(): source_config = { "source_type": "dict", "source_id": "my_src", - "content_hash": "semantic_v1:abc123", - "pipeline_hash": "semantic_v1:def456", + "content_hash": f"{SEMANTIC_HASH_LABEL}:abc123", + "pipeline_hash": f"{SEMANTIC_HASH_LABEL}:def456", "tag_schema": {"x": "int64"}, "data_schema": {"result": "int64"}, } proxy = _source_proxy_from_config(source_config) - assert proxy.content_hash().to_string() == "semantic_v1:abc123" + assert proxy.content_hash().to_string() == f"{SEMANTIC_HASH_LABEL}:abc123" # --------------------------------------------------------------------------- @@ -762,8 +763,8 @@ def test_function_node_stored_node_uri_from_descriptor(): descriptor = { "node_type": "function", "label": "fn", - "content_hash": "semantic_v1:abc", - "pipeline_hash": "semantic_v1:def", + "content_hash": f"{SEMANTIC_HASH_LABEL}:abc", + "pipeline_hash": f"{SEMANTIC_HASH_LABEL}:def", "table_scope": "pipeline_hash", "node_uri": ["add_one", "v0", "python.function.v0", "schema_repr"], "output_schema": {"tag": {"x": "int64"}, "data": {"result": "int64"}}, From 988a83612187efe0a153ef1d136cea537150391f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:32:10 -0800 Subject: [PATCH 5/8] test: pass the cache flags by name; assign unused results (ITL-975) Co-Authored-By: Claude Fable 5.2 --- .../test_logical_type_identifiers.py | 8 +++---- .../test_pipeline/test_stored_data_context.py | 23 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index daf903ac8..14db547c2 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -569,15 +569,15 @@ def test_python_type_of_an_unreadable_identifier_is_refused() -> None: assert strict.arrow_type_to_python_type(current.get_arrow_extension_type()) is Thing with pytest.raises(LogicalTypeIdentifierError): - strict.arrow_type_to_python_type(column_type) + _ = strict.arrow_type_to_python_type(column_type) with pytest.raises(LogicalTypeIdentifierError) as excinfo: - strict.arrow_schema_to_python_schema(table.schema) + _ = strict.arrow_schema_to_python_schema(table.schema) assert "'c'" in str(excinfo.value) # A member of a struct column is checked like a top-level column. with pytest.raises(LogicalTypeIdentifierError): - strict.arrow_type_to_python_type(pa.struct([pa.field("t", column_type)])) + _ = strict.arrow_type_to_python_type(pa.struct([pa.field("t", column_type)])) with pytest.raises(LogicalTypeIdentifierError) as excinfo: - strict.arrow_schema_to_python_schema( + _ = strict.arrow_schema_to_python_schema( pa.schema([pa.field("s", pa.struct([pa.field("t", column_type)]))]) ) assert "'s'" in str(excinfo.value) diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index f46a5ad79..522e00640 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -1364,19 +1364,16 @@ def test_cached_function_refuses_an_output_of_another_data_context( @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize( - "flags", - [ - {}, - {"skip_cache_lookup": True}, - {"skip_cache_lookup": True, "skip_cache_insert": True}, - ], + ("skip_lookup", "skip_insert"), + [(False, False), (True, False), (True, True)], ids=["default", "no_lookup", "no_lookup_no_insert"], ) def test_cached_function_refuses_an_input_of_another_data_context( self, store: Store, other_context: contexts.DataContext, - flags: dict[str, bool], + skip_lookup: bool, + skip_insert: bool, asynchronous: bool, ): """The input's data context is checked before the function is called (ITL-975). @@ -1390,9 +1387,17 @@ def test_cached_function_refuses_an_input_of_another_data_context( with pytest.raises(InputValidationError) as info: if asynchronous: - _ = asyncio.run(cached.async_call(data, **flags)) + _ = asyncio.run( + cached.async_call( + data, + skip_cache_lookup=skip_lookup, + skip_cache_insert=skip_insert, + ) + ) else: - _ = cached.call(data, **flags) + _ = cached.call( + data, skip_cache_lookup=skip_lookup, skip_cache_insert=skip_insert + ) _assert_names_both_contexts(info.value) assert calls == [] From c982e96bff2d401cfffa261193810f94ec0f7454 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:36:25 -0800 Subject: [PATCH 6/8] docs(plans): progress notes for ITL-975 Co-Authored-By: Claude Fable 5.2 --- superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md index 29fd9fe86..d15a2d045 100644 --- a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md +++ b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md @@ -152,4 +152,8 @@ def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> No ## Notes for a successor -(progress notes are appended here) +- 2026-11-12: Tasks 1-8 implemented and pushed (head 988a8361). PR #326 opened against main. + Full suite on 3.12: 8913 passed, 101 skipped, 18 xfailed. Golden regeneration: empty diff. + Item 1 also names the column in `arrow_schema_to_python_schema` (caller wraps with + `for_column`). Item 4 clears both endpoint variables in the `bucket` fixture. + Remaining: CI green, CodeRabbit round(s), `worker_done` with the reviewed sha. From 1d6125b5b2b50e5eda8c2c47b1cf9fa807dfc99a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 04:53:50 -0800 Subject: [PATCH 7/8] fix(converter): check nested identifiers before the type mapping cache (ITL-975) Arrow compares two instances of one extension class without their metadata, so a container remembered for a readable identifier answered for an unreadable one. Co-Authored-By: Claude Fable 5.2 --- .../semantic_types/universal_converter.py | 44 ++++++++++++++ .../2026-11-12-itl-975-hash-step-follow-up.md | 9 ++- .../test_logical_type_identifiers.py | 57 +++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 620cbd35f..7e169920a 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -908,7 +908,16 @@ def arrow_type_to_python_type(self, arrow_type: pa.DataType) -> DataType: This is the main entry point for Arrow → Python type conversion. Results are cached for performance. + + Raises: + LogicalTypeIdentifierError: If the type, or a member nested in it, has + the extension name of a registered logical type with metadata that + type does not read. """ + # Before the cache: Arrow compares two instances of one extension class + # without their metadata, so a container remembered for a readable + # identifier is equal to one whose member carries an unreadable identifier. + self._require_readable_members(arrow_type) try: if arrow_type in self._arrow_to_python_types: return self._arrow_to_python_types[arrow_type] @@ -923,6 +932,41 @@ def arrow_type_to_python_type(self, arrow_type: pa.DataType) -> DataType: pass # Unhashable type — skip caching. return python_type + def _require_readable_members(self, arrow_type: pa.DataType) -> None: + """Raise unless every registered extension type in ``arrow_type`` is readable. + + Walks the type itself, struct fields, list values and map keys and items. + + Args: + arrow_type: The Arrow type to check. + + Raises: + LogicalTypeIdentifierError: If an extension type whose name is registered + carries metadata the registered logical type does not read. + """ + if isinstance(arrow_type, pa.ExtensionType): + if self._logical_type_registry is not None: + lt = self._logical_type_registry.get_by_arrow_extension_name( + arrow_type.extension_name + ) + if lt is not None: + from orcapod.logical_types.identifier import require_readable + + require_readable(lt, arrow_type.__arrow_ext_serialize__()) + self._require_readable_members(arrow_type.storage_type) + elif pa.types.is_struct(arrow_type): + for field in arrow_type: + self._require_readable_members(field.type) + elif ( + pa.types.is_list(arrow_type) + or pa.types.is_large_list(arrow_type) + or pa.types.is_fixed_size_list(arrow_type) + ): + self._require_readable_members(arrow_type.value_type) + elif pa.types.is_map(arrow_type): + self._require_readable_members(arrow_type.key_type) + self._require_readable_members(arrow_type.item_type) + def get_logical_type_by_arrow_extension_name( self, arrow_extension_name: str ) -> "LogicalTypeProtocol | None": diff --git a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md index d15a2d045..5f33202ca 100644 --- a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md +++ b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md @@ -5,7 +5,7 @@ **Goal:** Fix the seven small findings left after PR #324's review (ITL-975), each with a regression test that fails first, without moving any golden value, stored path or stored schema. -**Architecture:** Three two-to-five line source fixes (a readable-metadata check in the +**Architecture:** Three two-to-five-line source fixes (a readable-metadata check in the Arrow-to-Python type mapping, a flag reset in `FunctionJobNode.set_ephemeral_store`, a data context check at the top of `CachedDataFunction.call` / `async_call`), three test-only hardenings, and one sentence in an existing changelog fragment. @@ -157,3 +157,10 @@ def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> No Item 1 also names the column in `arrow_schema_to_python_schema` (caller wraps with `for_column`). Item 4 clears both endpoint variables in the `bucket` fixture. Remaining: CI green, CodeRabbit round(s), `worker_done` with the reviewed sha. +- 2026-11-12, CodeRabbit round 1 (head c982e96b): one finding, reproduced. The cache in + `arrow_type_to_python_type` is keyed by the Arrow type, and Arrow compares two instances + of one extension class without their metadata, so a struct remembered for a readable + identifier answered for an unreadable one. Coordinator approved option A: walk the type + before the cache lookup (`_require_readable_members`), keep the check in + `_convert_arrow_to_python`. Stop rule after this push: repeats, base findings and style + notes are answered without a further push. diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index 14db547c2..271887a88 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -14,6 +14,7 @@ import pathlib import typing import uuid +from collections.abc import Callable from typing import Any import numpy as np @@ -583,6 +584,62 @@ def test_python_type_of_an_unreadable_identifier_is_refused() -> None: assert "'s'" in str(excinfo.value) +class _ParameterizedThingType(pa.ExtensionType): + """One extension class for ``thirdparty.thing`` whose metadata is a parameter. + + Arrow compares two instances of one extension class without their metadata, so + they are equal whatever identifiers they carry. + """ + + def __init__(self, metadata: bytes) -> None: + """Store ``metadata`` as the serialized extension metadata.""" + self._metadata = metadata + super().__init__(pa.large_string(), "thirdparty.thing") + + def __arrow_ext_serialize__(self) -> bytes: + """Return the metadata given at construction.""" + return self._metadata + + @classmethod + def __arrow_ext_deserialize__( + cls, storage_type: pa.DataType, serialized: bytes + ) -> "_ParameterizedThingType": + """Rebuild the type from its serialized metadata.""" + return cls(serialized) + + +@pytest.mark.parametrize( + "container", + [ + lambda member: pa.struct([pa.field("t", member)]), + lambda member: pa.large_list(pa.struct([pa.field("t", member)])), + lambda member: pa.map_(pa.large_string(), pa.struct([pa.field("t", member)])), + ], + ids=["struct", "list_of_struct", "map_of_struct"], +) +def test_python_type_of_an_unreadable_member_is_refused_after_a_readable_one( + container: Callable[[pa.DataType], pa.DataType], +) -> None: + """A mapping remembered for a readable identifier never answers for an unreadable + one (ITL-975): two containers that differ only in a member's metadata are equal + to Arrow, and the identifier is still checked for each. + """ + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + readable = container(_ParameterizedThingType(b'{"v":2}')) + unreadable = container(_ParameterizedThingType(b'{"v":1}')) + assert readable == unreadable # what makes a lookup by Arrow type alone unsafe + + # The readable container is mapped first, through both entry points. + _ = strict.arrow_type_to_python_type(readable) + _ = strict.arrow_schema_to_python_schema(pa.schema([pa.field("c", readable)])) + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.arrow_type_to_python_type(unreadable) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + _ = strict.arrow_schema_to_python_schema(pa.schema([pa.field("c", unreadable)])) + assert "'c'" in str(excinfo.value) + + def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> None: """An older identifier the type declares readable maps to the registered class.""" lenient = _converter( From d01847d958cc455dc1549c83745143dee2cd515a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 05:18:04 -0800 Subject: [PATCH 8/8] fix(converter): check nested identifiers before both Arrow-keyed caches, unions included (ITL-975) Co-Authored-By: Claude Fable 5.2 --- changelog.d/ITL-975.md | 4 +- .../semantic_types/universal_converter.py | 35 ++++++--- .../2026-11-12-itl-975-hash-step-follow-up.md | 10 +++ .../test_logical_type_identifiers.py | 75 ++++++++++++++++++- 4 files changed, 109 insertions(+), 15 deletions(-) diff --git a/changelog.d/ITL-975.md b/changelog.d/ITL-975.md index 7d3245b44..f257caea4 100644 --- a/changelog.d/ITL-975.md +++ b/changelog.d/ITL-975.md @@ -3,7 +3,9 @@ - `arrow_type_to_python_type` and `arrow_schema_to_python_schema` no longer name a logical type's Python class for a column (or a member of a struct column) whose extension metadata that type does not read: they raise `LogicalTypeIdentifierError`, as reading the values - already did (ITL-975). + already did. The check, and the one made before values are decoded + (`arrow_table_to_python_dicts`), also holds for a type nested in a struct, list, map or + union when an equal-looking type with readable metadata was converted before (ITL-975). - `CachedDataFunction.call` and `async_call` refuse an input of another data context before the function is called, also with `skip_cache_lookup=True` (ITL-975). - A side-effecting function node checks an ephemeral result store attached with diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index ee12c9d74..28b679490 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -937,7 +937,15 @@ def arrow_type_to_python_type(self, arrow_type: pa.DataType) -> DataType: def _require_readable_members(self, arrow_type: pa.DataType) -> None: """Raise unless every registered extension type in ``arrow_type`` is readable. - Walks the type itself, struct fields, list values and map keys and items. + Walks the type itself and every type nested in it: the storage of an + extension type, the fields of a struct or a union, the values of a list, the + keys and items of a map, and the values of a dictionary type. + + This runs before every lookup in a cache keyed by an Arrow type + (``_arrow_to_python_types``, ``_arrow_to_python_converters`` and, behind the + first, ``_struct_signature_to_typeddict``): Arrow compares two instances of + one extension class without their metadata, so such a key does not tell a + readable identifier from an unreadable one. Args: arrow_type: The Arrow type to check. @@ -956,18 +964,12 @@ def _require_readable_members(self, arrow_type: pa.DataType) -> None: require_readable(lt, arrow_type.__arrow_ext_serialize__()) self._require_readable_members(arrow_type.storage_type) - elif pa.types.is_struct(arrow_type): - for field in arrow_type: - self._require_readable_members(field.type) - elif ( - pa.types.is_list(arrow_type) - or pa.types.is_large_list(arrow_type) - or pa.types.is_fixed_size_list(arrow_type) - ): + elif pa.types.is_dictionary(arrow_type): self._require_readable_members(arrow_type.value_type) - elif pa.types.is_map(arrow_type): - self._require_readable_members(arrow_type.key_type) - self._require_readable_members(arrow_type.item_type) + else: + # Struct and union fields, a list's value field, a map's entries field. + for index in range(arrow_type.num_fields): + self._require_readable_members(arrow_type.field(index).type) def get_logical_type_by_arrow_extension_name( self, arrow_extension_name: str @@ -1233,7 +1235,16 @@ def get_arrow_to_python_converter( This creates and caches conversion functions for optimal performance during data conversion operations. + + Raises: + LogicalTypeIdentifierError: If the type, or a member nested in it, has + the extension name of a registered logical type with metadata that + type does not read. """ + # Before the cache, as in ``arrow_type_to_python_type``: a converter + # remembered for a readable identifier must not decode values stored under + # an unreadable one. + self._require_readable_members(arrow_type) try: if arrow_type in self._arrow_to_python_converters: return self._arrow_to_python_converters[arrow_type] diff --git a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md index 5f33202ca..483b8d9d8 100644 --- a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md +++ b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md @@ -164,3 +164,13 @@ def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> No before the cache lookup (`_require_readable_members`), keep the check in `_convert_arrow_to_python`. Stop rule after this push: repeats, base findings and style notes are answered without a further push. +- 2026-11-12, CodeRabbit round 2 (head ce97810a, after merging origin/main): three threads. + (1) the walk skipped unions: it now visits every nested type (`num_fields` / `field(i)`, + extension storage, dictionary values). (2) base finding, included on the coordinator's + decision: `get_arrow_to_python_converter` returned a cached value converter before the + readable check; the walk now runs before that cache too. (3) unquoted annotation. + Sweep of caches keyed by an Arrow type in `universal_converter.py`: + `_arrow_to_python_types` (walk before lookup), `_arrow_to_python_converters` (walk before + lookup), `_struct_signature_to_typeddict` (reached only from `_convert_arrow_to_python`, + behind `arrow_type_to_python_type`'s walk). The other caches are keyed by Python types. + After this push the stop rule applies. diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index 271887a88..37ff00b59 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -603,7 +603,7 @@ def __arrow_ext_serialize__(self) -> bytes: @classmethod def __arrow_ext_deserialize__( cls, storage_type: pa.DataType, serialized: bytes - ) -> "_ParameterizedThingType": + ) -> _ParameterizedThingType: """Rebuild the type from its serialized metadata.""" return cls(serialized) @@ -614,8 +614,14 @@ def __arrow_ext_deserialize__( lambda member: pa.struct([pa.field("t", member)]), lambda member: pa.large_list(pa.struct([pa.field("t", member)])), lambda member: pa.map_(pa.large_string(), pa.struct([pa.field("t", member)])), + lambda member: pa.dense_union( + [pa.field("i", pa.int64()), pa.field("t", member)] + ), + lambda member: pa.sparse_union( + [pa.field("i", pa.int64()), pa.field("t", member)] + ), ], - ids=["struct", "list_of_struct", "map_of_struct"], + ids=["struct", "list_of_struct", "map_of_struct", "dense_union", "sparse_union"], ) def test_python_type_of_an_unreadable_member_is_refused_after_a_readable_one( container: Callable[[pa.DataType], pa.DataType], @@ -640,6 +646,71 @@ def test_python_type_of_an_unreadable_member_is_refused_after_a_readable_one( assert "'c'" in str(excinfo.value) +def _parameterized_thing_column(metadata: bytes) -> "pa.Array[Any]": + """One ``thirdparty.thing`` value typed by ``_ParameterizedThingType(metadata)``.""" + return pa.ExtensionArray.from_storage( + _ParameterizedThingType(metadata), pa.array(["a"], pa.large_string()) + ) + + +def _struct_column(member: "pa.Array[Any]") -> "pa.Array[Any]": + """A struct column with ``member`` as its field ``t``.""" + return pa.StructArray.from_arrays([member], fields=[pa.field("t", member.type)]) + + +def _list_of_struct_column(member: "pa.Array[Any]") -> "pa.Array[Any]": + """A one-row list column whose element is a struct with ``member`` as field ``t``.""" + return pa.LargeListArray.from_arrays( + pa.array([0, 1], pa.int64()), _struct_column(member) + ) + + +@pytest.mark.parametrize( + "column", + [_struct_column, _list_of_struct_column], + ids=["struct", "list_of_struct"], +) +def test_values_of_an_unreadable_member_are_not_decoded_after_a_readable_one( + column: Callable[["pa.Array[Any]"], "pa.Array[Any]"], +) -> None: + """A value converter remembered for a readable identifier never decodes values + stored under an unreadable one (ITL-975): the two columns' types are equal to Arrow. + """ + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + readable = pa.table({"c": column(_parameterized_thing_column(b'{"v":2}'))}) + unreadable = pa.table({"c": column(_parameterized_thing_column(b'{"v":1}'))}) + assert readable.schema.field("c").type == unreadable.schema.field("c").type + + # The readable table is converted first, so its converter is remembered. + assert len(strict.arrow_table_to_python_dicts(readable)) == 1 + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.arrow_table_to_python_dicts(unreadable) + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.get_arrow_to_python_converter(unreadable.schema.field("c").type) + + +@pytest.mark.parametrize("mode", ["dense", "sparse"]) +def test_value_converter_of_an_unreadable_union_member_is_refused_after_a_readable_one( + mode: str, +) -> None: + """The same holds for a member of a union type (ITL-975).""" + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + + def union(metadata: bytes) -> pa.DataType: + """A union of ``int64`` and a ``thirdparty.thing`` carrying ``metadata``.""" + fields = [ + pa.field("i", pa.int64()), + pa.field("t", _ParameterizedThingType(metadata)), + ] + return pa.dense_union(fields) if mode == "dense" else pa.sparse_union(fields) + + _ = strict.get_arrow_to_python_converter(union(b'{"v":2}')) + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.get_arrow_to_python_converter(union(b'{"v":1}')) + + def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> None: """An older identifier the type declares readable maps to the registered class.""" lenient = _converter(