diff --git a/changelog.d/ITL-944.md b/changelog.d/ITL-944.md index 2b22afdc2..8b801eec1 100644 --- a/changelog.d/ITL-944.md +++ b/changelog.d/ITL-944.md @@ -124,7 +124,8 @@ read its tables as results, and computes every input again into its own tables. **Migration:** none is needed to run the function, operator and source nodes of a v0.2 store under v0.3, at the price of computing everything again; side-effect nodes fail - on such a store, and on a fresh store they perform every side effect again. A tool that converts old-layout tables is planned (ITL-913, ITL-914); until + on such a store, unless the store cannot list its tables (then the node warns and + runs), and on a fresh store they perform every side effect again. A tool that converts old-layout tables is planned (ITL-913, ITL-914); until then they are kept, untouched, for it. ### Added diff --git a/changelog.d/ITL-975.md b/changelog.d/ITL-975.md new file mode 100644 index 000000000..f257caea4 --- /dev/null +++ b/changelog.d/ITL-975.md @@ -0,0 +1,13 @@ +### Fixed + +- `arrow_type_to_python_type` and `arrow_schema_to_python_schema` no longer name a logical + type's Python class for a column (or a member of a struct column) whose extension metadata + that type does not read: they raise `LogicalTypeIdentifierError`, as reading the values + already did. The check, and the one made before values are decoded + (`arrow_table_to_python_dicts`), also holds for a type nested in a struct, list, map or + union when an equal-looking type with readable metadata was converted before (ITL-975). +- `CachedDataFunction.call` and `async_call` refuse an input of another data context before + the function is called, also with `skip_cache_lookup=True` (ITL-975). +- A side-effecting function node checks an ephemeral result store attached with + `set_ephemeral_store` for completions of another data context, also when the store is + attached after the node first read its stores (ITL-975). diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index c340086cc..15eaa2180 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -15,6 +15,7 @@ from orcapod.config import OrcapodConfig from orcapod.contexts import DataContext +from orcapod.contexts.stored import require_same_data_context from orcapod.core.base import TraceableBase from orcapod.core.datagrams import Datagram, Data from orcapod.hashing.hash_utils import ( @@ -1073,7 +1074,15 @@ def call( the underlying function. They are intentionally excluded from the cache key — such arguments are deterministic per row and pod, so the same input data always produces the same output. + + Raises: + InputValidationError: If ``data`` is of another data context than this + function's; the wrapped function has not been called then, whatever + ``skip_cache_lookup`` and ``skip_cache_insert`` are. """ + # Checked before anything else: with ``skip_cache_lookup`` the lookup, which + # checks it too, does not run, and the function must not be called. + require_same_data_context(data.data_context_key, self.data_context_key) output_data = None if not skip_cache_lookup: module_logger.info("Checking for cache...") @@ -1108,7 +1117,15 @@ async def async_call( skip_cache_insert: bool = False, **kwargs: Any, ) -> DataProtocol | None: - """Async counterpart of ``call`` with cache check and recording.""" + """Async counterpart of ``call`` with cache check and recording. + + Raises: + InputValidationError: If ``data`` is of another data context than this + function's; the wrapped function has not been called then. + """ + # Checked before anything else: with ``skip_cache_lookup`` the lookup, which + # checks it too, does not run, and the function must not be called. + require_same_data_context(data.data_context_key, self.data_context_key) output_data = None if not skip_cache_lookup: module_logger.info("Checking for cache...") diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index e6bd9ff2f..74104c0e6 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1284,6 +1284,11 @@ def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: store: The ``ArrowDatabaseProtocol`` to use for ephemeral result storage, or ``None`` to detach and revert to persistent-only writes. """ + # Another store has its own tables of other generations to account for, + # as in ``attach_databases``. + self._stale_tables_checked = False + self._current_tables_checked = False + self._unlisted_store_reported = False self.ephemeral_result_store = store if store is not None and self._function_pod is not None: self._ephemeral_cached_pod = CachedFunctionPod( diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index 2ebd21582..28b679490 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -910,7 +910,16 @@ def arrow_type_to_python_type(self, arrow_type: pa.DataType) -> DataType: This is the main entry point for Arrow → Python type conversion. Results are cached for performance. + + Raises: + LogicalTypeIdentifierError: If the type, or a member nested in it, has + the extension name of a registered logical type with metadata that + type does not read. """ + # Before the cache: Arrow compares two instances of one extension class + # without their metadata, so a container remembered for a readable + # identifier is equal to one whose member carries an unreadable identifier. + self._require_readable_members(arrow_type) try: if arrow_type in self._arrow_to_python_types: return self._arrow_to_python_types[arrow_type] @@ -925,6 +934,43 @@ def arrow_type_to_python_type(self, arrow_type: pa.DataType) -> DataType: pass # Unhashable type — skip caching. return python_type + def _require_readable_members(self, arrow_type: pa.DataType) -> None: + """Raise unless every registered extension type in ``arrow_type`` is readable. + + Walks the type itself and every type nested in it: the storage of an + extension type, the fields of a struct or a union, the values of a list, the + keys and items of a map, and the values of a dictionary type. + + This runs before every lookup in a cache keyed by an Arrow type + (``_arrow_to_python_types``, ``_arrow_to_python_converters`` and, behind the + first, ``_struct_signature_to_typeddict``): Arrow compares two instances of + one extension class without their metadata, so such a key does not tell a + readable identifier from an unreadable one. + + Args: + arrow_type: The Arrow type to check. + + Raises: + LogicalTypeIdentifierError: If an extension type whose name is registered + carries metadata the registered logical type does not read. + """ + if isinstance(arrow_type, pa.ExtensionType): + if self._logical_type_registry is not None: + lt = self._logical_type_registry.get_by_arrow_extension_name( + arrow_type.extension_name + ) + if lt is not None: + from orcapod.logical_types.identifier import require_readable + + require_readable(lt, arrow_type.__arrow_ext_serialize__()) + self._require_readable_members(arrow_type.storage_type) + elif pa.types.is_dictionary(arrow_type): + self._require_readable_members(arrow_type.value_type) + else: + # Struct and union fields, a list's value field, a map's entries field. + for index in range(arrow_type.num_fields): + self._require_readable_members(arrow_type.field(index).type) + def get_logical_type_by_arrow_extension_name( self, arrow_extension_name: str ) -> "LogicalTypeProtocol | None": @@ -989,10 +1035,20 @@ def arrow_schema_to_python_schema(self, arrow_schema: pa.Schema) -> Schema: Round-trip guarantee: - ``int`` → ``nullable=False`` → ``int`` - ``int | None`` → ``nullable=True`` → ``int | None`` + + Raises: + LogicalTypeIdentifierError: If a column (or a member of one) has the + extension name of a registered logical type with metadata that type + does not read; the error names the column. """ + from orcapod.logical_types.identifier import LogicalTypeIdentifierError + fields = {} for field in arrow_schema: - python_type = self.arrow_type_to_python_type(field.type) + try: + python_type = self.arrow_type_to_python_type(field.type) + except LogicalTypeIdentifierError as exc: + raise exc.for_column(field.name) from exc if field.nullable and python_type is not Any: python_type = python_type | None fields[field.name] = python_type @@ -1179,7 +1235,16 @@ def get_arrow_to_python_converter( This creates and caches conversion functions for optimal performance during data conversion operations. + + Raises: + LogicalTypeIdentifierError: If the type, or a member nested in it, has + the extension name of a registered logical type with metadata that + type does not read. """ + # Before the cache, as in ``arrow_type_to_python_type``: a converter + # remembered for a readable identifier must not decode values stored under + # an unreadable one. + self._require_readable_members(arrow_type) try: if arrow_type in self._arrow_to_python_converters: return self._arrow_to_python_converters[arrow_type] @@ -1353,6 +1418,11 @@ def _convert_arrow_to_python(self, arrow_type: pa.DataType) -> type | Any: arrow_type.extension_name ) if lt is not None: + # The registered class is named only for a column whose values that + # class reads: never by extension name alone. + from orcapod.logical_types.identifier import require_readable + + require_readable(lt, arrow_type.__arrow_ext_serialize__()) return lt.python_type # Handle basic types diff --git a/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md new file mode 100644 index 000000000..483b8d9d8 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-975-hash-step-follow-up.md @@ -0,0 +1,176 @@ +# ITL-975 Hash Step Follow-up Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the seven small findings left after PR #324's review (ITL-975), each with a +regression test that fails first, without moving any golden value, stored path or stored schema. + +**Architecture:** Three two-to-five-line source fixes (a readable-metadata check in the +Arrow-to-Python type mapping, a flag reset in `FunctionJobNode.set_ephemeral_store`, a data +context check at the top of `CachedDataFunction.call` / `async_call`), three test-only +hardenings, and one sentence in an existing changelog fragment. + +**Tech Stack:** Python, pyarrow, pytest, `uv run`. + +**Source of the fixes:** the evaluation report's revision plan +(`.../scratchpad/itl967_base_findings/report.md`; its items 24, 5, 22, 23, 3, 19, 20 are +items 1-7 of the issue). + +**Out of scope (do not touch):** per-row async checks, operator `forward`, cache-hit context +(ITL-951); read-only operator node (ITL-904). + +--- + +### Task 1 (issue item 1): Arrow type to Python type checks the identifier + +**Files:** +- Modify: `src/orcapod/semantic_types/universal_converter.py` (`_convert_arrow_to_python`, about line 1345) +- Test: `tests/test_logical_types/test_logical_type_identifiers.py` + +- [ ] Failing test, after `test_values_of_an_unreadable_identifier_are_not_decoded`: + +```python +def test_python_type_of_an_unreadable_identifier_is_refused() -> None: + """Mapping an Arrow type to a Python type checks the identifier, also when nested.""" + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + table = _typed_thing_column(b'{"v":1}') + column_type = table.schema.field("c").type + + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(column_type) + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_schema_to_python_schema(table.schema) + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(pa.struct([pa.field("t", column_type)])) + with pytest.raises(LogicalTypeIdentifierError): + strict.arrow_type_to_python_type(pa.large_list(column_type)) # if Arrow allows it + + +def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> None: + lenient = _converter(_fresh_context(_thing_logical_type(b'{"v":2}', also_reads=(b'{"v":1}',)))) + column_type = _typed_thing_column(b'{"v":1}').schema.field("c").type + assert lenient.arrow_type_to_python_type(column_type) is Thing +``` + +- [ ] Run: `uv run pytest tests/test_logical_types/test_logical_type_identifiers.py -k python_type_of -q` — first test FAILS (returns the class). +- [ ] Fix: before `return lt.python_type`: + +```python + if lt is not None: + # The schema names the registered class only for a column whose + # values that class reads (never by extension name alone). + from orcapod.logical_types.identifier import require_readable + + require_readable(lt, arrow_type.__arrow_ext_serialize__()) + return lt.python_type +``` + + Check whether a result cache keyed by Arrow type sits in front of this method; if so the + check must run before the cache is consulted or the cache must not hold a refused type. +- [ ] Run the file; PASS. Commit `fix(converter): ...`. + +### Task 2 (item 2): `set_ephemeral_store` resets the stale-table flags + +**Files:** +- Modify: `src/orcapod/core/nodes/function_node.py` (`FunctionJobNode.set_ephemeral_store`, about line 1268) +- Test: `tests/test_pipeline/test_stored_data_context.py`, class `TestSideEffectFunctionNodes` + +- [ ] Failing test `test_a_late_attached_ephemeral_store_is_checked_again`: build the `notify` + node on a fresh store; `get_all_records()` (the node checks its clean stores); build an + ephemeral store holding the result rows (`rdb_v2`) of a written store relabelled `OTHER` + with other keys; attach it through the public `set_ephemeral_store` and + `NodeConfig(is_result_ephemeral=True)` (through `PipelineJob.set_ephemeral_store` / + `apply_node_config` if the paths allow, else on the node); `run()` must raise + `SideEffectLogDataContextError` and `delivered == []`. +- [ ] Run; FAILS (node runs, delivers `[2, 4]`). +- [ ] Fix, at the top of `FunctionJobNode.set_ephemeral_store`: + +```python + # Another store has its own tables of other generations to account for. + self._stale_tables_checked = False + self._current_tables_checked = False + self._unlisted_store_reported = False +``` + +- [ ] Run class; PASS. Commit. + +### Task 3 (item 3): data context check before the cache bypass + +**Files:** +- Modify: `src/orcapod/core/data_function.py` (`CachedDataFunction.call`, `async_call`) +- Test: `tests/test_pipeline/test_stored_data_context.py`, class `TestInputOfAnotherDataContext` + +- [ ] Failing test, parametrized over sync/async and over + `{}`, `{"skip_cache_lookup": True}`, `{"skip_cache_lookup": True, "skip_cache_insert": True}`: + `CachedDataFunction(double.pod.data_function, result_database=store)`; call with + `Data({"x": 1}, data_context=other_context)`; expect `InputValidationError` naming both + contexts, `calls == []`, no table written. +- [ ] Run; the two bypass variants FAIL (function called). +- [ ] Fix: first statement of both methods: + +```python + # Checked before anything else: with ``skip_cache_lookup`` the lookup, which + # also checks it, does not run, and the function must not be called. + require_same_data_context(data.data_context_key, self.data_context_key) +``` + + (import from `orcapod.contexts.stored`; check for an import cycle.) +- [ ] Run; PASS. Commit. + +### Task 4 (item 4): env var in `test_list_table_paths.py` + +- [ ] Show failure: `FSSPEC_S3_ENDPOINT_URL=https://example.test uv run pytest tests/test_databases/test_list_table_paths.py -k http_endpoint -q` FAILS. +- [ ] Add `monkeypatch.delenv("FSSPEC_S3_ENDPOINT_URL", raising=False)` in + `test_an_http_endpoint_from_the_environment_follows_the_same_rule` (and check the sibling + tests of the class under the same variable; clear it wherever it changes the outcome). +- [ ] Same command PASSES. + +### Task 5 (item 5): `changelog.d/ITL-944.md` Migration paragraph + +- [ ] "side-effect nodes fail on such a store" gains "unless the store cannot list its + tables (then the node warns and runs)". `uv run python scripts/assemble_changelog.py --check`. + +### Task 6 (item 6): label literals in `test_serialization_helpers.py` + +- [ ] Replace the eight `"semantic_v1:..."` literals with strings built from + `SEMANTIC_HASH_LABEL` (`orcapod.hashing.semantic_hashing.semantic_hasher`), e.g. + `f"{SEMANTIC_HASH_LABEL}:abc123"`. Run the file. + +### Task 7 (item 7): key-list length in `test_wrong_hash_findings.py` + +- [ ] `sync_keys = stored_keys(...)`, `async_keys = stored_keys(...)`; + `assert len(sync_keys) == len(async_keys) == 2` then equality. Run the test. + +### Task 8: changelog fragment, verification, PR + +- [ ] `changelog.d/ITL-975.md`: `### Fixed` with two bullets (items 1 and 3; item 2 too). +- [ ] `uv run ruff format .`, `uv run ruff check .`, `uv run --with basedpyright basedpyright` (exit 0), + `uv run pytest tests/` on 3.12, golden regeneration with empty diff, + `scripts/assemble_changelog.py --check`, `scripts/design_issues.py --check`. +- [ ] Merge origin/main if moved, push, PR against main (`Fixes ITL-975`), link in Orca, + CodeRabbit rounds, `worker_done` with the reviewed sha. + +## Notes for a successor + +- 2026-11-12: Tasks 1-8 implemented and pushed (head 988a8361). PR #326 opened against main. + Full suite on 3.12: 8913 passed, 101 skipped, 18 xfailed. Golden regeneration: empty diff. + Item 1 also names the column in `arrow_schema_to_python_schema` (caller wraps with + `for_column`). Item 4 clears both endpoint variables in the `bucket` fixture. + Remaining: CI green, CodeRabbit round(s), `worker_done` with the reviewed sha. +- 2026-11-12, CodeRabbit round 1 (head c982e96b): one finding, reproduced. The cache in + `arrow_type_to_python_type` is keyed by the Arrow type, and Arrow compares two instances + of one extension class without their metadata, so a struct remembered for a readable + identifier answered for an unreadable one. Coordinator approved option A: walk the type + before the cache lookup (`_require_readable_members`), keep the check in + `_convert_arrow_to_python`. Stop rule after this push: repeats, base findings and style + notes are answered without a further push. +- 2026-11-12, CodeRabbit round 2 (head ce97810a, after merging origin/main): three threads. + (1) the walk skipped unions: it now visits every nested type (`num_fields` / `field(i)`, + extension storage, dictionary values). (2) base finding, included on the coordinator's + decision: `get_arrow_to_python_converter` returned a cached value converter before the + readable check; the walk now runs before that cache too. (3) unquoted annotation. + Sweep of caches keyed by an Arrow type in `universal_converter.py`: + `_arrow_to_python_types` (walk before lookup), `_arrow_to_python_converters` (walk before + lookup), `_struct_signature_to_typeddict` (reached only from `_convert_arrow_to_python`, + behind `arrow_type_to_python_type`'s walk). The other caches are keyed by Python types. + After this push the stop rule applies. diff --git a/tests/test_databases/test_list_table_paths.py b/tests/test_databases/test_list_table_paths.py index b1d081d90..95654bde9 100644 --- a/tests/test_databases/test_list_table_paths.py +++ b/tests/test_databases/test_list_table_paths.py @@ -208,6 +208,10 @@ def filesystem(protocol: str, **options: object) -> DirFileSystem: return DirFileSystem(path=str(tmp_path), fs=LocalFileSystem()) monkeypatch.setattr(fsspec, "filesystem", filesystem) + # The endpoint is also read from the environment: these tests must not depend + # on what the machine running them has set (ITL-975). + monkeypatch.delenv("FSSPEC_S3_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) _fill(DeltaTableDatabase(tmp_path / "bucket" / "root")) return calls diff --git a/tests/test_hashing/test_wrong_hash_findings.py b/tests/test_hashing/test_wrong_hash_findings.py index 37a8589c1..093562627 100644 --- a/tests/test_hashing/test_wrong_hash_findings.py +++ b/tests/test_hashing/test_wrong_hash_findings.py @@ -705,7 +705,11 @@ async def run_async() -> tuple[Any, Any]: return db, node async_db, async_node = asyncio.run(run_async()) - assert stored_keys(sync_db, sync_node) == stored_keys(async_db, async_node) + sync_keys = stored_keys(sync_db, sync_node) + async_keys = stored_keys(async_db, async_node) + # One key per input row on each path: two equal empty lists must not pass. + assert len(sync_keys) == len(async_keys) == 2 + assert sync_keys == async_keys # --------------------------------------------------------------------------- diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index 24ce53934..37ff00b59 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -14,6 +14,7 @@ import pathlib import typing import uuid +from collections.abc import Callable from typing import Any import numpy as np @@ -555,6 +556,173 @@ def test_values_of_an_unreadable_identifier_are_not_decoded() -> None: strict.arrow_table_to_python_dicts(table) +def test_python_type_of_an_unreadable_identifier_is_refused() -> None: + """Mapping an Arrow type to a Python type checks the identifier (ITL-975). + + The registered class is never named by extension name alone: a schema must not + promise ``Thing`` for a column whose values will be refused. + """ + current = _thing_logical_type(b'{"v":2}') + strict = _converter(_fresh_context(current)) + table = _typed_thing_column(b'{"v":1}') + column_type = table.schema.field("c").type + # The current identifier maps to the class; having mapped it changes nothing below. + assert strict.arrow_type_to_python_type(current.get_arrow_extension_type()) is Thing + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.arrow_type_to_python_type(column_type) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + _ = strict.arrow_schema_to_python_schema(table.schema) + assert "'c'" in str(excinfo.value) + # A member of a struct column is checked like a top-level column. + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.arrow_type_to_python_type(pa.struct([pa.field("t", column_type)])) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + _ = strict.arrow_schema_to_python_schema( + pa.schema([pa.field("s", pa.struct([pa.field("t", column_type)]))]) + ) + assert "'s'" in str(excinfo.value) + + +class _ParameterizedThingType(pa.ExtensionType): + """One extension class for ``thirdparty.thing`` whose metadata is a parameter. + + Arrow compares two instances of one extension class without their metadata, so + they are equal whatever identifiers they carry. + """ + + def __init__(self, metadata: bytes) -> None: + """Store ``metadata`` as the serialized extension metadata.""" + self._metadata = metadata + super().__init__(pa.large_string(), "thirdparty.thing") + + def __arrow_ext_serialize__(self) -> bytes: + """Return the metadata given at construction.""" + return self._metadata + + @classmethod + def __arrow_ext_deserialize__( + cls, storage_type: pa.DataType, serialized: bytes + ) -> _ParameterizedThingType: + """Rebuild the type from its serialized metadata.""" + return cls(serialized) + + +@pytest.mark.parametrize( + "container", + [ + lambda member: pa.struct([pa.field("t", member)]), + lambda member: pa.large_list(pa.struct([pa.field("t", member)])), + lambda member: pa.map_(pa.large_string(), pa.struct([pa.field("t", member)])), + lambda member: pa.dense_union( + [pa.field("i", pa.int64()), pa.field("t", member)] + ), + lambda member: pa.sparse_union( + [pa.field("i", pa.int64()), pa.field("t", member)] + ), + ], + ids=["struct", "list_of_struct", "map_of_struct", "dense_union", "sparse_union"], +) +def test_python_type_of_an_unreadable_member_is_refused_after_a_readable_one( + container: Callable[[pa.DataType], pa.DataType], +) -> None: + """A mapping remembered for a readable identifier never answers for an unreadable + one (ITL-975): two containers that differ only in a member's metadata are equal + to Arrow, and the identifier is still checked for each. + """ + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + readable = container(_ParameterizedThingType(b'{"v":2}')) + unreadable = container(_ParameterizedThingType(b'{"v":1}')) + assert readable == unreadable # what makes a lookup by Arrow type alone unsafe + + # The readable container is mapped first, through both entry points. + _ = strict.arrow_type_to_python_type(readable) + _ = strict.arrow_schema_to_python_schema(pa.schema([pa.field("c", readable)])) + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.arrow_type_to_python_type(unreadable) + with pytest.raises(LogicalTypeIdentifierError) as excinfo: + _ = strict.arrow_schema_to_python_schema(pa.schema([pa.field("c", unreadable)])) + assert "'c'" in str(excinfo.value) + + +def _parameterized_thing_column(metadata: bytes) -> "pa.Array[Any]": + """One ``thirdparty.thing`` value typed by ``_ParameterizedThingType(metadata)``.""" + return pa.ExtensionArray.from_storage( + _ParameterizedThingType(metadata), pa.array(["a"], pa.large_string()) + ) + + +def _struct_column(member: "pa.Array[Any]") -> "pa.Array[Any]": + """A struct column with ``member`` as its field ``t``.""" + return pa.StructArray.from_arrays([member], fields=[pa.field("t", member.type)]) + + +def _list_of_struct_column(member: "pa.Array[Any]") -> "pa.Array[Any]": + """A one-row list column whose element is a struct with ``member`` as field ``t``.""" + return pa.LargeListArray.from_arrays( + pa.array([0, 1], pa.int64()), _struct_column(member) + ) + + +@pytest.mark.parametrize( + "column", + [_struct_column, _list_of_struct_column], + ids=["struct", "list_of_struct"], +) +def test_values_of_an_unreadable_member_are_not_decoded_after_a_readable_one( + column: Callable[["pa.Array[Any]"], "pa.Array[Any]"], +) -> None: + """A value converter remembered for a readable identifier never decodes values + stored under an unreadable one (ITL-975): the two columns' types are equal to Arrow. + """ + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + readable = pa.table({"c": column(_parameterized_thing_column(b'{"v":2}'))}) + unreadable = pa.table({"c": column(_parameterized_thing_column(b'{"v":1}'))}) + assert readable.schema.field("c").type == unreadable.schema.field("c").type + + # The readable table is converted first, so its converter is remembered. + assert len(strict.arrow_table_to_python_dicts(readable)) == 1 + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.arrow_table_to_python_dicts(unreadable) + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.get_arrow_to_python_converter(unreadable.schema.field("c").type) + + +@pytest.mark.parametrize("mode", ["dense", "sparse"]) +def test_value_converter_of_an_unreadable_union_member_is_refused_after_a_readable_one( + mode: str, +) -> None: + """The same holds for a member of a union type (ITL-975).""" + strict = _converter(_fresh_context(_thing_logical_type(b'{"v":2}'))) + + def union(metadata: bytes) -> pa.DataType: + """A union of ``int64`` and a ``thirdparty.thing`` carrying ``metadata``.""" + fields = [ + pa.field("i", pa.int64()), + pa.field("t", _ParameterizedThingType(metadata)), + ] + return pa.dense_union(fields) if mode == "dense" else pa.sparse_union(fields) + + _ = strict.get_arrow_to_python_converter(union(b'{"v":2}')) + + with pytest.raises(LogicalTypeIdentifierError): + _ = strict.get_arrow_to_python_converter(union(b'{"v":1}')) + + +def test_python_type_of_a_readable_older_identifier_is_the_current_class() -> None: + """An older identifier the type declares readable maps to the registered class.""" + lenient = _converter( + _fresh_context(_thing_logical_type(b'{"v":2}', also_reads=(b'{"v":1}',))) + ) + table = _typed_thing_column(b'{"v":1}') + + assert lenient.arrow_type_to_python_type(table.schema.field("c").type) is Thing + schema = lenient.arrow_schema_to_python_schema(table.schema) + assert schema["c"] == Thing | None + + def test_list_wrapper_written_before_identifiers_is_read() -> None: """A list column whose metadata records no element metadata is read as the current list type. diff --git a/tests/test_pipeline/test_serialization_helpers.py b/tests/test_pipeline/test_serialization_helpers.py index ce232e4e3..874ed8f46 100644 --- a/tests/test_pipeline/test_serialization_helpers.py +++ b/tests/test_pipeline/test_serialization_helpers.py @@ -20,6 +20,7 @@ serialize_schema, ) from orcapod.databases.in_memory_databases import InMemoryArrowDatabase +from orcapod.hashing.semantic_hashing.semantic_hasher import SEMANTIC_HASH_LABEL from orcapod.core.operators import Join, Batch from orcapod.types import Schema @@ -695,15 +696,15 @@ def test_source_proxy_from_node_descriptor_fields(): source_config = {"source_type": "dict", "source_id": "my_src"} node_descriptor = { - "content_hash": "semantic_v1:abc123", - "pipeline_hash": "semantic_v1:def456", + "content_hash": f"{SEMANTIC_HASH_LABEL}:abc123", + "pipeline_hash": f"{SEMANTIC_HASH_LABEL}:def456", "output_schema": { "tag": {"x": "int64"}, "data": {"result": "int64"}, }, } proxy = _source_proxy_from_config(source_config, node_descriptor=node_descriptor) - assert proxy.content_hash().to_string() == "semantic_v1:abc123" + assert proxy.content_hash().to_string() == f"{SEMANTIC_HASH_LABEL}:abc123" def test_source_proxy_from_config_backward_compat(): @@ -713,13 +714,13 @@ def test_source_proxy_from_config_backward_compat(): source_config = { "source_type": "dict", "source_id": "my_src", - "content_hash": "semantic_v1:abc123", - "pipeline_hash": "semantic_v1:def456", + "content_hash": f"{SEMANTIC_HASH_LABEL}:abc123", + "pipeline_hash": f"{SEMANTIC_HASH_LABEL}:def456", "tag_schema": {"x": "int64"}, "data_schema": {"result": "int64"}, } proxy = _source_proxy_from_config(source_config) - assert proxy.content_hash().to_string() == "semantic_v1:abc123" + assert proxy.content_hash().to_string() == f"{SEMANTIC_HASH_LABEL}:abc123" # --------------------------------------------------------------------------- @@ -762,8 +763,8 @@ def test_function_node_stored_node_uri_from_descriptor(): descriptor = { "node_type": "function", "label": "fn", - "content_hash": "semantic_v1:abc", - "pipeline_hash": "semantic_v1:def", + "content_hash": f"{SEMANTIC_HASH_LABEL}:abc", + "pipeline_hash": f"{SEMANTIC_HASH_LABEL}:def", "table_scope": "pipeline_hash", "node_uri": ["add_one", "v0", "python.function.v0", "schema_repr"], "output_schema": {"tag": {"x": "int64"}, "data": {"result": "int64"}}, diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 15e233c4c..522e00640 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -48,7 +48,7 @@ from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.side_effects import SideEffectPod, sink_pod from orcapod.sources import ArrowTableSource, DictSource -from orcapod.types import CacheMode +from orcapod.types import CacheMode, NodeConfig Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator Store = InMemoryArrowDatabase | DeltaTableDatabase @@ -1134,6 +1134,39 @@ def test_a_newly_attached_store_is_checked_again( assert delivered == [] + @pytest.mark.parametrize("read_first", [False, True], ids=["unread", "read_first"]) + def test_a_late_attached_ephemeral_store_is_checked_again( + self, new_store: Callable[[], Store], read_first: bool + ): + """An ephemeral store attached after the node checked its stores is checked too + (ITL-975): completions of ``OTHER`` in it raise and nothing is delivered. + """ + relabelled = reopened(_notify_store(new_store, copy_as_other_context)) + ephemeral = new_store() + for path in relabelled.list_sources(): + if path[-1] == "rdb_v2": + rows = relabelled.get_all_records(path, record_id_column=RID) + assert rows is not None + # An ephemeral store holds the result table at the path relative to + # the job's result database. + assert path[:2] == (JOB_NAME, "_result") + ephemeral.add_records(path[2:], rows, record_id_column=RID) + ephemeral.flush() + assert [path[-1] for path in reopened(ephemeral).list_sources()] == ["rdb_v2"] + job = _notify_job(new_store()) + if read_first: + # The node checks its (clean) stores when it first reads them. + assert job.nodes["notify"].get_all_records() is None + + job.set_ephemeral_store(ephemeral) + job.apply_node_config(NodeConfig(is_result_ephemeral=True)) + with pytest.raises(SideEffectLogDataContextError) as info: + _ = job.run() + + assert delivered == [] + assert info.value.found_context == OTHER + assert [path[-1] for path in info.value.record_paths] == ["rdb_v2"] + def test_plain_function_node_in_the_same_store_is_only_reported( self, new_store: Callable[[], Store] ): @@ -1329,6 +1362,47 @@ def test_cached_function_refuses_an_output_of_another_data_context( _assert_names_both_contexts(info.value) assert reopened(store).list_sources() == [] + @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) + @pytest.mark.parametrize( + ("skip_lookup", "skip_insert"), + [(False, False), (True, False), (True, True)], + ids=["default", "no_lookup", "no_lookup_no_insert"], + ) + def test_cached_function_refuses_an_input_of_another_data_context( + self, + store: Store, + other_context: contexts.DataContext, + skip_lookup: bool, + skip_insert: bool, + asynchronous: bool, + ): + """The input's data context is checked before the function is called (ITL-975). + + Also when the cache lookup, which checks it too, is skipped. + """ + import asyncio + + cached = CachedDataFunction(double.pod.data_function, result_database=store) + data = Data({"x": 1}, data_context=other_context) + + with pytest.raises(InputValidationError) as info: + if asynchronous: + _ = asyncio.run( + cached.async_call( + data, + skip_cache_lookup=skip_lookup, + skip_cache_insert=skip_insert, + ) + ) + else: + _ = cached.call( + data, skip_cache_lookup=skip_lookup, skip_cache_insert=skip_insert + ) + + _assert_names_both_contexts(info.value) + assert calls == [] + assert reopened(store).list_sources() == [] + def test_cached_source_keeps_its_data_context_on_cached_rows(self, store: Store): """Rows read back from the cache carry the source's data context, not the default.""" table = pa.table({"k": ["a", "b"], "x": [1, 2]})