From 53a2d5d02a1e057524494466d1a5d2a4f2196a93 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 18:46:55 -0800 Subject: [PATCH 01/36] refactor(logical_types): share the Arrow IPC helpers between frame types Plan for ITL-578 included. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/logical_types/arrow_ipc.py | 42 +++ src/orcapod/logical_types/pandas_type.py | 39 +-- ...0-itl-578-polars-dataframe-logical-type.md | 274 ++++++++++++++++++ 3 files changed, 321 insertions(+), 34 deletions(-) create mode 100644 src/orcapod/logical_types/arrow_ipc.py create mode 100644 superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md diff --git a/src/orcapod/logical_types/arrow_ipc.py b/src/orcapod/logical_types/arrow_ipc.py new file mode 100644 index 000000000..0a9455eed --- /dev/null +++ b/src/orcapod/logical_types/arrow_ipc.py @@ -0,0 +1,42 @@ +"""Arrow IPC stream helpers shared by the frame-valued logical types. + +``LogicalPandasDataFrame``, ``LogicalPandasSeries`` and ``LogicalPolarsDataFrame`` all store +their values as Arrow IPC stream bytes in a ``large_binary`` column. These two functions are +that envelope. +""" + +from __future__ import annotations + +import io +from typing import Any + +import pyarrow as pa +import pyarrow.ipc + + +def table_to_ipc_bytes(table: pa.Table) -> bytes: + """Serialise an Arrow Table to IPC stream bytes. + + Args: + table: The Arrow Table to serialise. + + Returns: + Raw bytes in Arrow IPC stream format. + """ + buf = io.BytesIO() + with pa.ipc.new_stream(buf, table.schema) as writer: + writer.write_table(table) + return buf.getvalue() + + +def ipc_bytes_to_table(data: Any) -> pa.Table: + """Deserialise IPC stream bytes to an Arrow Table. + + Args: + data: Raw bytes (or buffer) containing an Arrow IPC stream. + + Returns: + The deserialised Arrow Table. + """ + buf = io.BytesIO(bytes(data)) + return pa.ipc.open_stream(buf).read_all() diff --git a/src/orcapod/logical_types/pandas_type.py b/src/orcapod/logical_types/pandas_type.py index 2bb962155..ea0cbff93 100644 --- a/src/orcapod/logical_types/pandas_type.py +++ b/src/orcapod/logical_types/pandas_type.py @@ -13,14 +13,13 @@ from __future__ import annotations -import io from typing import TYPE_CHECKING, Any import pandas as pd import polars as pl import pyarrow as pa -import pyarrow.ipc +from orcapod.logical_types.arrow_ipc import ipc_bytes_to_table, table_to_ipc_bytes from orcapod.logical_types.base_logical_type import BaseLogicalType from orcapod.logical_types.registry import ( make_arrow_extension_type, @@ -34,34 +33,6 @@ _SERIES_UNNAMED_SENTINEL = "__pandas_series_unnamed__" -def _table_to_ipc_bytes(table: pa.Table) -> bytes: - """Serialise an Arrow Table to IPC stream bytes. - - Args: - table: The Arrow Table to serialise. - - Returns: - Raw bytes in Arrow IPC stream format. - """ - buf = io.BytesIO() - with pa.ipc.new_stream(buf, table.schema) as writer: - writer.write_table(table) - return buf.getvalue() - - -def _ipc_bytes_to_table(data: Any) -> pa.Table: - """Deserialise IPC stream bytes to an Arrow Table. - - Args: - data: Raw bytes (or buffer) containing an Arrow IPC stream. - - Returns: - The deserialised Arrow Table. - """ - buf = io.BytesIO(bytes(data)) - return pa.ipc.open_stream(buf).read_all() - - class LogicalPandasDataFrame(BaseLogicalType): """Logical type for ``pd.DataFrame``. @@ -146,7 +117,7 @@ def python_to_storage( f"Check for columns with mixed or non-Arrow-serialisable types. " f"Original error: {exc}" ) from exc - return _table_to_ipc_bytes(table) + return table_to_ipc_bytes(table) def storage_to_python( self, storage_value: Any, converter: TypeConverterProtocol | None = None @@ -160,7 +131,7 @@ def storage_to_python( Returns: A ``pd.DataFrame`` with columns, dtypes, and index as originally stored. """ - return _ipc_bytes_to_table(storage_value).to_pandas() + return ipc_bytes_to_table(storage_value).to_pandas() class LogicalPandasSeries(BaseLogicalType): @@ -249,7 +220,7 @@ def python_to_storage( f"Check for non-Arrow-serialisable values. " f"Original error: {exc}" ) from exc - return _table_to_ipc_bytes(table) + return table_to_ipc_bytes(table) def storage_to_python( self, storage_value: Any, converter: TypeConverterProtocol | None = None @@ -263,7 +234,7 @@ def storage_to_python( Returns: A ``pd.Series`` with its original name and index restored. """ - df = _ipc_bytes_to_table(storage_value).to_pandas() + df = ipc_bytes_to_table(storage_value).to_pandas() series = df.iloc[:, 0] if series.name == _SERIES_UNNAMED_SENTINEL: series = series.rename(None) diff --git a/superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md b/superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md new file mode 100644 index 000000000..51cc7bfa2 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-578-polars-dataframe-logical-type.md @@ -0,0 +1,274 @@ +# ITL-578 Polars DataFrame Logical Type Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Register `polars.DataFrame` as an Orcapod logical type so a function pod can take and +return a `pl.DataFrame`, and the value is stored, read back, hashed and cached. + +**Architecture:** A new `LogicalPolarsDataFrame` (extension name `polars.dataframe`, storage +`large_binary`) stores the frame as Arrow IPC stream bytes, the same envelope as +`LogicalPandasDataFrame`. The IPC helpers move from `pandas_type.py` to a shared +`logical_types/arrow_ipc.py` that both types import. The type is wired into the default data +context (`contexts/data/v0.1.json`) like pandas and numpy. Hashing goes through a semantic +handler whose design is **pending a developer decision** (Task 6). + +**Tech Stack:** polars 1.41.2, pyarrow 23.0.1, starfix 0.4.0, pytest. + +--- + +## Findings that shape the plan (investigation, November 10, 2026) + +Scratch scripts live in the session scratchpad; the numbers below come from them. + +1. **Storage round trip is exact** with `df.to_arrow(compat_level=pl.CompatLevel.oldest())`, + `combine_chunks()`, Arrow IPC stream, then `pl.from_arrow(...)`. Every dtype checked + returns with an identical Polars schema and `assert_frame_equal` passes: Int8–Int64, + UInt8–UInt64, Float16/32/64, String, Binary, Boolean, Categorical, Enum (category order kept, + via the `_PL_ENUM_VALUES2` field metadata), Decimal(p, s), Datetime (unit and time zone), + Date, Time, Duration (unit), List, Array (also 2-D), Struct (also with a nested Enum), Null, + an empty frame with a schema, and a zero-column frame. + `compat_level=oldest` is pinned so strings are written as `large_string`, never + `string_view`, whatever the Polars default becomes. +2. **Not round-trippable, so rejected at write time:** + - `Int128` / `UInt128`: `to_arrow` raises `ArrowInvalid: unsupported format string '_pli128'`. + - `Object`: `to_arrow` silently writes `fixed_size_binary[8]` (pointer bytes) and reads back + as `Binary`, which is silent corruption. + - Polars extension dtypes (`pl.BaseExtension`, e.g. an Orcapod `Extension('numpy.ndarray')` + column inside the frame): nested logical values inside a frame value are a separate design + question; reject now, report as a follow-up. + The check walks the dtype tree (List, Array, Struct children) so a nested `Object` is also + caught, and the error names the top-level column and the offending dtype. +3. **The IPC bytes are not canonical.** Equal frames give different bytes when the chunk layout + differs (fixable with `combine_chunks`) and when the bytes under null slots differ (not + fixable without rebuilding buffers). The bytes also carry Polars-version-specific field + metadata (`_PL_CATEGORICAL2`, `_PL_ENUM_VALUES2`). With no handler registered, the semantic + visitor hashes the stored bytes, so a handler is required. +4. **Starfix alone is not enough for a frame value.** `StarfixArrowHasher.hash_table` (what + `PandasDataFrameHandler` uses) is chunk-independent but, by starfix design, ignores column + order and struct field order and hashes a dictionary column as its value type + (Categorical == Enum(any categories) == String). It raises `NotImplementedError` for + `fixed_size_list` (Polars Array) and `null`. The existing pandas handler has the same + collisions on main (reported to the coordinator, out of scope here). + +The hash design (option A, B or C in the coordinator question of November 10, 2026) is held +for the developer. Task 6 is blocked until that answer arrives; everything else proceeds. + +## File structure + +| File | Responsibility | +|---|---| +| Create `src/orcapod/logical_types/arrow_ipc.py` | `table_to_ipc_bytes`, `ipc_bytes_to_table` (moved from `pandas_type.py`, unchanged bodies) | +| Modify `src/orcapod/logical_types/pandas_type.py` | Import the helpers from `arrow_ipc` | +| Create `src/orcapod/logical_types/polars_type.py` | `LogicalPolarsDataFrame`, `check_polars_schema_storable` | +| Modify `src/orcapod/logical_types/__init__.py` | Export `LogicalPolarsDataFrame` | +| Modify `src/orcapod/contexts/data/v0.1.json` | Register the logical type (+ handler after Task 6), changelog line | +| Modify `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` | `PolarsDataFrameHandler` (Task 6, design pending) | +| Create `tests/test_logical_types/test_polars_type.py` | Unit tests: protocol, per-dtype round trip, rejections | +| Modify `tests/test_logical_types/test_roundtrips.py` | Builtin round trip over Parquet and Delta (mirrors pandas) | +| Create `tests/test_core/function_pod/test_polars_dataframe_values.py` | Public-interface integration tests | +| Create `tests/test_hashing/test_polars_handlers.py` | Hash properties (Task 6) | +| Modify `docs/concepts/function-pods.md` | "Value types" section listing built-in logical types incl. `pl.DataFrame` | +| Modify `DESIGN_ISSUES.md` | One appended entry for ITL-578 | + +No docs page lists pandas or numpy today (the brief assumed one). The plan adds a short +"Value types" section to `docs/concepts/function-pods.md` and reports this to the coordinator. + +--- + +### Task 1: Move the IPC helpers to a shared module (refactor, no behavior change) + +**Files:** Create `src/orcapod/logical_types/arrow_ipc.py`; modify `pandas_type.py`. + +- [ ] **Step 1:** Create `arrow_ipc.py` with `table_to_ipc_bytes(table: pa.Table) -> bytes` and + `ipc_bytes_to_table(data: Any) -> pa.Table`, bodies copied verbatim from + `pandas_type._table_to_ipc_bytes` / `_ipc_bytes_to_table`, Google docstrings. +- [ ] **Step 2:** In `pandas_type.py`, delete the two private functions and + `from orcapod.logical_types.arrow_ipc import ipc_bytes_to_table, table_to_ipc_bytes`; + update the four call sites. +- [ ] **Step 3:** Run `uv run pytest tests/test_logical_types/test_pandas_type.py tests/test_hashing/test_pandas_handlers.py -q` — expect all pass. +- [ ] **Step 4:** Commit `refactor(logical_types): share the Arrow IPC helpers between frame types`. + +### Task 2: Failing regression test for the decoration error + +**Files:** Create `tests/test_core/function_pod/test_polars_dataframe_values.py`. + +- [ ] **Step 1:** Write + +```python +import polars as pl +from polars.testing import assert_frame_equal + +from orcapod import function_pod # verify the public import path when writing + + +def test_pod_can_return_a_polars_dataframe(): + @function_pod(output_keys="df") + def make_frame(n: int) -> pl.DataFrame: + return pl.DataFrame({"i": list(range(n))}) + + assert make_frame is not None +``` + +- [ ] **Step 2:** Run it; expect `TypeError: No LogicalType or LogicalTypeFactory registered for `. + +### Task 3: `LogicalPolarsDataFrame` storage (TDD) + +**Files:** Create `src/orcapod/logical_types/polars_type.py`, +`tests/test_logical_types/test_polars_type.py`. + +- [ ] **Step 1: Tests first.** In `test_polars_type.py`: + - protocol: `isinstance(LogicalPolarsDataFrame(), LogicalTypeProtocol)`, + `logical_type_name == "polars.dataframe"`, `python_type is pl.DataFrame`, + Arrow extension name `polars.dataframe`, storage `pa.large_binary()`, Polars extension name. + - `python_to_storage` returns `bytes`. + - Parametrized round trip, one column per case, asserting + `back.schema == df.schema` **and** `assert_frame_equal(back, df)`: Int8 with null, UInt8, + UInt16, UInt32, UInt64, Float16, Float32, String with null, Binary with null, Boolean, + Categorical, Enum(["y","x","z"]), Decimal(10,2) and Decimal(38,2), Datetime("ms", + "America/New_York"), Datetime("ns"), Date, Time, Duration("ms"), List(Int64) with null, + List(String) with empty list, List(Categorical), Array(Int16, 2), Array(Int16, (1, 2)), + Struct({"a": Int64, "b": String}) with null, Struct({"e": Enum}), Null. + - A wide frame of all the above in one frame: column order and names preserved. + - Empty frame with schema (Int8 + Struct(Enum)): schema and zero height preserved. + - Zero-column frame `pl.DataFrame()`. + - Chunked frame (`pl.concat([...], rechunk=False)`, assert `n_chunks() == 2`) round trips. + - Rejections (`pytest.raises(ValueError, match=...)` matching column name and dtype): + `Object` column, `Int128`, `UInt128`, `List(Object)` nested, `Struct({"o": Object})` + nested, a Polars extension dtype column (`pl.Series([b"x"]).ext.to(LogicalNumpyArray().get_polars_extension_type())`). + - Non-DataFrame input (`pl.Series`, `pd.DataFrame`) raises `TypeError`. +- [ ] **Step 2:** Run `uv run pytest tests/test_logical_types/test_polars_type.py -q`; expect import failure. +- [ ] **Step 3: Implement** `polars_type.py`: + +```python +_UNSUPPORTED_LEAVES = (pl.Object, pl.Int128, pl.UInt128) + +def _find_unstorable_dtype(dtype: pl.DataType) -> pl.DataType | None: + """Return the first nested dtype that cannot round-trip, or None.""" + if isinstance(dtype, pl.BaseExtension): + return dtype + if isinstance(dtype, _UNSUPPORTED_LEAVES) or dtype in _UNSUPPORTED_LEAVES: + return dtype + if isinstance(dtype, (pl.List, pl.Array)): + return _find_unstorable_dtype(dtype.inner) + if isinstance(dtype, pl.Struct): + for field in dtype.fields: + found = _find_unstorable_dtype(field.dtype) + if found is not None: + return found + return None + +def check_polars_schema_storable(schema: pl.Schema) -> None: + """Raise ValueError naming the column and dtype that cannot round-trip exactly.""" + for name, dtype in schema.items(): + bad = _find_unstorable_dtype(dtype) + if bad is not None: + raise ValueError( + f"LogicalPolarsDataFrame: column {name!r} has dtype {dtype} " + f"(contains {bad}), which cannot be stored losslessly as Arrow IPC. " + "Supported: integer up to 64 bits, float, string, binary, boolean, " + "categorical, enum, decimal, date, time, datetime, duration, null, and " + "list/array/struct of those." + ) + +class LogicalPolarsDataFrame(BaseLogicalType): + # extension name "polars.dataframe": library-qualified like "pandas.dataframe" and + # "numpy.ndarray"; names the Python type users write, so stored schemas self-document. + ... + def python_to_storage(self, value, converter=None) -> bytes: + if not isinstance(value, pl.DataFrame): + raise TypeError(...) + check_polars_schema_storable(value.schema) + table = value.to_arrow(compat_level=pl.CompatLevel.oldest()).combine_chunks() + return table_to_ipc_bytes(table) + + def storage_to_python(self, storage_value, converter=None) -> pl.DataFrame: + frame = pl.from_arrow(ipc_bytes_to_table(storage_value)) + assert isinstance(frame, pl.DataFrame) # a Table always gives a DataFrame + return frame +``` + + (Extension class caching mirrors `LogicalPandasDataFrame` exactly.) +- [ ] **Step 4:** Run the unit tests; expect pass. +- [ ] **Step 5:** Commit `feat(logical_types): add polars.DataFrame logical type with Arrow IPC storage`. + +### Task 4: Register in the default data context + +**Files:** `contexts/data/v0.1.json`, `logical_types/__init__.py`. + +- [ ] **Step 1:** Before editing, record reference hashes with a scratch script (pipeline hash + and content hash of a small pod pipeline over `int`, `pd.DataFrame`, `np.ndarray` values, and + `get_default_context().semantic_hasher.hash_object(...)` of a few objects) to + `scratchpad/hashes_before.json`. +- [ ] **Step 2:** Add `{"_class": "orcapod.logical_types.polars_type.LogicalPolarsDataFrame", "_config": {}}` + after `LogicalPandasSeries`; append a changelog line; export from `__init__.py`. +- [ ] **Step 3:** Run Task 2's test; expect pass. Re-run the hash script; assert identical + to `hashes_before.json` (acceptance: existing hashes unchanged). Run + `uv run pytest tests/test_hashing tests/test_logical_types -q`. +- [ ] **Step 4:** Add the builtin round trip to `test_roundtrips.py` (Parquet + Delta), modelled + on `test_builtin_dataframe_round_trip`, with a frame holding Enum, Datetime(tz) and Struct. +- [ ] **Step 5:** Commit `feat(contexts): register polars.DataFrame in the default data context`. + +### Task 5: Public-interface integration tests (storage paths) + +**File:** `tests/test_core/function_pod/test_polars_dataframe_values.py`. +Every test drives pods/operators by calling them and reads results through +`iter_data()` / `as_table()` / node `get_all_records()`, then converts through the public +stream API. Frames are compared with `assert_frame_equal` **and** schema equality. + +- [ ] Pod output: `n: int -> pl.DataFrame` over a 3-row source; each emitted value is a + `pl.DataFrame` equal to the expected frame (Enum + Datetime(tz) + Struct + List columns). +- [ ] Pod input: source column of frames (built via `DictSource`/`ListSource` with an explicit + `pl.DataFrame` schema type) → `f(df: pl.DataFrame) -> int` returns `df.height`. +- [ ] Chained pods: produce → transform (`df.with_columns(...)`) → consume. +- [ ] `Batch` and `GroupBy` over a frame-valued data column: result column type is the logical + list type (`list[polars.dataframe]`), every member frame equal and in documented order. +- [ ] `Join` pass-through: frame column joined against another stream on the tag; frame unchanged. +- [ ] `PipelineJob` with `SyncPipelineOrchestrator` and `AsyncPipelineOrchestrator`, each over + `InMemoryArrowDatabase`, `DeltaTableDatabase(tmp_path)` and + `ConnectorArrowDatabase(SQLiteConnector(tmp_path / "db.sqlite"))`: records read back from the + node equal the produced frames. +- [ ] Cross-process cache hit: run a pipeline over a Delta store in a subprocess + (`sys.executable -c`, pattern from `tests/test_hashing/test_cross_process_stability.py`), + then in a second subprocess re-run it; the second run computes nothing (function call + counter via a marker file stays at the first-run count) and the frames read back are equal. + This test also needs Task 6 (the cache key is the hash). +- [ ] Commit `test(function_pod): polars.DataFrame values through pods, operators and databases`. + +### Task 6: Hash handler — BLOCKED on the developer's decision + +Do not implement until the coordinator answers. Tests to write first whatever the option: +`tests/test_hashing/test_polars_handlers.py` + +- equal frames hash identically regardless of chunk layout and of null-slot payload + (`[1, None, 3]` vs `[1, 2, 3]` nulled by `when/then`); +- differ in a value, a dtype (Int64 vs Int32; String vs Categorical vs Enum(a,b) vs + Enum(a,b,c)), a column name, column order, struct field order → different hashes; +- `Array` and `Null` columns hash without error; +- a frame read back from storage hashes the same as the original; +- cross-process stability: same hash in two interpreters with different `PYTHONHASHSEED`; +- a golden hex digest for one fixed frame, pinned in the test, to detect future drift. + +Then register `PolarsDataFrameHandler` in `v0.1.json` under `{"_type": "polars.DataFrame"}` +and in `register_builtin_python_type_handlers`; re-run the Task 4 hash comparison. + +### Task 7: Docs and design log + +- [ ] Add a "Value types" section to `docs/concepts/function-pods.md`: a table of the built-in + logical types (Python type, extension name, storage), including `polars.DataFrame`, the + rejected dtypes, and an executed sample of a pod returning a `pl.DataFrame` with its printed + output in `# ...` comments. +- [ ] `uv run pytest tests/test_docs -q` (doc samples + `mkdocs build --strict`). +- [ ] Append one DESIGN_ISSUES.md entry (ITL-578) recording the storage format, rejected + dtypes and the hash design. +- [ ] Commit `docs: document polars.DataFrame as a function pod value type`. + +### Task 8: Finish + +- [ ] `uv run --with ruff ruff format . && uv run --with ruff ruff check .`, + `uv run --with basedpyright basedpyright` (exit code 0), `uv run pytest tests/`. +- [ ] Merge `origin/main`, re-run the suite if it moved, push, open PR (`Fixes ITL-578`), + `orca linear attach`, `orca worktree set --worktree current --pr `, worker_done. + +Out of scope, reported in worker_done: `pl.LazyFrame`; `pl.Series` (held with the hash +decision); the pandas handler's column-order / dtype collisions; nested logical (extension) +dtypes inside a frame. From 9fd86b7aa9f45d7966f8370f9213981bc55dd623 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 18:48:46 -0800 Subject: [PATCH 02/36] feat(logical_types): add polars.DataFrame logical type with Arrow IPC storage (ITL-578) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/logical_types/__init__.py | 3 + src/orcapod/logical_types/polars_type.py | 189 +++++++++++++ tests/test_logical_types/test_polars_type.py | 265 +++++++++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 src/orcapod/logical_types/polars_type.py create mode 100644 tests/test_logical_types/test_polars_type.py diff --git a/src/orcapod/logical_types/__init__.py b/src/orcapod/logical_types/__init__.py index d5b50341c..313526379 100644 --- a/src/orcapod/logical_types/__init__.py +++ b/src/orcapod/logical_types/__init__.py @@ -47,6 +47,7 @@ from .directory_type import LogicalDirectory # ITL-451 from .numpy_type import LogicalNumpyArray # ITL-460 from .pandas_type import LogicalPandasDataFrame, LogicalPandasSeries # PLT-1869 +from .polars_type import LogicalPolarsDataFrame # ITL-578 __all__ = [ "LogicalTypeProtocol", @@ -83,4 +84,6 @@ # PLT-1869 "LogicalPandasDataFrame", "LogicalPandasSeries", + # ITL-578 + "LogicalPolarsDataFrame", ] diff --git a/src/orcapod/logical_types/polars_type.py b/src/orcapod/logical_types/polars_type.py new file mode 100644 index 000000000..1a58a2d12 --- /dev/null +++ b/src/orcapod/logical_types/polars_type.py @@ -0,0 +1,189 @@ +"""``polars.DataFrame`` logical type for orcapod. + +``LogicalPolarsDataFrame`` maps ``pl.DataFrame`` <-> Arrow ``large_binary`` using an Arrow +IPC stream as the storage envelope, the same envelope ``LogicalPandasDataFrame`` uses. + +The frame is exported with ``compat_level=pl.CompatLevel.oldest()``, so strings and binaries +are written as ``large_string`` / ``large_binary`` (never the view types) whatever the +installed Polars defaults to, and its chunks are combined into a single record batch. Reading +the stream back with ``pl.from_arrow`` restores the original Polars schema exactly: column +order and names, integer width and signedness, float width, ``Categorical``, ``Enum`` with its +category order, ``Decimal`` precision and scale, ``Datetime`` unit and time zone, ``Duration`` +unit, ``List``, ``Array`` shape, ``Struct`` field order, and ``Null``. + +Dtypes that cannot round-trip exactly are rejected when the value is written, naming the +column and dtype: ``Object`` (Arrow export would silently store raw pointer bytes), +``Int128`` / ``UInt128`` (pyarrow cannot import them), and Polars extension dtypes (a logical +value nested inside a frame value is not supported). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import polars as pl +import pyarrow as pa + +from orcapod.logical_types.arrow_ipc import ipc_bytes_to_table, table_to_ipc_bytes +from orcapod.logical_types.base_logical_type import BaseLogicalType +from orcapod.logical_types.registry import ( + make_arrow_extension_type, + make_polars_extension_type, +) + +if TYPE_CHECKING: + from orcapod.logical_types.protocols import TypeConverterProtocol + + +_UNSTORABLE_DTYPES: tuple[type[pl.DataType], ...] = (pl.Object, pl.Int128, pl.UInt128) + + +def _find_unstorable_dtype(dtype: pl.DataType) -> pl.DataType | None: + """Return the first dtype inside ``dtype`` that cannot round-trip exactly. + + Walks ``List``, ``Array`` and ``Struct`` children. + + Args: + dtype: A Polars dtype, possibly nested. + + Returns: + The offending (possibly nested) dtype, or ``None`` when every part of ``dtype`` can be + stored losslessly. + """ + if isinstance(dtype, pl.BaseExtension): + return dtype + if isinstance(dtype, _UNSTORABLE_DTYPES): + return dtype + if isinstance(dtype, (pl.List, pl.Array)): + return _find_unstorable_dtype(dtype.inner) + if isinstance(dtype, pl.Struct): + for field in dtype.fields: + found = _find_unstorable_dtype(field.dtype) + if found is not None: + return found + return None + + +def check_polars_schema_storable(schema: pl.Schema) -> None: + """Raise if any column of ``schema`` cannot be stored and read back exactly. + + Args: + schema: The schema of the frame about to be stored. + + Raises: + ValueError: If a column's dtype is, or contains, ``Object``, ``Int128``, ``UInt128`` + or a Polars extension dtype. The message names the column and its dtype. + """ + for name, dtype in schema.items(): + bad = _find_unstorable_dtype(dtype) + if bad is None: + continue + nested = "" if bad is dtype else f" (contains {bad})" + raise ValueError( + f"LogicalPolarsDataFrame: column {name!r} has dtype {dtype}{nested}, which " + "cannot be stored and read back exactly. Supported dtypes are integers up to " + "64 bits, floats, String, Binary, Boolean, Categorical, Enum, Decimal, Date, " + "Time, Datetime, Duration, Null, and List, Array or Struct of those. Convert " + "the column explicitly before returning the frame." + ) + + +class LogicalPolarsDataFrame(BaseLogicalType): + """Logical type for ``pl.DataFrame``. + + Stores frames as Arrow ``large_binary`` holding an Arrow IPC stream (see the module + docstring for the exact export settings and the rejected dtypes). + + The extension name ``"polars.dataframe"`` is library-qualified, following the + ``"pandas.dataframe"`` and ``"numpy.ndarray"`` precedent: it names the Python type users + annotate with, so a stored Arrow schema documents itself. + + Example: + >>> lt = LogicalPolarsDataFrame() + >>> df = pl.DataFrame({"x": [1, 2]}, schema={"x": pl.Int8}) + >>> lt.storage_to_python(lt.python_to_storage(df)).schema + Schema([('x', Int8)]) + """ + + _arrow_ext_class = make_arrow_extension_type("polars.dataframe", pa.large_binary()) + _arrow_ext: pa.ExtensionType | None = None + _polars_ext_class = make_polars_extension_type( + "polars.dataframe", pa.large_binary() + ) + _polars_ext: pl.BaseExtension | None = None + + logical_type_name: str = "polars.dataframe" + python_type: type = pl.DataFrame + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the cached Arrow extension type for ``pl.DataFrame``. + + Returns: + A ``pa.ExtensionType`` instance with extension name ``"polars.dataframe"`` and + storage type ``pa.large_binary()``. + """ + if LogicalPolarsDataFrame._arrow_ext is None: + LogicalPolarsDataFrame._arrow_ext = ( + LogicalPolarsDataFrame._arrow_ext_class() + ) + return LogicalPolarsDataFrame._arrow_ext + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the cached Polars extension type for ``pl.DataFrame``. + + Returns: + A ``pl.BaseExtension`` instance registered under ``"polars.dataframe"``. + """ + if LogicalPolarsDataFrame._polars_ext is None: + LogicalPolarsDataFrame._polars_ext = ( + LogicalPolarsDataFrame._polars_ext_class() + ) + return LogicalPolarsDataFrame._polars_ext + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None = None + ) -> bytes: + """Serialise a ``pl.DataFrame`` to Arrow IPC stream bytes. + + Args: + value: A ``pl.DataFrame`` whose dtypes can all round-trip exactly. + converter: Ignored. Present for protocol conformance. + + Returns: + Raw bytes of an Arrow IPC stream holding the frame as one record batch. + + Raises: + TypeError: If ``value`` is not a ``pl.DataFrame``. + ValueError: If a column's dtype cannot round-trip exactly (see + ``check_polars_schema_storable``). + """ + if not isinstance(value, pl.DataFrame): + raise TypeError( + f"LogicalPolarsDataFrame: expected a pl.DataFrame, got {type(value)!r}" + ) + check_polars_schema_storable(value.schema) + table = value.to_arrow(compat_level=pl.CompatLevel.oldest()).combine_chunks() + return table_to_ipc_bytes(table) + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None = None + ) -> pl.DataFrame: + """Reconstruct a ``pl.DataFrame`` from Arrow IPC stream bytes. + + Args: + storage_value: Raw bytes as stored in Arrow ``large_binary``. + converter: Ignored. Present for protocol conformance. + + Returns: + The ``pl.DataFrame`` with the schema it was stored with. + + Raises: + TypeError: If the stored stream does not decode to a ``pl.DataFrame``. + """ + frame = pl.from_arrow(ipc_bytes_to_table(storage_value)) + if not isinstance(frame, pl.DataFrame): + raise TypeError( + "LogicalPolarsDataFrame: stored value did not decode to a pl.DataFrame " + f"(got {type(frame)!r})" + ) + return frame diff --git a/tests/test_logical_types/test_polars_type.py b/tests/test_logical_types/test_polars_type.py new file mode 100644 index 000000000..88ddef104 --- /dev/null +++ b/tests/test_logical_types/test_polars_type.py @@ -0,0 +1,265 @@ +"""Tests for ``LogicalPolarsDataFrame`` (ITL-578). + +The round trip is type strict: every case asserts that the Polars schema read back equals +the one written, not only that the values compare equal. +""" + +from __future__ import annotations + +import datetime as dt +import decimal +from collections.abc import Callable + +import pandas as pd +import polars as pl +import pyarrow as pa +import pytest +from polars.testing import assert_frame_equal + +from orcapod.logical_types import LogicalNumpyArray, LogicalPolarsDataFrame +from orcapod.logical_types.protocols import LogicalTypeProtocol + + +def _round_trip(df: pl.DataFrame) -> pl.DataFrame: + lt = LogicalPolarsDataFrame() + return lt.storage_to_python(lt.python_to_storage(df)) + + +def _assert_exact(back: pl.DataFrame, df: pl.DataFrame) -> None: + assert back.schema == df.schema + assert_frame_equal(back, df) + + +# One column per case. Each builder returns a Series whose dtype is the one under test. +_COLUMN_CASES: dict[str, Callable[[], pl.Series]] = { + "int8_null": lambda: pl.Series([1, None, -3], dtype=pl.Int8), + "int16": lambda: pl.Series([1, 2, 3], dtype=pl.Int16), + "int32": lambda: pl.Series([1, 2, 3], dtype=pl.Int32), + "int64": lambda: pl.Series([1, 2, 3], dtype=pl.Int64), + "uint8": lambda: pl.Series([1, 2, 255], dtype=pl.UInt8), + "uint16": lambda: pl.Series([1, 2, 3], dtype=pl.UInt16), + "uint32": lambda: pl.Series([1, 2, 3], dtype=pl.UInt32), + "uint64": lambda: pl.Series([1, 2, 2**64 - 1], dtype=pl.UInt64), + "float16": lambda: pl.Series([1.0, 2.5, None], dtype=pl.Float16), + "float32": lambda: pl.Series([1.0, None, 3.5], dtype=pl.Float32), + "float64": lambda: pl.Series([1.0, float("nan"), None], dtype=pl.Float64), + "string_null": lambda: pl.Series(["a", None, ""], dtype=pl.String), + "binary_null": lambda: pl.Series([b"x", None, b""], dtype=pl.Binary), + "boolean": lambda: pl.Series([True, None, False], dtype=pl.Boolean), + "categorical": lambda: pl.Series(["b", "a", None], dtype=pl.Categorical), + "enum": lambda: pl.Series(["x", "y", None], dtype=pl.Enum(["y", "x", "z"])), + "decimal_10_2": lambda: pl.Series( + [decimal.Decimal("1.23"), None, decimal.Decimal("-9.99")], + dtype=pl.Decimal(10, 2), + ), + "decimal_38_2": lambda: pl.Series( + [decimal.Decimal("1.23"), None, decimal.Decimal("0.01")], + dtype=pl.Decimal(38, 2), + ), + "datetime_ms_tz": lambda: pl.Series( + [dt.datetime(2020, 1, 1, 12), None, dt.datetime(2021, 6, 1)], + dtype=pl.Datetime("ms", "America/New_York"), + ), + "datetime_ns": lambda: pl.Series( + [dt.datetime(2020, 1, 1), None, dt.datetime(1970, 1, 1)], + dtype=pl.Datetime("ns"), + ), + "datetime_us_utc": lambda: pl.Series( + [dt.datetime(2020, 1, 1), None, dt.datetime(1999, 12, 31)], + dtype=pl.Datetime("us", "UTC"), + ), + "date": lambda: pl.Series([dt.date(2020, 1, 1), None, dt.date(1900, 1, 1)]), + "time": lambda: pl.Series([dt.time(1, 2, 3), None, dt.time(23, 59)]), + "duration_ms": lambda: pl.Series( + [dt.timedelta(seconds=1), None, dt.timedelta(days=-1)], + dtype=pl.Duration("ms"), + ), + "duration_ns": lambda: pl.Series( + [dt.timedelta(microseconds=1), None, dt.timedelta(0)], + dtype=pl.Duration("ns"), + ), + "list_int_null": lambda: pl.Series([[1, 2], None, []], dtype=pl.List(pl.Int64)), + "list_string": lambda: pl.Series([["a"], [], [None]], dtype=pl.List(pl.String)), + "list_categorical": lambda: pl.Series( + [["a"], [], ["b", "a"]], dtype=pl.List(pl.Categorical) + ), + "list_list_uint8": lambda: pl.Series( + [[[1], []], None, [[2, 3]]], dtype=pl.List(pl.List(pl.UInt8)) + ), + "array_int16": lambda: pl.Series( + [[1, 2], [3, 4], None], dtype=pl.Array(pl.Int16, 2) + ), + "array_2d": lambda: pl.Series( + [[[1, 2]], [[3, 4]], [[5, 6]]], dtype=pl.Array(pl.Int16, (1, 2)) + ), + "struct_null": lambda: pl.Series( + [{"a": 1, "b": "x"}, None, {"a": None, "b": "z"}], + dtype=pl.Struct({"a": pl.Int64, "b": pl.String}), + ), + "struct_field_order": lambda: pl.Series( + [{"z": 1, "a": 2}, {"z": 3, "a": 4}, None], + dtype=pl.Struct({"z": pl.Int32, "a": pl.Int32}), + ), + "struct_enum": lambda: pl.Series( + [{"e": "x"}, None, {"e": "y"}], + dtype=pl.Struct({"e": pl.Enum(["x", "y"])}), + ), + "struct_list_datetime": lambda: pl.Series( + [{"t": [dt.datetime(2020, 1, 1)]}, {"t": []}, None], + dtype=pl.Struct({"t": pl.List(pl.Datetime("us", "Europe/Berlin"))}), + ), + "null": lambda: pl.Series([None, None, None], dtype=pl.Null), +} + + +class TestLogicalPolarsDataFrameProtocol: + def test_satisfies_protocol(self) -> None: + assert isinstance(LogicalPolarsDataFrame(), LogicalTypeProtocol) + + def test_logical_type_name(self) -> None: + assert LogicalPolarsDataFrame().logical_type_name == "polars.dataframe" + + def test_python_type(self) -> None: + assert LogicalPolarsDataFrame().python_type is pl.DataFrame + + def test_arrow_extension_name(self) -> None: + ext = LogicalPolarsDataFrame().get_arrow_extension_type() + assert ext.extension_name == "polars.dataframe" + + def test_arrow_storage_type_is_large_binary(self) -> None: + ext = LogicalPolarsDataFrame().get_arrow_extension_type() + assert ext.storage_type == pa.large_binary() + + def test_polars_extension_name(self) -> None: + ext = LogicalPolarsDataFrame().get_polars_extension_type() + assert ext.ext_name() == "polars.dataframe" + + def test_extension_types_are_cached(self) -> None: + a, b = LogicalPolarsDataFrame(), LogicalPolarsDataFrame() + assert a.get_arrow_extension_type() is b.get_arrow_extension_type() + assert a.get_polars_extension_type() is b.get_polars_extension_type() + + +class TestLogicalPolarsDataFrameStorage: + def test_storage_is_arrow_ipc_stream_bytes(self) -> None: + df = pl.DataFrame({"x": [1, 2]}) + stored = LogicalPolarsDataFrame().python_to_storage(df) + assert isinstance(stored, bytes) + table = pa.ipc.open_stream(stored).read_all() + assert table.column_names == ["x"] + + def test_strings_are_stored_as_large_string(self) -> None: + """The storage pins ``compat_level=oldest``: no ``string_view`` on disk.""" + df = pl.DataFrame({"s": ["a"], "b": [b"x"], "l": [["a"]]}) + stored = LogicalPolarsDataFrame().python_to_storage(df) + schema = pa.ipc.open_stream(stored).schema + assert schema.field("s").type == pa.large_string() + assert schema.field("b").type == pa.large_binary() + assert schema.field("l").type == pa.large_list(pa.large_string()) + + def test_chunked_frame_is_stored_as_one_batch(self) -> None: + df = pl.concat( + [pl.DataFrame({"x": [1]}), pl.DataFrame({"x": [2]})], rechunk=False + ) + assert df.n_chunks() == 2 + stored = LogicalPolarsDataFrame().python_to_storage(df) + batches = list(pa.ipc.open_stream(stored)) + assert len(batches) == 1 + + def test_non_dataframe_raises_type_error(self) -> None: + lt = LogicalPolarsDataFrame() + with pytest.raises(TypeError, match="pl.DataFrame"): + lt.python_to_storage(pl.Series("x", [1])) + with pytest.raises(TypeError, match="pl.DataFrame"): + lt.python_to_storage(pd.DataFrame({"x": [1]})) + + +class TestLogicalPolarsDataFrameRoundTrip: + @pytest.mark.parametrize("name", list(_COLUMN_CASES)) + def test_single_column_round_trip_is_exact(self, name: str) -> None: + df = pl.DataFrame({name: _COLUMN_CASES[name]()}) + _assert_exact(_round_trip(df), df) + + def test_wide_frame_keeps_column_order_and_names(self) -> None: + # Column names are deliberately not in sorted order. + names = list(reversed(_COLUMN_CASES)) + df = pl.DataFrame({name: _COLUMN_CASES[name]() for name in names}) + back = _round_trip(df) + assert back.columns == names + _assert_exact(back, df) + + def test_enum_category_order_is_kept(self) -> None: + df = pl.DataFrame({"e": pl.Series(["a"], dtype=pl.Enum(["c", "a", "b"]))}) + back = _round_trip(df) + assert back.schema["e"] == pl.Enum(["c", "a", "b"]) + assert back.schema["e"].categories.to_list() == ["c", "a", "b"] # pyright: ignore[reportAttributeAccessIssue] # Enum dtype has categories + + def test_empty_frame_keeps_schema(self) -> None: + df = pl.DataFrame( + schema={ + "i": pl.Int8, + "s": pl.Struct({"e": pl.Enum(["p", "q"])}), + "a": pl.Array(pl.Float32, 3), + "t": pl.Datetime("ms", "Asia/Tokyo"), + } + ) + back = _round_trip(df) + assert back.height == 0 + _assert_exact(back, df) + + def test_zero_column_frame(self) -> None: + df = pl.DataFrame() + back = _round_trip(df) + assert back.shape == (0, 0) + + def test_chunked_frame_round_trips(self) -> None: + df = pl.concat( + [pl.DataFrame({"x": [1, None]}), pl.DataFrame({"x": [3]})], rechunk=False + ) + _assert_exact(_round_trip(df), df) + + def test_sliced_frame_round_trips(self) -> None: + df = pl.DataFrame({"s": ["a", "b", "c", "d"], "l": [[1], [2], [3], [4]]}) + sliced = df.slice(1, 2) + _assert_exact(_round_trip(sliced), sliced) + + +class TestLogicalPolarsDataFrameRejectsLossyDtypes: + """Dtypes that cannot round-trip exactly are rejected at write time.""" + + @pytest.mark.parametrize( + ("series", "dtype_text"), + [ + (pl.Series("col", [object(), object()], dtype=pl.Object), "Object"), + (pl.Series("col", [1, 2], dtype=pl.Int128), "Int128"), + (pl.Series("col", [1, 2], dtype=pl.UInt128), "UInt128"), + ], + ) + def test_top_level_unsupported_dtype( + self, series: pl.Series, dtype_text: str + ) -> None: + df = pl.DataFrame([pl.Series("ok", [1, 2]), series]) + with pytest.raises(ValueError, match=rf"'col'.*{dtype_text}"): + LogicalPolarsDataFrame().python_to_storage(df) + + # Polars 1.41 cannot build an Object nested in a List or Struct, so nested rejection is + # exercised with Int128, which it can nest. + def test_int128_nested_in_list(self) -> None: + df = pl.DataFrame( + [pl.Series("nested", [[[1]]], dtype=pl.List(pl.Array(pl.Int128, 1)))] + ) + with pytest.raises(ValueError, match=r"'nested'.*contains Int128"): + LogicalPolarsDataFrame().python_to_storage(df) + + def test_int128_nested_in_struct(self) -> None: + df = pl.DataFrame( + [pl.Series("st", [{"v": 1}], dtype=pl.Struct({"v": pl.Int128}))] + ) + with pytest.raises(ValueError, match=r"'st'.*Int128"): + LogicalPolarsDataFrame().python_to_storage(df) + + def test_polars_extension_dtype(self) -> None: + ext = LogicalNumpyArray().get_polars_extension_type() + df = pl.DataFrame([pl.Series("arr", [b"x"], dtype=pl.Binary).ext.to(ext)]) + with pytest.raises(ValueError, match=r"'arr'.*numpy\.ndarray"): + LogicalPolarsDataFrame().python_to_storage(df) From 32dd5f726cfa4658fb7fa298b066b28a713acbf4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 18:50:23 -0800 Subject: [PATCH 03/36] feat(contexts): register polars.DataFrame in the default data context (ITL-578) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/contexts/data/v0.1.json | 7 ++- .../test_polars_dataframe_values.py | 21 +++++++++ tests/test_logical_types/test_roundtrips.py | 45 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/test_core/function_pod/test_polars_dataframe_values.py diff --git a/src/orcapod/contexts/data/v0.1.json b/src/orcapod/contexts/data/v0.1.json index f16b63289..4925f24f7 100644 --- a/src/orcapod/contexts/data/v0.1.json +++ b/src/orcapod/contexts/data/v0.1.json @@ -53,6 +53,10 @@ { "_class": "orcapod.logical_types.pandas_type.LogicalPandasSeries", "_config": {} + }, + { + "_class": "orcapod.logical_types.polars_type.LogicalPolarsDataFrame", + "_config": {} } ], "factories": [ @@ -162,7 +166,8 @@ "Added spikeinterface.core.SortingAnalyzer as a native value type via LogicalSISortingAnalyzer (large_string/JSON path reference {folder, format}) and SISortingAnalyzerHandler (SHA-256 of folder path string, phase 1); auto-registered when spikeinterface is installed via _optional entries in v0.1.json (ITL-469)", "Wired file_hasher reference into BasicDirectoryHasher so op.Directory hashing uses the same FileContentHasherProtocol as op.File hashing; required file_hasher constructor arg enforces explicit injection", "Moved SpikeInterface types (LogicalSIRecording, LogicalSISorting, LogicalSIMotion, LogicalSISortingAnalyzer) and their handlers to the standalone orcapod-extension-spikeinterface package; removed all SI _optional entries from this spec and SI deps from pyproject.toml; introduced OrcapodExtension protocol and op.register_extension() for normalized extension registration (ITL-473)", - "Registered SchemaHandler for orcapod.types.Schema in the context registry; moved handler dispatch before _is_structure check in hash_object so SchemaHandler takes priority over _expand_mapping for Schema objects (ITL-639)" + "Registered SchemaHandler for orcapod.types.Schema in the context registry; moved handler dispatch before _is_structure check in hash_object so SchemaHandler takes priority over _expand_mapping for Schema objects (ITL-639)", + "Added polars.DataFrame as a native value type via LogicalPolarsDataFrame (Arrow IPC stream / large_binary, exported with compat_level=oldest as one record batch); Object, Int128, UInt128 and Polars extension dtypes are rejected at write time (ITL-578)" ] } } diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py new file mode 100644 index 000000000..8158b3b43 --- /dev/null +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -0,0 +1,21 @@ +"""``pl.DataFrame`` values through the public interface (ITL-578). + +Function pods take and return ``pl.DataFrame``; the frames survive operators, orchestrators +and every database backend unchanged, dtype for dtype. +""" + +from __future__ import annotations + +import polars as pl + +from orcapod import function_pod + + +def test_pod_can_return_a_polars_dataframe() -> None: + """Declaring a pod with a ``pl.DataFrame`` return type no longer raises.""" + + @function_pod(output_keys="df") + def make_frame(n: int) -> pl.DataFrame: + return pl.DataFrame({"i": list(range(n))}) + + assert make_frame is not None diff --git a/tests/test_logical_types/test_roundtrips.py b/tests/test_logical_types/test_roundtrips.py index c576836f9..3d076b20e 100644 --- a/tests/test_logical_types/test_roundtrips.py +++ b/tests/test_logical_types/test_roundtrips.py @@ -498,6 +498,51 @@ def test_builtin_dataframe_round_trip( pd.testing.assert_frame_equal(recovered, df) +def test_builtin_polars_dataframe_round_trip( + storage_backend: _StorageBackend, tmp_path: Path +) -> None: + """pl.DataFrame round-trips through storage with extension name ``polars.dataframe``. + + The frame read back has exactly the schema it was written with (ITL-578). + """ + import datetime as dt + + import polars as pl + from polars.testing import assert_frame_equal + + df = pl.DataFrame( + { + "e": pl.Series(["b", "a"], dtype=pl.Enum(["b", "a", "c"])), + "t": pl.Series( + [dt.datetime(2020, 1, 1), None], dtype=pl.Datetime("ms", "UTC") + ), + "s": pl.Series( + [{"x": 1, "y": [1.5]}, None], + dtype=pl.Struct({"x": pl.UInt16, "y": pl.List(pl.Float32)}), + ), + } + ) + result, read_converter = _write_and_read( + {"frame": pl.DataFrame}, + [{"frame": df}], + storage_backend, + tmp_path, + ) + + field = result.schema.field("frame") + assert hasattr(field.type, "extension_name"), ( + f"Expected extension type on field 'frame', got plain type {field.type!r}" + ) + assert field.type.extension_name == "polars.dataframe" + + rows = read_converter.arrow_table_to_python_dicts(result) + assert len(rows) == 1 + recovered = rows[0]["frame"] + assert isinstance(recovered, pl.DataFrame) + assert recovered.schema == df.schema + assert_frame_equal(recovered, df) + + def test_builtin_series_round_trip( storage_backend: _StorageBackend, tmp_path: Path ) -> None: From 75c8b5520700bb72eeb7b0da1498464681561502 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 18:52:52 -0800 Subject: [PATCH 04/36] test(function_pod): polars.DataFrame values through pods, operators and databases (ITL-578) Co-Authored-By: Claude Opus 5.5 --- .../test_polars_dataframe_values.py | 271 +++++++++++++++++- 1 file changed, 267 insertions(+), 4 deletions(-) diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py index 8158b3b43..e22e60a76 100644 --- a/tests/test_core/function_pod/test_polars_dataframe_values.py +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -1,21 +1,284 @@ """``pl.DataFrame`` values through the public interface (ITL-578). Function pods take and return ``pl.DataFrame``; the frames survive operators, orchestrators -and every database backend unchanged, dtype for dtype. +and database backends unchanged, dtype for dtype. Every comparison checks the Polars schema +as well as the values. """ from __future__ import annotations +import datetime as dt +import json +import subprocess +import sys +import textwrap +from collections.abc import Callable +from pathlib import Path + import polars as pl +import pytest +from polars.testing import assert_frame_equal + +from orcapod import PipelineJob, function_pod +from orcapod.databases import ( + ConnectorArrowDatabase, + DeltaTableDatabase, + InMemoryArrowDatabase, + SQLiteConnector, +) +from orcapod.operators import Batch, GroupBy, Join +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.sources import DictSource + + +def _frame(n: int) -> pl.DataFrame: + """A frame of height ``n`` whose dtypes would all be lost by careless conversion.""" + return pl.DataFrame( + { + "i": pl.Series(list(range(n)), dtype=pl.Int8), + "e": pl.Series(["a"] * n, dtype=pl.Enum(["b", "a"])), + "t": pl.Series( + [dt.datetime(2020, 1, 1)] * n, dtype=pl.Datetime("ms", "UTC") + ), + "s": pl.Series( + [{"u": k, "l": [k]} for k in range(n)], + dtype=pl.Struct({"u": pl.UInt16, "l": pl.List(pl.Float32)}), + ), + } + ) + + +def _assert_exact(got: object, expected: pl.DataFrame) -> None: + assert isinstance(got, pl.DataFrame) + assert got.schema == expected.schema + assert_frame_equal(got, expected) + + +@function_pod(output_keys="df") +def make_frame(n: int) -> pl.DataFrame: + return _frame(n) + + +@function_pod(output_keys="df") +def add_row_index(df: pl.DataFrame) -> pl.DataFrame: + return df.with_row_index("row", offset=0) + + +@function_pod(output_keys="height") +def frame_height(df: pl.DataFrame) -> int: + return df.height -from orcapod import function_pod + +def _int_source() -> DictSource: + return DictSource( + [{"k": "a", "n": 1}, {"k": "b", "n": 3}], + tag_columns=["k"], + ) + + +def _frame_source() -> DictSource: + """Three frame-valued rows: two in group ``a``, one in group ``b``.""" + return DictSource( + [ + {"g": "a", "k": "1", "df": _frame(1)}, + {"g": "a", "k": "2", "df": _frame(2)}, + {"g": "b", "k": "3", "df": _frame(3)}, + ], + tag_columns=["g", "k"], + data_schema={"g": str, "k": str, "df": pl.DataFrame}, + ) def test_pod_can_return_a_polars_dataframe() -> None: """Declaring a pod with a ``pl.DataFrame`` return type no longer raises.""" @function_pod(output_keys="df") - def make_frame(n: int) -> pl.DataFrame: + def make(n: int) -> pl.DataFrame: return pl.DataFrame({"i": list(range(n))}) - assert make_frame is not None + assert make.pod is not None + + +class TestPodValues: + def test_pod_output(self) -> None: + stream = make_frame.pod(_int_source()) + got = {tag["k"]: data["df"] for tag, data in stream.iter_data()} + _assert_exact(got["a"], _frame(1)) + _assert_exact(got["b"], _frame(3)) + + def test_output_schema_names_the_python_type(self) -> None: + _, data_schema = make_frame.pod(_int_source()).output_schema() + assert data_schema["df"] is pl.DataFrame + + def test_pod_input(self) -> None: + stream = frame_height.pod(_frame_source()) + got = {tag["k"]: data["height"] for tag, data in stream.iter_data()} + assert got == {"1": 1, "2": 2, "3": 3} + + def test_chained_pods(self) -> None: + stream = frame_height.pod(add_row_index.pod(make_frame.pod(_int_source()))) + got = {tag["k"]: data["height"] for tag, data in stream.iter_data()} + assert got == {"a": 1, "b": 3} + + def test_chained_frame_is_transformed_exactly(self) -> None: + stream = add_row_index.pod(make_frame.pod(_int_source())) + got = {tag["k"]: data["df"] for tag, data in stream.iter_data()} + _assert_exact(got["b"], _frame(3).with_row_index("row", offset=0)) + + +class TestOperators: + def test_batch_over_frame_column(self) -> None: + stream = Batch(batch_size=2)(_frame_source()) + rows = [(tag["k"], data["df"]) for tag, data in stream.iter_data()] + assert [keys for keys, _ in rows] == [["1", "2"], ["3"]] + for got, n in zip(rows[0][1] + rows[1][1], [1, 2, 3], strict=True): + _assert_exact(got, _frame(n)) + + def test_batch_output_type_is_a_list_of_frames(self) -> None: + _, data_schema = Batch(batch_size=2)(_frame_source()).output_schema() + assert data_schema["df"] == list[pl.DataFrame] + + def test_group_by_over_frame_column(self) -> None: + stream = GroupBy(by=["g"])(_frame_source()) + got = {tag["g"]: data["df"] for tag, data in stream.iter_data()} + assert sorted(got) == ["a", "b"] + for frame, n in zip(got["a"], [1, 2], strict=True): + _assert_exact(frame, _frame(n)) + (only_b,) = got["b"] + _assert_exact(only_b, _frame(3)) + + def test_join_passes_frames_through(self) -> None: + weights = DictSource( + [{"g": "a", "k": "1", "w": 10}, {"g": "b", "k": "3", "w": 30}], + tag_columns=["g", "k"], + ) + stream = Join()(_frame_source(), weights) + got = {tag["k"]: (data["df"], data["w"]) for tag, data in stream.iter_data()} + assert sorted(got) == ["1", "3"] + _assert_exact(got["1"][0], _frame(1)) + _assert_exact(got["3"][0], _frame(3)) + assert (got["1"][1], got["3"][1]) == (10, 30) + + +_STORES: dict[str, Callable[[Path], ArrowDatabaseProtocol]] = { + "in_memory": lambda _: InMemoryArrowDatabase(), + "delta": lambda tmp: DeltaTableDatabase(tmp / "delta"), +} +_ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + + +class TestPipelineStores: + @pytest.mark.parametrize("store_name", list(_STORES)) + @pytest.mark.parametrize("orchestrator_name", list(_ORCHESTRATORS)) + def test_frames_stored_and_read_back( + self, store_name: str, orchestrator_name: str, tmp_path: Path + ) -> None: + job = PipelineJob(name="frames", store=_STORES[store_name](tmp_path)) + with job: + frames = make_frame.pod(_int_source(), label="make") + frame_height.pod(frames, label="height") + job.run(orchestrator=_ORCHESTRATORS[orchestrator_name]()) + + got = {tag["k"]: data["df"] for tag, data in job.nodes["make"].iter_data()} + _assert_exact(got["a"], _frame(1)) + _assert_exact(got["b"], _frame(3)) + heights = { + tag["k"]: data["height"] for tag, data in job.nodes["height"].iter_data() + } + assert heights == {"a": 1, "b": 3} + + @pytest.mark.xfail( + strict=True, + reason=( + "ITL-877: SQLiteConnector rejects every logical-typed column at write time " + "('does not preserve Arrow metadata'), for pandas, numpy and Path too; " + "not specific to pl.DataFrame." + ), + ) + def test_sqlite_store(self, tmp_path: Path) -> None: + connector = SQLiteConnector(str(tmp_path / "frames.sqlite")) + job = PipelineJob(name="frames", store=ConnectorArrowDatabase(connector)) + with job: + make_frame.pod(_int_source(), label="make") + job.run() + + got = {tag["k"]: data["df"] for tag, data in job.nodes["make"].iter_data()} + _assert_exact(got["a"], _frame(1)) + _assert_exact(got["b"], _frame(3)) + + +# Runs one pipeline over a Delta store and prints how many times each pod body ran, plus +# the frames read back. The pod bodies append to a marker file so a cache hit (body not +# run) is observable from the parent process. +_CROSS_PROCESS_SCRIPT = textwrap.dedent( + """ + import json, sys + from pathlib import Path + import polars as pl + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.sources import DictSource + + store_path, marker = Path(sys.argv[1]), Path(sys.argv[2]) + + @function_pod(output_keys="df") + def make_frame(n: int) -> pl.DataFrame: + with marker.open("a") as fh: + fh.write("make\\n") + return pl.DataFrame({ + "i": pl.Series(list(range(n)), dtype=pl.Int8), + "e": pl.Series(["a"] * n, dtype=pl.Enum(["b", "a"])), + }) + + @function_pod(output_keys="height") + def frame_height(df: pl.DataFrame) -> int: + with marker.open("a") as fh: + fh.write("height\\n") + return df.height + + source = DictSource([{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"]) + job = PipelineJob(name="frames", store=DeltaTableDatabase(store_path)) + with job: + frames = make_frame.pod(source, label="make") + frame_height.pod(frames, label="height") + job.run() + frames = {t["k"]: d["df"] for t, d in job.nodes["make"].iter_data()} + print(json.dumps({ + "frames": {k: f.write_json() for k, f in frames.items()}, + "schemas": {k: str(f.schema) for k, f in frames.items()}, + "heights": {t["k"]: d["height"] for t, d in job.nodes["height"].iter_data()}, + })) + """ +) + + +def _run_pipeline_process(store_path: Path, marker: Path) -> dict: + result = subprocess.run( + [sys.executable, "-c", _CROSS_PROCESS_SCRIPT, str(store_path), str(marker)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_cached_rerun_in_new_process_hits_the_cache(tmp_path: Path) -> None: + """A second process over the same Delta store recomputes nothing and reads equal frames.""" + store_path, marker = tmp_path / "store", tmp_path / "calls.txt" + + first = _run_pipeline_process(store_path, marker) + calls_after_first = marker.read_text().splitlines() + assert sorted(calls_after_first) == ["height", "height", "make", "make"] + + second = _run_pipeline_process(store_path, marker) + assert marker.read_text().splitlines() == calls_after_first # nothing re-ran + assert second == first + assert second["heights"] == {"a": 1, "b": 3} + assert second["schemas"]["b"] == str( + pl.Schema({"i": pl.Int8, "e": pl.Enum(["b", "a"])}) + ) From 504b4c19c201d1da20b5b1fcb7e290a2b2f0d30e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 18:53:32 -0800 Subject: [PATCH 05/36] docs(function_pods): list value types, including polars.DataFrame (ITL-578) Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8618052c..2413effee 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -228,6 +228,68 @@ A node from a job that has run also provides: - **`as_source()`** -- a `DerivedSource` backed by those stored records, usable as the input of another pipeline +## Value types + +A function pod's parameters and return values can be plain Python scalars (`int`, `float`, +`str`, `bool`, `bytes`), lists, dataclasses, pydantic models, and the built-in logical types +below. A logical type is stored as an Arrow extension type: the extension name is recorded +in the stored schema, so a value read back from a database comes back as the same Python +type. + +| Python type | Extension name | Arrow storage | +|---|---|---| +| `pathlib.Path` | `orcapod.path` | `large_string` | +| `upath.UPath` | `orcapod.upath` | `large_string` | +| `uuid.UUID` | `orcapod.uuid` | `large_binary` | +| `orcapod.File` | `orcapod.file` | `large_string` | +| `orcapod.Directory` | `orcapod.directory` | `large_string` | +| `numpy.ndarray` | `numpy.ndarray` | `large_binary` (`.npy` bytes) | +| `pandas.DataFrame` | `pandas.dataframe` | `large_binary` (Arrow IPC stream) | +| `pandas.Series` | `pandas.series` | `large_binary` (Arrow IPC stream) | +| `polars.DataFrame` | `polars.dataframe` | `large_binary` (Arrow IPC stream) | + +A `polars.DataFrame` comes back with exactly the schema it was stored with: column order and +names, integer and float widths, `Categorical`, `Enum` with its category order, `Decimal` +precision and scale, `Datetime` unit and time zone, `Duration` unit, `List`, `Array` shape, +`Struct` fields and `Null`. Dtypes that cannot be stored and read back exactly (`Object`, +`Int128`, `UInt128`, and Polars extension dtypes) raise `ValueError` naming the column when +the value is stored, so convert such a column explicitly before returning the frame. + +```python +import polars as pl + +from orcapod import function_pod +from orcapod.sources import DictSource + +@function_pod(output_keys="readings") +def load_readings(n_samples: int) -> pl.DataFrame: + return pl.DataFrame( + { + "sample": pl.Series(range(n_samples), dtype=pl.UInt16), + "voltage": pl.Series([0.5] * n_samples, dtype=pl.Float32), + } + ) + +@function_pod(output_keys="n_rows") +def count_rows(readings: pl.DataFrame) -> int: + return readings.height + +recordings = DictSource( + data=[{"session": "s1", "n_samples": 2}, {"session": "s2", "n_samples": 3}], + tag_columns=["session"], +) +readings = load_readings.pod(recordings) +for tag, data in readings.iter_data(): + print(tag["session"], data["readings"].schema) +# s1 Schema({'sample': UInt16, 'voltage': Float32}) +# s2 Schema({'sample': UInt16, 'voltage': Float32}) + +for tag, data in count_rows.pod(readings).iter_data(): + print(tag["session"], data["n_rows"]) +# s1 2 +# s2 3 +``` + ## Multiple input streams If you pass multiple streams to a function pod, they are automatically joined (using From cca88ca4e6343f480c25ef4edca03676c00e10cd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 18:55:32 -0800 Subject: [PATCH 06/36] fix(logical_types): type the polars dtype walker and polars value tests for basedpyright (ITL-578) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/logical_types/polars_type.py | 17 +++++++--- .../test_polars_dataframe_values.py | 32 +++++++++++++------ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/orcapod/logical_types/polars_type.py b/src/orcapod/logical_types/polars_type.py index 1a58a2d12..f110f0b84 100644 --- a/src/orcapod/logical_types/polars_type.py +++ b/src/orcapod/logical_types/polars_type.py @@ -32,24 +32,33 @@ ) if TYPE_CHECKING: + from polars._typing import PolarsDataType + from orcapod.logical_types.protocols import TypeConverterProtocol _UNSTORABLE_DTYPES: tuple[type[pl.DataType], ...] = (pl.Object, pl.Int128, pl.UInt128) -def _find_unstorable_dtype(dtype: pl.DataType) -> pl.DataType | None: +def _find_unstorable_dtype(dtype: PolarsDataType) -> PolarsDataType | None: """Return the first dtype inside ``dtype`` that cannot round-trip exactly. Walks ``List``, ``Array`` and ``Struct`` children. Args: - dtype: A Polars dtype, possibly nested. + dtype: A Polars dtype instance or class, possibly nested. Returns: The offending (possibly nested) dtype, or ``None`` when every part of ``dtype`` can be stored losslessly. """ + if isinstance(dtype, type): + # A bare dtype class has no children to walk. + return ( + dtype + if issubclass(dtype, (pl.BaseExtension, *_UNSTORABLE_DTYPES)) + else None + ) if isinstance(dtype, pl.BaseExtension): return dtype if isinstance(dtype, _UNSTORABLE_DTYPES): @@ -125,7 +134,7 @@ def get_arrow_extension_type(self) -> pa.ExtensionType: if LogicalPolarsDataFrame._arrow_ext is None: LogicalPolarsDataFrame._arrow_ext = ( LogicalPolarsDataFrame._arrow_ext_class() - ) + ) # pyright: ignore[reportCallIssue] # synthesized subclass takes no constructor arguments return LogicalPolarsDataFrame._arrow_ext def get_polars_extension_type(self) -> pl.BaseExtension: @@ -137,7 +146,7 @@ def get_polars_extension_type(self) -> pl.BaseExtension: if LogicalPolarsDataFrame._polars_ext is None: LogicalPolarsDataFrame._polars_ext = ( LogicalPolarsDataFrame._polars_ext_class() - ) + ) # pyright: ignore[reportCallIssue] # synthesized subclass takes no constructor arguments return LogicalPolarsDataFrame._polars_ext def python_to_storage( diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py index e22e60a76..e5ca18a5c 100644 --- a/tests/test_core/function_pod/test_polars_dataframe_values.py +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -14,6 +14,7 @@ import textwrap from collections.abc import Callable from pathlib import Path +from typing import Any, cast import polars as pl import pytest @@ -56,6 +57,13 @@ def _assert_exact(got: object, expected: pl.DataFrame) -> None: assert_frame_equal(got, expected) +def _frame_list(value: object) -> list[pl.DataFrame]: + """Narrow a list-valued data column to a list of frames.""" + assert isinstance(value, list) + assert all(isinstance(item, pl.DataFrame) for item in value) + return cast(list[pl.DataFrame], value) + + @function_pod(output_keys="df") def make_frame(n: int) -> pl.DataFrame: return _frame(n) @@ -80,12 +88,13 @@ def _int_source() -> DictSource: def _frame_source() -> DictSource: """Three frame-valued rows: two in group ``a``, one in group ``b``.""" + rows: list[dict[str, Any]] = [ + {"g": "a", "k": "1", "df": _frame(1)}, + {"g": "a", "k": "2", "df": _frame(2)}, + {"g": "b", "k": "3", "df": _frame(3)}, + ] return DictSource( - [ - {"g": "a", "k": "1", "df": _frame(1)}, - {"g": "a", "k": "2", "df": _frame(2)}, - {"g": "b", "k": "3", "df": _frame(3)}, - ], + rows, tag_columns=["g", "k"], data_schema={"g": str, "k": str, "df": pl.DataFrame}, ) @@ -133,7 +142,8 @@ def test_batch_over_frame_column(self) -> None: stream = Batch(batch_size=2)(_frame_source()) rows = [(tag["k"], data["df"]) for tag, data in stream.iter_data()] assert [keys for keys, _ in rows] == [["1", "2"], ["3"]] - for got, n in zip(rows[0][1] + rows[1][1], [1, 2, 3], strict=True): + members = _frame_list(rows[0][1]) + _frame_list(rows[1][1]) + for got, n in zip(members, [1, 2, 3], strict=True): _assert_exact(got, _frame(n)) def test_batch_output_type_is_a_list_of_frames(self) -> None: @@ -142,7 +152,9 @@ def test_batch_output_type_is_a_list_of_frames(self) -> None: def test_group_by_over_frame_column(self) -> None: stream = GroupBy(by=["g"])(_frame_source()) - got = {tag["g"]: data["df"] for tag, data in stream.iter_data()} + got = { + str(tag["g"]): _frame_list(data["df"]) for tag, data in stream.iter_data() + } assert sorted(got) == ["a", "b"] for frame, n in zip(got["a"], [1, 2], strict=True): _assert_exact(frame, _frame(n)) @@ -155,7 +167,9 @@ def test_join_passes_frames_through(self) -> None: tag_columns=["g", "k"], ) stream = Join()(_frame_source(), weights) - got = {tag["k"]: (data["df"], data["w"]) for tag, data in stream.iter_data()} + got = { + str(tag["k"]): (data["df"], data["w"]) for tag, data in stream.iter_data() + } assert sorted(got) == ["1", "3"] _assert_exact(got["1"][0], _frame(1)) _assert_exact(got["3"][0], _frame(3)) @@ -257,7 +271,7 @@ def frame_height(df: pl.DataFrame) -> int: ) -def _run_pipeline_process(store_path: Path, marker: Path) -> dict: +def _run_pipeline_process(store_path: Path, marker: Path) -> dict[str, Any]: result = subprocess.run( [sys.executable, "-c", _CROSS_PROCESS_SCRIPT, str(store_path), str(marker)], capture_output=True, From 7d0e21e814a97560d2c372d5d41b470ade3a1bd1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 19:10:40 -0800 Subject: [PATCH 07/36] feat(hashing): hash polars values by an Orcapod dtype descriptor plus starfix values; add polars.Series (ITL-578) PolarsDataFrameHandler / PolarsSeriesHandler hash sha256(versioned canonical dtype descriptor + starfix digest of a hash-only projection), independent of chunking, slice offsets, null-slot bytes, column order and struct field order. Also keep a column named "" on read (pl.from_arrow renames it to column_0). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/contexts/data/v0.1.json | 8 +- .../semantic_hashing/builtin_handlers.py | 71 +++ src/orcapod/logical_types/__init__.py | 3 +- src/orcapod/logical_types/polars_type.py | 388 ++++++++++++++-- ...0-itl-578-polars-dataframe-logical-type.md | 34 +- .../test_polars_dataframe_values.py | 29 ++ .../test_polars_dataframe_hashing.py | 439 ++++++++++++++++++ tests/test_logical_types/test_polars_type.py | 61 ++- 8 files changed, 998 insertions(+), 35 deletions(-) create mode 100644 tests/test_hashing/test_polars_dataframe_hashing.py diff --git a/src/orcapod/contexts/data/v0.1.json b/src/orcapod/contexts/data/v0.1.json index 4925f24f7..f24fa1a4e 100644 --- a/src/orcapod/contexts/data/v0.1.json +++ b/src/orcapod/contexts/data/v0.1.json @@ -57,6 +57,10 @@ { "_class": "orcapod.logical_types.polars_type.LogicalPolarsDataFrame", "_config": {} + }, + { + "_class": "orcapod.logical_types.polars_type.LogicalPolarsSeries", + "_config": {} } ], "factories": [ @@ -125,6 +129,8 @@ [{"_type": "numpy.ndarray"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.NumpyArrayHandler", "_config": {}}], [{"_type": "pandas.DataFrame"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PandasDataFrameHandler", "_config": {}}], [{"_type": "pandas.Series"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PandasSeriesHandler", "_config": {}}], + [{"_type": "polars.DataFrame"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PolarsDataFrameHandler", "_config": {}}], + [{"_type": "polars.Series"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.PolarsSeriesHandler", "_config": {}}], [{"_type": "orcapod.types.Schema"}, {"_class": "orcapod.hashing.semantic_hashing.builtin_handlers.SchemaHandler", "_config": {"type_converter": {"_ref": "type_converter"}}}] ] } @@ -167,7 +173,7 @@ "Wired file_hasher reference into BasicDirectoryHasher so op.Directory hashing uses the same FileContentHasherProtocol as op.File hashing; required file_hasher constructor arg enforces explicit injection", "Moved SpikeInterface types (LogicalSIRecording, LogicalSISorting, LogicalSIMotion, LogicalSISortingAnalyzer) and their handlers to the standalone orcapod-extension-spikeinterface package; removed all SI _optional entries from this spec and SI deps from pyproject.toml; introduced OrcapodExtension protocol and op.register_extension() for normalized extension registration (ITL-473)", "Registered SchemaHandler for orcapod.types.Schema in the context registry; moved handler dispatch before _is_structure check in hash_object so SchemaHandler takes priority over _expand_mapping for Schema objects (ITL-639)", - "Added polars.DataFrame as a native value type via LogicalPolarsDataFrame (Arrow IPC stream / large_binary, exported with compat_level=oldest as one record batch); Object, Int128, UInt128 and Polars extension dtypes are rejected at write time (ITL-578)" + "Added polars.DataFrame as a native value type via LogicalPolarsDataFrame (Arrow IPC stream / large_binary, exported with compat_level=oldest as one record batch); Object, Int128, UInt128 and Polars extension dtypes are rejected at write time; added polars.Series via LogicalPolarsSeries (one-column frame, same envelope); PolarsDataFrameHandler / PolarsSeriesHandler hash sha256(versioned Orcapod dtype descriptor + starfix digest of the values), independent of chunking, null-slot bytes, column order and struct field order (ITL-578)" ] } } diff --git a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py index 0fe51ed3f..a0a9d823f 100644 --- a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py +++ b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py @@ -494,6 +494,72 @@ def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: return self._get_arrow_hasher().hash_table(table) +class PolarsDataFrameHandler: + """Hasher for ``pl.DataFrame``: Orcapod's dtype descriptor plus a starfix value digest. + + Delegates to ``polars_frame_content_hash`` in ``orcapod.logical_types.polars_type``, + whose module docstring defines the scheme. The hash ignores chunking, slice offsets, the + bytes under null slots, column order and struct field order, and distinguishes every + dtype parameter (Enum categories and their order, Decimal precision and scale, Datetime + unit and time zone, Duration unit, Array shape). + """ + + def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: + """Return the content hash of ``obj``. + + Args: + obj: A ``pl.DataFrame`` instance. + hasher: Ignored. Present for protocol conformance. + + Returns: + A ``ContentHash`` with method ``"sha256"``. + + Raises: + TypeError: If ``obj`` is not a ``pl.DataFrame``. + ValueError: If a column's dtype is outside the descriptor grammar. + """ + import polars as _pl + + from orcapod.logical_types.polars_type import polars_frame_content_hash + + if not isinstance(obj, _pl.DataFrame): + raise TypeError( + f"PolarsDataFrameHandler: expected pl.DataFrame, got {type(obj)!r}" + ) + return polars_frame_content_hash(obj) + + +class PolarsSeriesHandler: + """Hasher for ``pl.Series``: its name, Orcapod dtype descriptor and starfix value digest. + + Delegates to ``polars_series_content_hash`` in ``orcapod.logical_types.polars_type``. + """ + + def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: + """Return the content hash of ``obj``. + + Args: + obj: A ``pl.Series`` instance. + hasher: Ignored. Present for protocol conformance. + + Returns: + A ``ContentHash`` with method ``"sha256"``. + + Raises: + TypeError: If ``obj`` is not a ``pl.Series``. + ValueError: If the dtype is outside the descriptor grammar. + """ + import polars as _pl + + from orcapod.logical_types.polars_type import polars_series_content_hash + + if not isinstance(obj, _pl.Series): + raise TypeError( + f"PolarsSeriesHandler: expected pl.Series, got {type(obj)!r}" + ) + return polars_series_content_hash(obj) + + def register_builtin_python_type_handlers( registry: "HandlerRegistryProtocol", file_hasher: Any = None, @@ -613,6 +679,11 @@ def register_builtin_python_type_handlers( registry.register(_pd.DataFrame, PandasDataFrameHandler(arrow_hasher)) registry.register(_pd.Series, PandasSeriesHandler(arrow_hasher)) + import polars as _pl + + registry.register(_pl.DataFrame, PolarsDataFrameHandler()) + registry.register(_pl.Series, PolarsSeriesHandler()) + logger.debug( "register_builtin_python_type_handlers: registered %d hashers", len(registry), diff --git a/src/orcapod/logical_types/__init__.py b/src/orcapod/logical_types/__init__.py index 313526379..639ffa49e 100644 --- a/src/orcapod/logical_types/__init__.py +++ b/src/orcapod/logical_types/__init__.py @@ -47,7 +47,7 @@ from .directory_type import LogicalDirectory # ITL-451 from .numpy_type import LogicalNumpyArray # ITL-460 from .pandas_type import LogicalPandasDataFrame, LogicalPandasSeries # PLT-1869 -from .polars_type import LogicalPolarsDataFrame # ITL-578 +from .polars_type import LogicalPolarsDataFrame, LogicalPolarsSeries # ITL-578 __all__ = [ "LogicalTypeProtocol", @@ -86,4 +86,5 @@ "LogicalPandasSeries", # ITL-578 "LogicalPolarsDataFrame", + "LogicalPolarsSeries", ] diff --git a/src/orcapod/logical_types/polars_type.py b/src/orcapod/logical_types/polars_type.py index f110f0b84..7752b7064 100644 --- a/src/orcapod/logical_types/polars_type.py +++ b/src/orcapod/logical_types/polars_type.py @@ -1,7 +1,11 @@ -"""``polars.DataFrame`` logical type for orcapod. +"""``polars.DataFrame`` and ``polars.Series`` logical types for orcapod. +Storage +------- ``LogicalPolarsDataFrame`` maps ``pl.DataFrame`` <-> Arrow ``large_binary`` using an Arrow IPC stream as the storage envelope, the same envelope ``LogicalPandasDataFrame`` uses. +``LogicalPolarsSeries`` stores a ``pl.Series`` the same way, as a one-column frame named +after the Series (a Polars Series name is always a ``str``, so no sentinel is needed). The frame is exported with ``compat_level=pl.CompatLevel.oldest()``, so strings and binaries are written as ``large_string`` / ``large_binary`` (never the view types) whatever the @@ -15,14 +19,57 @@ column and dtype: ``Object`` (Arrow export would silently store raw pointer bytes), ``Int128`` / ``UInt128`` (pyarrow cannot import them), and Polars extension dtypes (a logical value nested inside a frame value is not supported). + +Content hash +------------ +The stored IPC bytes are not canonical (the bytes under null slots, and Polars' own field +metadata, vary for equal frames), so the content hash is computed from the frame itself by +``polars_frame_content_hash`` / ``polars_series_content_hash``, which the semantic hashers +``PolarsDataFrameHandler`` / ``PolarsSeriesHandler`` call. The hash is:: + + sha256(descriptor + b"\\x00" + starfix_digest) + +``starfix_digest`` is ``starfix.ArrowDigester.hash_table`` of a hash-only Arrow projection of +the values (see ``_hash_projection``). Like every Orcapod table hash it ignores column order +and struct field order, and hashes a dictionary-encoded column by its values. + +``descriptor`` is canonical JSON (keys sorted, separators ``","`` and ``":"``, UTF-8) that +carries what starfix does not see, in a grammar Orcapod owns (never Polars' ``repr`` or its +``_PL_*`` metadata):: + + frame: {"v": 1, "kind": "polars.dataframe", "height": , + "columns": {: , ...}} + series: {"v": 1, "kind": "polars.series", "name": , "height": , + "dtype": } + + := "int8" | "int16" | "int32" | "int64" + | "uint8" | "uint16" | "uint32" | "uint64" + | "float16" | "float32" | "float64" + | "boolean" | "string" | "binary" | "date" | "time" | "null" + | "categorical" + | {"enum": [, ...]} categories in their order + | {"decimal": [, ]} + | {"datetime": [,