From b30160543ce5219111e4d73b076ed52171c28136 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:15:54 -0800 Subject: [PATCH 1/7] docs(plans): ITL-1054 reproduction findings and plan Co-Authored-By: Claude Opus 5.5 --- ...6-11-14-itl-1054-pandas-refuse-at-write.md | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md diff --git a/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md b/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md new file mode 100644 index 000000000..2be1dd0fc --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md @@ -0,0 +1,129 @@ +# ITL-1054: pandas values, refuse at write what storage cannot restore — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A pandas `DataFrame` / `Series` that storage would change silently (MultiIndex levels +reordered) or could not read back is refused with `ValueError` naming the column or level, by the +value hash and by `python_to_storage`, through one shared check. + +**Architecture:** One function in `orcapod/hashing/pandas_value_hash.py` (the module +`logical_types/pandas_type.py` already imports from) takes the Arrow schema storage would write and +the frame, returns the empty frame storage restores for that schema, and raises `ValueError` for +every value storage would not restore. `python_to_storage` (frame and Series) and the hash's +`_values_table` / `_in_canonical_column_order` both call it. No identifier, label, or golden value +changes. + +**Tech stack:** pandas 2.2.3 (locked; `<3`), pyarrow `>=22.0.0` (locked 23.0.1). + +**Status:** step 1 (reproduce, search) done; WAITING for the coordinator's reply to the list and +plan below (sent with `orca orchestration ask`). + +--- + +## Findings (reproduced November 14, 2026) + +Scripts: the session scratchpad `repro/` (direct.py, pipeline.py, search.py, classify.py, +probe.py, emptycheck.py); not committed. + +pandas 2.2.3; pyarrow 22.0.0, 23.0.1 (locked) and 26.0.1 (newest on PyPI; the lock allows any +`>=22`): **identical results on all three.** + +### Case (1): written, read back changed, nothing raises + +`pd.MultiIndex.from_tuples([("p", 7), ("q", 8)], names=[1, 0])` reads back as +`[(7, "p"), (8, "q")]`, names `["0", "1"]`: levels reordered. Frame and Series; logical type +directly and through a `PipelineJob` with `InMemoryArrowDatabase` and `DeltaTableDatabase`. The +hash is "stable" (`hash(v) == hash(read(write(v)))`) because the hash describes the restored +form; so `v` hashes equal to a genuinely different frame (the reordered one): a collision of an +identity property (`index_level_order`). + +Root cause (pyarrow `pandas_compat._get_index_level_values`): pyarrow fetches level `i` with +`index.get_level_values(i)`, and pandas resolves an integer as a level NAME first. So any +MultiIndex level name equal (by `==`, so `1`, `1.0`, `True`) to another level's position +reorders. Over 1342 accepted name combinations (2 and 3 levels; names from ints, floats, +bools, text, None), 202 reorder; the check "level `i` fetched by `get_level_values(i)` is not +level `i`" predicts every one, with zero disagreements. + +### Case (2): written, read raises + +Through a `PipelineJob`: the result row is written to the result table (`rdb_v2`), the run's +read-back fails and is reported as a row failure (`RowFailuresWarning`), and the cached row +stays unreadable on every rerun. + +Unreadable shapes found (frame and Series alike): + +- column (or Series values) of `pd.ArrowDtype`: `list`, `large_list`, `fixed_size_list`, + `list_view`, `struct`, `map`, `dictionary`, `run_end_encoded`, `dense_union`, + `fixed_size_binary`; +- index level of `pd.ArrowDtype`: `list`, `large_list`, `fixed_size_list`, `struct`, `map`, + `dictionary`, `fixed_size_binary`, `float16` ("float16 indexes are not supported"); +- column axis (any level of it): categorical, period, timedelta (any unit), interval, + `datetime.date` objects, `pd.ArrowDtype` timestamp / duration / bool / date32 / decimal. + +`schema.empty_table().to_pandas()` (no rows converted) raises exactly when the full read +raises: 1122 of 1122 convertible values agree, on all three pyarrow versions. + +### Search: matrix with only the first write allowed to refuse + +1026 values (every value of `tests/test_hashing/test_pandas_storage_matrix.py`): + +- refused at the first write: 10 (existing refusals); +- read fails: 9 (the column-axis and `ArrowDtype` shapes above); +- second round trip fails: 0; hash unstable: 0; +- read back differs from the original (`assert_frame_equal(check_exact=True)`): 726. Every one + is a property PR #339 declared not identity because storage does not restore it (hash + computed on the restored form): `RangeIndex` kind; index dtype (extension / object / string + -> NumPy or object); index name type (`1` -> `"1"`); `object` column dtype (ints -> `int64`, + `[1, None]` -> `float64`, datetimes -> `datetime64`); `object` list / dict / tuple / set + elements (tuples and sets read back as arrays); categorical category dtype; mixed column + labels -> text; fixed-offset tz class; string storage (`string[pyarrow]` -> `string[python]`); + `ArrowDtype` string -> `string`; `ArrowDtype` timestamp index -> `DatetimeIndex`; + `ArrowDtype` decimal -> `object` (precision lost); `attrs` JSON form. Only case (1) is an + unaccounted change. + +### Out of scope, reported (not fixed) + +- `pd.ArrowDtype` `string_view`, `binary_view`, `month_day_nano_interval` columns: storage + writes and reads them, the hash refuses them (`Hashing data of type ... is not supported`). +- An index level of `list_view` / `run_end_encoded` / union / view types raises + `NotImplementedError` (not `ValueError`) at write; `complex128` raises + `ArrowNotImplementedError`, sparse `TypeError`. +- `ArrowDtype` decimal precision lost by storage (developer note on ITL-1054: "decide here"). + +## Design + +`require_restorable(schema, frame) -> pd.DataFrame` in `pandas_value_hash.py` (replaces +`pandas_type._require_restorable_axes` and the silent fallback in `_restored_axes`): + +1. repeated field names -> `ValueError` (existing); +2. MultiIndex level `i` resolved by name by `frame.index.get_level_values(i)` (pyarrow's own + call) -> `ValueError` naming the level, its name and the reason ("integer level names that are + also level positions are read back in another order"); +3. `schema.empty_table().to_pandas()` raises -> locate the culprit (column axis level, column, + index level; each probed with a zero-row frame of that part alone) and raise `ValueError` + naming it, its dtype and storage's error; +4. other number of columns / index levels -> `ValueError` (existing); +5. distinct labels that read back duplicated -> `ValueError` (existing). + +Returns the restored empty frame, which `_restored_axes` reads its axes from. + +The hash calls it in `_values_table` and `_in_canonical_column_order`; storage in both +`python_to_storage`s. The bare `"arrow"` descriptor term becomes unreachable (a nested, +dictionary or run-end `ArrowDtype` column is refused first); the grammar text says so. No +digest of any accepted value changes; identifiers and labels unchanged. + +Tightened matrix test: only the first write may refuse; read and second round trip must not +raise; hash stable; plus a hash-independent positional check (see the ask). + +## Tasks (after the coordinator's reply) + +- [ ] Task 1: failing tests: `tests/test_hashing/test_pandas_refused_at_write.py` (each shape, + frame and Series, hash and `python_to_storage`, message names column/level), pipeline test + (in-memory and Delta: the pod's row fails at write, nothing unreadable is stored), tightened + matrix. Run, show failures. +- [ ] Task 2: `require_restorable` + callers; run targeted tests. +- [ ] Task 3: golden suite unchanged; identifier tests unchanged. +- [ ] Task 4: docs (`docs/reference/hashing.md` section 10 "What is refused", executed sample), + `changelog.d/ITL-1054.md`, `design_issues/ITL-1054-.md`. +- [ ] Task 5: ruff, basedpyright, targeted pytest (one process, 3.12); push; draft PR. From da3027dac6df132be1312ded9137c50cf9675002 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:18:55 -0800 Subject: [PATCH 2/7] test(pandas): failing tests for values refused at write (ITL-1054) Co-Authored-By: Claude Opus 5.5 --- .../test_pandas_refused_at_write.py | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 tests/test_hashing/test_pandas_refused_at_write.py diff --git a/tests/test_hashing/test_pandas_refused_at_write.py b/tests/test_hashing/test_pandas_refused_at_write.py new file mode 100644 index 000000000..d9b43a76b --- /dev/null +++ b/tests/test_hashing/test_pandas_refused_at_write.py @@ -0,0 +1,276 @@ +"""A pandas value storage would not restore is refused when it is written (ITL-1054). + +Storage writes a pandas value with pyarrow and reads it back with ``to_pandas()``. Two kinds of +value used to be written although storage could not restore them: + +- **Changed without an error.** pyarrow fetches index level ``i`` with + ``index.get_level_values(i)``, which pandas resolves as a level *name* first. A MultiIndex whose + level names equal other levels' positions (``names=[1, 0]``) was written with its levels in + another order, read back changed, and hashed like the reordered frame. +- **Not readable.** A nested, dictionary or fixed-size ``pd.ArrowDtype`` column or index level, and + a categorical, period, timedelta, interval, date or ``pd.ArrowDtype`` column axis, were written, + and the read raised. In a pipeline the computed result stayed stored and unreadable. + +Both are now refused, with a ``ValueError`` naming the column or level, by ``python_to_storage`` +and by the value hash alike (one shared check), on every supported pyarrow version. +""" + +from __future__ import annotations + +import datetime as dt +import decimal +import warnings +from pathlib import Path +from typing import Any + +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.logical_types.pandas_type import LogicalPandasDataFrame, LogicalPandasSeries +from orcapod.sources import DictSource + + +def _arrow_values(array: pa.Array) -> pd.Series: + """A Series named ``"s"`` of ``pd.ArrowDtype`` over ``array``.""" + return pd.Series(pd.arrays.ArrowExtensionArray(pa.chunked_array([array])), name="s") + + +def _multi(names: list[Any]) -> pd.MultiIndex: + """A two-row, two-level index with the given level names.""" + return pd.MultiIndex.from_tuples([("p", 7), ("q", 8)], names=names) + + +# pd.ArrowDtype values storage writes but cannot read back (as a column and as an index level). +UNREADABLE_ARROW: dict[str, pa.Array] = { + "list": pa.array([[1, 2], None], pa.list_(pa.int64())), + "large_list": pa.array([[1, 2], None], pa.large_list(pa.int64())), + "fixed_size_list": pa.array([[1, 2], None], pa.list_(pa.int64(), 2)), + "struct": pa.array([{"u": 1}, None], pa.struct([("u", pa.int64())])), + "map": pa.array([[("k", 1)], None], pa.map_(pa.string(), pa.int64())), + "dictionary": pa.array(["x", "y"]).dictionary_encode(), + "fixed_size_binary": pa.array([b"ab", None], pa.binary(2)), +} +# Readable as an index level? No for these either; readable as a column but refused at write +# by Arrow already for an index level. +UNREADABLE_ARROW_COLUMN_ONLY: dict[str, pa.Array] = { + "list_view": pa.array([[1, 2], None], pa.list_view(pa.int64())), + "run_end_encoded": pc.run_end_encode(pa.array([1, 1])), + "dense_union": pa.UnionArray.from_dense( + pa.array([0, 1], pa.int8()), + pa.array([0, 0], pa.int32()), + [pa.array([1]), pa.array(["x"])], + ), +} + +# Column axes storage writes but cannot read back. +UNREADABLE_COLUMN_AXES: dict[str, pd.Index] = { + "categorical": pd.CategoricalIndex(["a", "b"]), + "period": pd.period_range("2020-01", periods=2, freq="M"), + "timedelta": pd.TimedeltaIndex(["1s", "2s"]), + "timedelta s": pd.TimedeltaIndex(["1s", "2s"]).as_unit("s"), + "interval": pd.interval_range(0, 2), + "dates": pd.Index([dt.date(2020, 1, 1), dt.date(2020, 1, 2)], dtype=object), + "arrow timestamp": pd.Index( + pd.array( + pd.to_datetime(["2020-01-01", "2020-01-02"]), + dtype=pd.ArrowDtype(pa.timestamp("us")), + ) + ), + "arrow duration": pd.Index( + pd.array( + [dt.timedelta(seconds=1), dt.timedelta(seconds=2)], + dtype=pd.ArrowDtype(pa.duration("s")), + ) + ), + "arrow bool": pd.Index(pd.array([True, False], dtype="bool[pyarrow]")), + "arrow date32": pd.Index( + pd.array( + [dt.date(2020, 1, 1), dt.date(2020, 1, 2)], dtype=pd.ArrowDtype(pa.date32()) + ) + ), + "arrow decimal": pd.Index( + pd.array( + [decimal.Decimal("1.5"), decimal.Decimal("2.5")], + dtype=pd.ArrowDtype(pa.decimal128(5, 2)), + ) + ), + "multi, categorical level": pd.MultiIndex.from_arrays( + [pd.CategoricalIndex(["a", "b"]), pd.Index([1, 2])] + ), +} + + +def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: + """Every refused value, with a pattern its message must match (the part it names).""" + cases: dict[str, tuple[pd.DataFrame | pd.Series, str]] = {} + for names in ([1, 0], [1.0, 0], [True, 0], [0, 2, 1], [1, 0, "x"]): + tuples = [("p", 7, 1.5)[: len(names)], ("q", 8, 2.5)[: len(names)]] + index = pd.MultiIndex.from_tuples(tuples, names=names) + cases[f"frame, level names {names}"] = ( + pd.DataFrame({"a": [10, 20]}, index=index), + r"index level \d+ .*another order", + ) + cases[f"series, level names {names}"] = ( + pd.Series([10, 20], name="s", index=index), + r"index level \d+ .*another order", + ) + for kind, array in {**UNREADABLE_ARROW, **UNREADABLE_ARROW_COLUMN_ONLY}.items(): + values = _arrow_values(array) + cases[f"frame, arrow {kind} column"] = ( + values.to_frame(name="col"), + r"column 'col'.*cannot (be read back|be converted)", + ) + cases[f"series, arrow {kind} values"] = ( + values, + r"column 's'.*cannot (be read back|be converted)", + ) + for kind, array in UNREADABLE_ARROW.items(): + cases[f"frame, arrow {kind} index level"] = ( + pd.DataFrame({"a": [1, 2]}, index=pd.Index(_arrow_values(array).array)), + r"index level 0.*cannot be read back", + ) + cases["frame, arrow float16 index level"] = ( + pd.DataFrame( + {"a": [1, 2]}, + index=pd.Index(_arrow_values(pa.array([1.5, 2.5], pa.float16())).array), + ), + r"index level 0.*cannot be read back", + ) + for kind, axis in UNREADABLE_COLUMN_AXES.items(): + cases[f"frame, {kind} column axis"] = ( + pd.DataFrame([[1, 2], [3, 4]], columns=axis), + r"column axis.*cannot be read back", + ) + return cases + + +REFUSED = _refused() + +# Neighbours of the refused values that storage restores: they stay accepted. +ACCEPTED: dict[str, pd.DataFrame | pd.Series] = { + "level names [0, 1] (in order)": pd.DataFrame({"a": [1, 2]}, index=_multi([0, 1])), + "level names [5, 7]": pd.DataFrame({"a": [1, 2]}, index=_multi([5, 7])), + "level names ['1', '0'] (text)": pd.DataFrame( + {"a": [1, 2]}, index=_multi(["1", "0"]) + ), + "level names [0, 'x']": pd.Series([1, 2], name="s", index=_multi([0, "x"])), + "column axis level names [1, 0]": pd.DataFrame( + [[1, 2]], columns=pd.MultiIndex.from_tuples([("a", 1), ("b", 2)], names=[1, 0]) + ), + "arrow string column": _arrow_values(pa.array(["x", None])).to_frame(), + "arrow large_binary values": _arrow_values(pa.array([b"x", None], pa.large_binary())), + "arrow timestamp tz index": pd.DataFrame( + {"a": [1, 2]}, + index=pd.Index( + _arrow_values( + pa.array([dt.datetime(2020, 1, 1), None], pa.timestamp("s", tz="UTC")) + ).array + ), + ), + "Int64 column axis": pd.DataFrame([[1, 2]], columns=pd.Index([0, 1], dtype="Int64")), + "datetime column axis": pd.DataFrame( + [[1, 2]], columns=pd.DatetimeIndex(["2020-01-01", "2020-01-02"], tz="UTC") + ), +} + + +def _logical(value: pd.DataFrame | pd.Series) -> LogicalPandasDataFrame | LogicalPandasSeries: + """The logical type that stores ``value``.""" + if isinstance(value, pd.Series): + return LogicalPandasSeries() + return LogicalPandasDataFrame() + + +def _hash(value: Any) -> Any: + """The semantic hash of ``value`` under the default data context.""" + return get_default_context().semantic_hasher.hash_object(value) + + +@pytest.fixture(autouse=True) +def _quiet() -> Any: + """pyarrow warns about non-text index names; not what these tests are about.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + yield + + +@pytest.mark.parametrize("name", list(REFUSED)) +def test_python_to_storage_refuses(name: str) -> None: + """``python_to_storage`` raises ``ValueError`` naming the column or level.""" + value, pattern = REFUSED[name] + with pytest.raises(ValueError, match=pattern): + _logical(value).python_to_storage(value) + + +@pytest.mark.parametrize("name", list(REFUSED)) +def test_hash_refuses_with_the_same_error(name: str) -> None: + """The value hash refuses the value with the same message as storage.""" + value, _ = REFUSED[name] + with pytest.raises(ValueError) as stored: + _logical(value).python_to_storage(value) + with pytest.raises(ValueError) as hashed: + _hash(value) + assert str(hashed.value) == str(stored.value) + + +@pytest.mark.parametrize("name", list(ACCEPTED)) +def test_restorable_neighbours_are_accepted(name: str) -> None: + """A neighbour storage restores is written and read back, in order, with the same hash.""" + value = ACCEPTED[name] + logical = _logical(value) + back = logical.storage_to_python(logical.python_to_storage(value)) + assert list(back.index) == list(value.index) + assert [None if n is None else str(n) for n in back.index.names] == [ + None if n is None else str(n) for n in value.index.names + ] + assert _hash(back) == _hash(value) + + +def test_level_names_that_are_positions_name_the_level() -> None: + """The message names the level, its name, and the level its name resolves to.""" + value = pd.DataFrame({"a": [10, 20]}, index=_multi([1, 0])) + with pytest.raises(ValueError, match=r"index level 0 \(name 1\)") as raised: + LogicalPandasDataFrame().python_to_storage(value) + assert "level 1" in str(raised.value) + + +@function_pod(output_keys="df") +def _make_reordered_frame(n: int) -> pd.DataFrame: + """A frame whose MultiIndex level names are the other level's position.""" + return pd.DataFrame({"a": [n, n + 1]}, index=_multi([1, 0])) + + +@function_pod(output_keys="df") +def _make_unreadable_frame(n: int) -> pd.DataFrame: + """A frame with a ``pd.ArrowDtype`` list column.""" + return _arrow_values(pa.array([[n], None], pa.list_(pa.int64()))).to_frame("col") + + +@pytest.mark.parametrize("store_name", ["in_memory", "delta"]) +@pytest.mark.parametrize("pod", [_make_reordered_frame, _make_unreadable_frame]) +def test_pipeline_stores_nothing_for_a_refused_frame( + store_name: str, pod: Any, tmp_path: Path +) -> None: + """In a pipeline the row fails at write: nothing is stored, nothing reads back changed.""" + store = ( + InMemoryArrowDatabase() + if store_name == "in_memory" + else DeltaTableDatabase(tmp_path / "delta") + ) + source = DictSource([{"k": "x", "n": 1}], tag_columns=["k"]) + job = PipelineJob(name="refused", database=store) + with job: + pod.pod(source, label="make") + with pytest.warns(Warning, match="1 of 1 row failed"): + job.run() + assert list(job.nodes["make"].iter_data()) == [] + stored_rows = 0 + for path in store.list_table_paths(): + table = store.get_all_records(path) + stored_rows += 0 if table is None else table.num_rows + assert stored_rows == 0 From af9c82a2d745173743da6b3e717b6bd479f9104f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:34:23 -0800 Subject: [PATCH 3/7] fix(pandas)!: refuse at write the values storage would change or could not read back (ITL-1054) One check, require_restorable, shared by python_to_storage and the pandas value hash, refuses with ValueError naming the column or level: MultiIndex levels pyarrow would write in another order (level names that are also level positions), and values storage cannot read back (nested, dictionary, fixed-size or union pd.ArrowDtype columns and index levels, a float16 Arrow index, categorical, period, timedelta, interval, date or pd.ArrowDtype column axes). Arrow conversion errors become ValueError naming the part. The matrix test lets only the first write refuse and checks columns and index levels by position. The struct_field_order pandas ignore option is removed: it acted only on refused struct columns. No identifier, label or golden value moves. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1054.md | 53 +++ design_issues/ITL-1054-PV3.md | 29 ++ docs/reference/hashing.md | 77 +++-- src/orcapod/hashing/pandas_value_hash.py | 321 ++++++++++++++---- src/orcapod/logical_types/pandas_type.py | 96 +----- ...6-11-14-itl-1054-pandas-refuse-at-write.md | 20 +- tests/test_hashing/test_pandas_ignore_list.py | 57 ++-- .../test_pandas_refused_at_write.py | 37 +- .../test_pandas_storage_matrix.py | 112 +++++- .../test_pandas_value_identity.py | 52 ++- 10 files changed, 590 insertions(+), 264 deletions(-) create mode 100644 changelog.d/ITL-1054.md create mode 100644 design_issues/ITL-1054-PV3.md diff --git a/changelog.d/ITL-1054.md b/changelog.d/ITL-1054.md new file mode 100644 index 000000000..1936e780a --- /dev/null +++ b/changelog.d/ITL-1054.md @@ -0,0 +1,53 @@ +### Breaking changes and migration + +#### pandas values storage would not restore are refused at write (ITL-1054) + +A pandas `DataFrame` or `Series` that storage would change, or could not read back, is now +refused when it is written, with a `ValueError` that names the column, index level or +column-axis level and the reason. The value hash refuses the same values with the same message, +through one shared check. These values used to be written: + +- **Written, then read back changed, with no error:** a MultiIndex whose level names are also + level positions (`names=[1, 0]`; `1.0` and `True` count as `1`). pyarrow wrote it with the + levels swapped, and the hash did not tell the two apart. +- **Written, then failed to read back:** a `pd.ArrowDtype` column or Series of a list, large + list, fixed-size list, list view, struct, map, dictionary, run-end, union or fixed-size binary + type. An index level of a `pd.ArrowDtype` list, large list, fixed-size list, struct, map, + dictionary, fixed-size binary or `float16` type. A column axis (any level) that is + categorical, period, timedelta, interval, `datetime.date` objects, or a `pd.ArrowDtype` + timestamp, duration, bool, date or decimal. This includes a Series named with a + `pd.Timedelta`, `pd.Period` or `datetime.date`. In a pipeline, such a result stayed in the + result table and could not be loaded on any rerun. + +A value Arrow cannot convert (for example `complex128` or a sparse column) now raises +`ValueError` naming the column, where it used to raise `ArrowNotImplementedError`, `TypeError` +or `NotImplementedError`. + +No hash of a value that is still accepted changes, and the logical type identifiers are +unchanged. Rows already stored before this change are not migrated: a value of the first kind +cannot be told apart from a correct one, and a value of the second kind cannot be read. The +migration tool reports both (ITL-1048). + +#### The pandas ignore option `struct_field_order` is removed (ITL-1054) + +`struct_field_order` is no longer an option of `orcapod.pandas.dataframe` or +`orcapod.pandas.series`. It acted only on `pd.ArrowDtype` struct columns, which are now +refused. A data context that names it under `hash_ignore` is refused with the unknown-option +error, and so is `ignore=["struct_field_order"]` on the logical type or its handler. No shipped +data context named it, so no identifier or hash moves. The pandas types are provisional until +the v0.4 audit (ITL-1069). That audit is where struct columns and this option come back +together, if storage learns to restore them. The Polars types keep their `struct_field_order` +option. + +### Fixed + +- The pandas storage round-trip test lets only the first write refuse a value. A read failure + of a written value, a failed second round trip, or a column or index level read back in + another position now fails the test, where it used to count as unsupported input. + (ITL-1054) + +### Known limitations + +- A `pd.ArrowDtype` decimal column is read back as an `object` column of `Decimal` values. + Every value comes back exactly, digits and scale included. The dtype and its declared + precision do not, so two precisions hash equal. (ITL-1054; left for a developer decision) diff --git a/design_issues/ITL-1054-PV3.md b/design_issues/ITL-1054-PV3.md new file mode 100644 index 000000000..38c207532 --- /dev/null +++ b/design_issues/ITL-1054-PV3.md @@ -0,0 +1,29 @@ +## `src/orcapod/hashing/semantic_hashing/builtin_handlers.py` — pandas value hash identity + +### PV3 — pandas values written that storage changed silently or could not read back +**Status:** resolved +**Severity:** high +**Issue:** ITL-1054 + +A pandas `DataFrame` or `Series` was written whenever pyarrow could convert it, but two kinds of +written value did not come back. First, a MultiIndex whose level names are also level positions +(`names=[1, 0]`; `1.0` and `True` count as `1`) was written with its levels swapped: pyarrow +fetches level `i` with `index.get_level_values(i)`, which pandas resolves as a level name first. +The value read back was another value, nothing raised, and the hash (computed on the restored +form) gave both the same digest. Second, a nested, dictionary, fixed-size or union +`pd.ArrowDtype` column or index level, a `float16` Arrow index level, and a categorical, period, +timedelta, interval, date or `pd.ArrowDtype` column axis were written and then failed to read +back. In a pipeline, the computed result stayed in the result table and could not be loaded on +any rerun. The storage round-trip matrix test hid the second kind: it treated a failed read +like an input storage does not support. + +**Fix:** one check, `require_restorable(schema, frame)` in +`orcapod/hashing/pandas_value_hash.py`, shared by `python_to_storage` and the value hash. It +refuses both kinds at write with a `ValueError` naming the column, index level or column-axis +level. A failed read is detected from the schema alone (`schema.empty_table().to_pandas()`), +which agreed with the full read on every value tried, on pyarrow 22.0.0, 23.0.1 and 26.0.1. +`pandas_frame_to_arrow` wraps every Arrow conversion error into `ValueError`, naming the part +that fails. The matrix test now lets only the first write refuse, and it checks each column and +index level by position. The `struct_field_order` ignore option is removed, because it acted +only on the refused struct columns. Identifiers, labels and every golden value are unchanged. +Rows already stored with either kind are left for the migration tool (ITL-1048). diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 260b355b2..135f6a47d 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -1452,10 +1452,11 @@ series: {"kind": "orcapod.pandas.series", "height": , "attrs": , | {"arrow": } pd.ArrowDtype of a non-nested Arrow type: str(type), "large_binary", "null", "timestamp[ns, tz=UTC]" - | "arrow" pd.ArrowDtype of a nested, dictionary or - run-end type: the type is in D only ``` +A nested, dictionary or run-end `pd.ArrowDtype` has no term: storage cannot read such a +column back, so the value is refused (see "What is refused" below). + `kind` is the name of the value's logical type (its Arrow extension name). **The hash is strict.** Everything about a value that can be changed is part of its @@ -1468,7 +1469,6 @@ the value that was written. `PANDAS_IDENTITY_PROPERTIES` in |---|---|---| | Column order | yes | `D` (ordered) | | MultiIndex level order (frame and Series) | yes | `D` (ordered) | -| Struct field order in a `pd.ArrowDtype` struct column, at every depth | yes | `D` (ordered) | | Values, row order, row count, Arrow type (int width, datetime unit and timezone) | yes | `D` | | Index values, index level names (as the text storage writes), index dtype (as storage restores it) | yes | `D`, descriptor | | `columns.name` / `columns.names` | yes | descriptor | @@ -1538,19 +1538,17 @@ More consequences of the same rule: decimal column and the all-null or empty `object` column is `{"numpy": "object"}`, so they **hash equal**; the height still counts, and a `string`-dtype column of `None` (descriptor `"string"`) stays different. Other all-null or empty columns (Arrow string, timestamp, - integer, list or struct, `Int64`, `string`, categorical) keep their own types and are + integer, `Int64`, `string`, categorical) keep their own types and are unchanged. A `pd.ArrowDtype` of a non-nested Arrow type is described by its Arrow type text, so `pd.ArrowDtype(pa.null())` (`{"arrow": "null"}`) never hashes like a `pd.ArrowDtype(pa.large_string())` column of `None`, and `binary` and `large_binary` columns (which storage restores as themselves, although D unifies large and non-large variants) hash differently. A decimal column is the exception that shows the rule: storage - restores it as `object`, so two precisions hash equal. A nested `pd.ArrowDtype` (list, - struct, map, union), a dictionary or a run-end type keeps the bare term `"arrow"`; none - reads back today, and ITL-1054 decides what identifies one (so a `list` and a `large_list` - column hash equal until then). - A `pd.ArrowDtype` of a nested type with a `null` child (`list`, `struct`) - is still refused by the hash, as before. + restores it as `object` (each value comes back exactly, digits and scale included; the + dtype and its declared precision do not), so two precisions hash equal. A nested + `pd.ArrowDtype` (list, struct, map, union), a dictionary or a run-end type is refused: + storage cannot read it back (ITL-1054). - **A timezone is the text storage writes for it** (`"UTC"`, `"Europe/Berlin"`, `"+01:00"`), whatever class holds it: `datetime.timezone(timedelta(hours=1))` is read back as another class of the same offset, and both hash equal. @@ -1568,15 +1566,57 @@ warning, or raise `TypeError`, depending on its version). With `attrs` in the ig the hash does not read them (it never copies them either, so `attrs` that cannot be copied do not matter then). -Storage refuses a value it would corrupt. Column labels and index names that collide -once written as text (the index name `1` with a column `"1"`; integer names of MultiIndex -levels) would be written as two fields of one name, and the value read back would have lost -its index. Distinct labels that read back equal (the labels `True` and `False` both read -back `True`) would come back as duplicates. `python_to_storage` raises `ValueError` in both -cases. - A label or dtype outside the grammar raises `ValueError` naming it (a dtype error names its -column). A value that cannot be converted to Arrow raises `ValueError`. +column). + +**What is refused.** Storage refuses, when the value is written, a value it would not +restore, and the hash refuses the same values with the same `ValueError` (one check, +`require_restorable` in `orcapod/hashing/pandas_value_hash.py`; ITL-1054). The message names +the column, index level or column-axis level and the reason. Refused, on every supported +pyarrow version: + +- **MultiIndex levels that would be written in another order.** pyarrow fetches index level + `i` with `index.get_level_values(i)`, which pandas resolves as a level *name* first. So + level names that are also level positions (`names=[1, 0]`; `1.0` and `True` count as `1`) + would be written, and read back, with the levels swapped, and nothing would raise. +- **Values storage cannot read back.** A `pd.ArrowDtype` column (or Series) of a list, + large list, fixed-size list, list view, struct, map, dictionary, run-end or union type, + or of fixed-size binary; an index level of a `pd.ArrowDtype` list, large list, fixed-size + list, struct, map, dictionary, fixed-size binary or `float16` type; a column axis (any + level of it) that is categorical, period, timedelta, interval, `datetime.date` objects, or + a `pd.ArrowDtype` timestamp, duration, bool, date or decimal, which includes a Series + named with a `pd.Timedelta`, `pd.Period` or `datetime.date`. +- **Labels and names that collide once written as text** (the index name `1` with a column + `"1"`; integer names of MultiIndex levels): two fields of one name, and the value read + back would have lost its index. Distinct labels that read back equal (the labels `True` + and `False` both read back `True`) would come back as duplicates. +- **Values Arrow cannot convert** (mixed or non-Arrow values, `complex128`, sparse columns, + an index of a type Arrow does not support). + +Before v0.3 such values were written: a value of the first kind reads back changed, and one +of the second kind cannot be read at all. + +```python +import pandas as pd +from orcapod.contexts import get_default_context +from orcapod.logical_types.pandas_type import LogicalPandasDataFrame + +semantic_hasher = get_default_context().semantic_hasher +swapped = pd.DataFrame( + {"a": [10, 20]}, + index=pd.MultiIndex.from_tuples([("p", 7), ("q", 8)], names=[1, 0]), +) +try: + LogicalPandasDataFrame().python_to_storage(swapped) +except ValueError as error: + print(str(error).split(":")[0]) +# index level 0 (name 1) +try: + semantic_hasher.hash_object(swapped) +except ValueError as error: + print(str(error).split(":")[0]) +# index level 0 (name 1) +``` **What a data context can make the hash ignore.** Nothing is ignored by default. The registration file (`orcapod/contexts/logical_types.json`, `hash_ignore`) can name, per type, @@ -1588,7 +1628,6 @@ The preimage keeps its shape: an ignored property is hashed with a fixed value. |---|---|---| | `column_order` | DataFrame | the columns are hashed sorted by the canonical JSON of their typed labels | | `series_name` | Series | `name` is hashed as `null` | -| `struct_field_order` | both | starfix's `ordered_struct_fields` option is off for `D` | | `attrs` | both | `attrs` is hashed as `{}` | Every other property always counts, and the properties storage does not restore never do; diff --git a/src/orcapod/hashing/pandas_value_hash.py b/src/orcapod/hashing/pandas_value_hash.py index 05720950f..9325d3be1 100644 --- a/src/orcapod/hashing/pandas_value_hash.py +++ b/src/orcapod/hashing/pandas_value_hash.py @@ -27,9 +27,10 @@ (``PANDAS_DATAFRAME_OPTIONS``, ``PANDAS_SERIES_OPTIONS``; the list is saved in the logical type's extension metadata, ``orcapod.logical_types.ignore_list``). The preimage keeps its shape: an ignored property is hashed with a fixed value. ``attrs`` becomes ``{}``, -``series_name`` becomes ``null``, ``column_order`` hashes the columns sorted by the canonical -JSON of their labels, and ``struct_field_order`` turns off starfix's struct field order -option. Nothing is ignored by default. +``series_name`` becomes ``null``, and ``column_order`` hashes the columns sorted by the +canonical JSON of their labels. Nothing is ignored by default. (``struct_field_order`` is not +an option: it acted only on a ``pd.ArrowDtype`` struct column, which storage cannot read back +and so is refused, ITL-1054.) Hashing what storage restores ----------------------------- @@ -98,9 +99,6 @@ | {"arrow": } pd.ArrowDtype of a non-nested Arrow type: str(type), e.g. "large_binary", "decimal128(10, 2)", "null" - | "arrow" pd.ArrowDtype of a nested, dictionary - or run-end type: the type is in the - digest only (ITL-1054) ```` is ``attrs`` as the JSON object it is stored as: ``{1: "x"}`` is ``{"1": "x"}`` and a tuple is a list. ``attrs`` with an entry JSON cannot write (a ``set``, @@ -110,12 +108,25 @@ A period alias is pandas' own period frequency string (``"M"``, ``"Q-DEC"``, as in ``Period.freqstr``). A label or dtype outside the grammar raises ``ValueError``. + +What is refused +--------------- +A value storage would write but not restore is refused with ``ValueError`` by the hash and by +storage alike, through one check, ``require_restorable`` (ITL-1054): MultiIndex levels that +would be written in another order (level names that are also level positions, +``names=[1, 0]``), and values storage cannot read back (a nested, dictionary or fixed-size +``pd.ArrowDtype`` column or index level; a categorical, period, timedelta, interval, date or +``pd.ArrowDtype`` column axis), besides labels and names that collide once written as text. +A value Arrow cannot convert is refused by ``pandas_frame_to_arrow``. Both messages name the +column, index level or column-axis level. """ from __future__ import annotations import datetime as dt import json +from collections import Counter +from collections.abc import Callable, Iterator from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal @@ -174,13 +185,6 @@ class PandasIdentityProperty: PandasIdentityProperty( "index_level_order", True, "digest", "order of the levels of a MultiIndex" ), - PandasIdentityProperty( - "struct_field_order", - True, - "digest", - "order of the fields of a pd.ArrowDtype struct column, at every depth", - option=True, - ), PandasIdentityProperty( "values", True, "digest", "the values, their row order and the row count" ), @@ -331,14 +335,10 @@ class PandasIdentityProperty: a new logical type identifier. """ -PANDAS_DATAFRAME_OPTIONS: frozenset[str] = frozenset( - {"attrs", "column_order", "struct_field_order"} -) +PANDAS_DATAFRAME_OPTIONS: frozenset[str] = frozenset({"attrs", "column_order"}) """The properties a data context can configure a ``pd.DataFrame`` hash to ignore.""" -PANDAS_SERIES_OPTIONS: frozenset[str] = frozenset( - {"attrs", "series_name", "struct_field_order"} -) +PANDAS_SERIES_OPTIONS: frozenset[str] = frozenset({"attrs", "series_name"}) """The properties a data context can configure a ``pd.Series`` hash to ignore.""" _NOTHING_IGNORED: frozenset[str] = frozenset() @@ -447,18 +447,12 @@ def pandas_dtype_descriptor(dtype: Any) -> Any: } } if isinstance(dtype, pd.ArrowDtype): - arrow_type = dtype.pyarrow_dtype - if ( - pa.types.is_nested(arrow_type) - or pa.types.is_dictionary(arrow_type) - or pa.types.is_run_end_encoded(arrow_type) - ): - # No nested Arrow dtype reads back today; ITL-1054 decides what identifies one. - return "arrow" # Storage restores the exact Arrow type, but the values digest unifies large and # non-large variants, so the type text is the term (``null`` included: the digest sees an all-null ``large_string`` there, see # ``_restored_form``, and the term keeps it apart from a ``large_string`` column). - return {"arrow": str(arrow_type)} + # A nested, dictionary or run-end Arrow type never reaches a hash: storage cannot + # read such a column back, so ``require_restorable`` refuses the value (ITL-1054). + return {"arrow": str(dtype.pyarrow_dtype)} if isinstance(dtype, pd.StringDtype): # The storage (python / pyarrow) is not kept by the stored form, so it is not # part of the term. @@ -662,31 +656,234 @@ def _axes_of(frame: pd.DataFrame) -> _StoredAxes: ) +def _level_by_position(axis: pd.Index, position: int) -> pd.Index: + """Return level ``position`` of ``axis``, found by position only (``axis`` for a flat Index). + + ``MultiIndex.get_level_values`` resolves an integer as a level name first, so the levels are + looked up with every name cleared. The level keeps its own name. + """ + if not isinstance(axis, pd.MultiIndex): + return axis + unnamed = axis.set_names([None] * axis.nlevels) + return unnamed.get_level_values(position).rename(axis.names[position]) + + +def _parts(frame: pd.DataFrame, rows: int | None) -> Iterator[tuple[str, pd.DataFrame]]: + """Yield each part of ``frame`` storage writes, alone in a frame, with its description. + + The parts are the levels of the column axis, the columns, and the index levels, each by + position. A part's frame holds the first ``rows`` rows of it (all rows when ``None``); a + column-axis level is a zero-row frame with that level as its column labels. + """ + subset = frame if rows is None else frame.iloc[:rows] + columns = frame.columns + for position in range(columns.nlevels): + level = _level_by_position(columns, position) + where = ( + "column axis" if columns.nlevels == 1 else f"column axis level {position}" + ) + yield ( + f"{where} (dtype {level.dtype})", + pd.DataFrame(index=pd.RangeIndex(0), columns=level.unique()), + ) + for position in range(subset.shape[1]): + column = subset.iloc[:, [position]] + yield ( + f"column {columns[position]!r} (dtype {column.dtypes.iloc[0]})", + column.set_axis(pd.RangeIndex(1), axis=1).reset_index(drop=True), + ) + index = subset.index + for position in range(index.nlevels): + level = _level_by_position(index, position) + yield ( + f"index level {position} (name {index.names[position]!r}, dtype {level.dtype})", + pd.DataFrame(index=level), + ) + + +def _failing_part( + frame: pd.DataFrame, rows: int | None, attempt: Callable[[pd.DataFrame], object] +) -> str | None: + """Return the description of the first part of ``frame`` on which ``attempt`` raises. + + Args: + frame: The frame. + rows: How many rows of each part to try (all when ``None``); see ``_parts``. + attempt: The step that failed on the whole frame. + + Returns: + The description of the part, or ``None`` when every part alone succeeds. + """ + for description, part in _parts(frame, rows): + try: + attempt(part) + except Exception: # noqa: BLE001 - any failure of the step locates the part + return description + return None + + +def _converted(frame: pd.DataFrame) -> pa.Table: + """``pa.Table.from_pandas(frame, preserve_index=True)``: the table storage writes.""" + return pa.Table.from_pandas(frame, preserve_index=True) + + +def _read_back_empty(frame: pd.DataFrame) -> pd.DataFrame: + """The zero-row frame storage restores for ``frame``'s Arrow schema.""" + return _converted(frame).schema.empty_table().to_pandas() + + +def _with_part(part: str | None, message: str) -> str: + """``message`` prefixed by the part it is about, when the part is known.""" + return message if part is None else f"{part}: {message}" + + +def pandas_frame_to_arrow(frame: pd.DataFrame) -> pa.Table: + """Return the Arrow table storage writes for ``frame``, refusing a frame Arrow cannot hold. + + Storage and the value hash convert a frame with this one function, so they refuse the same + frames with the same message. + + Args: + frame: The frame (a Series is converted as its one-column frame). + + Returns: + ``pa.Table.from_pandas(frame, preserve_index=True)``. + + Raises: + ValueError: If Arrow cannot convert the frame (mixed or non-Arrow values, a dtype or an + index kind Arrow does not support). The message names the column, index level or + column-axis level when converting it alone fails too. + """ + try: + return _converted(frame) + except (pa.ArrowException, NotImplementedError, TypeError, ValueError) as exc: + part = _failing_part(frame, None, _converted) + raise ValueError( + _with_part( + part, + "cannot convert to Arrow; check for columns with mixed or " + f"non-Arrow-serialisable values. Original error: {exc}", + ) + ) from exc + + +def _require_levels_in_place(index: pd.Index) -> None: + """Raise if storage would write the levels of ``index`` in another order. + + pyarrow fetches index level ``i`` with ``index.get_level_values(i)``, and pandas resolves an + integer there as a level *name* first: with the names ``[1, 0]``, level 0 is fetched as the + level named 0 (position 1). The value would be written, and read back, with its levels in + another order, and nothing would raise. + + Raises: + ValueError: If a level would be written in another level's place; the message names it. + """ + if not isinstance(index, pd.MultiIndex): + return + names = list(index.names) + for position in range(index.nlevels): + # pandas' own rule for an integer level argument: a matching name wins (by ==, so 1.0 + # and True match 1), else the position. + fetched = names.index(position) if position in names else position + if fetched != position: + raise ValueError( + f"index level {position} (name {names[position]!r}): storage would write " + f"level {fetched} (name {names[fetched]!r}) in its place, because pyarrow " + f"fetches level {position} by index.get_level_values({position}), which " + f"pandas resolves as the level named {position}; the value would be read " + "back with its levels in another order. Rename the index levels (integer " + "level names that are also level positions are the cause)." + ) + + +def require_restorable(schema: pa.Schema, frame: pd.DataFrame) -> pd.DataFrame: + """Return the zero-row frame storage restores for ``frame``, refusing what it would not. + + The one check, shared by storage (``python_to_storage``) and the value hash, of a value + storage would write but not restore. Such a value is refused, before anything is written: + + - Two fields get one name (the index name ``1`` beside the column ``"1"``; integer names of + MultiIndex levels): the frame read back would have lost its index. + - The levels of a MultiIndex would be written in another order (level names that are also + level positions, ``names=[1, 0]``): the frame read back would differ, silently. + - The frame read back cannot be built: a nested, dictionary or fixed-size ``pd.ArrowDtype`` + column or index level; a categorical, period, timedelta, interval, date or + ``pd.ArrowDtype`` column axis. + - The frame read back would have another number of columns or index levels, or distinct + column labels that read back equal (``True`` and ``False`` are written as text). + + No rows are converted: what storage restores is read from the frame it builds for + ``schema`` alone. + + Args: + schema: The schema of ``pandas_frame_to_arrow(frame)``. + frame: The frame about to be stored or hashed. + + Returns: + The zero-row frame storage restores for ``schema``: its axes, labels, names and the + dtypes the schema decides. + + Raises: + ValueError: If storage would not restore the frame; the message names the column, + index level or column-axis level and the reason. + """ + counts = Counter(schema.names) + repeated = sorted(name for name, count in counts.items() if count > 1) + if repeated: + raise ValueError( + f"column labels and index names collide once written as text: {repeated}. " + "Storage could not restore the index. Rename the index levels (integer " + "level names are a common cause) or the columns." + ) + _require_levels_in_place(frame.index) + try: + restored = schema.empty_table().to_pandas() + except Exception as exc: # noqa: BLE001 - pandas and pyarrow raise many kinds here + part = _failing_part(frame, 0, _read_back_empty) + raise ValueError( + _with_part( + part, + "storage would write this value but it cannot be read back " + f"({type(exc).__name__}: {exc}).", + ) + ) from exc + if ( + restored.shape[1] != frame.shape[1] + or restored.index.nlevels != frame.index.nlevels + ): + raise ValueError( + "the value read back from storage would have " + f"{restored.shape[1]} column(s) and {restored.index.nlevels} index " + f"level(s), not {frame.shape[1]} and {frame.index.nlevels}. Rename the " + "columns or the index levels." + ) + if frame.columns.is_unique and not restored.columns.is_unique: + labels = sorted({repr(label) for label in restored.columns}) + raise ValueError( + "distinct column labels would read back from storage as duplicates: " + f"{labels}. Rename the columns (labels of mixed or bool type are written " + "as text)." + ) + return restored + + def _restored_axes(schema: pa.Schema, df: pd.DataFrame) -> _StoredAxes: """Return the axes storage restores for a frame whose Arrow schema is ``schema``. They are read from the frame storage builds for the schema (no rows are converted), so - the rule is storage's own. A frame storage cannot read back with its columns and index - levels (storage refuses to write such a frame) is described by its own axes. + the rule is storage's own. Args: - schema: The schema of ``pa.Table.from_pandas(df, preserve_index=True)``. + schema: The schema of ``pandas_frame_to_arrow(df)``. df: The frame. Returns: The axes of the frame once read back from storage. + + Raises: + ValueError: If storage would not restore the frame (``require_restorable``). """ - try: - restored = schema.empty_table().to_pandas() - except (TypeError, ValueError, NotImplementedError, pa.ArrowException): - return _axes_of(df) - if ( - restored.shape[1] != df.shape[1] - or restored.index.nlevels != df.index.nlevels - or (df.columns.is_unique and not restored.columns.is_unique) - ): - return _axes_of(df) - return _axes_of(restored) + return _axes_of(require_restorable(schema, df)) def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: @@ -700,14 +897,10 @@ def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: from storage. Raises: - ValueError: If a column's dtype is outside the grammar (the error names the column) - or a column cannot be converted to Arrow. + ValueError: If a column's dtype is outside the grammar (the error names the column), + the frame cannot be converted to Arrow, storage would not restore it + (``require_restorable``), or a label or name is outside the grammar. """ - # The frame's own labels and names must be in the grammar, although the descriptor - # holds them in the form storage restores: storage writes some as text, and the text of - # an arbitrary object (its ``repr``, with a memory address) must never reach a hash. - for label in (*df.columns, *df.columns.names, *df.index.names): - pandas_label_descriptor(label) for label, dtype in zip(df.columns, df.dtypes): if dtype != np.dtype(object): # A dtype outside the grammar is reported by name, before Arrow refuses it. @@ -715,13 +908,14 @@ def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: pandas_dtype_descriptor(dtype) except ValueError as exc: raise ValueError(f"column {label!r}: {exc}") from exc - try: - table = pa.Table.from_pandas(df, preserve_index=True) - except pa.ArrowInvalid as exc: - raise ValueError( - "cannot convert to Arrow; check for columns with mixed or " - f"non-Arrow-serialisable values. Original error: {exc}" - ) from exc + table = pandas_frame_to_arrow(df) + # Refuses what storage would not restore, with storage's own message (ITL-1054). + axes = _restored_axes(table.schema, df) + # The frame's own labels and names must be in the grammar, although the descriptor + # holds them in the form storage restores: storage writes some as text, and the text of + # an arbitrary object (its ``repr``, with a memory address) must never reach a hash. + for label in (*df.columns, *df.columns.names, *df.index.names): + pandas_label_descriptor(label) width = df.shape[1] index = df.index dtypes = [ @@ -731,7 +925,6 @@ def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: names = [f"c{position}" for position in range(width)] + [ f"i{position}" for position in range(index.nlevels) ] - axes = _restored_axes(table.schema, df) for position, schema_dtype in enumerate(axes.schema_dtypes): own = dtypes[position] if own != np.dtype(object) and not isinstance(own, pd.CategoricalDtype): @@ -851,11 +1044,7 @@ def _in_canonical_column_order(df: pd.DataFrame) -> pd.DataFrame: The sort is on the terms of the labels as storage restores them, so a frame and the frame read back from storage are sorted alike. """ - try: - schema = pa.Table.from_pandas(df.iloc[:0], preserve_index=True).schema - except pa.ArrowInvalid as exc: - raise ValueError(f"cannot convert to Arrow. Original error: {exc}") from exc - labels = _restored_axes(schema, df).column_labels + labels = _restored_axes(pandas_frame_to_arrow(df).schema, df).column_labels keys = [canonical_json(pandas_label_descriptor(label)) for label in labels] order = sorted(range(len(keys)), key=keys.__getitem__) return df.iloc[:, order] @@ -933,9 +1122,7 @@ def pandas_frame_content_hash( bare = _in_canonical_column_order(bare) values, stored = _values_table(bare) descriptor = _frame_descriptor(df, stored, ignore) - digest = arrow_hasher.hash_table_ordered( - values, struct_field_order="struct_field_order" not in ignore - ).digest + digest = arrow_hasher.hash_table_ordered(values).digest return labelled_content_hash(PANDAS_DATAFRAME_HASH_LABEL, descriptor, digest) @@ -963,7 +1150,5 @@ def pandas_series_content_hash( """ values, stored = _values_table(_series_as_stored_frame(series)) descriptor = _series_descriptor(series, stored, ignore) - digest = arrow_hasher.hash_table_ordered( - values, struct_field_order="struct_field_order" not in ignore - ).digest + digest = arrow_hasher.hash_table_ordered(values).digest return labelled_content_hash(PANDAS_SERIES_HASH_LABEL, descriptor, digest) diff --git a/src/orcapod/logical_types/pandas_type.py b/src/orcapod/logical_types/pandas_type.py index 167f4f2bd..c6a1d8f45 100644 --- a/src/orcapod/logical_types/pandas_type.py +++ b/src/orcapod/logical_types/pandas_type.py @@ -14,7 +14,6 @@ from __future__ import annotations import io -from collections import Counter from typing import TYPE_CHECKING, Any import pandas as pd @@ -26,6 +25,8 @@ PANDAS_SERIES_OPTIONS, PANDAS_UNNAMED_SERIES_COLUMN, pandas_attrs_as_stored, + pandas_frame_to_arrow, + require_restorable, ) from orcapod.logical_types.ignore_list import IgnoreListLogicalType @@ -48,61 +49,6 @@ """ -def _require_restorable_axes(table: pa.Table, frame: pd.DataFrame) -> None: - """Raise if the frame read back from ``table`` would not have ``frame``'s axes. - - Arrow writes some column labels and every index level name as text. Two cases corrupt - a value silently, so they are refused before anything is written: - - - Two fields get one name: the index name ``1`` beside the column ``"1"``, or integer - names of MultiIndex levels. The frame read back has lost its index (the levels come - back as columns). - - Distinct labels read back equal: the labels ``True`` and ``False`` are written as - text and both read back ``True``. - - The check builds the frame storage would restore for the table's schema; no rows are - converted. - - Args: - table: ``pa.Table.from_pandas(frame, preserve_index=True)``. - frame: The frame about to be stored. - - Raises: - ValueError: If a field name occurs twice (the message lists the names), or the - frame read back would have another number of columns or index levels, or - duplicate column labels the frame does not have. - """ - counts = Counter(table.schema.names) - repeated = sorted(name for name, count in counts.items() if count > 1) - if repeated: - raise ValueError( - f"column labels and index names collide once written as text: {repeated}. " - "Storage could not restore the index. Rename the index levels (integer " - "level names are a common cause) or the columns." - ) - try: - restored = table.schema.empty_table().to_pandas() - except (TypeError, ValueError, NotImplementedError, pa.ArrowException): - return # Not a case this check is about; reading reports it. - if ( - restored.shape[1] != frame.shape[1] - or restored.index.nlevels != frame.index.nlevels - ): - raise ValueError( - "the value read back from storage would have " - f"{restored.shape[1]} column(s) and {restored.index.nlevels} index " - f"level(s), not {frame.shape[1]} and {frame.index.nlevels}. Rename the " - "columns or the index levels." - ) - if frame.columns.is_unique and not restored.columns.is_unique: - labels = sorted({repr(label) for label in restored.columns}) - raise ValueError( - "distinct column labels would read back from storage as duplicates: " - f"{labels}. Rename the columns (labels of mixed or bool type are written " - "as text)." - ) - - def _table_to_ipc_bytes(table: pa.Table) -> bytes: """Serialise an Arrow Table to IPC stream bytes. @@ -175,20 +121,16 @@ def python_to_storage( schema, index, and data. Raises: - ValueError: If any column cannot be converted to an Arrow type, column - labels and index names would not read back as they are (they collide - once written as text), or ``attrs`` cannot be written as JSON. + ValueError: If the frame cannot be converted to Arrow, storage would not + restore it (labels and names that collide once written as text, + MultiIndex levels written in another order, a column, index level or + column-axis level that cannot be read back; see ``require_restorable``), + or ``attrs`` cannot be written as JSON. The message names the column or + level. """ pandas_attrs_as_stored(value.attrs) # refuses attrs that are not JSON - try: - table = pa.Table.from_pandas(value, preserve_index=True) - except pa.lib.ArrowInvalid as exc: - raise ValueError( - f"LogicalPandasDataFrame: cannot convert DataFrame to Arrow. " - f"Check for columns with mixed or non-Arrow-serialisable types. " - f"Original error: {exc}" - ) from exc - _require_restorable_axes(table, value) + table = pandas_frame_to_arrow(value) + require_restorable(table.schema, value) return _table_to_ipc_bytes(table) def storage_to_python( @@ -251,9 +193,10 @@ def python_to_storage( Raises: ValueError: If ``value.name`` equals the reserved sentinel, the Series - values cannot be converted to an Arrow type, the name and the index - names collide once written as text, or ``attrs`` cannot be written as - JSON. + cannot be converted to Arrow, storage would not restore it (see + ``require_restorable``: the name and the index names collide once written + as text, MultiIndex levels written in another order, values or an index + level that cannot be read back), or ``attrs`` cannot be written as JSON. """ if value.name == _SERIES_UNNAMED_SENTINEL: raise ValueError( @@ -264,15 +207,8 @@ def python_to_storage( col_name = value.name if value.name is not None else _SERIES_UNNAMED_SENTINEL pandas_attrs_as_stored(value.attrs) # refuses attrs that are not JSON df = value.to_frame(name=col_name) - try: - table = pa.Table.from_pandas(df, preserve_index=True) - except pa.lib.ArrowInvalid as exc: - raise ValueError( - f"LogicalPandasSeries: cannot convert Series to Arrow. " - f"Check for non-Arrow-serialisable values. " - f"Original error: {exc}" - ) from exc - _require_restorable_axes(table, df) + table = pandas_frame_to_arrow(df) + require_restorable(table.schema, df) return _table_to_ipc_bytes(table) def storage_to_python( diff --git a/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md b/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md index 2be1dd0fc..5365f5fc9 100644 --- a/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md +++ b/superpowers/plans/2026-11-14-itl-1054-pandas-refuse-at-write.md @@ -16,8 +16,16 @@ changes. **Tech stack:** pandas 2.2.3 (locked; `<3`), pyarrow `>=22.0.0` (locked 23.0.1). -**Status:** step 1 (reproduce, search) done; WAITING for the coordinator's reply to the list and -plan below (sent with `orca orchestration ask`). +**Status:** coordinator approved (Q1-Q5, all recommendations; then option B: remove the +`struct_field_order` ignore option). Implemented; draft PR next. + +Decisions (coordinator, November 14, 2026): Q1 refuse case 1 and every case-2 shape; the +documented lossy read-backs stay accepted (#339). Q2 matrix: only first write may refuse; read +and second round trip never raise; hash stable; per-position check of columns and index +levels. Q3 the hash refuses everything storage refuses (one shared check). Q4 `pd.ArrowDtype` +decimal left as is; report exactly what is preserved (values exact incl. scale; dtype and +declared precision lost). Q5 write-time conversion errors wrapped into ValueError naming the +part. struct_field_order: removed from both OPTIONS sets and the identity list (B). --- @@ -118,12 +126,12 @@ raise; hash stable; plus a hash-independent positional check (see the ask). ## Tasks (after the coordinator's reply) -- [ ] Task 1: failing tests: `tests/test_hashing/test_pandas_refused_at_write.py` (each shape, +- [x] Task 1: failing tests: `tests/test_hashing/test_pandas_refused_at_write.py` (each shape, frame and Series, hash and `python_to_storage`, message names column/level), pipeline test (in-memory and Delta: the pod's row fails at write, nothing unreadable is stored), tightened matrix. Run, show failures. -- [ ] Task 2: `require_restorable` + callers; run targeted tests. -- [ ] Task 3: golden suite unchanged; identifier tests unchanged. -- [ ] Task 4: docs (`docs/reference/hashing.md` section 10 "What is refused", executed sample), +- [x] Task 2: `require_restorable` + callers; run targeted tests. +- [x] Task 3: golden suite unchanged; identifier tests unchanged. +- [x] Task 4: docs (`docs/reference/hashing.md` section 10 "What is refused", executed sample), `changelog.d/ITL-1054.md`, `design_issues/ITL-1054-.md`. - [ ] Task 5: ruff, basedpyright, targeted pytest (one process, 3.12); push; draft PR. diff --git a/tests/test_hashing/test_pandas_ignore_list.py b/tests/test_hashing/test_pandas_ignore_list.py index ecc153a9b..8b2356afe 100644 --- a/tests/test_hashing/test_pandas_ignore_list.py +++ b/tests/test_hashing/test_pandas_ignore_list.py @@ -1,10 +1,11 @@ """What a data context can configure the pandas value hash to ignore (ITL-897, ITL-981). The hash of a ``pd.DataFrame`` / ``pd.Series`` is strict by default. A data context can -name properties to ignore, per type: ``column_order``, ``struct_field_order`` and ``attrs`` -for a frame; ``series_name``, ``struct_field_order`` and ``attrs`` for a Series. The list is -saved in the logical type's extension metadata, so two lists are two logical types. Storage -is the same under every list. +name properties to ignore, per type: ``column_order`` and ``attrs`` for a frame; +``series_name`` and ``attrs`` for a Series. The list is saved in the logical type's extension +metadata, so two lists are two logical types. Storage is the same under every list. +``struct_field_order`` is not an option: it acted only on a ``pd.ArrowDtype`` struct column, +which storage cannot read back and so is refused (ITL-1054). """ from __future__ import annotations @@ -46,19 +47,6 @@ def _with_attrs(value: Any) -> Any: return out -def _struct_series(order: list[str]) -> pd.Series: - """A ``pd.ArrowDtype`` struct Series whose fields are in ``order``.""" - types = {"u": pa.int64(), "v": pa.large_string()} - struct = pa.struct([(name, types[name]) for name in order]) - array = pa.array([{"u": 1, "v": "x"}], struct) - return pd.Series(array, dtype=pd.ArrowDtype(struct), name="s") - - -def _struct_frame(order: list[str]) -> pd.DataFrame: - """A frame with one ``pd.ArrowDtype`` struct column whose fields are in ``order``.""" - return pd.DataFrame({"s": _struct_series(order)}) - - def _reordered(frame: pd.DataFrame) -> pd.DataFrame: """``frame`` with its columns in reverse order.""" return frame.reindex(columns=list(frame.columns)[::-1]) @@ -67,12 +55,10 @@ def _reordered(frame: pd.DataFrame) -> pd.DataFrame: # option -> two values that differ only in that property FRAME_PAIRS: dict[str, tuple[pd.DataFrame, pd.DataFrame]] = { "column_order": (_frame(), _reordered(_frame())), - "struct_field_order": (_struct_frame(["u", "v"]), _struct_frame(["v", "u"])), "attrs": (_frame(), _with_attrs(_frame())), } SERIES_PAIRS: dict[str, tuple[pd.Series, pd.Series]] = { "series_name": (pd.Series([1, 2], name="a"), pd.Series([1, 2], name="b")), - "struct_field_order": (_struct_series(["u", "v"]), _struct_series(["v", "u"])), "attrs": (pd.Series([1, 2]), _with_attrs(pd.Series([1, 2]))), } @@ -178,16 +164,21 @@ def test_handler_and_logical_type_offer_the_same_options(self) -> None: """The handler and the logical type of one pandas type offer the same options.""" assert PandasDataFrameHandler.OPTIONS == LogicalPandasDataFrame.OPTIONS assert PandasSeriesHandler.OPTIONS == LogicalPandasSeries.OPTIONS - assert PandasDataFrameHandler.OPTIONS == { - "attrs", - "column_order", - "struct_field_order", - } - assert PandasSeriesHandler.OPTIONS == { - "attrs", - "series_name", - "struct_field_order", - } + assert PandasDataFrameHandler.OPTIONS == {"attrs", "column_order"} + assert PandasSeriesHandler.OPTIONS == {"attrs", "series_name"} + + def test_struct_field_order_is_not_an_option(self) -> None: + """``struct_field_order`` is refused: no value storage accepts has a struct column.""" + for refuse in ( + PandasDataFrameHandler, + PandasSeriesHandler, + LogicalPandasDataFrame, + LogicalPandasSeries, + ): + with pytest.raises( + ValueError, match="unknown property name 'struct_field_order'" + ): + refuse(ignore=["struct_field_order"]) class TestIdentifier: @@ -318,3 +309,11 @@ def test_an_unknown_name_in_the_configuration_is_refused( """A configured name that is not an option raises.""" with pytest.raises(ContextValidationError, match="flags"): _context(tmp_path, {FRAME_NAME: ["flags"]}) + + def test_struct_field_order_in_a_data_context_is_refused( + self, tmp_path: Path + ) -> None: + """A data context naming ``struct_field_order`` for a pandas type is refused (ITL-1054).""" + for name in (FRAME_NAME, SERIES_NAME): + with pytest.raises(ContextValidationError, match="struct_field_order"): + _context(tmp_path, {name: ["struct_field_order"]}) diff --git a/tests/test_hashing/test_pandas_refused_at_write.py b/tests/test_hashing/test_pandas_refused_at_write.py index d9b43a76b..5517ddc45 100644 --- a/tests/test_hashing/test_pandas_refused_at_write.py +++ b/tests/test_hashing/test_pandas_refused_at_write.py @@ -25,17 +25,19 @@ import pandas as pd import pyarrow as pa -import pyarrow.compute as pc import pytest from orcapod import PipelineJob, function_pod from orcapod.contexts import get_default_context from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase -from orcapod.logical_types.pandas_type import LogicalPandasDataFrame, LogicalPandasSeries +from orcapod.logical_types.pandas_type import ( + LogicalPandasDataFrame, + LogicalPandasSeries, +) from orcapod.sources import DictSource -def _arrow_values(array: pa.Array) -> pd.Series: +def _arrow_values(array: pa.Array[Any]) -> pd.Series: """A Series named ``"s"`` of ``pd.ArrowDtype`` over ``array``.""" return pd.Series(pd.arrays.ArrowExtensionArray(pa.chunked_array([array])), name="s") @@ -46,7 +48,7 @@ def _multi(names: list[Any]) -> pd.MultiIndex: # pd.ArrowDtype values storage writes but cannot read back (as a column and as an index level). -UNREADABLE_ARROW: dict[str, pa.Array] = { +UNREADABLE_ARROW: dict[str, pa.Array[Any]] = { "list": pa.array([[1, 2], None], pa.list_(pa.int64())), "large_list": pa.array([[1, 2], None], pa.large_list(pa.int64())), "fixed_size_list": pa.array([[1, 2], None], pa.list_(pa.int64(), 2)), @@ -57,9 +59,11 @@ def _multi(names: list[Any]) -> pd.MultiIndex: } # Readable as an index level? No for these either; readable as a column but refused at write # by Arrow already for an index level. -UNREADABLE_ARROW_COLUMN_ONLY: dict[str, pa.Array] = { +UNREADABLE_ARROW_COLUMN_ONLY: dict[str, pa.Array[Any]] = { "list_view": pa.array([[1, 2], None], pa.list_view(pa.int64())), - "run_end_encoded": pc.run_end_encode(pa.array([1, 1])), + "run_end_encoded": pa.RunEndEncodedArray.from_arrays( + pa.array([2], pa.int32()), pa.array([1]) + ), "dense_union": pa.UnionArray.from_dense( pa.array([0, 1], pa.int8()), pa.array([0, 0], pa.int32()), @@ -123,11 +127,11 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: values = _arrow_values(array) cases[f"frame, arrow {kind} column"] = ( values.to_frame(name="col"), - r"column 'col'.*cannot (be read back|be converted)", + r"column 'col'.*(cannot be read back|cannot convert)", ) cases[f"series, arrow {kind} values"] = ( values, - r"column 's'.*cannot (be read back|be converted)", + r"column 's'.*(cannot be read back|cannot convert)", ) for kind, array in UNREADABLE_ARROW.items(): cases[f"frame, arrow {kind} index level"] = ( @@ -163,23 +167,32 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: [[1, 2]], columns=pd.MultiIndex.from_tuples([("a", 1), ("b", 2)], names=[1, 0]) ), "arrow string column": _arrow_values(pa.array(["x", None])).to_frame(), - "arrow large_binary values": _arrow_values(pa.array([b"x", None], pa.large_binary())), + "arrow large_binary values": _arrow_values( + pa.array([b"x", None], pa.large_binary()) + ), "arrow timestamp tz index": pd.DataFrame( {"a": [1, 2]}, index=pd.Index( _arrow_values( - pa.array([dt.datetime(2020, 1, 1), None], pa.timestamp("s", tz="UTC")) + pa.array( + [dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 2)], + pa.timestamp("s", tz="UTC"), + ) ).array ), ), - "Int64 column axis": pd.DataFrame([[1, 2]], columns=pd.Index([0, 1], dtype="Int64")), + "Int64 column axis": pd.DataFrame( + [[1, 2]], columns=pd.Index([0, 1], dtype="Int64") + ), "datetime column axis": pd.DataFrame( [[1, 2]], columns=pd.DatetimeIndex(["2020-01-01", "2020-01-02"], tz="UTC") ), } -def _logical(value: pd.DataFrame | pd.Series) -> LogicalPandasDataFrame | LogicalPandasSeries: +def _logical( + value: pd.DataFrame | pd.Series, +) -> LogicalPandasDataFrame | LogicalPandasSeries: """The logical type that stores ``value``.""" if isinstance(value, pd.Series): return LogicalPandasSeries() diff --git a/tests/test_hashing/test_pandas_storage_matrix.py b/tests/test_hashing/test_pandas_storage_matrix.py index 02f58a8e3..03736441e 100644 --- a/tests/test_hashing/test_pandas_storage_matrix.py +++ b/tests/test_hashing/test_pandas_storage_matrix.py @@ -7,8 +7,11 @@ lists, each on its own, and over the products of representatives of each, for frames and Series. -A value storage does not accept (the write or the read raises) is outside the rule and is -left out; each test asserts that most of its matrix was accepted, so it cannot pass empty. +Only the first write may refuse a value: storage refuses at write, with ``ValueError``, what it +would not restore (ITL-1054), and such a value is left out. Every value written must read back, +survive a second round trip, hash the same after each, and hold each column and index level at +its own position. Each test asserts that most of its matrix was accepted, so it cannot pass +empty. """ from __future__ import annotations @@ -272,37 +275,112 @@ def _hash(value: Any) -> str: return get_default_context().semantic_hasher.hash_object(value).to_string() +def _logical(value: Any) -> LogicalPandasDataFrame | LogicalPandasSeries: + """The logical type that stores ``value``.""" + if isinstance(value, pd.Series): + return LogicalPandasSeries() + return LogicalPandasDataFrame() + + def _stored_and_read(value: Any) -> Any: """``value`` after the storage round trip of its logical type.""" - logical = ( - LogicalPandasSeries() - if isinstance(value, pd.Series) - else LogicalPandasDataFrame() - ) + logical = _logical(value) return logical.storage_to_python(logical.python_to_storage(value)) +def _level(axis: pd.Index, position: int) -> pd.Index: + """Level ``position`` of ``axis`` by position (``get_level_values`` tries names first).""" + if not isinstance(axis, pd.MultiIndex): + return axis + unnamed = axis.set_names([None] * axis.nlevels) + return unnamed.get_level_values(position).rename(axis.names[position]) + + +def _name_text(name: Any) -> Any: + """An index level name as storage writes it: its text, a tuple's parts each as text.""" + if name is None: + return None + if isinstance(name, tuple): + return str(tuple(_name_text(part) for part in name)) + return str(name) + + +def _positional_mismatches(value: Any, back: Any) -> list[str]: + """The parts of ``back`` that do not hold, at their position, what ``value`` holds there. + + Independent of the whole-value hash, which describes a value in the form storage + restores and so cannot see a part restored in another part's place (ITL-1054: MultiIndex + levels written in another order hashed like the reordered value). Each column and each + index level, taken by position, must hash like the original's part at that position; the + column labels (as text, per level) and the index level names (as text) must be in the + same order. + """ + mismatches: list[str] = [] + frame = value.to_frame() if isinstance(value, pd.Series) else value + frame_back = back.to_frame() if isinstance(back, pd.Series) else back + if frame.shape != frame_back.shape: + return [f"shape {frame.shape} -> {frame_back.shape}"] + for position in range(frame.shape[1]): + column = frame.iloc[:, position].reset_index(drop=True).rename(None) + column_back = frame_back.iloc[:, position].reset_index(drop=True).rename(None) + if _hash(column) != _hash(column_back): + mismatches.append(f"column {position}") + if isinstance(value, pd.DataFrame): + if value.columns.nlevels != back.columns.nlevels: + return [*mismatches, "column axis levels"] + for position in range(value.columns.nlevels): + labels = [str(label) for label in _level(value.columns, position)] + labels_back = [str(label) for label in _level(back.columns, position)] + if labels != labels_back: + mismatches.append(f"column axis level {position} labels") + if value.index.nlevels != back.index.nlevels: + return [*mismatches, "index levels"] + for position in range(value.index.nlevels): + level = pd.DataFrame(index=_level(value.index, position).rename(None)) + level_back = pd.DataFrame(index=_level(back.index, position).rename(None)) + if _hash(level) != _hash(level_back): + mismatches.append(f"index level {position}") + names = [_name_text(name) for name in value.index.names] + if names != [_name_text(name) for name in back.index.names]: + mismatches.append("index level names") + return mismatches + + def _unstable(values: dict[str, Any]) -> tuple[list[str], int]: - """Check the round-trip rule on every value storage accepts. + """Check the round-trip rule on every value storage accepts at the first write. + + Only the first write may refuse a value (that value is left out). Once written, a value + must read back, survive a second round trip, hash the same after each, and hold each part + at its own position (``_positional_mismatches``). A read that raises, a second round trip + that raises, or a changed value is a failure, never "unsupported input". Returns: - The names of the values whose hash changed through storage (or changed again on a - second round trip), and how many values storage accepted. + The names of the values that failed, each with the reason, and how many values the + first write accepted. """ - unstable: list[str] = [] + failed: list[str] = [] accepted = 0 with warnings.catch_warnings(): warnings.simplefilter("ignore") for name, value in values.items(): + logical = _logical(value) try: - back = _stored_and_read(value) - again = _stored_and_read(back) - except (ValueError, TypeError, NotImplementedError, pa.ArrowException): - continue # Storage does not accept this value. + stored = logical.python_to_storage(value) + except ValueError: + continue # Refused at write: storage would not restore this value. accepted += 1 + try: + back = logical.storage_to_python(stored) + again = _stored_and_read(back) + except Exception as exc: # noqa: BLE001 - every failure after the write counts + failed.append(f"{name}: {type(exc).__name__}: {exc}") + continue if not (_hash(value) == _hash(back) == _hash(again)): - unstable.append(name) - return unstable, accepted + failed.append(f"{name}: hash changed through storage") + mismatches = _positional_mismatches(value, back) + if mismatches: + failed.append(f"{name}: {mismatches}") + return failed, accepted def _frame(index: pd.Index, columns: pd.Index, values: Any) -> pd.DataFrame: diff --git a/tests/test_hashing/test_pandas_value_identity.py b/tests/test_hashing/test_pandas_value_identity.py index aa76449d8..e2fd6a234 100644 --- a/tests/test_hashing/test_pandas_value_identity.py +++ b/tests/test_hashing/test_pandas_value_identity.py @@ -47,14 +47,6 @@ def _multi_index() -> pd.MultiIndex: return pd.MultiIndex.from_tuples([(0, "p"), (1, "q")], names=["i", "j"]) -def _struct_frame(order: list[str]) -> pd.DataFrame: - """One ``pd.ArrowDtype`` struct column whose fields are in ``order``.""" - types = {"u": pa.int64(), "v": pa.large_string()} - struct = pa.struct([(name, types[name]) for name in order]) - array = pa.array([{"u": 1, "v": "x"}], struct) - return pd.DataFrame({"s": pd.Series(array, dtype=pd.ArrowDtype(struct))}) - - def _with_columns_name(df: pd.DataFrame, name: str) -> pd.DataFrame: """A copy of ``df`` whose column axis is named ``name``.""" out = df.copy() @@ -98,7 +90,6 @@ def _timestamp_labels(tz: str | None) -> pd.DataFrame: _frame().set_index(_multi_index()), _frame().set_index(_multi_index()).swaplevel(), ), - "struct field order": (_struct_frame(["u", "v"]), _struct_frame(["v", "u"])), # labels and names "column label type 0 vs '0'": ( pd.DataFrame({0: [1, 2]}), @@ -324,7 +315,6 @@ def _timestamp_labels(tz: str | None) -> pd.DataFrame: PROPERTY_OF_PAIR: dict[str, str] = { "column order": "column_order", "MultiIndex level order": "index_level_order", - "struct field order": "struct_field_order", "column label type 0 vs '0'": "label_type", "MultiIndex columns vs flat labels": "label_type", "columns.name": "columns_names", @@ -462,7 +452,7 @@ def test_only_identity_properties_are_options(self) -> None: ) options = {p.name for p in PANDAS_IDENTITY_PROPERTIES if p.option} - assert options == {"column_order", "struct_field_order", "series_name", "attrs"} + assert options == {"column_order", "series_name", "attrs"} assert options == PANDAS_DATAFRAME_OPTIONS | PANDAS_SERIES_OPTIONS assert all(p.identity for p in PANDAS_IDENTITY_PROPERTIES if p.option) @@ -590,9 +580,16 @@ def test_ignored_attrs_are_never_copied(self) -> None: _hash(series) assert list(frame.attrs) == ["lock"] - def test_date_label_hashes(self) -> None: + def test_date_label_is_in_the_grammar(self) -> None: """A ``datetime.date`` label is in the grammar.""" - assert _hash(pd.DataFrame([[1]], columns=pd.Index([dt.date(2020, 1, 1)]))) + from orcapod.hashing.pandas_value_hash import pandas_label_descriptor + + assert pandas_label_descriptor(dt.date(2020, 1, 1)) == {"date": "2020-01-01"} + + def test_date_column_axis_is_refused(self) -> None: + """A column axis of dates cannot be read back from storage, so it is refused (ITL-1054).""" + with pytest.raises(ValueError, match="column axis.*cannot be read back"): + _hash(pd.DataFrame([[1]], columns=pd.Index([dt.date(2020, 1, 1)]))) STORABLE_FRAMES: dict[str, Any] = { @@ -936,15 +933,6 @@ def test_dtype_grammar(self) -> None: assert pandas_dtype_descriptor(pd.IntervalDtype("int64", "left")) == { "interval": [{"numpy": "int64"}, "left"] } - # The Arrow type itself, struct field order included, is in the ordered digest. - assert ( - pandas_dtype_descriptor( - pd.ArrowDtype( - pa.list_(pa.struct([("v", pa.string()), ("u", pa.int8())])) - ) - ) - == "arrow" - ) def _arrow_series(values: list[Any], arrow_type: pa.DataType) -> pd.Series: @@ -1008,25 +996,23 @@ def test_decimal_precision_is_not_identity(self, kind: str) -> None: assert _hash(built[0]) == _hash(built[1]) assert _hash(_stored_and_read(built[0])) == _hash(built[0]) - def test_nested_arrow_dtypes_keep_the_bare_term(self) -> None: - """A nested ``pd.ArrowDtype`` is described by the bare term, large or not. + def test_nested_arrow_dtypes_are_refused(self) -> None: + """A nested or dictionary ``pd.ArrowDtype`` is refused by the hash (ITL-1054). - ITL-1054 decides the identity of nested Arrow dtypes (none reads back today), so the - large and non-large list stay equal until then. + Storage cannot read such a column back, so no hash describes one: the hash refuses it + with storage's own message, and a ``list`` and a ``large_list`` column are never + confused. """ - from orcapod.hashing.pandas_value_hash import pandas_dtype_descriptor - for nested in ( pa.list_(pa.int64()), pa.large_list(pa.int64()), pa.struct([("s", pa.string())]), pa.map_(pa.string(), pa.int64()), + pa.dictionary(pa.int8(), pa.string()), ): - assert pandas_dtype_descriptor(pd.ArrowDtype(nested)) == "arrow" - values = [[1, 2], None] - assert _hash(_arrow_series(values, pa.list_(pa.int64()))) == _hash( - _arrow_series(values, pa.large_list(pa.int64())) - ) + empty = pa.array([], nested) + with pytest.raises(ValueError, match="cannot be read back"): + _hash(pd.Series(empty, dtype=pd.ArrowDtype(nested), name="s")) def test_leaf_terms(self) -> None: """The term of a leaf type is its Arrow type text (``null`` included).""" From eddff120883b5f61f874e03b4d36f2be8199795b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:15:56 -0800 Subject: [PATCH 4/7] fix(pandas): refuse tuple categories and column labels that read back changed (ITL-1054) CodeRabbit review of #383: require_restorable reads back table.slice(0, 0), which keeps each categorical's dictionary, so categories storage cannot restore (tuples read back as arrays) are refused at write; and it compares each column-axis level, as text, with the labels read back, so a False label (read back True by pyarrow) is refused. The column-order sort converts a zero-row frame only. Changelog notes the struct_field_order columns stored on unreleased main. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1054.md | 12 ++- design_issues/ITL-1054-PV3.md | 11 ++- docs/reference/hashing.md | 8 +- src/orcapod/hashing/pandas_value_hash.py | 87 ++++++++++++++----- src/orcapod/logical_types/pandas_type.py | 4 +- .../test_pandas_refused_at_write.py | 31 +++++++ 6 files changed, 119 insertions(+), 34 deletions(-) diff --git a/changelog.d/ITL-1054.md b/changelog.d/ITL-1054.md index 1936e780a..22bd199cb 100644 --- a/changelog.d/ITL-1054.md +++ b/changelog.d/ITL-1054.md @@ -16,8 +16,12 @@ through one shared check. These values used to be written: dictionary, fixed-size binary or `float16` type. A column axis (any level) that is categorical, period, timedelta, interval, `datetime.date` objects, or a `pd.ArrowDtype` timestamp, duration, bool, date or decimal. This includes a Series named with a - `pd.Timedelta`, `pd.Period` or `datetime.date`. In a pipeline, such a result stayed in the - result table and could not be loaded on any rerun. + `pd.Timedelta`, `pd.Period` or `datetime.date`. A categorical column, Series or index + whose categories are tuples (ITL-1115). In a pipeline, such a result stayed in the result + table and could not be loaded on any rerun. +- **Written, then read back with another column label, with no error:** the label `False` + (a one-column bool axis, a Series named `False`, or a `False` in a level of a MultiIndex + column axis) read back as `True` (ITL-1116). A value Arrow cannot convert (for example `complex128` or a sparse column) now raises `ValueError` naming the column, where it used to raise `ArrowNotImplementedError`, `TypeError` @@ -34,7 +38,9 @@ migration tool reports both (ITL-1048). `orcapod.pandas.series`. It acted only on `pd.ArrowDtype` struct columns, which are now refused. A data context that names it under `hash_ignore` is refused with the unknown-option error, and so is `ignore=["struct_field_order"]` on the logical type or its handler. No shipped -data context named it, so no identifier or hash moves. The pandas types are provisional until +data context named it, so no identifier or hash moves. A column stored on unreleased `main` +under a custom data context that named it is refused on read (`LogicalTypeIdentifierError`); +such results are recomputed under a data context without it. The pandas types are provisional until the v0.4 audit (ITL-1069). That audit is where struct columns and this option come back together, if storage learns to restore them. The Polars types keep their `struct_field_order` option. diff --git a/design_issues/ITL-1054-PV3.md b/design_issues/ITL-1054-PV3.md index 38c207532..1fe205588 100644 --- a/design_issues/ITL-1054-PV3.md +++ b/design_issues/ITL-1054-PV3.md @@ -12,16 +12,19 @@ fetches level `i` with `index.get_level_values(i)`, which pandas resolves as a l The value read back was another value, nothing raised, and the hash (computed on the restored form) gave both the same digest. Second, a nested, dictionary, fixed-size or union `pd.ArrowDtype` column or index level, a `float16` Arrow index level, and a categorical, period, -timedelta, interval, date or `pd.ArrowDtype` column axis were written and then failed to read -back. In a pipeline, the computed result stayed in the result table and could not be loaded on +timedelta, interval, date or `pd.ArrowDtype` column axis, and categories that are tuples, were +written and then failed to read back. A `False` column label (or Series name) read back as +`True`. In a pipeline, the computed result stayed in the result table and could not be loaded on any rerun. The storage round-trip matrix test hid the second kind: it treated a failed read like an input storage does not support. **Fix:** one check, `require_restorable(schema, frame)` in `orcapod/hashing/pandas_value_hash.py`, shared by `python_to_storage` and the value hash. It refuses both kinds at write with a `ValueError` naming the column, index level or column-axis -level. A failed read is detected from the schema alone (`schema.empty_table().to_pandas()`), -which agreed with the full read on every value tried, on pyarrow 22.0.0, 23.0.1 and 26.0.1. +level. A failed read is detected from a zero-row read (`table.slice(0, 0).to_pandas()`, which +keeps the categories), which agreed with the full read on every value tried, on pyarrow 22.0.0, +23.0.1 and 26.0.1, and column labels are compared as text with the labels read back. +Future support: tuple categories ITL-1115, `False` labels ITL-1116. `pandas_frame_to_arrow` wraps every Arrow conversion error into `ValueError`, naming the part that fails. The matrix test now lets only the first write refuse, and it checks each column and index level by position. The `struct_field_order` ignore option is removed, because it acted diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 135f6a47d..a60716f32 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -1585,7 +1585,13 @@ pyarrow version: list, struct, map, dictionary, fixed-size binary or `float16` type; a column axis (any level of it) that is categorical, period, timedelta, interval, `datetime.date` objects, or a `pd.ArrowDtype` timestamp, duration, bool, date or decimal, which includes a Series - named with a `pd.Timedelta`, `pd.Period` or `datetime.date`. + named with a `pd.Timedelta`, `pd.Period` or `datetime.date`; a categorical column, Series + or index whose categories are tuples (they read back as arrays, which cannot be + categories; ITL-1115). +- **Column labels that would read back as another label.** pyarrow restores a bool label + from its text, so the label `False` (a one-column bool axis, a Series named `False`, a + `False` in a level of a MultiIndex column axis) reads back `True` (ITL-1116). Labels are + compared as text, so mixed-type labels, which storage writes as text, are not a change. - **Labels and names that collide once written as text** (the index name `1` with a column `"1"`; integer names of MultiIndex levels): two fields of one name, and the value read back would have lost its index. Distinct labels that read back equal (the labels `True` diff --git a/src/orcapod/hashing/pandas_value_hash.py b/src/orcapod/hashing/pandas_value_hash.py index 9325d3be1..4366b74a8 100644 --- a/src/orcapod/hashing/pandas_value_hash.py +++ b/src/orcapod/hashing/pandas_value_hash.py @@ -116,7 +116,9 @@ would be written in another order (level names that are also level positions, ``names=[1, 0]``), and values storage cannot read back (a nested, dictionary or fixed-size ``pd.ArrowDtype`` column or index level; a categorical, period, timedelta, interval, date or -``pd.ArrowDtype`` column axis), besides labels and names that collide once written as text. +``pd.ArrowDtype`` column axis; tuple categories), column labels that would read back as +another label (``False`` reads back ``True``), and labels and names that collide once written +as text. A value Arrow cannot convert is refused by ``pandas_frame_to_arrow``. Both messages name the column, index level or column-axis level. """ @@ -727,9 +729,9 @@ def _converted(frame: pd.DataFrame) -> pa.Table: return pa.Table.from_pandas(frame, preserve_index=True) -def _read_back_empty(frame: pd.DataFrame) -> pd.DataFrame: - """The zero-row frame storage restores for ``frame``'s Arrow schema.""" - return _converted(frame).schema.empty_table().to_pandas() +def _read_back(frame: pd.DataFrame) -> pd.DataFrame: + """The frame storage restores for ``frame`` (a zero-row part, see ``_parts``).""" + return _converted(frame).to_pandas() def _with_part(part: str | None, message: str) -> str: @@ -796,7 +798,39 @@ def _require_levels_in_place(index: pd.Index) -> None: ) -def require_restorable(schema: pa.Schema, frame: pd.DataFrame) -> pd.DataFrame: +def _require_labels_restored(columns: pd.Index, restored: pd.Index) -> None: + """Raise if a column label would read back from storage as another label. + + Labels are compared as text, level by level and position by position: storage writes + labels of mixed type as text (``0`` beside ``"b"`` reads back ``"0"``), and that is no + change. A label whose text changes is: pyarrow restores a bool level by casting the + text, so ``False`` reads back ``True``. + + Args: + columns: The column axis of the frame. + restored: The column axis of the frame storage restores, with as many levels. + + Raises: + ValueError: If a label's text changes; the message names the level and the label. + """ + for position in range(columns.nlevels): + written = _level_by_position(columns, position) + read = _level_by_position(restored, position) + for label, back in zip(written, read): + if str(label) != str(back): + where = ( + "column axis" + if columns.nlevels == 1 + else f"column axis level {position}" + ) + raise ValueError( + f"{where}: the label {label!r} would read back from storage as " + f"{back!r}. Rename the columns (pyarrow restores a bool label from " + "its text, so False reads back True)." + ) + + +def require_restorable(table: pa.Table, frame: pd.DataFrame) -> pd.DataFrame: """Return the zero-row frame storage restores for ``frame``, refusing what it would not. The one check, shared by storage (``python_to_storage``) and the value hash, of a value @@ -808,26 +842,28 @@ def require_restorable(schema: pa.Schema, frame: pd.DataFrame) -> pd.DataFrame: level positions, ``names=[1, 0]``): the frame read back would differ, silently. - The frame read back cannot be built: a nested, dictionary or fixed-size ``pd.ArrowDtype`` column or index level; a categorical, period, timedelta, interval, date or - ``pd.ArrowDtype`` column axis. - - The frame read back would have another number of columns or index levels, or distinct - column labels that read back equal (``True`` and ``False`` are written as text). + ``pd.ArrowDtype`` column axis; a categorical whose categories cannot be restored + (tuples read back as arrays, which cannot be categories). + - The frame read back would have another number of columns or index levels, distinct + column labels that read back equal, or a column label that reads back as another label + (``False`` reads back ``True``). - No rows are converted: what storage restores is read from the frame it builds for - ``schema`` alone. + No rows are converted: what storage restores is read from ``table.slice(0, 0)``, which + keeps the schema and the dictionary (the categories) of each categorical column. Args: - schema: The schema of ``pandas_frame_to_arrow(frame)``. + table: ``pandas_frame_to_arrow(frame)``. frame: The frame about to be stored or hashed. Returns: - The zero-row frame storage restores for ``schema``: its axes, labels, names and the - dtypes the schema decides. + The zero-row frame storage restores for ``table``: its axes, labels, names, the + categories and the dtypes the schema decides. Raises: ValueError: If storage would not restore the frame; the message names the column, index level or column-axis level and the reason. """ - counts = Counter(schema.names) + counts = Counter(table.schema.names) repeated = sorted(name for name, count in counts.items() if count > 1) if repeated: raise ValueError( @@ -837,9 +873,9 @@ def require_restorable(schema: pa.Schema, frame: pd.DataFrame) -> pd.DataFrame: ) _require_levels_in_place(frame.index) try: - restored = schema.empty_table().to_pandas() + restored = table.slice(0, 0).to_pandas() except Exception as exc: # noqa: BLE001 - pandas and pyarrow raise many kinds here - part = _failing_part(frame, 0, _read_back_empty) + part = _failing_part(frame, 0, _read_back) raise ValueError( _with_part( part, @@ -864,17 +900,18 @@ def require_restorable(schema: pa.Schema, frame: pd.DataFrame) -> pd.DataFrame: f"{labels}. Rename the columns (labels of mixed or bool type are written " "as text)." ) + _require_labels_restored(frame.columns, restored.columns) return restored -def _restored_axes(schema: pa.Schema, df: pd.DataFrame) -> _StoredAxes: - """Return the axes storage restores for a frame whose Arrow schema is ``schema``. +def _restored_axes(table: pa.Table, df: pd.DataFrame) -> _StoredAxes: + """Return the axes storage restores for a frame whose Arrow table is ``table``. - They are read from the frame storage builds for the schema (no rows are converted), so - the rule is storage's own. + They are read from the zero-row frame storage builds for the table (no rows are + converted), so the rule is storage's own. Args: - schema: The schema of ``pandas_frame_to_arrow(df)``. + table: ``pandas_frame_to_arrow(df)``, or the same for a zero-row ``df``. df: The frame. Returns: @@ -883,7 +920,7 @@ def _restored_axes(schema: pa.Schema, df: pd.DataFrame) -> _StoredAxes: Raises: ValueError: If storage would not restore the frame (``require_restorable``). """ - return _axes_of(require_restorable(schema, df)) + return _axes_of(require_restorable(table, df)) def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: @@ -910,7 +947,7 @@ def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: raise ValueError(f"column {label!r}: {exc}") from exc table = pandas_frame_to_arrow(df) # Refuses what storage would not restore, with storage's own message (ITL-1054). - axes = _restored_axes(table.schema, df) + axes = _restored_axes(table, df) # The frame's own labels and names must be in the grammar, although the descriptor # holds them in the form storage restores: storage writes some as text, and the text of # an arbitrary object (its ``repr``, with a memory address) must never reach a hash. @@ -1044,7 +1081,9 @@ def _in_canonical_column_order(df: pd.DataFrame) -> pd.DataFrame: The sort is on the terms of the labels as storage restores them, so a frame and the frame read back from storage are sorted alike. """ - labels = _restored_axes(pandas_frame_to_arrow(df).schema, df).column_labels + # Only the labels are needed: a zero-row frame has them, and its conversion is cheap. + empty = df.iloc[:0] + labels = _restored_axes(pandas_frame_to_arrow(empty), empty).column_labels keys = [canonical_json(pandas_label_descriptor(label)) for label in labels] order = sorted(range(len(keys)), key=keys.__getitem__) return df.iloc[:, order] diff --git a/src/orcapod/logical_types/pandas_type.py b/src/orcapod/logical_types/pandas_type.py index c6a1d8f45..5049ca2da 100644 --- a/src/orcapod/logical_types/pandas_type.py +++ b/src/orcapod/logical_types/pandas_type.py @@ -130,7 +130,7 @@ def python_to_storage( """ pandas_attrs_as_stored(value.attrs) # refuses attrs that are not JSON table = pandas_frame_to_arrow(value) - require_restorable(table.schema, value) + require_restorable(table, value) return _table_to_ipc_bytes(table) def storage_to_python( @@ -208,7 +208,7 @@ def python_to_storage( pandas_attrs_as_stored(value.attrs) # refuses attrs that are not JSON df = value.to_frame(name=col_name) table = pandas_frame_to_arrow(df) - require_restorable(table.schema, df) + require_restorable(table, df) return _table_to_ipc_bytes(table) def storage_to_python( diff --git a/tests/test_hashing/test_pandas_refused_at_write.py b/tests/test_hashing/test_pandas_refused_at_write.py index 5517ddc45..e73d8e53b 100644 --- a/tests/test_hashing/test_pandas_refused_at_write.py +++ b/tests/test_hashing/test_pandas_refused_at_write.py @@ -150,6 +150,32 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: pd.DataFrame([[1, 2], [3, 4]], columns=axis), r"column axis.*cannot be read back", ) + tuples = pd.Categorical([(1, 2), (3, 4)]) + cases["frame, tuple categories"] = ( + pd.DataFrame({"a": tuples}), + r"column 'a'.*cannot be read back", + ) + cases["series, tuple categories"] = ( + pd.Series(tuples, name="s"), + r"column 's'.*cannot be read back", + ) + cases["frame, tuple categories index"] = ( + pd.DataFrame({"a": [1, 2]}, index=pd.CategoricalIndex(tuples)), + r"index level 0.*cannot be read back", + ) + cases["frame, column label False"] = ( + pd.DataFrame([[7]], columns=pd.Index([False])), + r"column axis: the label False would read back from storage as True", + ) + cases["series, name False"] = ( + pd.Series([7], name=False), + r"column axis: the label False would read back from storage as True", + ) + cases["frame, column level holding False"] = ( + pd.DataFrame([[7]], columns=pd.MultiIndex.from_tuples([("a", False)])), + r"column axis level 1: the label .*False.* would read back from storage as " + r".*True", + ) return cases @@ -187,6 +213,11 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: "datetime column axis": pd.DataFrame( [[1, 2]], columns=pd.DatetimeIndex(["2020-01-01", "2020-01-02"], tz="UTC") ), + "column label True": pd.DataFrame([[7]], columns=pd.Index([True])), + "series named True": pd.Series([7], name=True), + "text and int labels": pd.DataFrame([[1, 2, 3]], columns=["a", "False", 0]), + "int column axis": pd.DataFrame([[1, 2]], columns=[0, 1]), + "string categories": pd.DataFrame({"a": pd.Categorical(["x", "y"])}), } From 7eb19b697a1efd7635d79eb4a53e15402c635972 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:21:06 -0800 Subject: [PATCH 5/7] test(pandas): type the column axes of the accepted neighbours (ITL-1054) Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/test_pandas_refused_at_write.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_hashing/test_pandas_refused_at_write.py b/tests/test_hashing/test_pandas_refused_at_write.py index e73d8e53b..01059c034 100644 --- a/tests/test_hashing/test_pandas_refused_at_write.py +++ b/tests/test_hashing/test_pandas_refused_at_write.py @@ -215,8 +215,10 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: ), "column label True": pd.DataFrame([[7]], columns=pd.Index([True])), "series named True": pd.Series([7], name=True), - "text and int labels": pd.DataFrame([[1, 2, 3]], columns=["a", "False", 0]), - "int column axis": pd.DataFrame([[1, 2]], columns=[0, 1]), + "text and int labels": pd.DataFrame( + [[1, 2, 3]], columns=pd.Index(["a", "False", 0]) + ), + "int column axis": pd.DataFrame([[1, 2]], columns=pd.Index([0, 1])), "string categories": pd.DataFrame({"a": pd.Categorical(["x", "y"])}), } From 61cfc81c38b87e8b7eabe8e124ed091be827696d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:49:19 -0800 Subject: [PATCH 6/7] fix(pandas): refuse index levels whose values or names would not read back (ITL-1054) CodeRabbit review of #383, round 2: require_restorable converts each index level once, in full, so a time64[ns] index holding sub-microsecond values (read fails on the values alone) is refused; and it compares each index level name, as the text storage writes, with the name read back, so a name pyarrow drops (__index_level___) is refused. The part locator builds each part inside its try, so a part pandas cannot build alone (a float16 Arrow column axis) is named instead of raising TypeError. The hash runs storage's checks before its grammar checks, so both refuse with one message (interval categories). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1054.md | 11 +- design_issues/ITL-1054-PV3.md | 11 +- docs/reference/hashing.md | 25 ++-- src/orcapod/hashing/pandas_value_hash.py | 129 +++++++++++++++--- .../test_pandas_refused_at_write.py | 74 +++++++++- 5 files changed, 210 insertions(+), 40 deletions(-) diff --git a/changelog.d/ITL-1054.md b/changelog.d/ITL-1054.md index 22bd199cb..173116695 100644 --- a/changelog.d/ITL-1054.md +++ b/changelog.d/ITL-1054.md @@ -12,16 +12,19 @@ through one shared check. These values used to be written: levels swapped, and the hash did not tell the two apart. - **Written, then failed to read back:** a `pd.ArrowDtype` column or Series of a list, large list, fixed-size list, list view, struct, map, dictionary, run-end, union or fixed-size binary - type. An index level of a `pd.ArrowDtype` list, large list, fixed-size list, struct, map, + type, or of an Arrow extension type (`uuid`, `json`, `bool8`, a fixed-shape tensor). An index level of a `pd.ArrowDtype` list, large list, fixed-size list, struct, map, dictionary, fixed-size binary or `float16` type. A column axis (any level) that is categorical, period, timedelta, interval, `datetime.date` objects, or a `pd.ArrowDtype` - timestamp, duration, bool, date or decimal. This includes a Series named with a + timestamp, duration, time, bool, date, decimal or unsigned integer. This includes a Series named with a `pd.Timedelta`, `pd.Period` or `datetime.date`. A categorical column, Series or index - whose categories are tuples (ITL-1115). In a pipeline, such a result stayed in the result + whose categories are tuples (ITL-1115) or intervals. A `pd.ArrowDtype` `time64[ns]` index + level holding a value that is not a whole number of microseconds (ITL-1118; index levels + are now converted once, in full, at write). In a pipeline, such a result stayed in the result table and could not be loaded on any rerun. - **Written, then read back with another column label, with no error:** the label `False` (a one-column bool axis, a Series named `False`, or a `False` in a level of a MultiIndex - column axis) read back as `True` (ITL-1116). + column axis) read back as `True` (ITL-1116); an index level named like pyarrow's generated + field names (`__index_level_0__`) read back unnamed (ITL-1119). A value Arrow cannot convert (for example `complex128` or a sparse column) now raises `ValueError` naming the column, where it used to raise `ArrowNotImplementedError`, `TypeError` diff --git a/design_issues/ITL-1054-PV3.md b/design_issues/ITL-1054-PV3.md index 1fe205588..513e43e30 100644 --- a/design_issues/ITL-1054-PV3.md +++ b/design_issues/ITL-1054-PV3.md @@ -13,8 +13,9 @@ The value read back was another value, nothing raised, and the hash (computed on form) gave both the same digest. Second, a nested, dictionary, fixed-size or union `pd.ArrowDtype` column or index level, a `float16` Arrow index level, and a categorical, period, timedelta, interval, date or `pd.ArrowDtype` column axis, and categories that are tuples, were -written and then failed to read back. A `False` column label (or Series name) read back as -`True`. In a pipeline, the computed result stayed in the result table and could not be loaded on +written and then failed to read back, and so was a `time64[ns]` index level holding +sub-microsecond values. A `False` column label (or Series name) read back as `True`, and an +index level named `__index_level___` read back unnamed. In a pipeline, the computed result stayed in the result table and could not be loaded on any rerun. The storage round-trip matrix test hid the second kind: it treated a failed read like an input storage does not support. @@ -23,8 +24,10 @@ like an input storage does not support. refuses both kinds at write with a `ValueError` naming the column, index level or column-axis level. A failed read is detected from a zero-row read (`table.slice(0, 0).to_pandas()`, which keeps the categories), which agreed with the full read on every value tried, on pyarrow 22.0.0, -23.0.1 and 26.0.1, and column labels are compared as text with the labels read back. -Future support: tuple categories ITL-1115, `False` labels ITL-1116. +23.0.1 and 26.0.1, together with a full conversion of each index level (the one failure found +that depends on values). Column labels and index level names are compared as text with those +read back. Future support: tuple categories ITL-1115, `False` labels ITL-1116, sub-microsecond +`time64[ns]` index ITL-1118, generated-looking index names ITL-1119. `pandas_frame_to_arrow` wraps every Arrow conversion error into `ValueError`, naming the part that fails. The matrix test now lets only the first write refuse, and it checks each column and index level by position. The `struct_field_order` ignore option is removed, because it acted diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index a60716f32..2f407a533 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -1581,13 +1581,21 @@ pyarrow version: would be written, and read back, with the levels swapped, and nothing would raise. - **Values storage cannot read back.** A `pd.ArrowDtype` column (or Series) of a list, large list, fixed-size list, list view, struct, map, dictionary, run-end or union type, - or of fixed-size binary; an index level of a `pd.ArrowDtype` list, large list, fixed-size - list, struct, map, dictionary, fixed-size binary or `float16` type; a column axis (any - level of it) that is categorical, period, timedelta, interval, `datetime.date` objects, or - a `pd.ArrowDtype` timestamp, duration, bool, date or decimal, which includes a Series - named with a `pd.Timedelta`, `pd.Period` or `datetime.date`; a categorical column, Series - or index whose categories are tuples (they read back as arrays, which cannot be - categories; ITL-1115). + of fixed-size binary, or of an Arrow extension type (`uuid`, `json`, `bool8`, a fixed-shape + tensor); an index level of a `pd.ArrowDtype` list, large list, fixed-size list, struct, + map, dictionary, fixed-size binary or `float16` type; a column axis (any level of it) that + is categorical, period, timedelta, interval, `datetime.date` objects, or a `pd.ArrowDtype` + timestamp, duration, time, bool, date, decimal or unsigned integer, which includes a + Series named with a `pd.Timedelta`, `pd.Period` or `datetime.date`; a categorical column, + Series or index whose categories are tuples (they read back as arrays, which cannot be + categories; ITL-1115) or intervals. +- **Index levels whose values cannot be read back.** A `pd.ArrowDtype` `time64[ns]` index + level holding a value that is not a whole number of microseconds (ITL-1118). The failure + depends on the values, so each index level is converted once, in full, when a value is + written or hashed; columns are checked without converting any row. +- **Index level names that would read back as another name.** pyarrow drops a name of the + form `__index_level___` (its own name for an unnamed level), so such a level reads back + unnamed (ITL-1119). Names are compared as the text storage writes for them. - **Column labels that would read back as another label.** pyarrow restores a bool label from its text, so the label `False` (a one-column bool axis, a Series named `False`, a `False` in a level of a MultiIndex column axis) reads back `True` (ITL-1116). Labels are @@ -1597,7 +1605,8 @@ pyarrow version: back would have lost its index. Distinct labels that read back equal (the labels `True` and `False` both read back `True`) would come back as duplicates. - **Values Arrow cannot convert** (mixed or non-Arrow values, `complex128`, sparse columns, - an index of a type Arrow does not support). + a categorical of periods, a `float16` column axis, an index of a type Arrow does not + support). Before v0.3 such values were written: a value of the first kind reads back changed, and one of the second kind cannot be read at all. diff --git a/src/orcapod/hashing/pandas_value_hash.py b/src/orcapod/hashing/pandas_value_hash.py index 4366b74a8..fb1905ac1 100644 --- a/src/orcapod/hashing/pandas_value_hash.py +++ b/src/orcapod/hashing/pandas_value_hash.py @@ -116,9 +116,10 @@ would be written in another order (level names that are also level positions, ``names=[1, 0]``), and values storage cannot read back (a nested, dictionary or fixed-size ``pd.ArrowDtype`` column or index level; a categorical, period, timedelta, interval, date or -``pd.ArrowDtype`` column axis; tuple categories), column labels that would read back as -another label (``False`` reads back ``True``), and labels and names that collide once written -as text. +``pd.ArrowDtype`` column axis; tuple categories; a ``time64[ns]`` index level holding +sub-microsecond values), column labels and index level names that would read back as another +label or name (``False`` reads back ``True``; ``"__index_level_0__"`` reads back unnamed), and +labels and names that collide once written as text. A value Arrow cannot convert is refused by ``pandas_frame_to_arrow``. Both messages name the column, index level or column-axis level. """ @@ -130,6 +131,7 @@ from collections import Counter from collections.abc import Callable, Iterator from dataclasses import dataclass +from functools import partial from typing import TYPE_CHECKING, Any, Literal import numpy as np @@ -670,12 +672,15 @@ def _level_by_position(axis: pd.Index, position: int) -> pd.Index: return unnamed.get_level_values(position).rename(axis.names[position]) -def _parts(frame: pd.DataFrame, rows: int | None) -> Iterator[tuple[str, pd.DataFrame]]: - """Yield each part of ``frame`` storage writes, alone in a frame, with its description. +def _parts( + frame: pd.DataFrame, rows: int | None +) -> Iterator[tuple[str, Callable[[], pd.DataFrame]]]: + """Yield each part of ``frame`` storage writes, with a function building it alone in a frame. The parts are the levels of the column axis, the columns, and the index levels, each by position. A part's frame holds the first ``rows`` rows of it (all rows when ``None``); a - column-axis level is a zero-row frame with that level as its column labels. + column-axis level is a zero-row frame with that level as its column labels. Building a + part can itself fail (pandas cannot make every Index a column axis), so it is a function. """ subset = frame if rows is None else frame.iloc[:rows] columns = frame.columns @@ -686,23 +691,33 @@ def _parts(frame: pd.DataFrame, rows: int | None) -> Iterator[tuple[str, pd.Data ) yield ( f"{where} (dtype {level.dtype})", - pd.DataFrame(index=pd.RangeIndex(0), columns=level.unique()), + partial(_column_axis_alone, level), ) for position in range(subset.shape[1]): - column = subset.iloc[:, [position]] yield ( - f"column {columns[position]!r} (dtype {column.dtypes.iloc[0]})", - column.set_axis(pd.RangeIndex(1), axis=1).reset_index(drop=True), + f"column {columns[position]!r} (dtype {subset.dtypes.iloc[position]})", + partial(_column_alone, subset, position), ) index = subset.index for position in range(index.nlevels): level = _level_by_position(index, position) yield ( f"index level {position} (name {index.names[position]!r}, dtype {level.dtype})", - pd.DataFrame(index=level), + partial(pd.DataFrame, index=level), ) +def _column_axis_alone(level: pd.Index) -> pd.DataFrame: + """A zero-row frame whose column labels are the distinct labels of ``level``.""" + return pd.DataFrame(index=pd.RangeIndex(0), columns=level.unique()) + + +def _column_alone(frame: pd.DataFrame, position: int) -> pd.DataFrame: + """Column ``position`` of ``frame`` alone, with a plain column label and row index.""" + column = frame.iloc[:, [position]] + return column.set_axis(pd.RangeIndex(1), axis=1).reset_index(drop=True) + + def _failing_part( frame: pd.DataFrame, rows: int | None, attempt: Callable[[pd.DataFrame], object] ) -> str | None: @@ -714,11 +729,12 @@ def _failing_part( attempt: The step that failed on the whole frame. Returns: - The description of the part, or ``None`` when every part alone succeeds. + The description of the part, or ``None`` when every part alone succeeds. A part + that cannot even be built alone is the failing part. """ - for description, part in _parts(frame, rows): + for description, build in _parts(frame, rows): try: - attempt(part) + attempt(build()) except Exception: # noqa: BLE001 - any failure of the step locates the part return description return None @@ -830,6 +846,69 @@ def _require_labels_restored(columns: pd.Index, restored: pd.Index) -> None: ) +def _name_as_written(name: Any) -> Any: + """Return an index level name as storage writes it: its text, a tuple's parts each as text. + + ``None`` stays ``None``. + """ + if name is None: + return None + if isinstance(name, tuple): + return str(tuple(_name_as_written(part) for part in name)) # pyright: ignore[reportUnknownVariableType] + return str(name) + + +def _require_index_names_restored(index: pd.Index, restored: pd.Index) -> None: + """Raise if an index level name would read back from storage as another name. + + Storage writes a level name as text, so ``1`` reads back ``"1"``: that is no change. A + name whose text changes is: pyarrow drops a name that looks like its own generated field + name (``"__index_level_0__"``), and the level reads back unnamed. + + Args: + index: The index of the frame. + restored: The index of the frame storage restores, with as many levels. + + Raises: + ValueError: If a level's name changes; the message names the level and both names. + """ + for position, (name, back) in enumerate(zip(index.names, restored.names)): + if _name_as_written(name) != back: + raise ValueError( + f"index level {position} (name {name!r}): the name would read back from " + f"storage as {back!r}. Rename the index level (pyarrow drops a name of the " + "form '__index_level___')." + ) + + +def _require_index_values_restored(table: pa.Table, frame: pd.DataFrame) -> None: + """Raise if an index level's values cannot be converted back to pandas. + + Storage restores each index level from its Arrow column, and that conversion can fail on + the values alone, which a zero-row read never meets: a ``pd.ArrowDtype`` ``time64[ns]`` + level holding a value that is not a whole number of microseconds raises. So each index + level is converted once, in full. + + Args: + table: ``pandas_frame_to_arrow(frame)``: the columns, then one field per index level. + frame: The frame. + + Raises: + ValueError: If an index level cannot be converted; the message names it. + """ + width = frame.shape[1] + for position in range(frame.index.nlevels): + try: + table.column(width + position).to_pandas() + except Exception as exc: # noqa: BLE001 - pandas and pyarrow raise many kinds here + raise ValueError( + f"index level {position} (name {frame.index.names[position]!r}, dtype " + f"{_level_by_position(frame.index, position).dtype}): storage would write " + "this value but it cannot be read back " + f"({type(exc).__name__}: {exc})." + ) from exc + + def require_restorable(table: pa.Table, frame: pd.DataFrame) -> pd.DataFrame: """Return the zero-row frame storage restores for ``frame``, refusing what it would not. @@ -844,12 +923,16 @@ def require_restorable(table: pa.Table, frame: pd.DataFrame) -> pd.DataFrame: column or index level; a categorical, period, timedelta, interval, date or ``pd.ArrowDtype`` column axis; a categorical whose categories cannot be restored (tuples read back as arrays, which cannot be categories). + - An index level whose values cannot be converted back (a ``pd.ArrowDtype`` ``time64[ns]`` + level holding sub-microsecond values). - The frame read back would have another number of columns or index levels, distinct - column labels that read back equal, or a column label that reads back as another label - (``False`` reads back ``True``). + column labels that read back equal, a column label that reads back as another label + (``False`` reads back ``True``), or an index level name that reads back as another + name (``"__index_level_0__"`` reads back unnamed). - No rows are converted: what storage restores is read from ``table.slice(0, 0)``, which - keeps the schema and the dictionary (the categories) of each categorical column. + What storage restores is read from ``table.slice(0, 0)``, which keeps the schema and the + dictionary (the categories) of each categorical column; the only rows converted are the + index levels', once each. Args: table: ``pandas_frame_to_arrow(frame)``. @@ -901,6 +984,8 @@ def require_restorable(table: pa.Table, frame: pd.DataFrame) -> pd.DataFrame: "as text)." ) _require_labels_restored(frame.columns, restored.columns) + _require_index_names_restored(frame.index, restored.index) + _require_index_values_restored(table, frame) return restored @@ -938,16 +1023,16 @@ def _values_table(df: pd.DataFrame) -> tuple[pa.Table, _StoredForm]: the frame cannot be converted to Arrow, storage would not restore it (``require_restorable``), or a label or name is outside the grammar. """ + # Storage's refusals come first, with storage's own message, so the hash and + # ``python_to_storage`` refuse a value alike (ITL-1054); the grammar checks follow. + table = pandas_frame_to_arrow(df) + axes = _restored_axes(table, df) for label, dtype in zip(df.columns, df.dtypes): if dtype != np.dtype(object): - # A dtype outside the grammar is reported by name, before Arrow refuses it. try: pandas_dtype_descriptor(dtype) except ValueError as exc: raise ValueError(f"column {label!r}: {exc}") from exc - table = pandas_frame_to_arrow(df) - # Refuses what storage would not restore, with storage's own message (ITL-1054). - axes = _restored_axes(table, df) # The frame's own labels and names must be in the grammar, although the descriptor # holds them in the form storage restores: storage writes some as text, and the text of # an arbitrary object (its ``repr``, with a memory address) must never reach a hash. diff --git a/tests/test_hashing/test_pandas_refused_at_write.py b/tests/test_hashing/test_pandas_refused_at_write.py index 01059c034..b2a1bc18e 100644 --- a/tests/test_hashing/test_pandas_refused_at_write.py +++ b/tests/test_hashing/test_pandas_refused_at_write.py @@ -176,6 +176,50 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: r"column axis level 1: the label .*False.* would read back from storage as " r".*True", ) + cases["frame, arrow float16 column axis"] = ( + pd.DataFrame( + [[1, 2]], + columns=pd.Index(_arrow_values(pa.array([0.5, 1.5], pa.float16())).array), + ), + r"column axis.*cannot convert", + ) + cases["frame, arrow uuid (extension type) column"] = ( + _arrow_values(pa.array([b"0123456789abcdef"] * 2, pa.uuid())).to_frame("col"), + r"column 'col'.*cannot be read back", + ) + cases["frame, interval categories"] = ( + pd.DataFrame({"a": pd.Categorical(pd.interval_range(0, 2))}), + r"column 'a'.*cannot be read back", + ) + sub_microsecond = pd.Index( + pd.arrays.ArrowExtensionArray( + pa.array([1, 2000], pa.int64()).view(pa.time64("ns")) + ) + ) + cases["frame, time64[ns] index with sub-microsecond values"] = ( + pd.DataFrame({"a": [1, 2]}, index=sub_microsecond), + r"index level 0.*cannot be read back.*non-zero nanoseconds", + ) + cases["series, time64[ns] index with sub-microsecond values"] = ( + pd.Series([1, 2], name="s", index=sub_microsecond), + r"index level 0.*cannot be read back.*non-zero nanoseconds", + ) + cases["frame, index named like a generated field"] = ( + pd.DataFrame({"a": [1]}, index=pd.Index([7], name="__index_level_0__")), + r"index level 0 \(name '__index_level_0__'\): the name would read back from " + r"storage as None", + ) + cases["frame, MultiIndex level named like a generated field"] = ( + pd.DataFrame( + {"a": [1]}, + index=pd.MultiIndex.from_tuples([(7, 8)], names=["x", "__index_level_1__"]), + ), + r"index level 1 \(name '__index_level_1__'\): the name would read back", + ) + cases["series, index named like a generated field"] = ( + pd.Series([1], name="s", index=pd.Index([7], name="__index_level_0__")), + r"index level 0 \(name '__index_level_0__'\): the name would read back", + ) return cases @@ -220,6 +264,23 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: ), "int column axis": pd.DataFrame([[1, 2]], columns=pd.Index([0, 1])), "string categories": pd.DataFrame({"a": pd.Categorical(["x", "y"])}), + "time64[ns] index, whole microseconds": pd.DataFrame( + {"a": [1, 2]}, + index=pd.Index( + pd.arrays.ArrowExtensionArray( + pa.array([1000, 2000], pa.int64()).view(pa.time64("ns")) + ) + ), + ), + "int, tuple and timestamp index names": pd.DataFrame( + {"a": [1]}, + index=pd.MultiIndex.from_tuples( + [(7, 8, 9)], names=[5, ("a", 1), pd.Timestamp("2020-01-01")] + ), + ), + "index named like a field, but not generated": pd.DataFrame( + {"a": [1]}, index=pd.Index([7], name="__index_level__") + ), } @@ -232,6 +293,15 @@ def _logical( return LogicalPandasDataFrame() +def _name_text(name: Any) -> Any: + """An index level name as storage writes it: its text, a tuple's parts each as text.""" + if name is None: + return None + if isinstance(name, tuple): + return str(tuple(_name_text(part) for part in name)) + return str(name) + + def _hash(value: Any) -> Any: """The semantic hash of ``value`` under the default data context.""" return get_default_context().semantic_hasher.hash_object(value) @@ -271,8 +341,8 @@ def test_restorable_neighbours_are_accepted(name: str) -> None: logical = _logical(value) back = logical.storage_to_python(logical.python_to_storage(value)) assert list(back.index) == list(value.index) - assert [None if n is None else str(n) for n in back.index.names] == [ - None if n is None else str(n) for n in value.index.names + assert [_name_text(n) for n in back.index.names] == [ + _name_text(n) for n in value.index.names ] assert _hash(back) == _hash(value) From 3c6d8fe8278c2fa8bc50481ffba09245b3c2f7a9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 04:19:55 -0800 Subject: [PATCH 7/7] fix(pandas): refuse a MultiIndex whose levels cannot be assembled on read (ITL-1054) CodeRabbit review of #383, round 3: a MultiIndex level holding tuples or lists converts alone but reads back as arrays, and pd.MultiIndex.from_arrays cannot factorize them, so the read failed. require_restorable now assembles the converted index levels once, the way storage does, and refuses a failure with ValueError naming the level. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1054.md | 5 +- design_issues/ITL-1054-PV3.md | 11 ++-- docs/reference/hashing.md | 6 ++- src/orcapod/hashing/pandas_value_hash.py | 51 +++++++++++++------ .../test_pandas_refused_at_write.py | 20 ++++++++ 5 files changed, 69 insertions(+), 24 deletions(-) diff --git a/changelog.d/ITL-1054.md b/changelog.d/ITL-1054.md index 173116695..bf882a961 100644 --- a/changelog.d/ITL-1054.md +++ b/changelog.d/ITL-1054.md @@ -18,8 +18,9 @@ through one shared check. These values used to be written: timestamp, duration, time, bool, date, decimal or unsigned integer. This includes a Series named with a `pd.Timedelta`, `pd.Period` or `datetime.date`. A categorical column, Series or index whose categories are tuples (ITL-1115) or intervals. A `pd.ArrowDtype` `time64[ns]` index - level holding a value that is not a whole number of microseconds (ITL-1118; index levels - are now converted once, in full, at write). In a pipeline, such a result stayed in the result + level holding a value that is not a whole number of microseconds (ITL-1118), and a + MultiIndex level holding tuples or lists (ITL-1120); index levels are now converted once, in + full, at write. In a pipeline, such a result stayed in the result table and could not be loaded on any rerun. - **Written, then read back with another column label, with no error:** the label `False` (a one-column bool axis, a Series named `False`, or a `False` in a level of a MultiIndex diff --git a/design_issues/ITL-1054-PV3.md b/design_issues/ITL-1054-PV3.md index 513e43e30..3b171c965 100644 --- a/design_issues/ITL-1054-PV3.md +++ b/design_issues/ITL-1054-PV3.md @@ -13,8 +13,8 @@ The value read back was another value, nothing raised, and the hash (computed on form) gave both the same digest. Second, a nested, dictionary, fixed-size or union `pd.ArrowDtype` column or index level, a `float16` Arrow index level, and a categorical, period, timedelta, interval, date or `pd.ArrowDtype` column axis, and categories that are tuples, were -written and then failed to read back, and so was a `time64[ns]` index level holding -sub-microsecond values. A `False` column label (or Series name) read back as `True`, and an +written and then failed to read back, and so were a `time64[ns]` index level holding +sub-microsecond values and a MultiIndex level holding tuples or lists. A `False` column label (or Series name) read back as `True`, and an index level named `__index_level___` read back unnamed. In a pipeline, the computed result stayed in the result table and could not be loaded on any rerun. The storage round-trip matrix test hid the second kind: it treated a failed read like an input storage does not support. @@ -24,10 +24,11 @@ like an input storage does not support. refuses both kinds at write with a `ValueError` naming the column, index level or column-axis level. A failed read is detected from a zero-row read (`table.slice(0, 0).to_pandas()`, which keeps the categories), which agreed with the full read on every value tried, on pyarrow 22.0.0, -23.0.1 and 26.0.1, together with a full conversion of each index level (the one failure found -that depends on values). Column labels and index level names are compared as text with those +23.0.1 and 26.0.1, together with a full conversion of each index level and one assembly of a MultiIndex (the +failures found that depend on values). Column labels and index level names are compared as text with those read back. Future support: tuple categories ITL-1115, `False` labels ITL-1116, sub-microsecond -`time64[ns]` index ITL-1118, generated-looking index names ITL-1119. +`time64[ns]` index ITL-1118, generated-looking index names ITL-1119, tuple or list MultiIndex +levels ITL-1120 (which also carries the untested candidates for the v0.4 audit). `pandas_frame_to_arrow` wraps every Arrow conversion error into `ValueError`, naming the part that fails. The matrix test now lets only the first write refuse, and it checks each column and index level by position. The `struct_field_order` ignore option is removed, because it acted diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 2f407a533..912a8456d 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -1590,8 +1590,10 @@ pyarrow version: Series or index whose categories are tuples (they read back as arrays, which cannot be categories; ITL-1115) or intervals. - **Index levels whose values cannot be read back.** A `pd.ArrowDtype` `time64[ns]` index - level holding a value that is not a whole number of microseconds (ITL-1118). The failure - depends on the values, so each index level is converted once, in full, when a value is + level holding a value that is not a whole number of microseconds (ITL-1118), and a + MultiIndex level holding tuples or lists, which read back as arrays that cannot be the + values of a MultiIndex level (ITL-1120). These failures depend on the values, so each + index level is converted once, in full, and a MultiIndex is assembled once, when a value is written or hashed; columns are checked without converting any row. - **Index level names that would read back as another name.** pyarrow drops a name of the form `__index_level___` (its own name for an unnamed level), so such a level reads back diff --git a/src/orcapod/hashing/pandas_value_hash.py b/src/orcapod/hashing/pandas_value_hash.py index fb1905ac1..e2e2a1068 100644 --- a/src/orcapod/hashing/pandas_value_hash.py +++ b/src/orcapod/hashing/pandas_value_hash.py @@ -117,7 +117,7 @@ ``names=[1, 0]``), and values storage cannot read back (a nested, dictionary or fixed-size ``pd.ArrowDtype`` column or index level; a categorical, period, timedelta, interval, date or ``pd.ArrowDtype`` column axis; tuple categories; a ``time64[ns]`` index level holding -sub-microsecond values), column labels and index level names that would read back as another +sub-microsecond values; a MultiIndex level holding tuples or lists), column labels and index level names that would read back as another label or name (``False`` reads back ``True``; ``"__index_level_0__"`` reads back unnamed), and labels and names that collide once written as text. A value Arrow cannot convert is refused by ``pandas_frame_to_arrow``. Both messages name the @@ -882,31 +882,51 @@ def _require_index_names_restored(index: pd.Index, restored: pd.Index) -> None: def _require_index_values_restored(table: pa.Table, frame: pd.DataFrame) -> None: - """Raise if an index level's values cannot be converted back to pandas. + """Raise if the index cannot be rebuilt from its stored values. - Storage restores each index level from its Arrow column, and that conversion can fail on - the values alone, which a zero-row read never meets: a ``pd.ArrowDtype`` ``time64[ns]`` - level holding a value that is not a whole number of microseconds raises. So each index - level is converted once, in full. + Storage converts each index level from its Arrow column and, for a MultiIndex, assembles + the levels with ``pd.MultiIndex.from_arrays``. Both steps can fail on the values alone, + which a zero-row read never meets: a ``pd.ArrowDtype`` ``time64[ns]`` level holding a + value that is not a whole number of microseconds cannot be converted, and a level holding + tuples or lists reads back as arrays, which cannot be the values of a MultiIndex level. + So each index level is converted once, in full, and a MultiIndex is assembled once. Args: table: ``pandas_frame_to_arrow(frame)``: the columns, then one field per index level. frame: The frame. Raises: - ValueError: If an index level cannot be converted; the message names it. + ValueError: If an index level cannot be converted or assembled; the message names it. """ + + def refuse(position: int, exc: Exception) -> ValueError: + return ValueError( + f"index level {position} (name {frame.index.names[position]!r}, dtype " + f"{_level_by_position(frame.index, position).dtype}): storage would write " + f"this value but it cannot be read back ({type(exc).__name__}: {exc})." + ) + width = frame.shape[1] + levels: list[Any] = [] for position in range(frame.index.nlevels): try: - table.column(width + position).to_pandas() + levels.append(table.column(width + position).to_pandas()) except Exception as exc: # noqa: BLE001 - pandas and pyarrow raise many kinds here - raise ValueError( - f"index level {position} (name {frame.index.names[position]!r}, dtype " - f"{_level_by_position(frame.index, position).dtype}): storage would write " - "this value but it cannot be read back " - f"({type(exc).__name__}: {exc})." - ) from exc + raise refuse(position, exc) from exc + if len(levels) < 2: + return + try: + pd.MultiIndex.from_arrays(levels) + except Exception as exc: # noqa: BLE001 - pandas raises many kinds here + for position, level in enumerate(levels): + try: + pd.MultiIndex.from_arrays([level]) + except Exception as level_exc: # noqa: BLE001 - locates the level + raise refuse(position, level_exc) from exc + raise ValueError( + "the index: storage would write this value but its MultiIndex cannot be " + f"rebuilt ({type(exc).__name__}: {exc})." + ) from exc def require_restorable(table: pa.Table, frame: pd.DataFrame) -> pd.DataFrame: @@ -924,7 +944,8 @@ def require_restorable(table: pa.Table, frame: pd.DataFrame) -> pd.DataFrame: ``pd.ArrowDtype`` column axis; a categorical whose categories cannot be restored (tuples read back as arrays, which cannot be categories). - An index level whose values cannot be converted back (a ``pd.ArrowDtype`` ``time64[ns]`` - level holding sub-microsecond values). + level holding sub-microsecond values) or cannot be a MultiIndex level (tuples or lists, + which read back as arrays). - The frame read back would have another number of columns or index levels, distinct column labels that read back equal, a column label that reads back as another label (``False`` reads back ``True``), or an index level name that reads back as another diff --git a/tests/test_hashing/test_pandas_refused_at_write.py b/tests/test_hashing/test_pandas_refused_at_write.py index b2a1bc18e..900e61563 100644 --- a/tests/test_hashing/test_pandas_refused_at_write.py +++ b/tests/test_hashing/test_pandas_refused_at_write.py @@ -191,6 +191,17 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: pd.DataFrame({"a": pd.Categorical(pd.interval_range(0, 2))}), r"column 'a'.*cannot be read back", ) + tuple_level = pd.MultiIndex.from_tuples( + [((1, 2), 7), ((3, 4), 8)], names=["x", "y"] + ) + cases["frame, MultiIndex level holding tuples"] = ( + pd.DataFrame({"a": [1, 2]}, index=tuple_level), + r"index level 0 \(name 'x', dtype object\).*cannot be read back", + ) + cases["series, MultiIndex level holding tuples"] = ( + pd.Series([1, 2], name="s", index=tuple_level), + r"index level 0 \(name 'x', dtype object\).*cannot be read back", + ) sub_microsecond = pd.Index( pd.arrays.ArrowExtensionArray( pa.array([1, 2000], pa.int64()).view(pa.time64("ns")) @@ -278,6 +289,15 @@ def _refused() -> dict[str, tuple[pd.DataFrame | pd.Series, str]]: [(7, 8, 9)], names=[5, ("a", 1), pd.Timestamp("2020-01-01")] ), ), + "MultiIndex levels of bytes and Decimals": pd.DataFrame( + {"a": [1, 2]}, + index=pd.MultiIndex.from_arrays( + [ + pd.Index([b"x", b"y"]), + pd.Index([decimal.Decimal("1.5"), decimal.Decimal("2.5")]), + ] + ), + ), "index named like a field, but not generated": pd.DataFrame( {"a": [1]}, index=pd.Index([7], name="__index_level__") ),