From 11b567f57675eea6bc4a8ba24aa1acf0d44c6ba3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:55:08 -0800 Subject: [PATCH 01/61] docs(itl-897): investigation scripts for pandas hash identity and storage (WIP) Co-Authored-By: Claude Opus 5.5 --- .../itl_897/claim_a_equality_vs_hash.py | 351 ++++++++++++++++++ .../itl_897/claim_b_storage_roundtrip.py | 130 +++++++ 2 files changed, 481 insertions(+) create mode 100644 superpowers/scripts/itl_897/claim_a_equality_vs_hash.py create mode 100644 superpowers/scripts/itl_897/claim_b_storage_roundtrip.py diff --git a/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py b/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py new file mode 100644 index 000000000..fef933764 --- /dev/null +++ b/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py @@ -0,0 +1,351 @@ +"""ITL-897 claim (a): what pandas equality depends on vs what Orcapod's hash depends on. + +For each property, builds two values that differ only in that property and reports: + +* ``equals`` -- ``DataFrame.equals`` / ``Series.equals`` says the two are equal +* ``assert`` -- ``assert_frame_equal`` / ``assert_series_equal`` with default arguments passes +* ``hash==`` -- Orcapod's semantic hash (default data context) is the same for both +* ``stored`` -- the stored round trip (``LogicalPandasDataFrame`` / ``LogicalPandasSeries``, + Arrow IPC) gives back a value that keeps the property of the second value + +A row is a MISMATCH when pandas (``equals`` or the default ``assert``) says "different" but +the hash is equal (a cache collision), or pandas says "equal" on both checks but the hash +differs (a spurious cache miss). + +Run: ``uv run python superpowers/scripts/itl_897/claim_a_equality_vs_hash.py`` +""" + +from __future__ import annotations + +import warnings +from collections.abc import Callable +from typing import Any + +import numpy as np +import pandas as pd +from pandas.testing import assert_frame_equal, assert_series_equal + +from orcapod.contexts import get_default_context +from orcapod.logical_types.pandas_type import LogicalPandasDataFrame, LogicalPandasSeries + +warnings.simplefilter("ignore") +HASHER = get_default_context().semantic_hasher +LT_DF = LogicalPandasDataFrame() +LT_S = LogicalPandasSeries() + + +def _df(**kw: Any) -> pd.DataFrame: + return pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}, **kw) + + +def _with_attrs(df: Any, attrs: dict[str, Any]) -> Any: + out = df.copy() + out.attrs = attrs + return out + + +def _with_columns_name(df: pd.DataFrame, name: str) -> pd.DataFrame: + out = df.copy() + out.columns = out.columns.rename(name) + return out + + +def _no_dup_flag(df: pd.DataFrame) -> pd.DataFrame: + return df.set_flags(allows_duplicate_labels=False) + + +# (property, value_1, value_2, keeps(roundtripped_2) -> bool) +Keeps = Callable[[Any], bool] +FRAME_CASES: list[tuple[str, pd.DataFrame, pd.DataFrame, Keeps]] = [ + ( + "column order", + _df(), + _df()[["b", "a"]], + lambda r: list(r.columns) == ["b", "a"], + ), + ( + "row order", + _df(), + _df().iloc[::-1], + lambda r: list(r["a"]) == [2, 1], + ), + ( + "index values", + _df(), + _df(index=[10, 20]), + lambda r: list(r.index) == [10, 20], + ), + ( + "index name", + _df(), + _df(index=pd.RangeIndex(2, name="id")), + lambda r: r.index.name == "id", + ), + ( + "index type: RangeIndex vs Index[int64], same values", + _df(), + _df(index=pd.Index([0, 1], dtype="int64")), + lambda r: type(r.index) is pd.Index and r.index.dtype == "int64", + ), + ( + "index dtype: int64 vs float64, same values", + _df(index=pd.Index([0, 1], dtype="int64")), + _df(index=pd.Index([0.0, 1.0], dtype="float64")), + lambda r: r.index.dtype == "float64", + ), + ( + "MultiIndex level names", + _df(index=pd.MultiIndex.from_tuples([(0, "p"), (1, "q")])), + _df(index=pd.MultiIndex.from_tuples([(0, "p"), (1, "q")], names=["i", "j"])), + lambda r: list(r.index.names) == ["i", "j"], + ), + ( + "MultiIndex level order (swaplevel)", + _df(index=pd.MultiIndex.from_tuples([(0, "p"), (1, "q")], names=["i", "j"])), + _df( + index=pd.MultiIndex.from_tuples([(0, "p"), (1, "q")], names=["i", "j"]) + ).swaplevel(), + lambda r: list(r.index.names) == ["j", "i"], + ), + ( + "columns.name", + _df(), + _with_columns_name(_df(), "features"), + lambda r: r.columns.name == "features", + ), + ( + "column label type: 0 vs '0'", + pd.DataFrame({0: [1, 2]}), + pd.DataFrame({"0": [1, 2]}), + lambda r: list(r.columns) == ["0"], + ), + ( + "MultiIndex columns vs flat tuple-ish labels", + pd.DataFrame([[1, 2]], columns=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])), + pd.DataFrame([[1, 2]], columns=["('a', 'x')", "('a', 'y')"]), + lambda r: not isinstance(r.columns, pd.MultiIndex), + ), + ( + "column dtype: int64 vs float64", + pd.DataFrame({"a": [1, 2]}), + pd.DataFrame({"a": [1.0, 2.0]}), + lambda r: r["a"].dtype == "float64", + ), + ( + "column dtype: int64 vs int32", + pd.DataFrame({"a": np.array([1, 2], dtype="int64")}), + pd.DataFrame({"a": np.array([1, 2], dtype="int32")}), + lambda r: r["a"].dtype == "int32", + ), + ( + "categorical vs object string (ITL-887)", + pd.DataFrame({"a": ["x", "y"]}), + pd.DataFrame({"a": pd.Categorical(["x", "y"])}), + lambda r: isinstance(r["a"].dtype, pd.CategoricalDtype), + ), + ( + "categorical: category order", + pd.DataFrame({"a": pd.Categorical(["x", "y"], categories=["x", "y"])}), + pd.DataFrame({"a": pd.Categorical(["x", "y"], categories=["y", "x"])}), + lambda r: list(r["a"].cat.categories) == ["y", "x"], + ), + ( + "categorical: unused category", + pd.DataFrame({"a": pd.Categorical(["x", "y"], categories=["x", "y"])}), + pd.DataFrame({"a": pd.Categorical(["x", "y"], categories=["x", "y", "z"])}), + lambda r: list(r["a"].cat.categories) == ["x", "y", "z"], + ), + ( + "categorical: ordered flag", + pd.DataFrame({"a": pd.Categorical(["x", "y"], ordered=False)}), + pd.DataFrame({"a": pd.Categorical(["x", "y"], ordered=True)}), + lambda r: bool(r["a"].cat.ordered), + ), + ( + "nullable Int64 vs int64 (no nulls)", + pd.DataFrame({"a": pd.array([1, 2], dtype="int64")}), + pd.DataFrame({"a": pd.array([1, 2], dtype="Int64")}), + lambda r: str(r["a"].dtype) == "Int64", + ), + ( + "nullable boolean vs bool (no nulls)", + pd.DataFrame({"a": [True, False]}), + pd.DataFrame({"a": pd.array([True, False], dtype="boolean")}), + lambda r: str(r["a"].dtype) == "boolean", + ), + ( + "string[python] vs object str", + pd.DataFrame({"a": ["x", "y"]}), + pd.DataFrame({"a": pd.array(["x", "y"], dtype="string")}), + lambda r: str(r["a"].dtype) == "string", + ), + ( + "Int64 vs float64 NaN", + pd.DataFrame({"a": [1.0, np.nan]}), + pd.DataFrame({"a": pd.array([1, None], dtype="Int64")}), + lambda r: str(r["a"].dtype) == "Int64", + ), + ( + "datetime64 unit ns vs us", + pd.DataFrame({"t": pd.to_datetime(["2020-01-01"]).astype("datetime64[ns]")}), + pd.DataFrame({"t": pd.to_datetime(["2020-01-01"]).astype("datetime64[us]")}), + lambda r: str(r["t"].dtype) == "datetime64[us]", + ), + ( + "datetime tz: UTC vs naive (same wall clock)", + pd.DataFrame({"t": pd.to_datetime(["2020-01-01"])}), + pd.DataFrame({"t": pd.to_datetime(["2020-01-01"]).tz_localize("UTC")}), + lambda r: str(r["t"].dtype) == "datetime64[ns, UTC]", + ), + ( + "float -0.0 vs 0.0", + pd.DataFrame({"a": [0.0]}), + pd.DataFrame({"a": [-0.0]}), + lambda r: bool(np.signbit(r["a"].iloc[0])), + ), + ( + "attrs", + _df(), + _with_attrs(_df(), {"unit": "mV"}), + lambda r: r.attrs == {"unit": "mV"}, + ), + ( + "flags.allows_duplicate_labels", + _df(), + _no_dup_flag(_df()), + lambda r: r.flags.allows_duplicate_labels is False, + ), +] + +SERIES_CASES: list[tuple[str, pd.Series, pd.Series, Keeps]] = [ + ( + "Series name", + pd.Series([1, 2], name="a"), + pd.Series([1, 2], name="b"), + lambda r: r.name == "b", + ), + ( + "Series name None vs 'x'", + pd.Series([1, 2]), + pd.Series([1, 2], name="x"), + lambda r: r.name == "x", + ), + ( + "Series name 0 vs '0'", + pd.Series([1, 2], name=0), + pd.Series([1, 2], name="0"), + lambda r: r.name == "0" and isinstance(r.name, str), + ), + ( + "Series index name", + pd.Series([1, 2]), + pd.Series([1, 2], index=pd.RangeIndex(2, name="id")), + lambda r: r.index.name == "id", + ), + ( + "Series dtype int64 vs Int64", + pd.Series([1, 2], dtype="int64"), + pd.Series([1, 2], dtype="Int64"), + lambda r: str(r.dtype) == "Int64", + ), + ( + "Series categorical vs object", + pd.Series(["x", "y"]), + pd.Series(pd.Categorical(["x", "y"])), + lambda r: isinstance(r.dtype, pd.CategoricalDtype), + ), + ( + "Series attrs", + pd.Series([1, 2]), + _with_attrs(pd.Series([1, 2]), {"u": 1}), + lambda r: r.attrs == {"u": 1}, + ), +] + + +def _passes(check: Callable[[Any, Any], None], a: Any, b: Any) -> bool: + try: + check(a, b) + except AssertionError: + return False + return True + + +def _hash(value: Any) -> str: + try: + return HASHER.hash_object(value).to_string() + except Exception as exc: # noqa: BLE001 - report, do not stop the survey + return f"ERROR {type(exc).__name__}: {exc}" + + +def _roundtrip_keeps(lt: Any, value: Any, keeps: Keeps) -> str: + try: + back = lt.storage_to_python(lt.python_to_storage(value)) + except Exception as exc: # noqa: BLE001 + return f"ERROR {type(exc).__name__}" + return "yes" if keeps(back) else "NO" + + +def _report( + kind: str, + cases: list[tuple[str, Any, Any, Keeps]], + assert_fn: Callable[[Any, Any], None], + lt: Any, +) -> None: + print(f"\n## {kind}\n") + print("| property | equals | assert (defaults) | hash equal | stored keeps it | verdict |") + print("|---|---|---|---|---|---|") + for name, v1, v2, keeps in cases: + eq = bool(v1.equals(v2)) + asserted = _passes(assert_fn, v1, v2) + h1, h2 = _hash(v1), _hash(v2) + if h1.startswith("ERROR") or h2.startswith("ERROR"): + hash_eq = h1 if h1.startswith("ERROR") else h2 + verdict = "hash error" + else: + same = h1 == h2 + hash_eq = "yes" if same else "no" + pandas_differs = (not eq) or (not asserted) + pandas_same = eq and asserted + if pandas_differs and same: + verdict = "MISMATCH: collision" + elif pandas_same and not same: + verdict = "MISMATCH: spurious miss" + elif eq != asserted and not same: + verdict = "pandas split; hash distinguishes" + else: + verdict = "consistent" + stored = _roundtrip_keeps(lt, v2, keeps) + print( + f"| {name} | {'equal' if eq else 'differ'} | " + f"{'equal' if asserted else 'differ'} | {hash_eq} | {stored} | {verdict} |" + ) + + +EDGE_FRAMES: list[tuple[str, pd.DataFrame]] = [ + ("duplicate column labels ['a', 'a']", pd.DataFrame([[1, 2]], columns=["a", "a"])), + ("labels 1 and '1' in one frame", pd.DataFrame([[1, 2]], columns=[1, "1"])), + ("int label 0", pd.DataFrame({0: [1, 2]})), + ("zero columns, two rows", pd.DataFrame(index=[0, 1])), +] + + +def _edges() -> None: + print("\n## Edge values: hashable? stored round trip?\n") + print("| value | hash | stored round trip columns | stored equals original |") + print("|---|---|---|---|") + for name, df in EDGE_FRAMES: + h = _hash(df) + try: + back = LT_DF.storage_to_python(LT_DF.python_to_storage(df)) + cols, same = repr(list(back.columns)), str(bool(df.equals(back))) + except Exception as exc: # noqa: BLE001 + cols, same = f"ERROR {type(exc).__name__}", "-" + print(f"| {name} | {h if h.startswith('ERROR') else 'ok'} | {cols} | {same} |") + + +if __name__ == "__main__": + print(f"pandas {pd.__version__}") + _report("DataFrame", FRAME_CASES, assert_frame_equal, LT_DF) + _report("Series", SERIES_CASES, assert_series_equal, LT_S) + _edges() diff --git a/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py b/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py new file mode 100644 index 000000000..f7e1ea8d3 --- /dev/null +++ b/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py @@ -0,0 +1,130 @@ +"""ITL-897 claim (b): stored pandas frames keep column order; the cache key does not. + +Through the public interface, for the in-memory and Delta stores and the sync and async +orchestrators: + +1. ``make`` returns ``df`` with columns ``a, b`` (row ``fwd``) or ``b, a`` (row ``rev``). + Reading the node back gives each frame with its own column order (storage round trip). +2. ``first`` returns ``df.columns[0]`` from the stored frame. A correct cache gives + ``{"fwd": "a", "rev": "b"}``; on ``main`` one row reuses the other's cached result. + +Also prints what the Arrow IPC storage envelope records for a reordered frame. + +Run: ``uv run python superpowers/scripts/itl_897/claim_b_storage_roundtrip.py`` +""" + +from __future__ import annotations + +import json +import tempfile +import warnings +from pathlib import Path + +import pandas as pd +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.logical_types.pandas_type import LogicalPandasDataFrame +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.sources import DictSource + +warnings.simplefilter("ignore") + + +def _frame(reverse: bool) -> pd.DataFrame: + df = pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}) + return df[["b", "a"]] if reverse else df + + +@function_pod(output_keys="df") +def make(reverse: bool) -> pd.DataFrame: + return _frame(reverse) + + +@function_pod(output_keys="first") +def first(df: pd.DataFrame) -> str: + return str(df.columns[0]) + + +def _run(store_name: str, orchestrator_name: str, tmp: Path) -> None: + store = ( + InMemoryArrowDatabase() + if store_name == "in_memory" + else DeltaTableDatabase(tmp / f"delta_{orchestrator_name}") + ) + orchestrator = ( + SyncPipelineOrchestrator() + if orchestrator_name == "sync" + else AsyncPipelineOrchestrator() + ) + source = DictSource( + [{"k": "fwd", "reverse": False}, {"k": "rev", "reverse": True}], + tag_columns=["k"], + ) + job = PipelineJob(name="order", store=store) + with job: + frames = make.pod(source, label="make") + first.pod(frames, label="first") + job.run(orchestrator=orchestrator) + + stored = {t["k"]: list(d["df"].columns) for t, d in job.nodes["make"].iter_data()} + firsts = {t["k"]: d["first"] for t, d in job.nodes["first"].iter_data()} + order_kept = stored == {"fwd": ["a", "b"], "rev": ["b", "a"]} + cache_ok = firsts == {"fwd": "a", "rev": "b"} + print( + f"| {store_name} | {orchestrator_name} | {stored} | " + f"{'yes' if order_kept else 'NO'} | {firsts} | {'yes' if cache_ok else 'NO'} |" + ) + + +def _run_sequential(store_name: str, orchestrator_name: str, tmp: Path) -> None: + """Run ``fwd`` alone, then ``rev`` alone, on one store: a warm cache, no race.""" + store = ( + InMemoryArrowDatabase() + if store_name == "in_memory" + else DeltaTableDatabase(tmp / f"seq_{orchestrator_name}") + ) + got: dict[str, str] = {} + for key, reverse in (("fwd", False), ("rev", True)): + orchestrator = ( + SyncPipelineOrchestrator() + if orchestrator_name == "sync" + else AsyncPipelineOrchestrator() + ) + source = DictSource([{"k": key, "reverse": reverse}], tag_columns=["k"]) + job = PipelineJob(name="order_seq", store=store) + with job: + frames = make.pod(source, label="make") + first.pod(frames, label="first") + job.run(orchestrator=orchestrator) + got.update({t["k"]: d["first"] for t, d in job.nodes["first"].iter_data()}) + ok = got.get("rev") == "b" + print(f"| {store_name} | {orchestrator_name} | {got} | {'yes' if ok else 'NO'} |") + + +def _envelope() -> None: + lt = LogicalPandasDataFrame() + table = pa.ipc.open_stream(lt.python_to_storage(_frame(True))).read_all() + meta = json.loads(table.schema.metadata[b"pandas"]) + print("\nIPC envelope of the reversed frame:") + print(f" Arrow field order : {table.schema.names}") + print(f" pandas metadata : columns={[c['name'] for c in meta['columns']]}") + print(f" index_columns : {meta['index_columns']}") + + +if __name__ == "__main__": + print("| store | orchestrator | stored column order | order kept | first column | cache correct |") + print("|---|---|---|---|---|---|") + with tempfile.TemporaryDirectory() as tmp: + for store_name in ("in_memory", "delta"): + for orchestrator_name in ("sync", "async"): + _run(store_name, orchestrator_name, Path(tmp)) + print("\nSequential runs on one store (fwd, then rev):\n") + print("| store | orchestrator | first column seen across runs | rev correct |") + print("|---|---|---|---|") + for store_name in ("in_memory", "delta"): + for orchestrator_name in ("sync", "async"): + _run_sequential(store_name, orchestrator_name, Path(tmp)) + _envelope() From ac99a09c66f1088f7abba61f48ac6a01201f878b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:59:01 -0800 Subject: [PATCH 02/61] docs(itl-897): migration, golden-diff and storage scripts with the prototype hash (WIP) Co-Authored-By: Claude Opus 5.5 --- .../itl_897/claim_a_equality_vs_hash.py | 47 ++++- .../itl_897/claim_b_storage_roundtrip.py | 38 +++- .../scripts/itl_897/claim_d_golden_diff.py | 63 +++++++ .../scripts/itl_897/claim_d_migration.py | 162 ++++++++++++++++++ superpowers/scripts/itl_897/prototype_v1.py | 52 ++++++ 5 files changed, 358 insertions(+), 4 deletions(-) create mode 100644 superpowers/scripts/itl_897/claim_d_golden_diff.py create mode 100644 superpowers/scripts/itl_897/claim_d_migration.py create mode 100644 superpowers/scripts/itl_897/prototype_v1.py diff --git a/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py b/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py index fef933764..90ec2b684 100644 --- a/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py +++ b/superpowers/scripts/itl_897/claim_a_equality_vs_hash.py @@ -23,10 +23,14 @@ import numpy as np import pandas as pd +import pyarrow as pa from pandas.testing import assert_frame_equal, assert_series_equal from orcapod.contexts import get_default_context -from orcapod.logical_types.pandas_type import LogicalPandasDataFrame, LogicalPandasSeries +from orcapod.logical_types.pandas_type import ( + LogicalPandasDataFrame, + LogicalPandasSeries, +) warnings.simplefilter("ignore") HASHER = get_default_context().semantic_hasher @@ -50,6 +54,13 @@ def _with_columns_name(df: pd.DataFrame, name: str) -> pd.DataFrame: return out +def _arrow_struct_frame(order: list[str]) -> pd.DataFrame: + types = {"u": pa.int64(), "v": pa.large_string()} + struct = pa.struct([(name, types[name]) for name in order]) + array = pa.array([{"u": 1, "v": "x"}], struct) + return pd.DataFrame({"s": pd.Series(array, dtype=pd.ArrowDtype(struct))}) + + def _no_dup_flag(df: pd.DataFrame) -> pd.DataFrame: return df.set_flags(allows_duplicate_labels=False) @@ -121,7 +132,9 @@ def _no_dup_flag(df: pd.DataFrame) -> pd.DataFrame: ), ( "MultiIndex columns vs flat tuple-ish labels", - pd.DataFrame([[1, 2]], columns=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")])), + pd.DataFrame( + [[1, 2]], columns=pd.MultiIndex.from_tuples([("a", "x"), ("a", "y")]) + ), pd.DataFrame([[1, 2]], columns=["('a', 'x')", "('a', 'y')"]), lambda r: not isinstance(r.columns, pd.MultiIndex), ), @@ -203,6 +216,18 @@ def _no_dup_flag(df: pd.DataFrame) -> pd.DataFrame: pd.DataFrame({"a": [-0.0]}), lambda r: bool(np.signbit(r["a"].iloc[0])), ), + ( + "struct field order (ArrowDtype struct column)", + _arrow_struct_frame(["u", "v"]), + _arrow_struct_frame(["v", "u"]), + lambda r: [f.name for f in r["s"].dtype.pyarrow_dtype] == ["v", "u"], + ), + ( + "dict key order (object column of dicts)", + pd.DataFrame({"s": [{"u": 1, "v": "x"}]}), + pd.DataFrame({"s": [{"v": "x", "u": 1}]}), + lambda r: list(r["s"].iloc[0]) == ["v", "u"], + ), ( "attrs", _df(), @@ -236,6 +261,20 @@ def _no_dup_flag(df: pd.DataFrame) -> pd.DataFrame: pd.Series([1, 2], name="0"), lambda r: r.name == "0" and isinstance(r.name, str), ), + ( + "Series MultiIndex level order (swaplevel)", + pd.Series( + [1, 2], + index=pd.MultiIndex.from_tuples([(0, "p"), (1, "q")], names=["i", "j"]), + name="v", + ), + pd.Series( + [1, 2], + index=pd.MultiIndex.from_tuples([(0, "p"), (1, "q")], names=["i", "j"]), + name="v", + ).swaplevel(), + lambda r: list(r.index.names) == ["j", "i"], + ), ( "Series index name", pd.Series([1, 2]), @@ -293,7 +332,9 @@ def _report( lt: Any, ) -> None: print(f"\n## {kind}\n") - print("| property | equals | assert (defaults) | hash equal | stored keeps it | verdict |") + print( + "| property | equals | assert (defaults) | hash equal | stored keeps it | verdict |" + ) print("|---|---|---|---|---|---|") for name, v1, v2, keeps in cases: eq = bool(v1.equals(v2)) diff --git a/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py b/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py index f7e1ea8d3..e2e328ddf 100644 --- a/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py +++ b/superpowers/scripts/itl_897/claim_b_storage_roundtrip.py @@ -114,8 +114,43 @@ def _envelope() -> None: print(f" index_columns : {meta['index_columns']}") +def _prototype_hash_survives_storage() -> None: + """Under the prototype hash, a value and its stored round trip hash the same.""" + import sys + + sys.path.insert(0, str(Path(__file__).parent)) + import prototype_v1 + from claim_a_equality_vs_hash import FRAME_CASES, SERIES_CASES + + from orcapod.contexts import get_default_context + from orcapod.logical_types.pandas_type import LogicalPandasSeries + + prototype_v1.install() + hasher = get_default_context().semantic_hasher + lt_df, lt_s = LogicalPandasDataFrame(), LogicalPandasSeries() + bad: list[str] = [] + unstorable: list[str] = [] + values = [(n, v, lt_df) for n, a, b, _ in FRAME_CASES for v in (a, b)] + values += [(n, v, lt_s) for n, a, b, _ in SERIES_CASES for v in (a, b)] + for name, value, lt in values: + try: + back = lt.storage_to_python(lt.python_to_storage(value)) + except Exception as exc: # noqa: BLE001 - a storage failure is reported, not a hash + unstorable.append(f"{name}: {type(exc).__name__}") + continue + if hasher.hash_object(value) != hasher.hash_object(back): + bad.append(name) + print( + f"\nPrototype hash equal before and after storage: {len(values) - len(unstorable) - len(bad)}" + f" of {len(values) - len(unstorable)} storable values; differ: {bad or 'none'};" + f" not storable on main: {unstorable or 'none'}" + ) + + if __name__ == "__main__": - print("| store | orchestrator | stored column order | order kept | first column | cache correct |") + print( + "| store | orchestrator | stored column order | order kept | first column | cache correct |" + ) print("|---|---|---|---|---|---|") with tempfile.TemporaryDirectory() as tmp: for store_name in ("in_memory", "delta"): @@ -128,3 +163,4 @@ def _envelope() -> None: for orchestrator_name in ("sync", "async"): _run_sequential(store_name, orchestrator_name, Path(tmp)) _envelope() + _prototype_hash_survives_storage() diff --git a/superpowers/scripts/itl_897/claim_d_golden_diff.py b/superpowers/scripts/itl_897/claim_d_golden_diff.py new file mode 100644 index 000000000..4cc1d1381 --- /dev/null +++ b/superpowers/scripts/itl_897/claim_d_golden_diff.py @@ -0,0 +1,63 @@ +"""ITL-897 claim (d): which golden entries the proposed pandas hash moves. + +Runs the golden generator twice into scratch directories, once as released and once with the +prototype (``prototype_v1.install()``) patched in, each in its own process, and lists every key +whose value differs. ``tests/test_hashing/golden/data`` is never written. + +Run: ``uv run python superpowers/scripts/itl_897/claim_d_golden_diff.py`` +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +HERE = Path(__file__).resolve().parent +GENERATOR = ROOT / "tests/test_hashing/golden/generate_golden_hashes.py" + +RUNNER = """ +import runpy, sys +sys.path.insert(0, {here!r}) +if {patch!r}: + import prototype_v1 + prototype_v1.install() +sys.argv = [{gen!r}, "--output-dir", {out!r}] +runpy.run_path({gen!r}, run_name="__main__") +""" + + +def _generate(out: Path, patch: bool) -> dict[str, dict[str, object]]: + code = RUNNER.format(here=str(HERE), patch=patch, gen=str(GENERATOR), out=str(out)) + subprocess.run( + [sys.executable, "-c", code], check=True, cwd=ROOT, capture_output=True + ) + return {p.stem: json.loads(p.read_text()) for p in sorted(out.glob("*.json"))} + + +def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + released = _generate(Path(tmp) / "released", patch=False) + proposed = _generate(Path(tmp) / "proposed", patch=True) + pinned = { + p.stem: json.loads(p.read_text()) + for p in sorted((ROOT / "tests/test_hashing/golden/data").glob("*.json")) + } + assert released == pinned, "released run does not reproduce the pinned golden files" + total = sum(len(v) for v in pinned.values()) + moved = [ + (group, key) + for group, entries in pinned.items() + for key, value in entries.items() + if proposed[group].get(key) != value + ] + print(f"{len(moved)} of {total} pinned golden entries move:") + for group, key in moved: + print(f" {group}.json {key}") + + +if __name__ == "__main__": + main() diff --git a/superpowers/scripts/itl_897/claim_d_migration.py b/superpowers/scripts/itl_897/claim_d_migration.py new file mode 100644 index 000000000..16afa4dac --- /dev/null +++ b/superpowers/scripts/itl_897/claim_d_migration.py @@ -0,0 +1,162 @@ +"""ITL-897 claim (d): what a v0.2.0 store sees after the pandas hash changes. + +Two separate processes share one Delta store: + +1. ``old``: the released (v0.2.0 / ``main``) pandas handlers. +2. ``new``: the prototype hash in ``prototype_v1.py``, patched into the handlers in that + process only. + +Pipeline: ``make(reverse) -> df`` (pandas output; its input is a bool, so its cache key does +not involve a pandas hash), then ``first(df) -> df.columns[0]`` (pandas input, so its cache +key does). A third node, ``height(df)``, does not depend on column order. + +The script prints, per run, which pod bodies ran, what each node yields, and the raw row +counts in each node's tables, so we can see what recomputes, where old rows stay, and whether +anything looks dropped. + +Run: ``uv run python superpowers/scripts/itl_897/claim_d_migration.py`` +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +CHILD = textwrap.dedent( + r""" + import json, sys, warnings + from pathlib import Path + warnings.simplefilter("ignore") + import pandas as pd + + MODE, STORE, LOG = sys.argv[1], Path(sys.argv[2]), Path(sys.argv[3]) + EPHEMERAL = sys.argv[4] == "ephemeral" + + if MODE == "new": + sys.path.insert(0, sys.argv[5]) + import prototype_v1 + + prototype_v1.install() + + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + from orcapod.sources import DictSource + + def _log(event): + with LOG.open("a") as fh: + fh.write(json.dumps([MODE, event]) + "\n") + + @function_pod(output_keys="df") + def make(reverse: bool) -> pd.DataFrame: + _log("make") + df = pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}) + return df[["b", "a"]] if reverse else df + + @function_pod(output_keys="first") + def first(df: pd.DataFrame) -> str: + _log("first") + return str(df.columns[0]) + + @function_pod(output_keys="height") + def height(df: pd.DataFrame) -> int: + _log("height") + return len(df) + + source = DictSource( + [{"k": "fwd", "reverse": False}, {"k": "rev", "reverse": True}], + tag_columns=["k"], source_id="frames", + ) + store = DeltaTableDatabase(STORE) + job = PipelineJob(name="migration", store=store) + with job: + frames = make.pod(source, label="make") + first.pod(frames, label="first") + height.pod(frames, label="height") + if EPHEMERAL: + # make's results live only in this process's ephemeral store; a later process + # sees make's pipeline records only, with their stored output data hash. + from orcapod.databases import InMemoryArrowDatabase + from orcapod.types import NodeConfig + + job.nodes["make"].node_config = NodeConfig(is_result_ephemeral=True) + job.set_ephemeral_store(InMemoryArrowDatabase()) + job.run(orchestrator=SyncPipelineOrchestrator()) + + out = {} + for label in ("make", "first", "height"): + node = job.nodes[label] + rows = [] + for tag, data in node.iter_data(): + value = data[{"make": "df", "first": "first", "height": "height"}[label]] + rows.append([tag["k"], list(value.columns) if label == "make" else value]) + out[label] = sorted(rows, key=str) + print(json.dumps(out)) + """ +) + + +def _count_rows(store: Path) -> dict[str, int]: + """Row count of every Delta table under ``store``, as ``/``.""" + import deltalake + + counts: dict[str, int] = {} + for log_dir in sorted(store.rglob("_delta_log")): + table_dir = log_dir.parent + parts = table_dir.relative_to(store).parts # (job, [_result,] node, ...) + label = ( + f"{parts[2]}/result" if parts[1] == "_result" else f"{parts[1]}/pipeline" + ) + counts[label] = deltalake.DeltaTable(str(table_dir)).to_pyarrow_table().num_rows + return counts + + +def main(variant: str) -> None: + print(f"\n## variant: {variant} make results") + with tempfile.TemporaryDirectory() as tmp: + store, log, script = ( + Path(tmp) / "store", + Path(tmp) / "log", + Path(tmp) / "child.py", + ) + script.write_text(CHILD) + env = {**os.environ, "RUST_LOG": "error"} + for mode in ("old", "old", "new", "new"): + log.write_text("") if not log.exists() else None + before = log.read_text().count("\n") + proc = subprocess.run( + [ + sys.executable, + str(script), + mode, + str(store), + str(log), + variant, + str(Path(__file__).parent), + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + if proc.returncode != 0: + print(proc.stderr[-3000:]) + raise SystemExit(f"{mode} run failed") + ran = [ + json.loads(line)[1] for line in log.read_text().splitlines()[before:] + ] + print(f"\n### run with {mode} hash") + print(f"pod bodies run: {sorted(ran) or 'none'}") + for label, rows in json.loads(proc.stdout.strip().splitlines()[-1]).items(): + print(f" {label:6} yields {len(rows)} rows: {rows}") + print(f" Delta row counts: {_count_rows(store)}") + + +if __name__ == "__main__": + main("persistent") + main("ephemeral") diff --git a/superpowers/scripts/itl_897/prototype_v1.py b/superpowers/scripts/itl_897/prototype_v1.py new file mode 100644 index 000000000..6bbb6cb96 --- /dev/null +++ b/superpowers/scripts/itl_897/prototype_v1.py @@ -0,0 +1,52 @@ +"""Prototype of the proposed pandas hash, for the ITL-897 investigation scripts only. + +``install()`` patches ``PandasDataFrameHandler.handle`` / ``PandasSeriesHandler.handle`` in +the running process (never in the library) so that:: + + pandas_dataframe_v1 / pandas_series_v1 = + sha256(descriptor + b"\\x00" + values_digest) + +where ``descriptor`` is canonical JSON ``{"fields": [, ...], "kind": ...}`` +listing the fields of ``pa.Table.from_pandas(value, preserve_index=True)`` in order (data +columns, then index levels), and ``values_digest`` is today's Arrow hasher digest of that same +table. This is the minimal variant: it adds order only. It is not the final grammar. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +import pyarrow as pa + + +def _labelled(label: str, table: pa.Table, values: bytes) -> Any: + from orcapod.types import ContentHash + + descriptor = json.dumps( + {"kind": label, "fields": table.schema.names}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode() + return ContentHash(label, hashlib.sha256(descriptor + b"\x00" + values).digest()) + + +def install() -> None: + """Patch the pandas handlers in this process with the prototype hash.""" + from orcapod.hashing.semantic_hashing import builtin_handlers as bh + + def df_handle(self: Any, obj: Any, hasher: Any) -> Any: + table = pa.Table.from_pandas(obj, preserve_index=True) + values = self._get_arrow_hasher().hash_table(table).digest + return _labelled("pandas_dataframe_v1", table, values) + + def series_handle(self: Any, obj: Any, hasher: Any) -> Any: + name = obj.name if obj.name is not None else self._UNNAMED_SENTINEL + table = pa.Table.from_pandas(obj.to_frame(name=name), preserve_index=True) + values = self._get_arrow_hasher().hash_table(table).digest + return _labelled("pandas_series_v1", table, values) + + bh.PandasDataFrameHandler.handle = df_handle # type: ignore[method-assign] + bh.PandasSeriesHandler.handle = series_handle # type: ignore[method-assign] From f6ebaeca4ba0db77f03e38c1a5c3fe721b156792 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 02:03:32 -0800 Subject: [PATCH 03/61] docs(itl-897): draft investigation spec and progress notes (WIP) Co-Authored-By: Claude Opus 5.5 --- ...2026-11-11-itl-897-pandas-hash-identity.md | 20 ++ .../scripts/itl_897/claim_d_golden_diff.py | 55 ++- .../itl_897/claim_d_source_identity.py | 73 ++++ .../itl_897/claim_f_table_value_paths.py | 67 ++++ .../itl_897/claim_g_arrow_table_value.py | 89 +++++ superpowers/scripts/itl_897/prototype_v1.py | 39 ++ ...2026-11-11-itl-897-pandas-hash-identity.md | 337 ++++++++++++++++++ 7 files changed, 664 insertions(+), 16 deletions(-) create mode 100644 superpowers/plans/2026-11-11-itl-897-pandas-hash-identity.md create mode 100644 superpowers/scripts/itl_897/claim_d_source_identity.py create mode 100644 superpowers/scripts/itl_897/claim_f_table_value_paths.py create mode 100644 superpowers/scripts/itl_897/claim_g_arrow_table_value.py create mode 100644 superpowers/specs/2026-11-11-itl-897-pandas-hash-identity.md diff --git a/superpowers/plans/2026-11-11-itl-897-pandas-hash-identity.md b/superpowers/plans/2026-11-11-itl-897-pandas-hash-identity.md new file mode 100644 index 000000000..1649c97cf --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-897-pandas-hash-identity.md @@ -0,0 +1,20 @@ +# ITL-897 plan and progress notes + +Branch: `eywalker/itl-897-pandas-dataframe-hash-ignores-column-order` (from `origin/main` `b197544e`). +Coordinator task `task_25c74f4198f0`, dispatch `ctx_4af6270358e5`. + +## Step 1: investigation and proposal (no library code) + +- [x] Scripts in `superpowers/scripts/itl_897/`, one per claim; prototypes in `prototype_v1.py` + (patched into the script's process only). +- [x] Spec: `superpowers/specs/2026-11-11-itl-897-pandas-hash-identity.md`, sections (a) to (h). +- [x] Scope addition from the coordinator (Nov 11): `pa.Table` / `RecordBatch` values, claims (f), (g), (h). +- [ ] Send the proposal with `orca orchestration ask`; wait for decisions 1 to 4 in the spec. + +## Step 2: implementation (only after approval) + +To be written from the approved decisions with `sensei:writing-plans`. Expected outline: +failing tests first (reordered columns give a different hash; a cached pod returning +`df.columns[0]` recomputes, in-memory and Delta, sync and async; round trip keeps order), shared +labelled-hash helper, handlers, docs §1c, CHANGELOG `[0.3.0]`, DESIGN_ISSUES entry, golden +changes limited to the approved pandas entries plus additions. diff --git a/superpowers/scripts/itl_897/claim_d_golden_diff.py b/superpowers/scripts/itl_897/claim_d_golden_diff.py index 4cc1d1381..199c0d823 100644 --- a/superpowers/scripts/itl_897/claim_d_golden_diff.py +++ b/superpowers/scripts/itl_897/claim_d_golden_diff.py @@ -1,8 +1,8 @@ """ITL-897 claim (d): which golden entries the proposed pandas hash moves. Runs the golden generator twice into scratch directories, once as released and once with the -prototype (``prototype_v1.install()``) patched in, each in its own process, and lists every key -whose value differs. ``tests/test_hashing/golden/data`` is never written. +prototypes in ``prototype_v1.py`` patched in, each in its own process, and lists every key +whose value differs, for each variant in ``VARIANTS``. ``tests/test_hashing/golden/data`` is never written. Run: ``uv run python superpowers/scripts/itl_897/claim_d_golden_diff.py`` """ @@ -19,18 +19,29 @@ HERE = Path(__file__).resolve().parent GENERATOR = ROOT / "tests/test_hashing/golden/generate_golden_hashes.py" +VARIANTS = { + "pandas": "pandas_dataframe_v1 / pandas_series_v1 prototype only", + "table_naive": "arrow_table_v1 in ArrowTableHandler, datagrams and streams unchanged", + "table_separated": "arrow_table_v1 for values; datagram and stream identities separated", + "both_separated": "pandas prototype plus separated arrow_table_v1", +} + RUNNER = """ import runpy, sys sys.path.insert(0, {here!r}) -if {patch!r}: - import prototype_v1 +import prototype_v1 +if {patch!r} in ("pandas", "both_separated"): prototype_v1.install() +if {patch!r} == "table_naive": + prototype_v1.install_arrow_table(separated=False) +if {patch!r} in ("table_separated", "both_separated"): + prototype_v1.install_arrow_table(separated=True) sys.argv = [{gen!r}, "--output-dir", {out!r}] runpy.run_path({gen!r}, run_name="__main__") """ -def _generate(out: Path, patch: bool) -> dict[str, dict[str, object]]: +def _generate(out: Path, patch: str) -> dict[str, dict[str, object]]: code = RUNNER.format(here=str(HERE), patch=patch, gen=str(GENERATOR), out=str(out)) subprocess.run( [sys.executable, "-c", code], check=True, cwd=ROOT, capture_output=True @@ -40,23 +51,35 @@ def _generate(out: Path, patch: bool) -> dict[str, dict[str, object]]: def main() -> None: with tempfile.TemporaryDirectory() as tmp: - released = _generate(Path(tmp) / "released", patch=False) - proposed = _generate(Path(tmp) / "proposed", patch=True) + released = _generate(Path(tmp) / "released", patch="none") + proposed = { + variant: _generate(Path(tmp) / variant, patch=variant) + for variant in VARIANTS + } pinned = { p.stem: json.loads(p.read_text()) for p in sorted((ROOT / "tests/test_hashing/golden/data").glob("*.json")) } assert released == pinned, "released run does not reproduce the pinned golden files" total = sum(len(v) for v in pinned.values()) - moved = [ - (group, key) - for group, entries in pinned.items() - for key, value in entries.items() - if proposed[group].get(key) != value - ] - print(f"{len(moved)} of {total} pinned golden entries move:") - for group, key in moved: - print(f" {group}.json {key}") + for variant, description in VARIANTS.items(): + moved = [ + (group, key) + for group, entries in pinned.items() + for key, value in entries.items() + if proposed[variant][group].get(key) != value + ] + by_group: dict[str, int] = {} + for group, _ in moved: + by_group[group] = by_group.get(group, 0) + 1 + print(f"\n## {variant}: {description}") + print( + f"{len(moved)} of {total} pinned golden entries move; by file: {by_group}" + ) + for group, key in moved[:12]: + print(f" {group}.json {key}") + if len(moved) > 12: + print(f" ... and {len(moved) - 12} more") if __name__ == "__main__": diff --git a/superpowers/scripts/itl_897/claim_d_source_identity.py b/superpowers/scripts/itl_897/claim_d_source_identity.py new file mode 100644 index 000000000..17a215651 --- /dev/null +++ b/superpowers/scripts/itl_897/claim_d_source_identity.py @@ -0,0 +1,73 @@ +"""ITL-897 claim (d): row-level identities that embed a pandas value hash. + +Builds the same ``DictSource`` (one pandas column) before and after installing the prototype +pandas hash in this process, and reports which identities change: + +* the default ``source_id`` (the table hash, used when no ``source_id`` is given), which is + the value of the ``_tag_source_id`` system tag and an input to every record ID; +* record IDs (``_tag_record_id``), with and without an explicit ``source_id``; +* the per-row data hash; +* the source's ``content_hash`` and ``pipeline_hash``. + +Run: ``uv run python superpowers/scripts/itl_897/claim_d_source_identity.py`` +""" + +from __future__ import annotations + +import sys +import warnings +from pathlib import Path +from typing import Any + +import pandas as pd + +from orcapod.sources import DictSource + +warnings.simplefilter("ignore") +sys.path.insert(0, str(Path(__file__).parent)) +import prototype_v1 # noqa: E402 + + +def _rows() -> list[dict[str, Any]]: + return [ + {"k": "r1", "n": 1, "df": pd.DataFrame({"a": [1, 2], "b": ["x", "y"]})}, + {"k": "r2", "n": 2, "df": pd.DataFrame({"a": [3]})}, + ] + + +def _summary(source_id: str | None) -> dict[str, Any]: + source = DictSource(_rows(), tag_columns=["k"], source_id=source_id) + table = source.as_table(all_info=True) + system = { + c: table.column(c).to_pylist() + for c in table.column_names + if c.startswith("_tag") + } + data_hashes = [d.content_hash().to_string() for _, d in source.iter_data()] + return { + "system tags": system, + "row data hashes": data_hashes, + "content_hash": source.content_hash().to_string(), + "pipeline_hash": source.pipeline_hash().to_string(), + } + + +def main() -> None: + before = {sid: _summary(sid) for sid in (None, "explicit")} + prototype_v1.install() + after = {sid: _summary(sid) for sid in (None, "explicit")} + for sid in (None, "explicit"): + print(f"\n## source_id={sid!r}") + for key in before[sid]: + same = before[sid][key] == after[sid][key] + print(f" {key:16} {'unchanged' if same else 'CHANGES'}") + if key == "system tags" and not same: + for col, values in before[sid][key].items(): + if values != after[sid][key].get(col): + print( + f" {col.split('::')[0]}: {values} -> {after[sid][key].get(col)}" + ) + + +if __name__ == "__main__": + main() diff --git a/superpowers/scripts/itl_897/claim_f_table_value_paths.py b/superpowers/scripts/itl_897/claim_f_table_value_paths.py new file mode 100644 index 000000000..69d859269 --- /dev/null +++ b/superpowers/scripts/itl_897/claim_f_table_value_paths.py @@ -0,0 +1,67 @@ +"""ITL-897 claim (f): every runtime path by which a ``pa.Table`` / ``pa.RecordBatch`` reaches +the type-dispatched ``ArrowTableHandler``. + +Wraps ``ArrowTableHandler.handle`` (in this process only) to record who handed the table to +the semantic hasher: the nearest frame outside the hashing package, plus the +``identity_structure`` owner when there is one. Then runs the given pytest selection (default: +the whole ``tests/`` tree) and prints the distinct call sites with counts. + +Run: ``uv run python superpowers/scripts/itl_897/claim_f_table_value_paths.py [pytest args]`` +""" + +from __future__ import annotations + +import collections +import sys +from typing import Any + +import pytest + +from orcapod.hashing.semantic_hashing import builtin_handlers as bh + +SITES: collections.Counter[str] = collections.Counter() +_original = bh.ArrowTableHandler.handle + + +def _short(filename: str) -> str: + for marker in ("/src/", "/tests/"): + if marker in filename: + return marker.strip("/") + "/" + filename.split(marker, 1)[1] + return filename + + +def _site() -> str: + """Name the object whose identity carried the table, or the direct caller.""" + frame = sys._getframe(2) # noqa: SLF001 - diagnostic script + caller = None + while frame is not None: + code = frame.f_code + owner = frame.f_locals.get("self") + if code.co_name in ("content_hash", "pipeline_hash") and owner is not None: + return ( + f"{code.co_name}() of {type(owner).__module__}.{type(owner).__name__}" + ) + if caller is None and "/orcapod/hashing/" not in code.co_filename: + caller = f"{_short(code.co_filename)}:{code.co_name}" + frame = frame.f_back + return f"direct hash_object from {caller}" + + +def _recording_handle(self: Any, obj: Any, hasher: Any) -> Any: + SITES[_site()] += 1 + return _original(self, obj, hasher) + + +bh.ArrowTableHandler.handle = _recording_handle # type: ignore[method-assign] + + +if __name__ == "__main__": + args = sys.argv[1:] or ["tests/", "-q", "-p", "no:cacheprovider", "-x"] + code = pytest.main(args) + by_kind: dict[str, int] = collections.Counter() + print("\n## ArrowTableHandler call sites\n") + for site, count in sorted(SITES.items(), key=lambda kv: -kv[1]): + kind = "tests" if "from tests/" in site else "library objects" + by_kind[kind] += count + print(f"{count:8d} {site}") + print(f"\npytest exit code: {code}; calls by origin: {dict(by_kind)}") diff --git a/superpowers/scripts/itl_897/claim_g_arrow_table_value.py b/superpowers/scripts/itl_897/claim_g_arrow_table_value.py new file mode 100644 index 000000000..0241bcf9e --- /dev/null +++ b/superpowers/scripts/itl_897/claim_g_arrow_table_value.py @@ -0,0 +1,89 @@ +"""ITL-897 claim (g): what ``pa.Table`` equality depends on vs its value hash today. + +For each property, two tables differing only in it; reports ``Table.equals`` (default +``check_metadata=False``), ``Table.equals(check_metadata=True)``, and whether +``semantic_hasher.hash_object`` gives the same hash. The last section checks that a +``RecordBatch`` hashes like the one-batch ``Table`` with the same content. + +Run: ``uv run python superpowers/scripts/itl_897/claim_g_arrow_table_value.py`` +""" + +from __future__ import annotations + +import pyarrow as pa + +from orcapod.contexts import get_default_context + +HASHER = get_default_context().semantic_hasher + + +def _t(**columns: pa.Array) -> pa.Table: + return pa.table(columns) + + +AB = _t(a=pa.array([1, 2]), b=pa.array(["x", "y"])) +UV = pa.struct([("u", pa.int64()), ("v", pa.string())]) +VU = pa.struct([("v", pa.string()), ("u", pa.int64())]) + +CASES: list[tuple[str, pa.Table, pa.Table]] = [ + ("column order", AB, AB.select(["b", "a"])), + ("row order", AB, AB.take([1, 0])), + ("values", AB, _t(a=pa.array([1, 3]), b=pa.array(["x", "y"]))), + ("column name", AB, AB.rename_columns(["a", "c"])), + ( + "struct field order in a column", + _t(s=pa.array([{"u": 1, "v": "x"}], UV)), + _t(s=pa.array([{"v": "x", "u": 1}], VU)), + ), + ( + "struct field order in a list column", + _t(s=pa.array([[{"u": 1, "v": "x"}]], pa.list_(UV))), + _t(s=pa.array([[{"v": "x", "u": 1}]], pa.list_(VU))), + ), + ( + "int64 vs int32", + _t(a=pa.array([1], pa.int64())), + _t(a=pa.array([1], pa.int32())), + ), + ( + "string vs large_string", + _t(a=pa.array(["x"], pa.string())), + _t(a=pa.array(["x"], pa.large_string())), + ), + ( + "nullable vs non-nullable field", + AB, + AB.cast( + pa.schema([pa.field("a", pa.int64(), False), pa.field("b", pa.string())]) + ), + ), + ("chunking", AB, pa.concat_tables([AB.slice(0, 1), AB.slice(1, 1)])), + ("schema metadata", AB, AB.replace_schema_metadata({"k": "v"})), +] + + +def main() -> None: + print("| property | equals | equals(check_metadata) | hash equal | verdict |") + print("|---|---|---|---|---|") + for name, t1, t2 in CASES: + eq = t1.equals(t2) + eq_meta = t1.equals(t2, check_metadata=True) + same = HASHER.hash_object(t1) == HASHER.hash_object(t2) + if not eq and same: + verdict = "MISMATCH: collision" + elif eq and not same: + verdict = "MISMATCH: spurious miss" + else: + verdict = "consistent" + print(f"| {name} | {eq} | {eq_meta} | {'yes' if same else 'no'} | {verdict} |") + + batch = AB.to_batches()[0] + print( + "\nRecordBatch hashes like the one-batch Table: " + f"{HASHER.hash_object(batch) == HASHER.hash_object(AB)}" + ) + print(f"Label of a table value hash today: {HASHER.hash_object(AB).method!r}") + + +if __name__ == "__main__": + main() diff --git a/superpowers/scripts/itl_897/prototype_v1.py b/superpowers/scripts/itl_897/prototype_v1.py index 6bbb6cb96..cf9558de3 100644 --- a/superpowers/scripts/itl_897/prototype_v1.py +++ b/superpowers/scripts/itl_897/prototype_v1.py @@ -50,3 +50,42 @@ def series_handle(self: Any, obj: Any, hasher: Any) -> Any: bh.PandasDataFrameHandler.handle = df_handle # type: ignore[method-assign] bh.PandasSeriesHandler.handle = series_handle # type: ignore[method-assign] + + +def install_arrow_table(*, separated: bool) -> None: + """Patch ``ArrowTableHandler`` with an order-dependent ``arrow_table_v1`` prototype. + + The descriptor lists the table's top-level field names in order; the values digest is + today's Arrow hasher digest. With ``separated=True``, ``Datagram`` and producer-less + ``ArrowTableStream`` identities no longer route their table through the type-dispatched + handler: they return ``arrow_hasher.hash_table(table)`` (a terminal ``ContentHash``), + which is what the handler returns today, so stream and datagram hashes stay put. + """ + from orcapod.core.datagrams.datagram import Datagram + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + from orcapod.hashing.semantic_hashing import builtin_handlers as bh + + def table_handle(self: Any, obj: Any, hasher: Any) -> Any: + if isinstance(obj, pa.RecordBatch): + obj = pa.Table.from_batches([obj]) + values = self._get_arrow_hasher().hash_table(obj).digest + return _labelled("arrow_table_v1", obj, values) + + bh.ArrowTableHandler.handle = table_handle # type: ignore[method-assign] + if not separated: + return + + def datagram_identity(self: Any) -> Any: + return self.data_context.arrow_hasher.hash_table(self._ensure_data_table()) + + original_stream_identity = ArrowTableStream.identity_structure + + def stream_identity(self: Any) -> Any: + structure = original_stream_identity(self) + if self._producer is not None: + return structure + name, table, tag_columns = structure + return (name, self.data_context.arrow_hasher.hash_table(table), tag_columns) + + Datagram.identity_structure = datagram_identity # type: ignore[method-assign] + ArrowTableStream.identity_structure = stream_identity # type: ignore[method-assign] diff --git a/superpowers/specs/2026-11-11-itl-897-pandas-hash-identity.md b/superpowers/specs/2026-11-11-itl-897-pandas-hash-identity.md new file mode 100644 index 000000000..71ad32d5c --- /dev/null +++ b/superpowers/specs/2026-11-11-itl-897-pandas-hash-identity.md @@ -0,0 +1,337 @@ +# ITL-897: pandas DataFrame hash ignores column order (step 1: investigation and proposal) + +Status: proposal, awaiting the coordinator's decision. No library code is changed. +Base: `main` at `b197544e` (v0.2.0 + #291), pandas 2.2.3, pyarrow 23.0.1, Python 3.12. + +Developer decision (November 11, 2026): order is identity wherever the type's own equality +depends on order. Only `dict` and Orcapod streams ignore order, deliberately. Added the same day: +a `pa.Table` used as a value must be order dependent too. + +Every claim below has a script in `superpowers/scripts/itl_897/`, run with +`uv run python superpowers/scripts/itl_897/