From 2711ba1a0041ccf966efa2b3399323aa774bce69 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:24:35 -0800 Subject: [PATCH 1/5] docs(plans): ITL-1137 cause report and plan Co-Authored-By: Claude Opus 5.5 --- ...6-11-14-itl-1137-type-hash-registration.md | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1137-type-hash-registration.md diff --git a/superpowers/plans/2026-11-14-itl-1137-type-hash-registration.md b/superpowers/plans/2026-11-14-itl-1137-type-hash-registration.md new file mode 100644 index 000000000..52159c414 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1137-type-hash-registration.md @@ -0,0 +1,123 @@ +# ITL-1137: a type's hash must not depend on whether it was registered + +Branch `eywalker/itl-1137-a-pipeline-hash-over-a-user-class-depends`, from `origin/main` +`e4b4319c`. Draft PR against `main`, `Fixes ITL-1137`. + +## Status + +- [x] Cause established (below), sent to the coordinator by `ask`. +- [ ] Coordinator approval of the proposal. +- [ ] Failing tests first. +- [ ] Fix. +- [ ] Changelog fragment, design_issues entry, full suite, PR. + +## Cause + +`TypeObjectHandler.handle` (`hashing/semantic_hashing/builtin_handlers.py`) is the one place a +type object becomes a hash input. It asks the type converter `get_logical_type(obj)`, which is +a registry lookup only: + +- registered: `type:`, e.g. for an exact dataclass + `type:__main__.A@1ee123…;7b2263617465676f7279223a226f726361706f642e64617461636c617373…` + (extension name `name@sha256(record)` + metadata `{category, identity:{fields, kind, name}}`); +- not registered: `type:.`, e.g. `type:__main__.A`. + +So the unregistered value carries only the module path and qualified name: no kind, no fields. +Collisions (one process, nothing registered): + +| class | hash input | slot pipeline hash | +|---|---|---| +| exact `A(u: int, v: str)` | `type:__main__.A` | `397c08159f373f49` | +| exact `A(w: float)` (other fields) | `type:__main__.A` | `397c08159f373f49` | +| structural `A(u: int, v: str)` | `type:__main__.A` | `397c08159f373f49` | +| exact `B(u: int, v: str)` | `type:__main__.B` | `741bb18fb6155218` | + +After registration the four give `9d01e7b7…`, `0723367a…`, `6cddc2e5…`, `ca2138ba…`. + +Why nothing registers first: `SchemaHandler` checks translatability with +`python_type_to_arrow_type`, which does not register (it raises `Unsupported Python type` for an +unregistered dataclass), and a `Schema` nested inside a structure (a node's +`pipeline_identity_structure()` is the tuple `(tag_schema, data_schema)`) never reaches +`SchemaHandler` at all: `_expand_element` expands it as a Mapping (`_expand_mapping`), which +hashes each field type through `TypeObjectHandler` with no check. A top-level +`hash_object(Schema({"p": A}))` of an unregistered class raises `TypeError`. + +Registration happens only at: `FunctionPod.__init__` (`ensure_types_registered_for_schemas` +over input and output schemas), `UniversalTypeConverter.python_dicts_to_arrow_table` +(DictSource, ListSource, datagrams from Python values), reading Arrow data with extension types +(`register_storage_type` / `register_discovered_logical_types`, i.e. sources over Arrow tables, +database reads, loading a saved job's schema text when the class is already bound), +`output_conformance` (`get_logical_type_for_python_type`), and nested registration inside +`user_class_logical_type` / `user_class_identity`. + +Not affected: built-in and library logical types (`Path`, `UUID`, `File`, `np.ndarray`, +`pd.DataFrame`, `pl.DataFrame`, `torch.Tensor`) are registered when the data context loads: +same hash before and after. + +Affected in one process (before != after): exact dataclass, structural class, pydantic model, +nested exact and nested structural classes, `list[A]`, `A | None`. + +## Realistic sequences + +1. Pipeline with a `SourceNode` slot followed only by operators, job built before a source of + the class exists: operator node pipeline hash `0a1f900ab2da6e57` is memoized and kept after + bind and run; the same pipeline built after registration has `ea3272f4ec3bb58c`. Path + component `schema:` differs. +2. Pipeline slot `(p: A, x: int)` -> function pod `h(x: int)` (pod never names `A`): run once + with the pipeline built first, once with the source built first, on one Delta store. Two + pipeline tables for one node: + `demo/h//v0/python.function.v0/schema:efce4ce5…/pdb_v2` and `schema:bd9c930f…/pdb_v2`. + The result table (`_result/h/…/rdb_v2`) is shared, so the second run took `1 from store`; + the pipeline records (the node's stored view) are split. +3. A function pod annotated with the class, defined before the slot is hashed: stable from the + start (pod registers). +4. Save/load (DictSource -> MapData -> MapData, and DictSource -> pod; Delta store; load in a + fresh process): operator and function job nodes take their hashes from the saved file: + identical to the saving process, and `run(load_only=True)` after bind loads every stored + row. The loaded `SourceJobNode` differs: the saved type text + `extension` cannot be restored to a Python type unless the class is + registered in the loading process before `load` (importing the module is not enough), so + the slot's schema holds the text string and its pipeline hash is `8f42c0d1…` instead of the + saved `c7e3da4d…` (structural: `57b3cbad…` vs `756682dc…`). Registered before load: equal. + This is a separate mechanism (load cannot rebuild a user class from metadata); the + proposed fix does not change it. Joined-source system-tag names in a loaded job come from + the bound source, so they were equal in the test. + +## Which value is the value + +The registered one. Every golden case that hashes a user class calls +`register_logical_classes()` first (`semantic.json type.*`, `schema.json single_field.*`, +`datagram.json` dataclass / pydantic / structural groups). No golden entry holds an +unregistered value. + +## Stored data under the unstable value + +Paths that embed a pipeline hash: function pipeline tables (`pdb_v2`) and operator caches +(`odb_v1`) of nodes downstream of a slot hashed before registration, written by any main +build since the user-class identity (ITL-972). Not affected: result tables (`rdb_v2`, keyed by +the pod), and nodes over a source built from data (the source registers at construction). No +release holds this: v0.3.0 is not tagged, and v0.2 hashes differ from v0.3 anyway. + +## Proposal (recommended: A) + +A. `TypeObjectHandler` resolves the type through + `type_converter.get_logical_type_for_python_type(obj)` (registry lookup, else register via + the factory that covers it; `None` for primitives and classes no factory covers), and + `SchemaHandler` hashes each field type before its translatability check (so the check sees + the registered type). Registered classes hash exactly as today; the unregistered value can + no longer be produced for a class a factory covers. Prototype: all probes equal, golden + suite and `test_hash_label_rule.py` 1317 passed unchanged, hashing / user types / logical + types / semantic types 3759 passed. +B. Register in `SourceNodeBase.__init__` (as `FunctionPod` does). Covers slots only; other + schemas hashed before registration (pipeline load, operator predictions) stay exposed. +C. Route nested `Schema` objects through `SchemaHandler` and make it register. Fixes schemas + only, and makes nested schemas with an untranslatable type raise where they hash today. + +## Tests (after approval) + +`tests/test_hashing/test_type_hash_registration.py`: in one process, the hash of a fresh class +taken before registration equals the hash after (`SourceNode` pipeline hash and schema hash), +for exact dataclass, structural class, pydantic model, nested class, `list[A]`, `A | None`; +two unregistered classes of the same qualified name and different fields hash differently; a +fresh-process subprocess test: a pipeline built before a source of the class gives the same +node pipeline hashes as one built after. From 2c8f4930f2335fad27cea2099077d652a8f66490 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:30:26 -0800 Subject: [PATCH 2/5] test(hashing): a type hashes the same before and after registration (ITL-1137, failing) Co-Authored-By: Claude Opus 5.5 --- .../test_type_hash_registration.py | 410 ++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100644 tests/test_hashing/test_type_hash_registration.py diff --git a/tests/test_hashing/test_type_hash_registration.py b/tests/test_hashing/test_type_hash_registration.py new file mode 100644 index 000000000..0484d1d32 --- /dev/null +++ b/tests/test_hashing/test_type_hash_registration.py @@ -0,0 +1,410 @@ +"""A type's hash does not depend on whether its class was registered (ITL-1137). + +A user class (an ordinary dataclass, an ``orcapod.dataclass``, a pydantic model) gets its +logical type when something registers it: a function pod naming it, a source built from its +values. Before ITL-1137 the type hasher only looked the class up, so a schema hashed before +registration hashed the class by module path and qualified name, and the same schema hashed +after registration by its logical type identifier: two hashes for one pipeline node, two stored +tables for one node. Hashing a type now resolves its logical type, registering it when a factory +covers the class, so the hash before and after is the registered one, the value the golden suite +pins. + +Each test imports its classes from a module written for it alone, so no other test has +registered them; a class defined inside a function cannot be stored. +""" + +import importlib +import itertools +import json +import subprocess +import sys +import textwrap +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from orcapod import SourceNode +from orcapod.contexts import get_default_context +from orcapod.sources import DictSource +from orcapod.types import Schema + +_module_numbers = itertools.count() + + +@pytest.fixture +def fresh_module(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Any: + """Return a function that writes Python source to a new module and imports it.""" + monkeypatch.syspath_prepend(str(tmp_path)) + + def load(source: str) -> ModuleType: + name = f"itl1137_classes_{next(_module_numbers)}" + (tmp_path / f"{name}.py").write_text(textwrap.dedent(source)) + return importlib.import_module(name) + + return load + + +def _slot(data_type: Any) -> SourceNode: + """A slot with one tag column and one data column of ``data_type``.""" + return SourceNode( + "slot", tag_schema=Schema({"id": int}), data_schema=Schema({"p": data_type}) + ) + + +def _slot_hashes(data_type: Any) -> tuple[str, str]: + """The pipeline hash and content hash of a slot holding ``data_type``.""" + node = _slot(data_type) + return node.pipeline_hash().to_string(), node.content_hash().to_string() + + +def _schema_hash(data_type: Any) -> str: + """The hash of a schema holding ``data_type``, hashed on its own.""" + hasher = get_default_context().semantic_hasher + return hasher.hash_object(Schema({"p": data_type})).to_string() + + +def _register_by_building_a_source(data_type: Any, value: Any) -> DictSource: + """Build a source of ``data_type``: the source registers the class it holds.""" + return DictSource( + [{"id": 1, "p": value}], + tag_columns=["id"], + data_schema={"id": int, "p": data_type}, + source_id="s", + ) + + +# Each module defines ``TYPE``, the column type, and ``VALUE``, a value of it. +_KINDS = { + "exact dataclass": """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + label: str + + TYPE, VALUE = Point, Point(1, "a") + """, + "structural class": """ + import orcapod + + @orcapod.dataclass + class Point: + x: int + label: str + + TYPE, VALUE = Point, Point(1, "a") + """, + "pydantic model": """ + import pydantic + + class Point(pydantic.BaseModel): + x: int + label: str + + TYPE, VALUE = Point, Point(x=1, label="a") + """, + "nested exact": """ + import dataclasses + + @dataclasses.dataclass + class Inner: + x: int + + @dataclasses.dataclass + class Outer: + inner: Inner + label: str + + TYPE, VALUE = Outer, Outer(Inner(1), "a") + """, + "nested structural": """ + import orcapod + + @orcapod.dataclass + class Inner: + x: int + + @orcapod.dataclass + class Outer: + inner: Inner + label: str + + TYPE, VALUE = Outer, Outer(Inner(1), "a") + """, + "list of a dataclass": """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + + TYPE, VALUE = list[Point], [Point(1), Point(2)] + """, + "optional dataclass": """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + + TYPE, VALUE = Point | None, Point(1) + """, +} + + +@pytest.mark.parametrize("source", _KINDS.values(), ids=list(_KINDS)) +def test_a_slot_hashes_the_same_before_and_after_registration( + fresh_module: Any, source: str +) -> None: + """A slot's pipeline hash and content hash do not move when a source registers.""" + module = fresh_module(source) + before = _slot_hashes(module.TYPE) + built = _register_by_building_a_source(module.TYPE, module.VALUE) + assert _slot_hashes(module.TYPE) == before + # The value is the registered one: the slot's pipeline hash is the source's. + assert before[0] == built.pipeline_hash().to_string() + + +@pytest.mark.parametrize("source", _KINDS.values(), ids=list(_KINDS)) +def test_a_schema_hashes_the_same_before_and_after_registration( + fresh_module: Any, source: str +) -> None: + """A schema hashed on its own gives the registered value, also before registration.""" + module = fresh_module(source) + before = _schema_hash(module.TYPE) + _register_by_building_a_source(module.TYPE, module.VALUE) + assert _schema_hash(module.TYPE) == before + + +def test_a_function_pod_defined_after_the_slot_sees_the_same_slot_hash( + fresh_module: Any, +) -> None: + """A slot hashed before the pod that names its class keeps its hash afterwards.""" + module = fresh_module( + """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + """ + ) + early_slot = _slot(module.Point) + before = early_slot.pipeline_hash() + pods = fresh_module( + f""" + from orcapod import function_pod + from {module.__name__} import Point + + @function_pod(output_keys="y") + def read_x(p: Point) -> int: + return p.x + """ + ) + late_slot = _slot(module.Point) + assert late_slot.pipeline_hash() == before + early, late = pods.read_x.pod(early_slot), pods.read_x.pod(late_slot) + assert early.pipeline_hash() == late.pipeline_hash() + + +def test_two_classes_with_one_qualified_name_and_different_fields_hash_apart( + fresh_module: Any, +) -> None: + """The class's identity enters the hash, not only its module path and name.""" + source = """ + import dataclasses + import orcapod + + @dataclasses.dataclass + class Point: + x: int + label: str + + FIRST = Point + + @dataclasses.dataclass + class Point: + x: float + + SECOND = Point + + @orcapod.dataclass + class Point: + x: int + label: str + + THIRD = Point + """ + module = fresh_module(source) + classes = (module.FIRST, module.SECOND, module.THIRD) + assert len({_slot(cls).pipeline_hash() for cls in classes}) == 3 + + +def test_a_class_no_factory_covers_hashes_by_its_name_stably( + fresh_module: Any, +) -> None: + """A class Orcapod has no logical type for hashes as ``type:.``.""" + module = fresh_module( + """ + class Opaque: + pass + """ + ) + opaque = module.Opaque + hasher = get_default_context().semantic_hasher + expected = hasher.hash_object(f"type:{module.__name__}.Opaque") + assert hasher.hash_object(opaque) == expected + slot_hash = _slot(opaque).pipeline_hash() + assert hasher.hash_object(opaque) == expected + assert _slot(opaque).pipeline_hash() == slot_hash + assert get_default_context().type_converter.get_logical_type(opaque) is None + + +def test_hashing_a_type_has_no_effect_beyond_registering_it(fresh_module: Any) -> None: + """The first hash registers the class; hashing again registers nothing more.""" + module = fresh_module( + """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + """ + ) + converter = get_default_context().type_converter + hasher = get_default_context().semantic_hasher + # The registry has no public size; the test is about what hashing adds to it. + registry = converter._logical_type_registry # pyright: ignore[reportAttributeAccessIssue] + + def registry_size() -> tuple[int, int, int]: + return ( + len(registry._by_python_type), + len(registry._by_arrow_name), + len(registry._by_logical_name), + ) + + first = hasher.hash_object(module.Point) + logical_type = converter.get_logical_type(module.Point) + assert logical_type is not None + size = registry_size() + assert hasher.hash_object(module.Point) == first + _ = hasher.hash_object(Schema({"p": module.Point})) + _ = _slot(module.Point).pipeline_hash() + assert registry_size() == size + assert converter.get_logical_type(module.Point) is logical_type + + +# --------------------------------------------------------------------------- +# One store, two processes: a pipeline built before the source, then after it +# --------------------------------------------------------------------------- + +_CLASS_MODULE = textwrap.dedent( + """\ + import dataclasses + + + @dataclasses.dataclass + class Reading: + value: int + unit: str + """ +) + +_RUN_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + store, order = sys.argv[2], sys.argv[3] + + from readings import Reading + + from orcapod import Pipeline, PipelineJob, SourceNode, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.operators import MapData + from orcapod.sources import DictSource + from orcapod.types import CacheMode, Schema + + + @function_pod(output_keys="doubled") + def double(n: int) -> int: + return n * 2 + + + def source(): + return DictSource( + [{"id": 1, "r": Reading(3, "mV"), "n": 3}], + tag_columns=["id"], + data_schema={"id": int, "r": Reading, "n": int}, + source_id="readings", + ) + + + if order == "source_first": + bound = source() + pipeline = Pipeline("demo") + with pipeline: + slot = SourceNode( + "slot", + tag_schema=Schema({"id": int}), + data_schema=Schema({"r": Reading, "n": int}), + ) + MapData({"r": "reading"})(slot, label="rename") + double.pod(slot, label="double") + job = PipelineJob.from_pipeline(pipeline, database=DeltaTableDatabase(store + "/job")) + job.nodes["rename"].attach_databases( + pipeline_database=DeltaTableDatabase(store + "/operators"), + cache_mode=CacheMode.LOG, + ) + if order != "source_first": + bound = source() + job.bind(sources={"slot": bound}) + job.run() + summary = job.last_run.nodes["double"] + print(json.dumps({"computed": summary.computed, "from_store": summary.reused})) + """ +) + + +def _run(tmp_path: Path, order: str) -> dict[str, int]: + """Run the job in a fresh interpreter, building the pipeline or the source first.""" + result = subprocess.run( + [ + sys.executable, + str(tmp_path / "run_job.py"), + str(tmp_path), + str(tmp_path / "store"), + order, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_one_stored_table_per_node_whatever_is_built_first(tmp_path: Path) -> None: + """A job whose pods never name the class stores each node at one path.""" + from orcapod.databases import DeltaTableDatabase + + (tmp_path / "readings.py").write_text(_CLASS_MODULE) + (tmp_path / "run_job.py").write_text(_RUN_SCRIPT) + first = _run(tmp_path, "pipeline_first") + second = _run(tmp_path, "source_first") + third = _run(tmp_path, "pipeline_first") + assert first == {"computed": 1, "from_store": 0} + assert second == third == {"computed": 0, "from_store": 1} + + job_tables = DeltaTableDatabase(tmp_path / "store" / "job").list_table_paths() + pipeline_tables = [p for p in job_tables if p[-1] == "pdb_v2"] + assert len(pipeline_tables) == 1, pipeline_tables + operator_tables = DeltaTableDatabase( + tmp_path / "store" / "operators" + ).list_table_paths() + assert len(operator_tables) == 1, operator_tables From b036cca993f27b2e7d04530ee0b84ce2b4bd3ded Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:33:24 -0800 Subject: [PATCH 3/5] fix(hashing): a type hashes as its logical type before and after registration (ITL-1137) TypeObjectHandler only looked a class up in the logical type registry, so a schema hashed before a source or function pod registered its class hashed the class by module path and qualified name, and the same schema hashed later by its logical type identifier. It now resolves the type through get_logical_type_for_python_type, which registers a class a factory covers. SchemaHandler hashes each field type before its translatability check, so a schema hashed on its own no longer raises for a class not registered yet. Registered classes hash exactly as before; no golden value moves. Co-Authored-By: Claude Opus 5.5 --- .../hashing/semantic_hashing/__init__.py | 2 +- .../semantic_hashing/builtin_handlers.py | 49 ++++++++++++------- .../test_type_hash_registration.py | 8 ++- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/orcapod/hashing/semantic_hashing/__init__.py b/src/orcapod/hashing/semantic_hashing/__init__.py index 115538878..bfe05f9a6 100644 --- a/src/orcapod/hashing/semantic_hashing/__init__.py +++ b/src/orcapod/hashing/semantic_hashing/__init__.py @@ -12,7 +12,7 @@ UUIDHandler -- uuid.UUID → canonical bytes BytesHandler -- bytes/bytearray → hex string FunctionHandler -- callable → via FunctionInfoExtractorProtocol - TypeObjectHandler -- type objects → "type:." + TypeObjectHandler -- type objects → logical type identifier, else "type:." FileHandler -- orcapod.File → file content hash DirectoryHandler -- orcapod.Directory → recursive Merkle tree hash register_builtin_python_type_handlers -- populate a registry with all of the above diff --git a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py index 229228efb..d35b29b01 100644 --- a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py +++ b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py @@ -4,7 +4,7 @@ UUIDHandler -- uuid.UUID: 16-byte binary representation BytesHandler -- bytes/bytearray: hex string representation FunctionHandler -- callable with __code__: via FunctionInfoExtractorProtocol - TypeObjectHandler -- type objects: stable "type:." string + TypeObjectHandler -- type objects: logical type identifier, else "type:." SpecialFormHandler -- typing._SpecialForm GenericAliasHandler -- generic alias type annotations UnionTypeHandler -- types.UnionType (Python 3.10+ X | Y syntax) @@ -169,25 +169,34 @@ def _with_hashed_default( class TypeObjectHandler: """Hasher for type objects (classes passed as values). - Resolves types registered in the ``LogicalTypeRegistry`` exposed by - *type_converter* to their logical type identifier, the Arrow extension name - and extension metadata, written as ``LogicalTypeIdentifier.text`` + Resolves a type to its logical type in the ``LogicalTypeRegistry`` exposed by + *type_converter* and hashes its logical type identifier, the Arrow extension + name and extension metadata, written as ``LogicalTypeIdentifier.text`` (``"type:orcapod.file;7b2276657273696f6e223a317d"`` for ``op.File``: the name, ``;``, then the metadata ``{"version":1}`` in hex). A change of either part of the identifier therefore changes every type, schema, function and pipeline hash the type takes part in. - Falls back to ``"type:."`` for unregistered types or - when no ``type_converter`` is provided. The module is the canonical one - from ``canonical_type_name``, so standard-library types that moved into a - private submodule (``pathlib._local`` on Python 3.13) hash identically on - every Python version. + + A class not registered yet that a registered factory covers (a dataclass, + an ``orcapod.dataclass``, a pydantic model) is registered first, through + ``type_converter.get_logical_type_for_python_type``. So a type hashes the + same before and after anything else registers it (ITL-1137): hashing a + schema before the source or function pod that registers its class gives + the registered value, the one every stored path and golden value holds. + + Falls back to ``"type:."`` for a type with no logical + type (``int``, a class no factory covers) or when no ``type_converter`` is + provided. The module is the canonical one from ``canonical_type_name``, so + standard-library types that moved into a private submodule + (``pathlib._local`` on Python 3.13) hash identically on every Python + version. Args: type_converter: Optional ``TypeConverterProtocol``. When provided, - ``type_converter.get_logical_type(obj)`` is called to resolve - registered logical types to their identifier. When - ``None`` (the default), the fallback ``"type:."`` - serialisation is always used. + ``type_converter.get_logical_type_for_python_type(obj)`` resolves the + type to its logical type, registering it when a factory covers it. + When ``None`` (the default), the fallback + ``"type:."`` serialisation is always used. """ def __init__(self, type_converter: "TypeConverterProtocol | None" = None) -> None: @@ -199,7 +208,7 @@ def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: f"TypeObjectHandler: expected a type/class, got {type(obj)!r}" ) if self._type_converter is not None: - lt = self._type_converter.get_logical_type(obj) + lt = self._type_converter.get_logical_type_for_python_type(obj) if lt is not None: return f"type:{logical_type_identifier(lt).text}" return f"type:{canonical_type_name(obj)}" @@ -310,9 +319,11 @@ class SchemaHandler: not part of the structural identity used for caching or pipeline routing. When ``type_converter`` is provided, every field type is verified to be - Arrow-translatable before hashing. A type that cannot be converted raises - ``TypeError`` with a diagnostic message, catching unregistered types at hash time - rather than silently falling back to a potentially unstable Python module path. + Arrow-translatable. A type that cannot be converted raises ``TypeError`` with a + diagnostic message, catching unsupported types at hash time rather than silently + falling back to a Python module path. The check runs after the field type is + hashed, because hashing a type registers a class a factory covers + (``TypeObjectHandler``), and the check reads only registered types. Args: type_converter: Optional ``TypeConverterProtocol``. When provided, validates @@ -344,6 +355,8 @@ def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: raise TypeError(f"SchemaHandler: expected a Schema, got {type(obj)!r}") result: dict[str, ContentHash] = {} for field_name, python_type in obj.items(): + # Hashing the type registers a class a factory covers, so it comes first. + type_hash = hasher.hash_object(python_type) if self._type_converter is not None: try: self._type_converter.python_type_to_arrow_type(python_type) @@ -356,7 +369,7 @@ def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> Any: f"supported native type (int, str, float, bool, bytes, " f"datetime, date)." ) from exc - result[field_name] = hasher.hash_object(python_type) + result[field_name] = type_hash # Sort by field name for determinism — matches _expand_mapping's sort_keys. return dict(sorted(result.items())) diff --git a/tests/test_hashing/test_type_hash_registration.py b/tests/test_hashing/test_type_hash_registration.py index 0484d1d32..8916f8efd 100644 --- a/tests/test_hashing/test_type_hash_registration.py +++ b/tests/test_hashing/test_type_hash_registration.py @@ -357,11 +357,15 @@ def source(): ) MapData({"r": "reading"})(slot, label="rename") double.pod(slot, label="double") - job = PipelineJob.from_pipeline(pipeline, database=DeltaTableDatabase(store + "/job")) + database = DeltaTableDatabase(store) + job = PipelineJob.from_pipeline(pipeline, database=database.at("job")) job.nodes["rename"].attach_databases( - pipeline_database=DeltaTableDatabase(store + "/operators"), + pipeline_database=database.at("operators"), cache_mode=CacheMode.LOG, ) + # Look at the job before any source exists, as a user printing its nodes would. + for node in job.nodes.values(): + node.pipeline_hash() if order != "source_first": bound = source() job.bind(sources={"slot": bound}) From c977ab617802245a3d63705777227de457bbce39 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:36:08 -0800 Subject: [PATCH 4/5] fix(hashing): declare get_logical_type_for_python_type on the converter protocol; changelog and design issue (ITL-1137) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1137.md | 3 +++ design_issues/ITL-1137-H6.md | 25 +++++++++++++++++++ .../protocols/semantic_types_protocols.py | 4 +++ .../test_type_hash_registration.py | 6 ++--- 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 changelog.d/ITL-1137.md create mode 100644 design_issues/ITL-1137-H6.md diff --git a/changelog.d/ITL-1137.md b/changelog.d/ITL-1137.md new file mode 100644 index 000000000..e7981f17b --- /dev/null +++ b/changelog.d/ITL-1137.md @@ -0,0 +1,3 @@ +### Fixed + +- A schema that holds a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model, also nested in another class, inside `list[...]` or as `T | None`) hashes the same whether or not anything has registered the class yet (ITL-1137). A class gets its logical type when a function pod names it or a source is built from its values; before that, hashing a type looked the class up only and hashed it by module path and qualified name, with no kind and no fields, so two different classes of one name hashed alike and a pipeline hash taken before registration differed from the same hash taken after. Such a hash is kept by the node, so one pipeline could store one node at two paths, depending on what the process built first: a job looked at (a node's hash, path or preview) before its source existed, over a `SourceNode` slot followed by operators or by function pods that never name the class. Hashing a type now registers a class that a factory covers, so the hash before registration is the registered one; a registered class hashes exactly as before, and no golden value or stored path of a registered class moves. A class Orcapod has no logical type for still hashes by its name. A schema hashed on its own no longer raises `TypeError` for a class not registered yet. A store written by a development build of `main` before this fix may hold, for such a node, function pipeline tables (`pdb_v2`) and operator caches (`odb_v1`) under a `schema:` path component computed from the unregistered class; those tables are not read again (the node now reads its tables at the registered path), and function results (`rdb_v2`, keyed by the function) are not affected. No release wrote such tables. diff --git a/design_issues/ITL-1137-H6.md b/design_issues/ITL-1137-H6.md new file mode 100644 index 000000000..a07043206 --- /dev/null +++ b/design_issues/ITL-1137-H6.md @@ -0,0 +1,25 @@ +## `src/orcapod/hashing/semantic_hashing/` + +### H6 — A user class hashed before registration hashed by its name only +**Status:** resolved +**Severity:** high +**Issue:** ITL-1137 + +`TypeObjectHandler` only looked a class up in the logical type registry. A user class (an +ordinary dataclass, an `orcapod.dataclass`, a pydantic model, also nested or inside `list[...]` +or `T | None`) gets its logical type when something registers it: a function pod naming it, a +source built from its values. Before that, the class hashed as `type:.`, with +no kind and no fields, so two different classes of one qualified name hashed alike, and the same +schema hashed differently before and after registration. A `Schema` nested in an identity +structure (a node's `(tag_schema, data_schema)`) is expanded as a mapping and never reaches +`SchemaHandler`, so nothing registered the class first; a `Schema` hashed on its own raised +`TypeError` instead. A pipeline hash taken before registration (a job looked at before its +source exists, a slot followed only by operators or by pods that never name the class) was +memoized, so one node wrote function pipeline tables (`pdb_v2`) and operator caches (`odb_v1`) +at two paths depending on what the process built first. + +**Fix:** `TypeObjectHandler` resolves the type through +`type_converter.get_logical_type_for_python_type`, which registers a class a factory covers, and +`SchemaHandler` hashes each field type before its translatability check. A registered class +hashes exactly as before and no golden value moves; a class no factory covers still hashes as +`type:.`. Tests: `tests/test_hashing/test_type_hash_registration.py`. diff --git a/src/orcapod/protocols/semantic_types_protocols.py b/src/orcapod/protocols/semantic_types_protocols.py index 56b9cefb0..b09b3b3d7 100644 --- a/src/orcapod/protocols/semantic_types_protocols.py +++ b/src/orcapod/protocols/semantic_types_protocols.py @@ -56,3 +56,7 @@ def get_arrow_to_python_converter( def ensure_types_registered_for_schemas(self, *schemas: Schema) -> None: ... def get_logical_type(self, python_type: type) -> "LogicalTypeProtocol | None": ... + + def get_logical_type_for_python_type( + self, annotation: Any + ) -> "LogicalTypeProtocol | None": ... diff --git a/tests/test_hashing/test_type_hash_registration.py b/tests/test_hashing/test_type_hash_registration.py index 8916f8efd..505f1e327 100644 --- a/tests/test_hashing/test_type_hash_registration.py +++ b/tests/test_hashing/test_type_hash_registration.py @@ -40,7 +40,7 @@ def fresh_module(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Any: def load(source: str) -> ModuleType: name = f"itl1137_classes_{next(_module_numbers)}" - (tmp_path / f"{name}.py").write_text(textwrap.dedent(source)) + _ = (tmp_path / f"{name}.py").write_text(textwrap.dedent(source)) return importlib.import_module(name) return load @@ -397,8 +397,8 @@ def test_one_stored_table_per_node_whatever_is_built_first(tmp_path: Path) -> No """A job whose pods never name the class stores each node at one path.""" from orcapod.databases import DeltaTableDatabase - (tmp_path / "readings.py").write_text(_CLASS_MODULE) - (tmp_path / "run_job.py").write_text(_RUN_SCRIPT) + _ = (tmp_path / "readings.py").write_text(_CLASS_MODULE) + _ = (tmp_path / "run_job.py").write_text(_RUN_SCRIPT) first = _run(tmp_path, "pipeline_first") second = _run(tmp_path, "source_first") third = _run(tmp_path, "pipeline_first") From 3aef99bba25977f3f93d7e3540963790241b8321 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:24:15 -0800 Subject: [PATCH 5/5] fix(hashing): a parameter annotation resolves its class before and after registration (ITL-1137, review round 1) canonical_annotation_str, which canonicalizes each parameter annotation of a hashed function, used the non-registering get_logical_type, so a function whose parameter's class was not registered yet hashed by the class name: the signature hash a data function computes when it is built, before its pod registers the annotation types, depended on what was registered first. It now uses get_logical_type_for_python_type, as TypeObjectHandler does. Adds a fresh-process save/load test with the class registered before the load. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1137.md | 2 +- src/orcapod/hashing/hash_utils.py | 28 ++-- .../semantic_hashing/builtin_handlers.py | 3 +- .../test_type_hash_registration.py | 135 +++++++++++++++++- 4 files changed, 154 insertions(+), 14 deletions(-) diff --git a/changelog.d/ITL-1137.md b/changelog.d/ITL-1137.md index e7981f17b..75f095218 100644 --- a/changelog.d/ITL-1137.md +++ b/changelog.d/ITL-1137.md @@ -1,3 +1,3 @@ ### Fixed -- A schema that holds a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model, also nested in another class, inside `list[...]` or as `T | None`) hashes the same whether or not anything has registered the class yet (ITL-1137). A class gets its logical type when a function pod names it or a source is built from its values; before that, hashing a type looked the class up only and hashed it by module path and qualified name, with no kind and no fields, so two different classes of one name hashed alike and a pipeline hash taken before registration differed from the same hash taken after. Such a hash is kept by the node, so one pipeline could store one node at two paths, depending on what the process built first: a job looked at (a node's hash, path or preview) before its source existed, over a `SourceNode` slot followed by operators or by function pods that never name the class. Hashing a type now registers a class that a factory covers, so the hash before registration is the registered one; a registered class hashes exactly as before, and no golden value or stored path of a registered class moves. A class Orcapod has no logical type for still hashes by its name. A schema hashed on its own no longer raises `TypeError` for a class not registered yet. A store written by a development build of `main` before this fix may hold, for such a node, function pipeline tables (`pdb_v2`) and operator caches (`odb_v1`) under a `schema:` path component computed from the unregistered class; those tables are not read again (the node now reads its tables at the registered path), and function results (`rdb_v2`, keyed by the function) are not affected. No release wrote such tables. +- A schema that holds a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model, also nested in another class, inside `list[...]` or as `T | None`) hashes the same whether or not anything has registered the class yet (ITL-1137). A class gets its logical type when a function pod names it or a source is built from its values; before that, hashing a type looked the class up only and hashed it by module path and qualified name, with no kind and no fields, so two different classes of one name hashed alike and a pipeline hash taken before registration differed from the same hash taken after. Such a hash is kept by the node, so one pipeline could store one node at two paths, depending on what the process built first: a job looked at (a node's hash, path or preview) before its source existed, over a `SourceNode` slot followed by operators or by function pods that never name the class. Hashing a type now registers a class that a factory covers, so the hash before registration is the registered one; a registered class hashes exactly as before, and no golden value or stored path of a registered class moves. The same holds for a parameter annotated with such a class when a function is hashed (a function used as a value, and the function signature hash a function pod stores with every result row as `__pf_var_function_signature_hash`): a pod defined before its parameter's class was registered stored the name-based signature hash, one defined after it the registered one; now both store the registered one. The signature hash is a recorded column, not a lookup key, so stored results are found as before. A class Orcapod has no logical type for still hashes by its name. A schema hashed on its own no longer raises `TypeError` for a class not registered yet. A store written by a development build of `main` before this fix may hold, for such a node, function pipeline tables (`pdb_v2`) and operator caches (`odb_v1`) under a `schema:` path component computed from the unregistered class; those tables are not read again (the node now reads its tables at the registered path), and function results (`rdb_v2`, keyed by the function) are not affected. No release wrote such tables. diff --git a/src/orcapod/hashing/hash_utils.py b/src/orcapod/hashing/hash_utils.py index 856a93c5b..d59e24841 100644 --- a/src/orcapod/hashing/hash_utils.py +++ b/src/orcapod/hashing/hash_utils.py @@ -44,12 +44,16 @@ def canonical_annotation_str( ) -> str: """Return a stable, canonical string for a type annotation. - Resolves types registered in *type_converter* (e.g. ``orcapod.File``) to - their logical type identifier text (``LogicalTypeIdentifier.text``: the Arrow - extension name, ``;``, then the extension metadata in hex, e.g. - ``"orcapod.file;7b2276657273696f6e223a317d"``) via - ``type_converter.get_logical_type()``. Internal module relocations do not - change the string; a change of the type's identifier does. + Resolves a type that has a logical type in *type_converter* (e.g. + ``orcapod.File``) to its logical type identifier text + (``LogicalTypeIdentifier.text``: the Arrow extension name, ``;``, then the + extension metadata in hex, e.g. ``"orcapod.file;7b2276657273696f6e223a317d"``) + via ``type_converter.get_logical_type_for_python_type()``, which registers a + class a factory covers (a dataclass, an ``orcapod.dataclass``, a pydantic + model) that nothing has registered yet. So an annotation gives the same string + before and after its class is registered (ITL-1137), as ``TypeObjectHandler`` + does for a type hashed as a value. Internal module relocations do not change + the string; a change of the type's identifier does. For union types (both PEP 604 ``X | Y`` and ``typing.Union[X, Y]``), members are sorted byte-wise so that ``str | Path`` and ``Path | str`` @@ -66,16 +70,18 @@ def canonical_annotation_str( Args: annotation: A type annotation object. - type_converter: Optional ``TypeConverterProtocol``. When provided, - registered logical types resolve to their identifier text via - ``get_logical_type()``. + type_converter: Optional ``TypeConverterProtocol``. When provided, a + type with a logical type resolves to its identifier text via + ``get_logical_type_for_python_type()``, registering it if needed. Returns: A canonical string representation. """ - # Registered logical type: use its identifier (extension name and metadata) + # A type with a logical type: its identifier (extension name and metadata). + # Registers a class a factory covers, so the string never depends on whether + # something registered it first (ITL-1137). if type_converter is not None and isinstance(annotation, type): - lt = type_converter.get_logical_type(annotation) + lt = type_converter.get_logical_type_for_python_type(annotation) if lt is not None: return logical_type_identifier(lt).text diff --git a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py index d35b29b01..55f4f523f 100644 --- a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py +++ b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py @@ -777,7 +777,8 @@ def register_builtin_python_type_handlers( type_converter: Optional ``TypeConverterProtocol`` forwarded to ``TypeObjectHandler``, ``FunctionSignatureExtractor``, and ``SchemaHandler`` for stable canonical type-name resolution via - ``get_logical_type()`` and Arrow-translatability validation. + ``get_logical_type_for_python_type()`` and Arrow-translatability + validation. When ``None`` (the default), all three handlers fall back to the raw ``"type:."`` serialisation and skip Arrow validation. """ diff --git a/tests/test_hashing/test_type_hash_registration.py b/tests/test_hashing/test_type_hash_registration.py index 505f1e327..57024ccc5 100644 --- a/tests/test_hashing/test_type_hash_registration.py +++ b/tests/test_hashing/test_type_hash_registration.py @@ -25,7 +25,7 @@ import pytest -from orcapod import SourceNode +from orcapod import SourceNode, function_pod from orcapod.contexts import get_default_context from orcapod.sources import DictSource from orcapod.types import Schema @@ -297,6 +297,56 @@ def registry_size() -> tuple[int, int, int]: assert converter.get_logical_type(module.Point) is logical_type +def test_a_function_annotated_with_a_class_hashes_the_same_before_and_after( + fresh_module: Any, +) -> None: + """A parameter annotation resolves its class as a type hashed as a value does.""" + module = fresh_module( + """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + + def first_x(p: Point, rest: list[Point] | None = None) -> int: + return p.x + """ + ) + hasher = get_default_context().semantic_hasher + before = hasher.hash_object(module.first_x) + _ = _register_by_building_a_source(module.Point, module.Point(1)) + assert hasher.hash_object(module.first_x) == before + + +def test_a_data_function_stores_one_signature_hash_whatever_was_registered_first( + fresh_module: Any, +) -> None: + """The signature hash stored with every result row does not depend on the order.""" + module = fresh_module( + """ + import dataclasses + + @dataclasses.dataclass + class Point: + x: int + + def read_x(p: Point) -> int: + return p.x + """ + ) + + def stored_signature_hash() -> bytes: + pod = function_pod(output_keys="y")(module.read_x) + return pod.pod.data_function.get_function_variation_data()[ + "function_signature_hash" + ] + + # The first pod is built before anything has registered Point. + before = stored_signature_hash() + assert stored_signature_hash() == before + + # --------------------------------------------------------------------------- # One store, two processes: a pipeline built before the source, then after it # --------------------------------------------------------------------------- @@ -412,3 +462,86 @@ def test_one_stored_table_per_node_whatever_is_built_first(tmp_path: Path) -> No tmp_path / "store" / "operators" ).list_table_paths() assert len(operator_tables) == 1, operator_tables + + +_SAVE_LOAD_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + work, mode = sys.argv[2], sys.argv[3] + + from readings import Reading + + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.operators import MapData + from orcapod.sources import DictSource + + # Building the source registers Reading, in both processes before the job exists. + source = DictSource( + [{"id": 1, "r": Reading(3, "mV"), "n": 3}, {"id": 2, "r": Reading(4, "V"), "n": 4}], + tag_columns=["id"], + data_schema={"id": int, "r": Reading, "n": int}, + source_id="readings", + ) + if mode == "save": + + @function_pod(output_keys="unit") + def unit_of(r: Reading) -> str: + return r.unit + + job = PipelineJob(name="demo", database=DeltaTableDatabase(work + "/store")) + with job: + renamed = MapData({"n": "count"})(source, label="renamed") + unit_of.pod(renamed, label="unit_of") + job.run() + job.save(work + "/job.json") + reused = None + else: + job = PipelineJob.load(work + "/job.json") + job.bind(sources={"readings": source}) + job.run(load_only=True) + reused = {label: node.reused for label, node in job.last_run.nodes.items()} + hashes = { + label: [node.pipeline_hash().to_string(), node.content_hash().to_string()] + for label, node in sorted(job.nodes.items()) + } + print(json.dumps({"hashes": hashes, "reused": reused})) + """ +) + + +def test_a_saved_job_loads_in_a_fresh_process_with_the_saved_hashes( + tmp_path: Path, +) -> None: + """Every node of a job over a user class loads with its saved hashes and results. + + The class is registered before the load here (the source is built first); a job + loaded before anything registers its class is ITL-1138. + """ + _ = (tmp_path / "readings.py").write_text(_CLASS_MODULE) + _ = (tmp_path / "save_load.py").write_text(_SAVE_LOAD_SCRIPT) + + def run(mode: str) -> dict[str, Any]: + result = subprocess.run( + [ + sys.executable, + str(tmp_path / "save_load.py"), + str(tmp_path), + str(tmp_path), + mode, + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-3000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + saved = run("save") + loaded = run("load") + assert loaded["hashes"] == saved["hashes"] + assert loaded["reused"] == {"unit_of": 2}