diff --git a/.zed/rules b/.zed/rules index 5b81b2de9..e4759a29a 100644 --- a/.zed/rules +++ b/.zed/rules @@ -320,6 +320,11 @@ Developer, November 12, 2026 (ITL-981); see `docs/concepts/arrays.md` and `allow_pickle=False` on write and read; never raise `max_header_size`). Options: `memory_order`, `byte_order`, `struct_layout`. Where stored arrays must be ordered, order them by value hash (`storage_sort_key`), never by stored bytes. +- Library value types are provisional until the v0.4 audit (developer, November 13, 2026, + ITL-1068): their extension names are listed in `provisional` of + `contexts/logical_types.json`, and using one issues `ProvisionalHashWarning` once per type + per process. A new library value type must be added to that list. The list never enters a + hash, identifier or stored path. ## User classes: exact and structural diff --git a/CLAUDE.md b/CLAUDE.md index 0bebc5ade..6927abaa2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -331,6 +331,11 @@ Developer, November 12, 2026 (ITL-981); see `docs/concepts/arrays.md` and `allow_pickle=False` on write and read; never raise `max_header_size`). Options: `memory_order`, `byte_order`, `struct_layout`. Where stored arrays must be ordered, order them by value hash (`storage_sort_key`), never by stored bytes. +- Library value types are provisional until the v0.4 audit (developer, November 13, 2026, + ITL-1068): their extension names are listed in `provisional` of + `contexts/logical_types.json`, and using one issues `ProvisionalHashWarning` once per type + per process. A new library value type must be added to that list. The list never enters a + hash, identifier or stored path. ## User classes: exact and structural diff --git a/changelog.d/ITL-1068.md b/changelog.d/ITL-1068.md new file mode 100644 index 000000000..57c9ef49e --- /dev/null +++ b/changelog.d/ITL-1068.md @@ -0,0 +1,17 @@ +### Added + +- `orcapod.errors.ProvisionalHashWarning`, a `UserWarning` issued once per type per process + when a library value type is first used (ITL-1068). Silence it with + `warnings.filterwarnings("ignore", category=ProvisionalHashWarning)`. + +### Known limitations + +- The value hashes and logical type identifiers of the library value types are provisional: + `numpy.ndarray`, `torch.Tensor`, and pandas and polars `DataFrame` and `Series`. They will + be reviewed for v0.4 and may change then; results computed from such values under v0.3 may + then be computed again (nothing stored is lost or wrong). Using one of these types issues + `ProvisionalHashWarning` once per type per process: a value hashed, a source column, a + function pod parameter or output, or stored results read back. The types are listed in the + `provisional` list of `orcapod/contexts/logical_types.json`; the list changes no hash, + identifier or stored path. See "Provisional value hashes" in `docs/concepts/arrays.md` + (ITL-1068). diff --git a/design_issues/ITL-1068-PH1.md b/design_issues/ITL-1068-PH1.md new file mode 100644 index 000000000..7fce62f6d --- /dev/null +++ b/design_issues/ITL-1068-PH1.md @@ -0,0 +1,20 @@ +## `src/orcapod/logical_types/` + +### PH1 — Library value type hashes are provisional until the v0.4 audit +**Status:** in progress +**Severity:** medium +**Issue:** ITL-1068 + +The value hashes and logical type identifiers of the library value types +(`orcapod.numpy.ndarray`, `orcapod.torch.tensor`, `orcapod.pandas.dataframe`, +`orcapod.pandas.series`, `orcapod.polars.dataframe`, `orcapod.polars.series`) are not yet +guaranteed stable across releases. ITL-1069 audits them for v0.4 against the principle "for +anything maintained or yielded by another system, respect every distinction it has; make no +assumption about what matters semantically". Until then v0.3 warns whenever one is used. + +**Fix (interim, ITL-1068):** the types are listed by extension name in `provisional` of +`contexts/logical_types.json`. The logical type registry (`get_by_python_type`, +`get_by_arrow_extension_name`) and the value handler registry (`get_handler`) of each data +context hold that list and issue `orcapod.errors.ProvisionalHashWarning` once per type per +process (`logical_types/provisional.py`). The list enters no hash, identifier or stored path. +The entry is resolved when ITL-1069 settles the hashes and removes the names from the list. diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index 9cc1efce9..15aeb2eac 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -42,6 +42,36 @@ for tag, data in scale.pod(source).iter_data(): The v0.2 tables and rows stay on disk unchanged, readable with v0.2. A migration is planned after v0.3 (ITL-889). +## Provisional value hashes + +The value hashes and stored identities of the library value types are provisional in v0.3: +`numpy.ndarray`, `torch.Tensor`, and the pandas and polars `DataFrame` and `Series`. They +will be reviewed for v0.4 and may change then. Every other type, and the hashing of everything +else in a pipeline, is unaffected. + +Using one of these types issues `orcapod.errors.ProvisionalHashWarning` once per type per +process, on whichever use comes first: a value hashed, a source column of the type, a function +pod with a parameter or output of the type, or stored results of the type read back. It names +the type, for example `orcapod.numpy.ndarray`. + +In practice nothing needs to change. Use these types as described on this page. If v0.4 +changes how one of them is hashed, a result Orcapod computed under v0.3 from such a value gets +a different identity, so v0.4 computes it again instead of reusing it. Nothing stored is lost +or becomes wrong: the v0.3 rows stay where they are. Plan for that recomputation if a +pipeline is expensive. To silence the warning once you have taken note of it, filter its class +after importing Orcapod: + +```python +import warnings + +from orcapod.errors import ProvisionalHashWarning + +warnings.filterwarnings("ignore", category=ProvisionalHashWarning) +``` + +Python's `-W` option and `PYTHONWARNINGS` cannot name this class: they are read before +installed packages can be imported. + ## What counts as the same array Two arrays are the same value when everything below that counts is equal. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 107623b94..59a7a3d6e 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -485,6 +485,8 @@ moves no path. ### 1e. Polars DataFrame and Series values +This hash is provisional in v0.3 (see [section 8](#8-array-value-hash)). + A `polars.DataFrame` or `polars.Series` is a value-hashed logical type (`PolarsDataFrameHandler`, `PolarsSeriesHandler`; the algorithm is defined in `orcapod.logical_types.polars_type`). Inside a table its cell token is, like every @@ -914,6 +916,15 @@ in any data-level preimage; they serve logging, observability, and result inspec ## 8. Array Value Hash +**Provisional.** The value hashes of the library value types, `numpy.ndarray` (this section), +`torch.Tensor` ([below](#torch-tensor-value-hash)), polars `DataFrame` and `Series` +([1e](#1e-polars-dataframe-and-series-values)) and pandas `DataFrame` and `Series`, and their +logical type identifiers, are provisional in v0.3 and may change in v0.4. Using one issues +`ProvisionalHashWarning` once per type per process (see +[Provisional value hashes](../concepts/arrays.md#provisional-value-hashes)). The types are +declared in the `provisional` list of `orcapod/contexts/logical_types.json`, by extension name; +that list enters no hash, identifier or stored path. + **Value hash labels.** A type whose values are hashed by a dedicated handler labels its hash `_v`, for example `ndarray_v1` or `torch_tensor_v1`. The label is the `ContentHash.method`; the type's implementer chooses it. The bytes produced under an existing label never change. A new @@ -1376,6 +1387,8 @@ print(digest.method, digest.digest[:3].hex()) ## 10. pandas Value Hash (`pandas_dataframe_v1`, `pandas_series_v1`) +This hash is provisional in v0.3 (see [section 8](#8-array-value-hash)). + A `pandas.DataFrame` or `pandas.Series` value (a pod input or output, a value in a source column, or any object given to the semantic hasher) is hashed by `pandas_frame_content_hash` / `pandas_series_content_hash` in diff --git a/pytest.ini b/pytest.ini index 88d892c46..a7501fe1e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,3 +9,8 @@ markers = postgres: mark test as requiring a live PostgreSQL instance integration: mark test as an integration test (may be slower) minio: tests requiring a running MinIO instance (via testcontainers or MINIO_ENDPOINT env var) +# The library value types warn that their hashes are provisional (ITL-1068), once per +# type per process, so whichever test uses one first would show it. The tests of the +# warning record it explicitly. +filterwarnings = + ignore::orcapod.errors.ProvisionalHashWarning diff --git a/src/orcapod/contexts/logical_types.json b/src/orcapod/contexts/logical_types.json index 7088e7855..5aa387a47 100644 --- a/src/orcapod/contexts/logical_types.json +++ b/src/orcapod/contexts/logical_types.json @@ -1,5 +1,5 @@ { - "description": "Logical types registered in every data context, with the value handlers that hash their Python values. This file is not frozen: a logical type is versioned by its own identifier (Arrow extension name and metadata), not by the data context version, so adding one edits this file and no released data context manifest. hash_ignore lists, per extension name, the property names the value hash of a library value type ignores (empty: every property the storage keeps is part of the hash); the list is written into the type's extension metadata, so changing it changes the type's identifier. The logical type entry and the value handler entry of such a type both refer to their list as hash_ignore:.", + "description": "Logical types registered in every data context, with the value handlers that hash their Python values. This file is not frozen: a logical type is versioned by its own identifier (Arrow extension name and metadata), not by the data context version, so adding one edits this file and no released data context manifest. hash_ignore lists, per extension name, the property names the value hash of a library value type ignores (empty: every property the storage keeps is part of the hash); the list is written into the type's extension metadata, so changing it changes the type's identifier. The logical type entry and the value handler entry of such a type both refer to their list as hash_ignore:. provisional lists, by extension name, the types whose value hash and stored identity may still change (until the v0.4 audit): using one issues ProvisionalHashWarning once per type per process. Deleting a name stops its warning and changes nothing else; the list never enters an identifier, a hash or a stored path. A new library value type is declared provisional here.", "hash_ignore": { "orcapod.numpy.ndarray": [], "orcapod.pandas.dataframe": [], @@ -8,6 +8,14 @@ "orcapod.polars.series": [], "orcapod.torch.tensor": [] }, + "provisional": [ + "orcapod.numpy.ndarray", + "orcapod.pandas.dataframe", + "orcapod.pandas.series", + "orcapod.polars.dataframe", + "orcapod.polars.series", + "orcapod.torch.tensor" + ], "logical_type_registry": { "_class": "orcapod.logical_types.registry.LogicalTypeRegistry", "_config": { @@ -124,7 +132,10 @@ }, "category": "set" } - ] + ], + "provisional": { + "_ref": "provisional" + } } }, "value_handlers": [ diff --git a/src/orcapod/contexts/registry.py b/src/orcapod/contexts/registry.py index 983ae2551..020cafee1 100644 --- a/src/orcapod/contexts/registry.py +++ b/src/orcapod/contexts/registry.py @@ -373,6 +373,10 @@ def get_context(self, context_string: str | None = None) -> DataContext: # ``hash_ignore`` object is referred to as ``hash_ignore:``. _HASH_IGNORE_REF_PREFIX = "hash_ignore:" + # The registration file's ``provisional`` list (extension names of the types + # whose value hash is provisional, ITL-1068) is referred to by this name. + _PROVISIONAL_REF = "provisional" + def _uses_logical_type_registration(self, spec: dict[str, Any]) -> bool: """Whether ``spec`` takes its logical types from the registration file. @@ -395,7 +399,8 @@ def _load_logical_type_registration(self) -> dict[str, Any]: of ``[type, handler]`` object-spec pairs and, when present, its ``deferred_value_handlers`` list of handlers loaded on first use and its ``hash_ignore`` object: per extension name, the property names the - value hash of that type ignores. + value hash of that type ignores; and its ``provisional`` list: the + extension names of the types whose value hash is provisional. """ return _read_json_file(self.logical_types_file) @@ -433,6 +438,19 @@ def _create_context_from_spec(self, spec: dict[str, Any]) -> DataContext: f"names, got {ignore!r}" ) ref_lut[f"{self._HASH_IGNORE_REF_PREFIX}{name}"] = list(ignore) + # The types whose value hash is provisional are declared once, by + # extension name; the logical type registry takes the list as + # ``{"_ref": "provisional"}`` and the value handler registry below + # is given it. It never enters an identifier or a hash (ITL-1068). + provisional = registration.get(self._PROVISIONAL_REF, []) + if not isinstance(provisional, list) or not all( + isinstance(item, str) for item in provisional + ): + raise ValueError( + f"provisional must be a list of extension names, got " + f"{provisional!r}" + ) + ref_lut[self._PROVISIONAL_REF] = list(provisional) ref_lut[self._LOGICAL_TYPE_REGISTRY_REF] = parse_objectspec( registration["logical_type_registry"], ref_lut=ref_lut ) @@ -462,6 +480,7 @@ def _create_context_from_spec(self, spec: dict[str, Any]) -> DataContext: entry["handler"], parse_objectspec(entry.get("_config", {}), ref_lut=ref_lut), ) + handler_registry.declare_provisional(ref_lut[self._PROVISIONAL_REF]) return DataContext( context_key=context_key, diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index b5379e262..dc557f3c0 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -379,6 +379,25 @@ class StoreContentWarning(UserWarning): """ +class ProvisionalHashWarning(UserWarning): + """A value type was used whose value hash and stored identity are provisional. + + The library value types (``numpy.ndarray``, ``torch.Tensor``, pandas and polars + ``DataFrame`` and ``Series``) are hashed and stored with formats that may still + change in v0.4. A result computed now from such a value may then get another hash, + so it is computed again after the upgrade: nothing stored is lost or wrong. The + warning is issued once per type per process, on whichever use comes first: a value + hashed, a column converted to or from Arrow, a function pod annotated with the type, + a stored column read. Which types are provisional is declared in the ``provisional`` + list of ``orcapod/contexts/logical_types.json``. + + To silence it, call ``warnings.filterwarnings("ignore", + category=ProvisionalHashWarning)`` after importing Orcapod. Python's ``-W`` option + and ``PYTHONWARNINGS`` cannot name this class: they are read before installed + packages can be imported. + """ + + class EmptyDataAccessError(Exception): """Raised when a payload-access method is called on an ``EmptyData`` instance. diff --git a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py index d1617ada8..229228efb 100644 --- a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py +++ b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py @@ -38,9 +38,17 @@ ) from orcapod.hashing.semantic_hashing.encoded import Encoded from orcapod.hashing.semantic_hashing.function_info_extractors import ParameterDefault -from orcapod.logical_types.array_format import NDARRAY_OPTIONS +from orcapod.logical_types.array_format import NDARRAY_EXTENSION_NAME, NDARRAY_OPTIONS from orcapod.logical_types.identifier import logical_type_identifier from orcapod.logical_types.ignore_list import IgnoreListHandler +from orcapod.logical_types.pandas_type import ( + LogicalPandasDataFrame, + LogicalPandasSeries, +) +from orcapod.logical_types.polars_type import ( + POLARS_DATAFRAME_EXTENSION_NAME, + POLARS_SERIES_EXTENSION_NAME, +) from orcapod.types import ContentHash, Schema from orcapod.utils.canonical_type_names import canonical_type_name @@ -429,6 +437,7 @@ class NumpyArrayHandler(IgnoreListHandler): array is hashed (the data context's list for the type). """ + EXTENSION_NAME = NDARRAY_EXTENSION_NAME OPTIONS = NDARRAY_OPTIONS HASH_LABEL = "ndarray_v1" """``ContentHash.method`` of an ndarray hash.""" @@ -471,6 +480,7 @@ class PandasDataFrameHandler(IgnoreListHandler): the registry were constructed before the arrow hasher). """ + EXTENSION_NAME = LogicalPandasDataFrame.EXTENSION_NAME OPTIONS = PANDAS_DATAFRAME_OPTIONS HASH_LABEL = PANDAS_DATAFRAME_HASH_LABEL @@ -539,6 +549,7 @@ class PandasSeriesHandler(IgnoreListHandler): the registry were constructed before the arrow hasher). """ + EXTENSION_NAME = LogicalPandasSeries.EXTENSION_NAME OPTIONS = PANDAS_SERIES_OPTIONS HASH_LABEL = PANDAS_SERIES_HASH_LABEL @@ -636,6 +647,7 @@ class PolarsDataFrameHandler(IgnoreListHandler): ``struct_field_order``. """ + EXTENSION_NAME = POLARS_DATAFRAME_EXTENSION_NAME OPTIONS = frozenset({"column_order", "struct_field_order"}) HASH_LABEL = "polars_dataframe_v1" """``ContentHash.method`` of a ``pl.DataFrame`` hash.""" @@ -677,6 +689,7 @@ class PolarsSeriesHandler(IgnoreListHandler): ``struct_field_order``. """ + EXTENSION_NAME = POLARS_SERIES_EXTENSION_NAME OPTIONS = frozenset({"series_name", "struct_field_order"}) HASH_LABEL = "polars_series_v1" """``ContentHash.method`` of a ``pl.Series`` hash.""" diff --git a/src/orcapod/hashing/semantic_hashing/torch_handler.py b/src/orcapod/hashing/semantic_hashing/torch_handler.py index 9af8a41c6..554c24a0a 100644 --- a/src/orcapod/hashing/semantic_hashing/torch_handler.py +++ b/src/orcapod/hashing/semantic_hashing/torch_handler.py @@ -11,6 +11,7 @@ from orcapod.logical_types.ignore_list import IgnoreListHandler from orcapod.logical_types.torch_type import ( OPTIONS, + TORCH_TENSOR_EXTENSION_NAME, TORCH_TENSOR_HASH_LABEL, tensor_digest, ) @@ -29,6 +30,7 @@ class TorchTensorHandler(IgnoreListHandler): elements, also inside an untyped Python structure. """ + EXTENSION_NAME = TORCH_TENSOR_EXTENSION_NAME OPTIONS = OPTIONS HASH_LABEL = TORCH_TENSOR_HASH_LABEL diff --git a/src/orcapod/hashing/semantic_hashing/type_handler_registry.py b/src/orcapod/hashing/semantic_hashing/type_handler_registry.py index 61894efe6..314b80b56 100644 --- a/src/orcapod/hashing/semantic_hashing/type_handler_registry.py +++ b/src/orcapod/hashing/semantic_hashing/type_handler_registry.py @@ -9,9 +9,10 @@ import logging import threading -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any +from orcapod.logical_types.provisional import warn_provisional from orcapod.utils.deferred_types import load_object, qualified_name if TYPE_CHECKING: @@ -48,6 +49,8 @@ def __init__( self._deferred: dict[str, dict[str, Any]] = {} self._handlers: dict[type, "PythonTypeHandlerProtocol"] = {} self._lock = threading.RLock() + # Extension names of the types whose value hash is provisional (ITL-1068). + self._provisional: frozenset[str] = frozenset() if handlers: for entry in handlers: # Skip empty or incomplete pairs. When a handler pair in the context @@ -124,16 +127,38 @@ def unregister(self, target_type: type) -> bool: return True return False + def declare_provisional(self, extension_names: Iterable[str]) -> None: + """Declare the types whose value hash is provisional (ITL-1068). + + A handler whose ``EXTENSION_NAME`` is declared issues + ``ProvisionalHashWarning`` the first time in the process that ``get_handler`` + returns it for a value. The data context loader passes the ``provisional`` + list of ``contexts/logical_types.json``; the declaration never enters a hash. + + Args: + extension_names: The extension names declared provisional. They replace + any earlier declaration. + """ + with self._lock: + self._provisional = frozenset(extension_names) + def get_handler(self, obj: Any) -> "PythonTypeHandlerProtocol | None": """Look up the handler for *obj* using MRO-aware resolution. + A handler of a type declared provisional (``declare_provisional``) issues + ``ProvisionalHashWarning`` on its first use in the process. + Args: obj: The object for which a handler is needed. Returns: The registered ``PythonTypeHandlerProtocol``, or None. """ - return self.get_handler_for_type(type(obj)) + handler = self.get_handler_for_type(type(obj)) + extension_name = getattr(handler, "EXTENSION_NAME", None) + if extension_name is not None and extension_name in self._provisional: + warn_provisional(extension_name) + return handler def get_handler_for_type( self, target_type: type diff --git a/src/orcapod/logical_types/ignore_list.py b/src/orcapod/logical_types/ignore_list.py index 6ead1d736..1a4d0396b 100644 --- a/src/orcapod/logical_types/ignore_list.py +++ b/src/orcapod/logical_types/ignore_list.py @@ -408,12 +408,15 @@ def get_polars_extension_type(self) -> "pl.BaseExtension": class IgnoreListHandler: """Base of a value hasher that honours an ignore list. - A subclass sets ``OPTIONS`` and ``HASH_LABEL`` and implements ``digest``. - A bare value is hashed with the list the handler was configured with (the + A subclass sets ``EXTENSION_NAME``, ``OPTIONS`` and ``HASH_LABEL`` and + implements ``digest``. A bare value is hashed with the list the handler was configured with (the data context's). A cell of a typed column is hashed with the list of the column's own extension metadata, through ``hash_ignoring``. Attributes: + EXTENSION_NAME: The extension name of the logical type of the values, + the same constant as the logical type's. The handler registry checks + it against the types declared provisional (ITL-1068). OPTIONS: The closed set of property names the hash can ignore; the same set as the logical type's. HASH_LABEL: The label of the hash algorithm (``ContentHash.method``). @@ -425,6 +428,7 @@ class IgnoreListHandler: ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. """ + EXTENSION_NAME: ClassVar[str] OPTIONS: ClassVar[frozenset[str]] HASH_LABEL: ClassVar[str] diff --git a/src/orcapod/logical_types/provisional.py b/src/orcapod/logical_types/provisional.py new file mode 100644 index 000000000..553f0c76c --- /dev/null +++ b/src/orcapod/logical_types/provisional.py @@ -0,0 +1,74 @@ +"""The warning that a value type's hash is provisional (ITL-1068). + +The library value types are declared provisional in the ``provisional`` list of +``orcapod/contexts/logical_types.json``; the logical type registry and the value +handler registry of a data context hold that list and call ``warn_provisional`` +when they resolve a declared type. This module only keeps the once-per-type +bookkeeping, which is process-wide. +""" + +from __future__ import annotations + +import sys +import threading +import warnings +from types import FrameType + +from orcapod.errors import ProvisionalHashWarning + +_lock = threading.RLock() +_issued: set[str] = set() + + +def _outside_orcapod_stacklevel() -> int: + """Return the stacklevel of the first frame outside Orcapod, as seen from the caller. + + Level 1 is the caller of this function (``warn_provisional``). When every frame + is Orcapod's (a worker thread), the caller of ``warn_provisional`` is used. + """ + level = 1 + frame: FrameType | None = sys._getframe(1) # pyright: ignore[reportPrivateUsage] # as the warnings module does + while frame is not None: + module = str(frame.f_globals.get("__name__", "")) + if module != "orcapod" and not module.startswith("orcapod."): + return level + frame = frame.f_back + level += 1 + return 2 + + +def warn_provisional(extension_name: str) -> None: + """Issue ``ProvisionalHashWarning`` for a type, once per process. + + The type is recorded only after ``warnings.warn`` returns, so under an error + filter every use raises. The warning is attributed to the first frame outside + Orcapod, the code that used the type. + + Args: + extension_name: The Arrow extension name of a type declared provisional. + """ + if extension_name in _issued: + return + with _lock: + if extension_name in _issued: + return + warnings.warn( + f"{extension_name} values are provisional: their value hash and stored " + "identity may change in Orcapod v0.4, and results computed from them " + "before then may be recomputed after the upgrade (nothing stored is lost " + "or wrong). See 'Provisional value hashes' in docs/concepts/arrays.md. To " + "silence: warnings.filterwarnings('ignore', " + "category=orcapod.errors.ProvisionalHashWarning).", + ProvisionalHashWarning, + stacklevel=_outside_orcapod_stacklevel(), + ) + _issued.add(extension_name) + + +def reset_provisional_warnings() -> None: + """Forget which types have warned, so each warns again on its next use. + + A test hook: a test that checks the warning starts as a new process would. + """ + with _lock: + _issued.clear() diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index 3b50948df..b8a1ddc4e 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -14,6 +14,7 @@ from orcapod.errors import MissingOptionalDependencyError from orcapod.logical_types.identifier import LogicalTypeIdentifierError from orcapod.logical_types.identifier import LogicalTypeIdentifier +from orcapod.logical_types.provisional import warn_provisional from orcapod.logical_types.protocols import ( LogicalTypeProtocol, LogicalTypeFactoryProtocol, @@ -270,6 +271,13 @@ class LogicalTypeRegistry: for example the ignore list of its value hash). See ``orcapod.utils.deferred_types``. + An optional ``provisional`` collection names, by extension name, the types + whose value hash and stored identity may still change (ITL-1068): looking one + up (``get_by_python_type``, ``get_by_arrow_extension_name``) issues + ``ProvisionalHashWarning`` once per type per process. It is read from the + ``provisional`` list of ``contexts/logical_types.json`` and never enters an + identifier or a hash. + An optional ``factories`` list can also be passed to pre-register ``LogicalTypeFactoryProtocol`` instances at construction time. Each entry is a dict with keys ``factory`` (the factory instance), ``category`` (optional str), @@ -289,7 +297,10 @@ def __init__( logical_types: list[LogicalTypeProtocol] | None = None, factories: list[dict] | None = None, deferred_logical_types: list[dict[str, Any]] | None = None, + provisional: Iterable[str] = (), ) -> None: + # Extension names of the types whose value hash is provisional (ITL-1068). + self._provisional: frozenset[str] = frozenset(provisional) # Serialises loading a deferred type, so concurrent first lookups load it once. self._deferred_lock = threading.Lock() self._deferred_by_python_name: dict[str, dict[str, Any]] = {} @@ -458,6 +469,14 @@ def get_by_logical_name(self, name: str) -> LogicalTypeProtocol | None: def get_by_python_type(self, python_type: type) -> LogicalTypeProtocol | None: """Return the logical type for *python_type*, or ``None``. + A type declared provisional issues ``ProvisionalHashWarning`` on its first + lookup in the process (``_find_by_python_type`` gives the rules). + """ + return self._noted(self._find_by_python_type(python_type)) + + def _find_by_python_type(self, python_type: type) -> LogicalTypeProtocol | None: + """Return the logical type for *python_type*, or ``None``. + The most specific type wins: *python_type* itself, then each class of its MRO in order, and for each class a registered type before a deferred one (which is loaded), so a subclass registered with its own logical @@ -518,6 +537,31 @@ def get_by_arrow_extension_name( ) -> LogicalTypeProtocol | None: """Return the logical type registered under *arrow_name*, or ``None``. + A type declared provisional issues ``ProvisionalHashWarning`` on its first + lookup in the process. + + Raises: + MissingOptionalDependencyError: If *arrow_name* belongs to a deferred + logical type whose required module is not installed. + """ + return self._noted(self._find_by_arrow_extension_name(arrow_name)) + + def _noted( + self, logical_type: LogicalTypeProtocol | None + ) -> LogicalTypeProtocol | None: + """Warn when *logical_type* is declared provisional; return it unchanged.""" + if ( + logical_type is not None + and logical_type.logical_type_name in self._provisional + ): + warn_provisional(logical_type.logical_type_name) + return logical_type + + def _find_by_arrow_extension_name( + self, arrow_name: str + ) -> LogicalTypeProtocol | None: + """Return the logical type registered under *arrow_name*, or ``None``. + A deferred logical type (see the class docstring) is loaded on the first lookup of its extension name. diff --git a/superpowers/plans/2026-11-13-itl-1068-provisional-hash-warning.md b/superpowers/plans/2026-11-13-itl-1068-provisional-hash-warning.md new file mode 100644 index 000000000..bec375b20 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1068-provisional-hash-warning.md @@ -0,0 +1,103 @@ +# ITL-1068: warn that library value types are provisional + +Branch `eywalker/itl-1068-warn-that-library-value-types-numpy-torch`. Refs ITL-1069 (v0.4 +audit), ITL-1054. PR #339 (pandas handlers become `IgnoreListHandler`, pandas joins +`hash_ignore`) must be merged into this branch before the pandas part is finished. + +Status (2026-11-13): approved with notes (test "one line removes a type"; practical docs +guidance; ITL-1069 only in design_issues; API reference page for warnings: none exists). +Implemented and pushed: class, helper, declaration, both registries, tests, docs, changelog, +rules, design issue. Revision of (b): the declared list is held per data context (logical type +registry `_config` ref + `PythonTypeHandlerRegistry.declare_provisional`), not process-global, +so a context without a name does not warn for it; only the issued set is process-global. +Stored-read test reads through `PipelineJob.load` + node `as_table` (a raw Delta read never +resolves the logical type). Remaining: merge main once #339 lands, then add +`EXTENSION_NAME = LogicalPandasDataFrame.EXTENSION_NAME` / `LogicalPandasSeries...` to the +two pandas handlers in builtin_handlers.py and a one-line pointer in hashing.md section 10; +verified on a trial merge (41/41 new tests, hashing/logical_types/contexts suites green). + +## (a) Warning class + +`ProvisionalHashWarning(UserWarning)` in `orcapod/errors.py`, beside +`ValueConversionWarning`, `RowFailuresWarning`, `StoreContentWarning` (all `UserWarning` +subclasses; none re-exported at top level). Public as `orcapod.errors.ProvisionalHashWarning`. +`UserWarning`, not `FutureWarning`: `FutureWarning` means "an API you call is deprecated / +will change behaviour"; this is a stability notice about stored data, and the three existing +Orcapod warnings are `UserWarning` subclasses. Both show by default; the class is what a user +filters on. + +## (b) The one declaration + +Top-level key `"provisional"` in `contexts/logical_types.json` (not frozen), beside +`hash_ignore`: a list of extension names, one per line. Removing a type = deleting its line. + +Not a per-entry field: value handler entries are `[type, handler]` pairs with no slot, and a +`_config` key would be passed to the constructors, i.e. next to what builds the identifier. + +`contexts/registry._create_context_from_spec` validates it (list of strings) and calls +`logical_types/provisional.declare_provisional(names)`: a process-global set (the +once-per-process bookkeeping is process-global anyway). It is passed to no constructor, so it +cannot reach extension metadata, an identifier, a hash or a path. Proof: golden suite and +`test_hash_label_rule` unchanged; plus a test that the identifiers and value hashes of all six +types are equal with and without the declaration. + +## (c) Emit points + +Helper module `orcapod/logical_types/provisional.py`: + +- `declare_provisional(names)`, `warn_if_provisional(extension_name)`, + `reset_provisional_warnings()` (test hook: clears the issued set). +- `threading.RLock` around check / warn / record. The name is recorded only after + `warnings.warn` returns, so an error filter raises on every use (as `ValueConversionWarning` + does). +- stacklevel: walk `sys._getframe` to the first frame outside the `orcapod` package; fall + back to stacklevel 2 when none (worker threads). + +Call sites (every use resolves the type through one of these): + +1. `LogicalTypeRegistry.get_by_python_type` and `get_by_arrow_extension_name`: the only ways + the type converter (17 call sites in `universal_converter.py`) and the database hooks (2) + resolve a logical type. Covers source columns (Python values or Arrow extension columns), + function pod input/output annotations (resolved when the pod is built), Arrow to Python + conversion, reading stored columns. `get_by_logical_name` and registration do not warn. +2. `IgnoreListHandler.handle` and `hash_ignoring` (base of all six value handlers once #339 + is merged): a bare value hashed by the semantic hasher, which never touches the logical type + registry. Each handler gets `EXTENSION_NAME` taken from its logical type class's constant + (no new literal). + +## (d) Message + +" values are provisional in Orcapod 0.3: their value hash and stored identity may +change in v0.4, and results cached now may then be recomputed. See 'Provisional value hashes' +in docs/concepts/arrays.md. To silence: warnings.filterwarnings('ignore', +category=orcapod.errors.ProvisionalHashWarning)." + +## (e) What must not warn; CI noise + +- `import orcapod`: no lookups at import (subprocess test). +- Pipelines of primitive / Arrow / Path / user-class columns: never resolve these names (test). +- `pytest.ini`: `filterwarnings = ignore::orcapod.errors.ProvisionalHashWarning` (one class). + New tests: autouse fixture `reset_provisional_warnings()` and + `catch_warnings(record=True)` + `simplefilter("always")` / `pytest.warns`. +- Existing tests with strict filters over these types are order-dependent once the warning + exists (first use per process warns): `test_output_conformance.py` 488/511/524, + `test_array_value_format.py` 263, `test_tensor_conversion_warning.py`, + `test_job_run_summary.py` 151, `test_workshop_scenarios.py` 44. Each gets an explicit + `filterwarnings("ignore", category=ProvisionalHashWarning)` inside its block; no loosening. +- Docs samples run in a subprocess and compare stdout only; warnings go to stderr. + +## (f) Docs and changelog + +- `docs/concepts/arrays.md`: section "Provisional value hashes" (anchor named in the message). +- `docs/reference/hashing.md` section 8 (array, torch) and 1e (polars), pandas section: a note. +- `changelog.d/ITL-1068.md` under `### Known limitations`. +- `CLAUDE.md` + `.zed/rules`: one rule. + +## Steps (after approval) + +1. Failing tests `tests/test_logical_types/test_provisional_warning.py`. +2. Warning class, helper, declaration, loader, two emit sites. +3. Adjust the strict-filter tests listed above; pytest.ini filter. +4. Docs, changelog, CLAUDE.md, .zed/rules. +5. Merge main after #339; pandas handlers. +6. Targeted tests (3.12), push, PR, Copilot review. diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index a6f42a2fe..7e9a2b045 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -20,6 +20,7 @@ from orcapod import function_pod from orcapod.contexts import get_default_context +from orcapod.errors import ProvisionalHashWarning from orcapod.logical_types.numpy_type import dtype_to_tag from orcapod.operators import MergeJoin from orcapod.sources import ArrowTableSource, DictSource @@ -261,6 +262,9 @@ def test_dtype_metadata_is_dropped_without_a_warning() -> None: with warnings.catch_warnings(): warnings.simplefilter("error") + # The array type's own provisional-hash notice (ITL-1068) is not about + # dtype metadata; it is issued on the first use in a process. + warnings.filterwarnings("ignore", category=ProvisionalHashWarning) got = read_back(with_metadata) assert got.dtype.metadata is None assert got.tobytes() == plain.tobytes() diff --git a/tests/test_logical_types/test_provisional_hash_warning.py b/tests/test_logical_types/test_provisional_hash_warning.py new file mode 100644 index 000000000..e332d7d75 --- /dev/null +++ b/tests/test_logical_types/test_provisional_hash_warning.py @@ -0,0 +1,364 @@ +"""Library value types warn that their hashes are provisional (ITL-1068). + +The value hashes and stored identities of the library value types (numpy, torch, +pandas, polars) are not yet stable across releases; they are audited in v0.4. Until +then each of them issues ``ProvisionalHashWarning`` once per type per process, on +whichever path first uses it. Which types are provisional is declared once, in the +``provisional`` list of ``contexts/logical_types.json``. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import warnings +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import polars as pl +import pyarrow as pa +import pytest +import torch + +import orcapod.contexts as contexts_package +from orcapod import PipelineJob, function_pod +from orcapod.contexts import get_default_context +from orcapod.contexts.core import DataContext +from orcapod.contexts.registry import JSONDataContextRegistry +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import ProvisionalHashWarning + +# The test hook that forgets which types have warned; not a public entry point. +from orcapod.logical_types.provisional import reset_provisional_warnings +from orcapod.sources import DictSource + +REGISTRATION = Path(contexts_package.__file__).parent / "logical_types.json" + + +def _value(name: str) -> Any: + """A fresh value of the type whose extension name is ``name``.""" + if name == "orcapod.numpy.ndarray": + return np.arange(3.0) + if name == "orcapod.torch.tensor": + return torch.arange(3.0) + if name == "orcapod.pandas.dataframe": + return pd.DataFrame({"a": [1, 2]}) + if name == "orcapod.pandas.series": + return pd.Series([1, 2], name="a") + if name == "orcapod.polars.dataframe": + return pl.DataFrame({"a": [1, 2]}) + assert name == "orcapod.polars.series" + return pl.Series("a", [1, 2]) + + +PYTHON_TYPES: dict[str, type] = { + "orcapod.numpy.ndarray": np.ndarray, + "orcapod.torch.tensor": torch.Tensor, + "orcapod.pandas.dataframe": pd.DataFrame, + "orcapod.pandas.series": pd.Series, + "orcapod.polars.dataframe": pl.DataFrame, + "orcapod.polars.series": pl.Series, +} +NAMES = sorted(PYTHON_TYPES) + + +@pytest.fixture(autouse=True) +def _fresh_bookkeeping() -> Iterator[None]: + """Every test starts as a new process would: no type has warned yet.""" + reset_provisional_warnings() + yield + reset_provisional_warnings() + + +@contextmanager +def recorded() -> Iterator[list[warnings.WarningMessage]]: + """Record the ``ProvisionalHashWarning`` issued inside the block.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ProvisionalHashWarning) + found: list[warnings.WarningMessage] = [] + yield found + found.extend(w for w in caught if issubclass(w.category, ProvisionalHashWarning)) + + +def _warned_names(caught: list[warnings.WarningMessage]) -> list[str]: + return sorted( + name for w in caught for name in NAMES if f"{name} " in str(w.message) + ) + + +def _registration() -> dict[str, Any]: + return json.loads(REGISTRATION.read_text()) + + +def _context_with(tmp_path: Path, provisional: list[str] | None) -> DataContext: + """A data context whose registration declares ``provisional`` (None: no key).""" + registration = _registration() + if provisional is None: + del registration["provisional"] + else: + registration["provisional"] = provisional + custom = tmp_path / "logical_types.json" + custom.write_text(json.dumps(registration)) + return JSONDataContextRegistry(logical_types_file=custom).get_context() + + +def _quiet(action: Callable[[], Any]) -> Any: + """Run ``action`` without recording its warnings, then forget them.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ProvisionalHashWarning) + result = action() + reset_provisional_warnings() + return result + + +# --------------------------------------------------------------------------- +# The class and the declaration +# --------------------------------------------------------------------------- + + +def test_the_warning_is_a_public_user_warning() -> None: + assert issubclass(ProvisionalHashWarning, UserWarning) + assert ProvisionalHashWarning.__module__ == "orcapod.errors" + + +def test_the_six_library_value_types_are_declared_provisional() -> None: + assert sorted(_registration()["provisional"]) == NAMES + + +# --------------------------------------------------------------------------- +# Each entry path warns once per type +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", NAMES) +def test_hashing_a_bare_value_warns_once(name: str) -> None: + hasher = get_default_context().semantic_hasher + with recorded() as caught: + hasher.hash_object(_value(name)) + assert _warned_names(caught) == [name] + (warning,) = caught + # The warning points at the code that used the type, not at Orcapod. + assert warning.filename == __file__ + message = str(warning.message) + assert "v0.4" in message + assert "recomputed" in message + assert "ProvisionalHashWarning" in message + assert "docs/concepts/arrays.md" in message + + with recorded() as again: + hasher.hash_object(_value(name)) + hasher.hash_object(_value(name)) + assert again == [] + + +@pytest.mark.parametrize("name", NAMES) +def test_a_source_column_of_the_type_warns_once(name: str) -> None: + with recorded() as caught: + source = DictSource( + data=[{"k": i, "x": _value(name)} for i in range(3)], + tag_columns=["k"], + source_id="values", + ) + source.as_table() + list(source.iter_data()) + assert _warned_names(caught) == [name] + + with recorded() as again: + DictSource( + data=[{"k": 9, "x": _value(name)}], tag_columns=["k"], source_id="more" + ).as_table() + assert again == [] + + +def _producer(name: str) -> Any: + """A function pod ``make(n: int)`` whose output ``y`` is of the type ``name``.""" + + def make(n: int) -> Any: + return _value(name) + + make.__annotations__["return"] = PYTHON_TYPES[name] + return function_pod(output_keys="y")(make) + + +def _numbers() -> DictSource: + return DictSource( + data=[{"k": i, "n": i} for i in range(3)], tag_columns=["k"], source_id="n" + ) + + +@pytest.mark.parametrize("name", NAMES) +def test_a_function_pod_output_of_the_type_warns_once(name: str) -> None: + source = _quiet(_numbers) + with recorded() as caught: + pod = _producer(name) + pod.pod(source).as_table() + assert _warned_names(caught) == [name] + # Defining the pod is the use: the warning points at that line of user code. + assert caught[0].filename == __file__ + + +@pytest.mark.parametrize("name", NAMES) +def test_a_function_pod_input_of_the_type_warns_once(name: str) -> None: + def size(x: Any) -> int: + return len(x) + + size.__annotations__["x"] = PYTHON_TYPES[name] + with recorded() as caught: + function_pod(output_keys="n")(size) + assert _warned_names(caught) == [name] + + +@pytest.mark.parametrize("name", NAMES) +def test_reading_stored_results_of_the_type_warns_once( + name: str, tmp_path: Path +) -> None: + """A loaded job, whose pods are not saved, reads a stored column of the type.""" + + def run_and_save() -> None: + job = PipelineJob(name="values", store=DeltaTableDatabase(tmp_path / "store")) + with job: + _producer(name).pod(_numbers(), label="make") + job.run() + job.save(tmp_path / "job.json") + + _quiet(run_and_save) + with recorded() as caught: + loaded = PipelineJob.load(tmp_path / "job.json") + table = loaded.nodes["make"].as_table() + assert table.num_rows == 3 + assert _warned_names(caught) == [name] + + +def test_each_type_warns_once_in_one_pipeline() -> None: + """Six types used together: six warnings, one per type, however many rows.""" + with recorded() as caught: + source = DictSource( + data=[{"k": i, **{n: _value(n) for n in NAMES}} for i in range(4)], + tag_columns=["k"], + source_id="everything", + ) + source.as_table() + assert _warned_names(caught) == NAMES + + +# --------------------------------------------------------------------------- +# Behaviour of the warning +# --------------------------------------------------------------------------- + + +def test_an_error_filter_raises_on_every_use() -> None: + hasher = get_default_context().semantic_hasher + with warnings.catch_warnings(): + warnings.simplefilter("error", ProvisionalHashWarning) + for _ in range(2): + with pytest.raises(ProvisionalHashWarning): + hasher.hash_object(np.arange(2)) + + +def test_an_ignore_filter_silences_it() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + warnings.filterwarnings("ignore", category=ProvisionalHashWarning) + get_default_context().semantic_hasher.hash_object(np.arange(2)) + assert not [w for w in caught if issubclass(w.category, ProvisionalHashWarning)] + + +def test_concurrent_first_uses_warn_once() -> None: + hasher = get_default_context().semantic_hasher + barrier = threading.Barrier(8) + + def use() -> None: + barrier.wait() + hasher.hash_object(np.arange(3)) + + with recorded() as caught: + threads = [threading.Thread(target=use) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert _warned_names(caught) == ["orcapod.numpy.ndarray"] + + +# --------------------------------------------------------------------------- +# What does not warn +# --------------------------------------------------------------------------- + + +def test_importing_orcapod_does_not_warn() -> None: + code = ( + "import warnings\n" + "with warnings.catch_warnings(record=True) as caught:\n" + " warnings.simplefilter('always')\n" + " import orcapod\n" + " from orcapod.contexts import get_default_context\n" + " get_default_context()\n" + "print(sorted({w.category.__name__ for w in caught}))\n" + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert "ProvisionalHashWarning" not in proc.stdout, proc.stdout + + +def test_a_pipeline_without_these_types_does_not_warn(tmp_path: Path) -> None: + @function_pod(output_keys="y") + def scale(x: int, p: Path) -> float: + return x * 1.5 + + with recorded() as caught: + source = DictSource( + data=[{"k": i, "x": i, "p": tmp_path} for i in range(3)], + tag_columns=["k"], + source_id="plain", + ) + scale.pod(source).as_table() + get_default_context().semantic_hasher.hash_object([1, "a", 2.5]) + assert caught == [] + + +# --------------------------------------------------------------------------- +# The declaration is one line per type and changes no identity +# --------------------------------------------------------------------------- + + +def _identity(context: DataContext, name: str) -> tuple[str, bytes, str]: + arrow_type = context.type_converter.python_type_to_arrow_type(PYTHON_TYPES[name]) + assert isinstance(arrow_type, pa.ExtensionType) + return ( + arrow_type.extension_name, + arrow_type.__arrow_ext_serialize__(), + context.semantic_hasher.hash_object(_value(name)).to_string(), + ) + + +def test_removing_a_name_silences_that_type_only(tmp_path: Path) -> None: + kept = [n for n in NAMES if n != "orcapod.polars.series"] + context = _context_with(tmp_path, kept) + with recorded() as caught: + for name in NAMES: + context.semantic_hasher.hash_object(_value(name)) + assert _warned_names(caught) == kept + + +def test_the_declaration_changes_no_identifier_or_hash(tmp_path: Path) -> None: + undeclared = _quiet(lambda: _context_with(tmp_path, None)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ProvisionalHashWarning) + for name in NAMES: + assert _identity(undeclared, name) == _identity(get_default_context(), name) + + +def test_an_undeclared_type_never_warns(tmp_path: Path) -> None: + context = _context_with(tmp_path, []) + with recorded() as caught: + for name in NAMES: + context.semantic_hasher.hash_object(_value(name)) + context.type_converter.python_type_to_arrow_type(PYTHON_TYPES[name]) + assert caught == []