diff --git a/.zed/rules b/.zed/rules index a9740f662..5a4f84704 100644 --- a/.zed/rules +++ b/.zed/rules @@ -257,7 +257,11 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension - The identifier is declared in one place, the extension type returned by `get_arrow_extension_type()`, and read through `logical_types/identifier.py` (`logical_type_identifier`, `LogicalTypeIdentifier.text`). Never put a logical type's bare - name into a hash input: use the identifier text. + name into a hash input: use the identifier text. Exception (developer, November 12, 2026): + the leading type text inside a library value hash (`numpy.ndarray`, `torch.Tensor`, pandas, + polars) is the extension name alone, because the metadata holds the ignore list, which must + not enter the value's preimage; the full identifier (name plus metadata) still enters every + type, schema and cell hash. - Change a type's identifier when, and only when, an existing value would be stored or hashed differently. It is not a package version. Built-in types use `version_metadata(N)` (`{"version":N}`); a new built-in type must write one, never empty metadata. @@ -683,10 +687,14 @@ data. The rules: expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on every execution path (sync and async). - Function-output rule (developer, November 12, 2026): the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base and is stored as the base. + Function-output rule (developer, November 12, 2026): the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base. + What is stored for a subclass instance is decided by the logical type of the declared + class (developer, November 12, 2026, ITL-871): a plain user class is stored as the base + (the declared class's fields only; subclass-only fields are omitted and do not enter the + stored row hash), while the tensor type keeps the class (an `nn.Parameter` returned for + `-> torch.Tensor` is stored, read back and hashed as an `nn.Parameter`). Structural classes additionally accept equivalent structural identities. Numeric - narrowing and unrelated container types require explicit conversion. Base storage - omits subclass-only fields, and those fields do not enter the stored row hash. + narrowing and unrelated container types require explicit conversion. - Every output column takes the large-normalized form of its input type (arrow_utils.normalize_to_large_types: string to large_string, binary to large_binary, list diff --git a/CLAUDE.md b/CLAUDE.md index c3dc23676..c6a84eb91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -268,7 +268,11 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension - The identifier is declared in one place, the extension type returned by `get_arrow_extension_type()`, and read through `logical_types/identifier.py` (`logical_type_identifier`, `LogicalTypeIdentifier.text`). Never put a logical type's bare - name into a hash input: use the identifier text. + name into a hash input: use the identifier text. Exception (developer, November 12, 2026): + the leading type text inside a library value hash (`numpy.ndarray`, `torch.Tensor`, pandas, + polars) is the extension name alone, because the metadata holds the ignore list, which must + not enter the value's preimage; the full identifier (name plus metadata) still enters every + type, schema and cell hash. - Change a type's identifier when, and only when, an existing value would be stored or hashed differently. It is not a package version. Built-in types use `version_metadata(N)` (`{"version":N}`); a new built-in type must write one, never empty metadata. @@ -725,10 +729,14 @@ data. The rules: expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on every execution path (sync and async). - Function-output rule (developer, November 12, 2026): the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base and is stored as the base. + Function-output rule (developer, November 12, 2026): the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base. + What is stored for a subclass instance is decided by the logical type of the declared + class (developer, November 12, 2026, ITL-871): a plain user class is stored as the base + (the declared class's fields only; subclass-only fields are omitted and do not enter the + stored row hash), while the tensor type keeps the class (an `nn.Parameter` returned for + `-> torch.Tensor` is stored, read back and hashed as an `nn.Parameter`). Structural classes additionally accept equivalent structural identities. Numeric - narrowing and unrelated container types require explicit conversion. Base storage - omits subclass-only fields, and those fields do not enter the stored row hash. + narrowing and unrelated container types require explicit conversion. - Every output column takes the large-normalized form of its input type (`arrow_utils.normalize_to_large_types`: `string` to `large_string`, `binary` to `large_binary`, `list` diff --git a/changelog.d/ITL-871.md b/changelog.d/ITL-871.md new file mode 100644 index 000000000..bf8bd9697 --- /dev/null +++ b/changelog.d/ITL-871.md @@ -0,0 +1,77 @@ +### Added + +#### PyTorch tensors as data values (ITL-871) + +- **`torch.Tensor` is a value type**: a source column, a function pod parameter or return value, + stored, cached and re-run, sync and async, in memory and in Delta stores, and inside + `list[...]` (`Batch`, `GroupBy`, `MergeJoin`). + PyTorch is optional: support works whenever `torch` can be imported, however it was + installed; the new extra `orcapod[torch]` (`torch>=2.6`) installs it from PyPI. PyTorch 2.6 + is the first version whose restricted loader reads `uint16`, `uint32` and `uint64` + tensors. Tested with PyTorch 2.15. The PyTorch build must support the installed NumPy, because a tensor's + elements are hashed through NumPy. + `import orcapod` never imports PyTorch: the type is set up on first use, and reading a + stored tensor column without PyTorch raises `orcapod.errors.MissingOptionalDependencyError`. +- **Storage is PyTorch's own serialization.** Extension type `orcapod.torch.tensor` over + `large_binary`: one cell holds what `torch.save` writes for one tensor. A cell is read with + `torch.load(map_location="cpu", weights_only=True)`, PyTorch's restricted loader, always + with both arguments explicit and with no unrestricted fallback. Stored and read back: + dtype, shape, elements, `requires_grad`, strides (transposed and `channels_last` layouts), + the lazy conjugate and negative bits, and the class (`torch.Tensor` or + `torch.nn.Parameter`). Not stored: the device (a tensor reads back on the CPU) and the + autograd history (`grad_fn`, `.grad`; a tensor reads back as a leaf). +- **Only the passed elements are stored.** A tensor that does not cover its underlying buffer + exactly (a slice, a step slice, an expanded view) is stored as + `value.clone(memory_format=torch.preserve_format)`, so the rest of the buffer is never + written; such a view has the clone's strides and its lazy bits are resolved. The caller's + tensor is never modified. +- **The hash is strict and defined by Orcapod**, never on the stored bytes. + `TorchTensorHandler` returns `ContentHash("torch_tensor_v1", D)`, where + `D = SHA-256(lp("orcapod.torch.tensor") || E || requires_grad || conj_bit || neg_bit || ndim + || strides || lp(class name))` and `E` is the array value digest of the elements + (`docs/reference/hashing.md` §8). Dtype, shape and every element bit always count. + `requires_grad`, the memory layout, the lazy conjugate and negative bits and the class count + by default. Device and autograd history never count, because storage does not keep them. A + tensor and an ndarray with equal elements never hash the same, also inside an untyped + Python structure. +- **What the hash ignores is an option of the type.** The five optional properties + (`requires_grad`, `memory_layout`, `conj_bit`, `neg_bit`, `subclass`) can be named in the + type's ignore list: `hash_ignore` in `orcapod/contexts/logical_types.json`, empty by + default. The list is in the extension metadata (`{"ignore":[],"version":1}`), so it is part + of the logical type identifier, and a data context refuses to read a tensor column stored + under another list. An unknown option name raises when a tensor is first used. There is no + user API for the list in v0.3. +- **Dtypes.** The NumPy-shared dtypes, plus `bfloat16`, `complex32` (real then imaginary + `float16`) and `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, + `float8_e8m0fnu`, hashed as raw bits under the PyTorch dtype name. +- **`orcapod.errors.ValueConversionWarning`** (a `UserWarning`) reports a tensor on a non-CPU + device or with autograd history (a `grad_fn`), which is stored as a CPU leaf tensor. It + names the function and output (or the source and column), what was converted and how to + avoid it (`value.detach().cpu()`). A function warns once per output and conversion, however + many rows it processes; `DictSource` and `ListSource` warn once per column with the count. + A leaf tensor with `requires_grad=True` and an `nn.Parameter` do not warn. After + `warnings.simplefilter("error", ValueConversionWarning)` every conversion raises (Python's + `-W` option cannot name the class: it is read before installed packages are importable). +- **Refused at write time**, naming the column and reason: sparse, mkldnn, quantized, nested + and meta tensors, named tensors, other dtypes (sub-byte and packed types, `bcomplex32`), + and subclasses other than `torch.nn.Parameter` (the restricted loader cannot read them + back). Such a subclass is not hashed as a tensor either: a subclass with its own logical + type needs its own value handler. +- **Declared outputs.** A pod declared `-> torch.Tensor` must return a tensor; an ndarray or a + number raises `OutputTypeError`. A `torch.nn.Parameter` is accepted and is stored, read + back and hashed as an `nn.Parameter`. A pod declared with a subclass must return that + subclass. +- **Registration on first use.** `LogicalTypeRegistry(deferred_logical_types=...)` and + `PythonTypeHandlerRegistry.register_deferred(...)` name a value type by its qualified class + name and extension name and load it on first use, with the configuration (the ignore list) + given in the entry. The tensor entries are in `orcapod/contexts/logical_types.json` + (`deferred_logical_types`, `deferred_value_handlers`); the `std:v1:default` data context + file is unchanged. No existing hash, schema or table path changes. + +### Fixed + +- **A declared output class with a logical type requires an instance of it (ITL-895).** The + output check required `isinstance` only for `np.ndarray`; for any other class with a + registered logical type a wrong value was caught only if its conversion failed, so a plain + tensor returned for `-> torch.nn.Parameter` was accepted. Such a value now raises + `OutputTypeError` naming the function, the output, the declared type and the returned type. diff --git a/design_issues/ITL-871-TT1.md b/design_issues/ITL-871-TT1.md new file mode 100644 index 000000000..6e1952485 --- /dev/null +++ b/design_issues/ITL-871-TT1.md @@ -0,0 +1,38 @@ +## `src/orcapod/logical_types/torch_type.py` — PyTorch tensors + +### TT1 — `torch.Tensor` could not be a data value +**Status:** resolved +**Severity:** medium +**Issue:** ITL-871 (design: the module docstring of `logical_types/torch_type.py`; history: `superpowers/specs/2026-11-11-torch-tensor-v0.3.md`) + +A pod annotated with `torch.Tensor` could not be defined, and the semantic hasher refused +tensors. Users converted at every pod boundary by hand. + +**Fix:** `LogicalTorchTensor` (`orcapod.torch.tensor`, `large_binary`) stores a tensor with +PyTorch's own serialization (`torch.save`; read with +`torch.load(map_location="cpu", weights_only=True)`, never unrestricted), and +`TorchTensorHandler` hashes it under `torch_tensor_v1` with a strict, Orcapod-defined hash: +the array digest of the elements, then `requires_grad`, the lazy conjugate and negative bits, +the strides and the class. Those five can be ignored through the type's ignore list (the +shared mechanism of ITL-981, `logical_types/ignore_list.py`). Both classes register on first +use through deferred entries in `contexts/logical_types.json`, so `import orcapod` never +imports PyTorch, and support depends only on `torch` being importable. + +Developer decisions (November 12, 2026): storage and hash are decided separately; a property +storage cannot restore (device, autograd history) never counts in the hash; a tensor that does +not cover its buffer is stored as `clone(memory_format=torch.preserve_format)` and its +optional properties are hashed after that clone; extension name `orcapod.torch.tensor`, which +is also the leading text of the hash; label `torch_tensor_v1`; minimum `torch>=2.6` +(`uint16`/`uint32`/`uint64` cannot be loaded with `weights_only=True` before it). Earlier +decisions that stand (November 11, 2026): the dtype list (the NumPy-shared ones plus +`bfloat16`, `complex32` and the float8 family as raw bits); `ValueConversionWarning`, issued +once per function output or source column and conversion; the convenience extra +`orcapod[torch]`. The dev dependency group pulls the extra, with CPU-only wheels on Linux, so +the required CI always runs the torch tests. A guard test fails if torch is missing under CI. + +Known limits: within one in-memory run a downstream pod receives the very tensor the upstream +pod returned (device and autograd history included); only stored or read-back values are in +the stored form. NumPy arrays behave the same. A lazy negative view made by public PyTorch +code (`x.conj().imag`) never covers its buffer, so its `neg_bit` is always stored and hashed +as 0. A non-default ignore list works at the type converter and hasher level only: datagrams +find their data context by key (ITL-981). diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index ae1ab6187..9cc1efce9 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -1,4 +1,4 @@ -# NumPy Arrays +# Arrays and Tensors A `numpy.ndarray` can be a data value like any other: a source column, a function pod parameter or a return value. Storage and hashing are two separate things: @@ -235,3 +235,206 @@ implementation that does not use Orcapod are in the [Hashing Reference](../reference/hashing.md#8-array-value-hash). A PyTorch tensor type shares the element digest and has its own name and properties, so an array and a tensor with equal values have different hashes. + +## PyTorch tensors + +A `torch.Tensor` can be a data value too. PyTorch is an optional dependency. Tensor support +works whenever PyTorch can be imported, however it was installed: `pip install "orcapod[torch]"` +installs it from PyPI, or install the build that suits your hardware yourself (see +pytorch.org; on Linux the PyPI build includes CUDA, and pytorch.org also offers smaller +CPU-only builds). Orcapod needs PyTorch 2.6 or later, the first version whose restricted +loader (below) reads `uint16`, `uint32` and `uint64` tensors, and is tested with PyTorch 2.15. Orcapod +hashes a tensor's elements through NumPy (`Tensor.numpy()`), so your PyTorch build must +support the NumPy version you have installed: some older builds cannot use NumPy 2 and fail +with "Numpy is not available". If you see that error, upgrade PyTorch or install a NumPy +version your PyTorch build supports. + +Importing Orcapod never imports PyTorch. The tensor type is set up the first time Orcapod +meets a tensor or a `torch.Tensor` annotation, or reads a stored tensor column. Reading a +stored tensor column without PyTorch installed raises +`orcapod.errors.MissingOptionalDependencyError`, which names the missing package. + +```python +import torch + +from orcapod import function_pod +from orcapod.sources import DictSource + +@function_pod(output_keys="scaled") +def scale_tensor(signal: torch.Tensor) -> torch.Tensor: + return signal * 2 + +tensors = DictSource( + data=[{"trial": "t1", "signal": torch.arange(6, dtype=torch.float32).reshape(2, 3)}], + tag_columns=["trial"], + source_id="tensor_trials", +) +for tag, data in scale_tensor.pod(tensors).iter_data(): + print(tag["trial"], data["scaled"].dtype, tuple(data["scaled"].shape)) +# t1 torch.float32 (2, 3) +``` + +**A tensor is a different type from an array.** The column's extension name is +`orcapod.torch.tensor` and the hash label is `torch_tensor_v1`. A tensor and an ndarray with +equal dtype, shape and values never hash the same: not as values, not inside a `dict` or +`list`, not as columns, schemas or types. A pod that declares `x: torch.Tensor` cannot take an +`np.ndarray` column (or the reverse). Convert at the pod boundary when you need to: +`torch.from_numpy(a)` or `t.numpy()`. + +### How a tensor is stored + +Storage and hash are decided separately. A tensor is stored with PyTorch's own serialization: +one cell holds the bytes `torch.save` writes for that tensor, in a `large_binary` column. It +is read with `torch.load(..., map_location="cpu", weights_only=True)`. That is PyTorch's +restricted loader: it rebuilds tensors and runs no other pickled code. Orcapod always passes +both arguments, never falls back to an unrestricted load, and never adds a class to the +loader's allow list; the environment variable `TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD` does not +change that. A cell that is not a tensor archive raises an error naming the column. + +What a stored tensor keeps: + +| Property | Stored and read back? | +|---|---| +| dtype, shape, element bits | yes | +| `requires_grad` | yes | +| Memory layout: strides (transposed views, `channels_last`) | yes, for a tensor that covers its buffer (below) | +| Lazy conjugate and negative bits (`t.conj()`) | yes, for a tensor that covers its buffer | +| Class: `torch.Tensor` or `torch.nn.Parameter` | yes | +| Device (CUDA, MPS) | no: a stored tensor is read on the CPU | +| Autograd history: `grad_fn`, `.grad`, hooks | no: a stored tensor is read as a leaf | + +**Only the elements you pass are stored.** `torch.save` writes the whole buffer under a +tensor, so a 3-element slice of a 1,000,000-element tensor would store all million elements. +When a tensor does not cover its underlying buffer exactly (a slice, a step slice such as +`t[::2]`, an expanded view), Orcapod stores `value.clone(memory_format=torch.preserve_format)` +instead. The clone holds just the elements of the view. It has PyTorch's strides for that +clone, and a lazy conjugate or negative bit is resolved into the elements: `x.conj().imag`, +which is a negative view of every second number of `x`, is stored as plain negated values. +A tensor that covers its buffer (`x.conj()`, `x.transpose(0, 1)`) is stored as it is. + +Orcapod never modifies your tensor. One stored tensor costs about 1.5 KB more than its +elements, so a column of many very small tensors is larger than the same data in one tensor. + +### What counts as the same tensor + +The hash is defined by Orcapod and never reads the stored bytes. It is strict: by default two +tensors hash the same only when all of these are equal. + +| Property | In the hash | Can be ignored (option name) | +|---|---|---| +| dtype, shape, element bits (`-0.0` and NaN payloads count) | always | no | +| `requires_grad` | by default | `requires_grad` | +| Memory layout (strides) | by default | `memory_layout` | +| Lazy conjugate bit | by default | `conj_bit` | +| Lazy negative bit | by default | `neg_bit` | +| Class (`torch.Tensor` or `torch.nn.Parameter`) | by default | `subclass` | +| Device, autograd history, `.grad`, position inside a larger buffer | never | | + +A property that storage does not keep is never in the hash: otherwise a value read back from +a store would hash differently from the value that was written, and every cached result would +be computed again. For the same reason the optional properties are those of the *stored* +tensor: a step slice hashes like its contiguous clone. + +The properties the hash ignores are the type's **ignore list**. It is empty by default. It is +configured per data context (`hash_ignore` in `orcapod/contexts/logical_types.json`) and is +written in the column's extension metadata, `{"ignore":[],"version":1}`, so it is part of the +type's [identifier](logical-type-identifiers.md): columns with different lists are different +types, and a data context refuses to read a tensor column stored under another list. An +ignored property is still stored and read back; it only stops counting in the hash. Orcapod +v0.3 has no user API for the list. + +```python +from orcapod.contexts import get_default_context + +hasher = get_default_context().semantic_hasher +t = torch.arange(6, dtype=torch.float32).reshape(2, 3) +print(hasher.hash_object(t) == hasher.hash_object(t.clone())) +# True +print(hasher.hash_object(t.T) == hasher.hash_object(t.T.contiguous())) +# False +print(hasher.hash_object(t) == hasher.hash_object(t.clone().requires_grad_(True))) +# False +print(hasher.hash_object(t) == hasher.hash_object(torch.nn.Parameter(t, requires_grad=False))) +# False +print(hasher.hash_object(t).digest == hasher.hash_object(t.numpy()).digest) +# False +print(hasher.hash_object(t).to_string()) +# torch_tensor_v1:... +``` + +The byte-level definition is in the +[Hashing Reference](../reference/hashing.md#torch-tensor-value-hash). + +### Device and autograd history + +A pod may return a GPU tensor, or one computed from tensors that track gradients (it has a +`grad_fn`), and a source may hold one. Storage keeps neither the device nor the history, so +Orcapod stores a CPU leaf tensor and issues `orcapod.errors.ValueConversionWarning`, which +names the function and output (or the source and column), says what was converted, and how to +avoid it: return or pass `value.detach().cpu()`. A function warns once per output and +conversion, however many rows it processes; a source warns once per column, with the number +of values converted. A CPU tensor without a `grad_fn` never warns, also when it has +`requires_grad=True` or is an `nn.Parameter`: those are stored. To treat the conversion as an +error, call `warnings.simplefilter("error", ValueConversionWarning)` in your program (import +the class from `orcapod.errors`); every conversion then raises, and in a pipeline that row +fails. Python's `-W` command-line option and `PYTHONWARNINGS` cannot name this class: they are +read before installed packages can be imported, and Python ignores the filter. + +```python +import warnings + +from orcapod.errors import ValueConversionWarning + +@function_pod(output_keys="weights") +def fit(signal: torch.Tensor) -> torch.Tensor: + return signal.clone().requires_grad_(True) * 0.5 + +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(fit.pod(tensors).iter_data()) +print(caught[0].category is ValueConversionWarning) +# True +print(caught[0].message) +# Function 'fit' output 'weights' returned a torch tensor with autograd history (a grad_fn); Orcapod stored a CPU tensor without autograd history, and the returned tensor is unchanged. Return value.detach().cpu() to avoid this warning. +``` + +**What comes back.** A tensor read from a store or a source is a CPU leaf tensor with the +stored dtype, shape, elements, strides, `requires_grad`, lazy bits and class (a `bfloat16` +tensor reads back as `bfloat16`, an `nn.Parameter` as an `nn.Parameter`). Within a single +in-memory run, though, the next pod receives the very object the previous pod returned, on its +device and with its autograd history. The same holds for NumPy arrays (a Fortran-ordered array +passes through in Fortran order). Write pods that do not depend on the device or the autograd +history of their inputs. + +### Dtypes and refusals + +`bool`, `int8` to `int64`, `uint8` to `uint64`, `float16`, `float32`, `float64`, `complex64` +and `complex128` are hashed with the NumPy tags and bytes from the table above. The dtypes +NumPy does not have are hashed as their raw bits, each element little-endian, under the +PyTorch dtype name as the tag: + +| Dtype | Element bytes | +|---|---| +| `bfloat16` | 2: the bfloat16 bit pattern | +| `complex32` | 4: the real part, then the imaginary part, each a `float16` | +| `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `float8_e8m0fnu` | 1: the float8 bit pattern | + +A stored dtype that your PyTorch does not have (`float8_e8m0fnu` needs PyTorch 2.7) raises an +error naming the column when it is read; everything else still works. These are refused when +they are written, with an `UnsupportedValueError` that names the column and the reason: + +| Tensor | What to do | +|---|---| +| sparse or `mkldnn` layout | `.to_dense()` | +| quantized | `.dequantize()` | +| nested (jagged) | store the parts as separate tensors | +| on the `meta` device | it holds no data | +| named dimensions (PyTorch versions that have them) | `.rename(None)` | +| sub-byte and packed dtypes (`int1` to `int7`, `uint1` to `uint7`, `float4_e2m1fn_x2`, `bits*`), `bcomplex32` | convert to a supported dtype | +| a subclass other than `torch.nn.Parameter` | `.as_subclass(torch.Tensor)`, or register a logical type **and** a value handler for the class (PyTorch's restricted loader cannot read it back, and the tensor hash, which includes the class, is defined for `torch.Tensor` and `nn.Parameter` only) | + +A function pod that declares `-> torch.Tensor` must return a tensor. An `np.ndarray` or a +Python number raises `OutputTypeError` that names the function and the output. Return +`torch.as_tensor(...)` instead. An `nn.Parameter` is a tensor, so it is accepted; it is +stored, read back and hashed as an `nn.Parameter`. A pod that declares `-> torch.nn.Parameter` +must return one. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2c5da734e..ec8e69073 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -219,7 +219,7 @@ the other rows are computed and stored. With `error_policy="fail_fast"` the sync orchestrator stops the run; the asynchronous orchestrator does not stop for row failures yet (ITL-879, see below). -The function-output rule is: the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base and is stored as the base. +The function-output rule is: the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base. What is stored for a subclass instance is decided by the logical type of the declared class: a user class is stored as the base (the declared class's fields only), while a `torch.nn.Parameter` returned for `-> torch.Tensor` is stored, read back and hashed as an `nn.Parameter`. | Declared | Returned | Result | |---|---|---| diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index 00a494bf5..d7d69daf4 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -78,6 +78,7 @@ Orcapod's own types follow the same contract as yours. Their metadata is the JSO | `orcapod.File` | `orcapod.file` | `{"version":1}` | | `orcapod.Directory` | `orcapod.directory` | `{"version":1}` | | `numpy.ndarray` | `orcapod.numpy.ndarray` | `{"ignore":[],"version":1}` | +| `torch.Tensor` (optional, needs PyTorch) | `orcapod.torch.tensor` | `{"ignore":[],"version":1}` | | `pandas.DataFrame` | `pandas.dataframe` | `{"version":1}` | | `pandas.Series` | `pandas.series` | `{"version":1}` | | `polars.DataFrame` | `orcapod.polars.dataframe` | `{"ignore":[],"version":1}` | @@ -102,7 +103,8 @@ also when empty. So one type under two lists is two logical types: the schema ha a join of two such columns is an error, and a data context reads only columns stored under its own list (anything else raises `LogicalTypeIdentifierError`, naming the column and both lists). Storage is the same under every list. `numpy.ndarray` is the first such type; its -property names are in [NumPy Arrays](arrays.md#the-ignore-list). +property names are in [NumPy Arrays](arrays.md#the-ignore-list). `torch.Tensor` is another; +its names are in [PyTorch tensors](arrays.md#what-counts-as-the-same-tensor). ### Native containers: `list[T]` and `set[T]` diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 00f170816..19b042f60 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -57,6 +57,7 @@ The labels of v0.3: | `arrow_v1` | the Arrow table and schema hash (§9) | `ARROW_HASH_LABEL`, `hashing/arrow_hashers.py` | | `directory_v1` | the directory hash: a SHA-256 Merkle tree over the entries | `DIRECTORY_HASH_LABEL`, `hashing/directory_hashers.py` | | `ndarray_v1` | the array value hash (§8) | `NumpyArrayHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | +| `torch_tensor_v1` | the torch tensor value hash (§8): the element digest, then `requires_grad`, the lazy conjugate and negative bits, the strides and the class | `TORCH_TENSOR_HASH_LABEL`, `logical_types/torch_type.py` | | `polars_expr_v1` | the Polars expression hash (§1c) | `POLARS_EXPR_HASH_LABEL`, `hashing/semantic_hashing/polars_expr.py` | | `polars_dataframe_v1` | the Polars DataFrame value hash (§1e) | `PolarsDataFrameHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | | `polars_series_v1` | the Polars Series value hash (§1e) | `PolarsSeriesHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | @@ -912,7 +913,7 @@ in any data-level preimage; they serve logging, observability, and result inspec ## 8. Array Value Hash **Value hash labels.** A type whose values are hashed by a dedicated handler labels its hash -`_v`, for example `ndarray_v1`. The label is the `ContentHash.method`; the type's +`_v`, for example `ndarray_v1` or `torch_tensor_v1`. The label is the `ContentHash.method`; the type's implementer chooses it. The bytes produced under an existing label never change. A new definition gets a new label, so a hash always says which definition produced it. The label is not part of any hash input (see [The hash label rule](#the-hash-label-rule)). @@ -934,7 +935,8 @@ E = SHA-256( lp(T) || u64le(n) || i64le(d0) || ... || i64le(dn-1) || u64le(len(B ``` `E` covers the dtype, the shape and every element bit, and nothing else. It is shared with -other array types (a tensor type computes the same `E`). +other array types: a `torch.Tensor` computes the same `E`, see +[Torch tensor value hash](#torch-tensor-value-hash) below. **Step 2, the value hash `D`.** @@ -1076,6 +1078,99 @@ print(hasher.hash_object(np.array([(1, 0.5)], dtype=aligned)).to_string()) # ndarray_v1:a47c34b71f1d8c812f52d3a3b78ef6dbafb3b9adb320123b885cbacf72fa50b8 ``` +### Torch tensor value hash + +A `torch.Tensor` value (since v0.3, ITL-871) is hashed by `TorchTensorHandler`, which returns +`ContentHash(method="torch_tensor_v1", digest=D)`. The hash is defined on the tensor, never on +its stored bytes (a cell is a `torch.save` archive, whose bytes PyTorch does not promise to +keep across versions). It has two steps. + +Step 1 is the element digest `E` of the array value hash above, over `(T, shape, B)`, +where `T` is the PyTorch dtype name (`float32`, `bfloat16`), and `B` holds the logical +elements (lazy conjugate and negative bits applied) in C order, little-endian. The dtypes +NumPy does not have give their raw bits: `bfloat16` the 16-bit patterns, `complex32` the real +then the imaginary `float16` of each element, and the `float8` tags (`float8_e4m3fn`, +`float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `float8_e8m0fnu`) one byte per element. + +Step 2, the value hash: + +``` +lp(x) = u64le(len(utf8(x))) || utf8(x) +P = lp("orcapod.torch.tensor") + || E (32 bytes) + || u8(requires_grad) || u8(conj_bit) || u8(neg_bit) + || u64le(n) || i64le(s0) || ... || i64le(sn-1) + || lp(class name) +D = SHA-256(P) +``` + +`s0 ... sn-1` are the strides, in elements, and the class name is `torch.Tensor` or +`torch.nn.parameter.Parameter`. The leading text is the type's extension name; it keeps a +tensor apart from every other array type, so a tensor and an ndarray with equal elements +never share a hash, also inside an untyped Python structure. + +The five properties of step 2 are read from the tensor **as it is stored** +([Arrays and Tensors](../concepts/arrays.md#how-a-tensor-is-stored)): a tensor that does not +cover its buffer is stored as `clone(memory_format=torch.preserve_format)`, so its strides are +those of the clone and its lazy bits are 0. This is what makes +`hash(read(write(v))) == hash(v)`. + +Each of the five properties has an option name: `requires_grad`, `memory_layout`, `conj_bit`, +`neg_bit`, `subclass`. A name in the type's ignore list (the `ignore` array of the extension +metadata, `{"ignore":[...],"version":1}`) writes that property's slot with its canonical +value instead of the real one: `0`, the strides of a C-contiguous tensor of that shape (the +last is 1, each earlier one is the next stride times the next dimension's size, a size of 0 +counting as 1), `0`, `0`, `torch.Tensor`. The slot is always present and the list itself is +not in `P`. So values that differ only in ignored properties hash the same, and with all five +ignored `D` equals the strict `D` of the plain contiguous tensor. A bare value is hashed with +the list of the data context; a stored cell with the list of its own column. The cell token is +` || "::" || D`, where the identifier text (the extension name, `;`, the +metadata in hex) holds the list: for the default, +`b"orcapod.torch.tensor;7b2269676e6f7265223a5b5d2c2276657273696f6e223a317d::" + D`. + +| Variation | Same `D` by default? | +|---|---| +| Dtype, shape, any element bit | no | +| `requires_grad` | no | +| Strides of a tensor that covers its buffer (transposed, `channels_last`) | no | +| Lazy conjugate or negative bit of a tensor that covers its buffer | no | +| `torch.Tensor` against `torch.nn.Parameter` | no | +| Device (CPU, CUDA, MPS) | yes | +| Autograd history (`grad_fn`, `.grad`) | yes | +| A view of a larger buffer against its `clone(memory_format=torch.preserve_format)` | yes | + +A reference implementation with the standard library only, with `lp` and `element_digest` +from the array reference above: + +```python +def torch_tensor_hash( + element: bytes, + requires_grad: bool, + conj_bit: bool, + neg_bit: bool, + strides: tuple[int, ...], + class_name: str, +) -> str: + p = lp("orcapod.torch.tensor") + element + p += bytes([requires_grad, conj_bit, neg_bit]) + struct.pack("=3.0"] spiraldb = [ "pyspiral>=0.16.33", ] -all = ["orcapod[redis]", "orcapod[ray]", "orcapod[postgresql]", "orcapod[spiraldb]"] +# Convenience extra that installs PyTorch. Orcapod's torch.Tensor support depends only on +# torch being importable, however it was installed (ITL-871). 2.6 is the first version whose +# restricted loader (torch.load(weights_only=True)) reads uint16, uint32 and uint64 tensors. +torch = ["torch>=2.6"] +all = [ + "orcapod[redis]", + "orcapod[ray]", + "orcapod[postgresql]", + "orcapod[spiraldb]", + "orcapod[torch]", +] [project.scripts] orcapod = "orcapod.cli:app" @@ -98,8 +108,21 @@ dev = [ "typing_extensions", "mkdocs-material>=9.7.5", "mkdocstrings[python]>=1.0.3", + # torch for the torch.Tensor tests: the default `uv run` sync (the org-required pytest + # and typecheck jobs) installs the dev group but no extras (ITL-871). + "orcapod[torch]", ] +[[tool.uv.index]] +# CPU-only PyTorch wheels, so Linux CI never downloads the CUDA stack (ITL-871). Only uv +# uses this; `pip install orcapod[torch]` installs torch from PyPI as usual. +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[tool.uv.sources] +torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }] + [tool.ruff] target-version = "py311" diff --git a/src/orcapod/contexts/logical_types.json b/src/orcapod/contexts/logical_types.json index d9f5b25d8..13b654285 100644 --- a/src/orcapod/contexts/logical_types.json +++ b/src/orcapod/contexts/logical_types.json @@ -3,7 +3,8 @@ "hash_ignore": { "orcapod.numpy.ndarray": [], "orcapod.polars.dataframe": [], - "orcapod.polars.series": [] + "orcapod.polars.series": [], + "orcapod.torch.tensor": [] }, "logical_type_registry": { "_class": "orcapod.logical_types.registry.LogicalTypeRegistry", @@ -62,6 +63,19 @@ } } ], + "deferred_logical_types": [ + { + "python_type": "torch.Tensor", + "extension_name": "orcapod.torch.tensor", + "logical_type": "orcapod.logical_types.torch_type.LogicalTorchTensor", + "requires": "torch", + "_config": { + "ignore": { + "_ref": "hash_ignore:orcapod.torch.tensor" + } + } + } + ], "factories": [ { "factory": { @@ -204,5 +218,16 @@ } } ] + ], + "deferred_value_handlers": [ + { + "python_type": "torch.Tensor", + "handler": "orcapod.hashing.semantic_hashing.torch_handler.TorchTensorHandler", + "_config": { + "ignore": { + "_ref": "hash_ignore:orcapod.torch.tensor" + } + } + } ] } diff --git a/src/orcapod/contexts/registry.py b/src/orcapod/contexts/registry.py index 0bc706954..983ae2551 100644 --- a/src/orcapod/contexts/registry.py +++ b/src/orcapod/contexts/registry.py @@ -393,7 +393,8 @@ def _load_logical_type_registration(self) -> dict[str, Any]: Returns: Its ``logical_type_registry`` object spec, its ``value_handlers`` list of ``[type, handler]`` object-spec pairs and, when present, its - ``hash_ignore`` object: per extension name, the property names the + ``deferred_value_handlers`` list of handlers loaded on first use and + its ``hash_ignore`` object: per extension name, the property names the value hash of that type ignores. """ return _read_json_file(self.logical_types_file) @@ -453,6 +454,14 @@ def _create_context_from_spec(self, spec: dict[str, Any]) -> DataContext: pair = parse_objectspec(pair_spec, ref_lut=ref_lut) if pair and len(pair) == 2 and all(x is not None for x in pair): handler_registry.register(pair[0], pair[1]) + # Handlers of optional value types are named, not built: the + # library is imported when a value of the type is first hashed. + for entry in registration.get("deferred_value_handlers", []): + handler_registry.register_deferred( + entry["python_type"], + entry["handler"], + parse_objectspec(entry.get("_config", {}), ref_lut=ref_lut), + ) return DataContext( context_key=context_key, diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index b4e47c4c3..3f085e0ef 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -37,6 +37,7 @@ from orcapod.utils.canonical_type_names import stabilize_type_names from orcapod.utils.git_utils import get_git_info_for_python_object from orcapod.utils.lazy_module import LazyModule +from orcapod.utils.tensor_conversions import describe, tensor_conversions, warn_once from orcapod.protocols.observability_protocols import DataExecutionLoggerProtocol @@ -454,6 +455,8 @@ def __init__( assert function_name is not None self._function_name = function_name + # (output key, conversion) pairs already reported with ValueConversionWarning. + self._reported_conversions: set[tuple[str, str]] = set() super().__init__( label=label, @@ -638,6 +641,17 @@ def _build_output_data(self, values: Any) -> DataProtocol: mismatch.describe(), mismatch.hint, ) + conversions = tensor_conversions(value) + if conversions: + warn_once( + self._reported_conversions, + key, + conversions, + f"Function {self._function_name!r} output {key!r} returned a " + f"torch tensor {describe(conversions)}; Orcapod stored a CPU " + "tensor without autograd history, and the returned tensor is " + "unchanged. Return value.detach().cpu() to avoid this warning.", + ) def combine(*components: tuple[str, ...]) -> str: inner_parsed = [":".join(component) for component in components] diff --git a/src/orcapod/core/sources/dict_source.py b/src/orcapod/core/sources/dict_source.py index 9cb7cb2bc..8b6cecd82 100644 --- a/src/orcapod/core/sources/dict_source.py +++ b/src/orcapod/core/sources/dict_source.py @@ -9,6 +9,7 @@ from orcapod.utils import arrow_utils from orcapod.utils.schema_utils import _normalize_column_list from orcapod.utils.lazy_module import LazyModule +from orcapod.utils.tensor_conversions import warn_source_tensor_conversions if TYPE_CHECKING: import pyarrow as pa @@ -41,14 +42,15 @@ def __init__( # Accept either a Python SchemaLike (Mapping[str, type]) or a # pa.Schema directly via data_schema; route to the converter accordingly. + rows = [dict(row) for row in data] if isinstance(data_schema, pa.Schema): arrow_table = self.data_context.type_converter.python_dicts_to_arrow_table( - [dict(row) for row in data], + rows, arrow_schema=data_schema, ) else: arrow_table = self.data_context.type_converter.python_dicts_to_arrow_table( - [dict(row) for row in data], + rows, python_schema=data_schema, ) if data_schema is None: @@ -72,6 +74,7 @@ def __init__( self._tag_columns = result.tag_columns if self._source_id is None: self._source_id = result.source_id + warn_source_tensor_conversions(self._source_id, rows) def to_config(self, db_registry=None) -> dict[str, Any]: """Serialize metadata-only config (data is not serializable).""" diff --git a/src/orcapod/core/sources/list_source.py b/src/orcapod/core/sources/list_source.py index a0f49716e..f0ecbd525 100644 --- a/src/orcapod/core/sources/list_source.py +++ b/src/orcapod/core/sources/list_source.py @@ -9,6 +9,7 @@ from orcapod.types import SchemaLike from orcapod.utils import arrow_utils from orcapod.utils.canonical_type_names import stabilize_type_names +from orcapod.utils.tensor_conversions import warn_source_tensor_conversions if TYPE_CHECKING: pass @@ -98,6 +99,7 @@ def __init__( self._stream = result.stream if self._source_id is None: self._source_id = result.source_id + warn_source_tensor_conversions(self._source_id, rows) def _hash_tag_function(self) -> str: """Produce a stable hash string for the tag function.""" diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 1923bbffc..f7b604d6d 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -76,6 +76,16 @@ class UnsupportedValueError(ValueError): """ +class MissingOptionalDependencyError(ImportError): + """A stored value needs an optional dependency that is not installed. + + Raised when a column's Arrow extension type belongs to a value type that is + registered lazily (for example ``orcapod.torch.tensor``, which needs PyTorch) + and its library cannot be imported. The message names the extension type and + the missing module. + """ + + class DataFunctionUnavailableError(RuntimeError): """Raised when a data function proxy is invoked without a bound function. @@ -85,6 +95,22 @@ class DataFunctionUnavailableError(RuntimeError): """ +class ValueConversionWarning(UserWarning): + """Orcapod accepted a value and stored a converted form of it. + + Example: a torch tensor on a GPU or with autograd history (a ``grad_fn``), which + is stored as a CPU tensor without that history. The message names who gave the value (a function pod's output, a source's + column), what was converted and how to avoid the conversion. A value passed in by + the caller is never modified. The code that issues the warning issues it once per + (function or source, output or column, conversion), however many rows a run has, + and does not rely on Python's warning registry to do so. To make it an error, call + ``warnings.simplefilter("error", ValueConversionWarning)``: every conversion then + raises, on every row, and in a pipeline that row fails. Python's ``-W`` option + and ``PYTHONWARNINGS`` cannot name this class: they are read before installed + packages can be imported. + """ + + def type_text(annotation: object) -> str: """Return a short, readable name for a type or annotation, for error messages. diff --git a/src/orcapod/hashing/semantic_hashing/torch_handler.py b/src/orcapod/hashing/semantic_hashing/torch_handler.py new file mode 100644 index 000000000..9af8a41c6 --- /dev/null +++ b/src/orcapod/hashing/semantic_hashing/torch_handler.py @@ -0,0 +1,49 @@ +"""Semantic hash handler for ``torch.Tensor`` (ITL-871). + +Loaded on first use through the ``deferred_value_handlers`` entry of +``contexts/logical_types.json``, so importing Orcapod never imports torch. +""" + +from __future__ import annotations + +from typing import Any + +from orcapod.logical_types.ignore_list import IgnoreListHandler +from orcapod.logical_types.torch_type import ( + OPTIONS, + TORCH_TENSOR_HASH_LABEL, + tensor_digest, +) + + +class TorchTensorHandler(IgnoreListHandler): + """Hasher for ``torch.Tensor``: the value hash ``D`` under ``torch_tensor_v1``. + + The hash is strict by default: dtype, shape and every element bit always + count, and so do ``requires_grad``, the memory layout (strides), the lazy + conjugate and negative bits and the class (``torch.Tensor`` or + ``torch.nn.Parameter``) unless the ignore list names them. Device and + autograd history never count: storage does not restore them. The formula is + in ``orcapod.logical_types.torch_type``. The preimage starts with the + extension name, so a tensor never hashes like an ndarray with the same + elements, also inside an untyped Python structure. + """ + + OPTIONS = OPTIONS + HASH_LABEL = TORCH_TENSOR_HASH_LABEL + + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return the value hash of a tensor, ignoring the named properties. + + Args: + obj: A ``torch.Tensor`` or ``torch.nn.Parameter``. + ignore: The property names to ignore; a subset of ``OPTIONS``. + + Returns: + The 32-byte digest (``torch_type.tensor_digest``). + + Raises: + TypeError: If ``obj`` is not a ``torch.Tensor``. + UnsupportedValueError: If the tensor cannot be stored. + """ + return tensor_digest(obj, ignore) diff --git a/src/orcapod/hashing/semantic_hashing/type_handler_registry.py b/src/orcapod/hashing/semantic_hashing/type_handler_registry.py index ffef6019a..61894efe6 100644 --- a/src/orcapod/hashing/semantic_hashing/type_handler_registry.py +++ b/src/orcapod/hashing/semantic_hashing/type_handler_registry.py @@ -9,8 +9,11 @@ import logging import threading +from collections.abc import Mapping from typing import TYPE_CHECKING, Any +from orcapod.utils.deferred_types import load_object, qualified_name + if TYPE_CHECKING: from orcapod.protocols.hashing_protocols import ( ArrowHasherProtocol, @@ -41,6 +44,8 @@ def __init__( handlers: Optional list of ``(target_type, hasher)`` pairs to register at construction time. """ + # Handlers loaded on first use, by qualified class name (``register_deferred``). + self._deferred: dict[str, dict[str, Any]] = {} self._handlers: dict[type, "PythonTypeHandlerProtocol"] = {} self._lock = threading.RLock() if handlers: @@ -54,6 +59,28 @@ def __init__( target_type, handler = entry self.register(target_type, handler) + def register_deferred( + self, python_type: str, handler: str, config: Mapping[str, Any] | None = None + ) -> None: + """Register a handler that is loaded on first use (ITL-871). + + Neither the type nor the handler is imported until a lookup meets a + class whose MRO contains *python_type*. A deferred handler already + named for *python_type* is replaced. See ``orcapod.utils.deferred_types``. + + Args: + python_type: Qualified class name, for example ``"torch.Tensor"``. + handler: Dotted path of the handler class. + config: Keyword arguments the handler class is instantiated with + (for example the ignore list of the value hash); none by default. + """ + with self._lock: + self._deferred[python_type] = { + "python_type": python_type, + "handler": handler, + "config": dict(config or {}), + } + def register(self, target_type: type, handler: "PythonTypeHandlerProtocol") -> None: """Register a hasher for a specific Python type. @@ -106,27 +133,19 @@ def get_handler(self, obj: Any) -> "PythonTypeHandlerProtocol | None": Returns: The registered ``PythonTypeHandlerProtocol``, or None. """ - obj_type = type(obj) - with self._lock: - handler = self._handlers.get(obj_type) - if handler is not None: - return handler - for base in obj_type.__mro__[1:]: - handler = self._handlers.get(base) - if handler is not None: - logger.debug( - "PythonTypeHandlerRegistry: resolved hasher for %s via base %s", - obj_type.__name__, - base.__name__, - ) - return handler - return None + return self.get_handler_for_type(type(obj)) def get_handler_for_type( self, target_type: type ) -> "PythonTypeHandlerProtocol | None": """Look up the handler for a *type object* (rather than an instance). + Walks ``target_type.__mro__`` once, most-derived class first. For each + class a registered handler is taken first, then a deferred handler named + for that class (``register_deferred``), which is loaded and registered. So + the handler of the most-derived class wins, whether it is registered or + deferred. Comparing qualified names imports nothing. + Args: target_type: The type to look up. @@ -134,14 +153,28 @@ def get_handler_for_type( The registered ``PythonTypeHandlerProtocol``, or None. """ with self._lock: - handler = self._handlers.get(target_type) - if handler is not None: - return handler - for base in target_type.__mro__[1:]: + for base in getattr(target_type, "__mro__", (target_type,)): handler = self._handlers.get(base) if handler is not None: return handler - return None + entry = self._deferred.get(qualified_name(base)) + if entry is not None: + return self._load_deferred(entry) + return None + + def _load_deferred(self, entry: dict[str, Any]) -> "PythonTypeHandlerProtocol": + """Load and register a deferred handler. Callers hold ``self._lock``. + + Args: + entry: The deferred entry (``register_deferred``). + + Returns: + The newly registered handler. + """ + handler = load_object(entry["handler"])(**entry["config"]) + self.register(load_object(entry["python_type"]), handler) + del self._deferred[entry["python_type"]] + return handler def has_handler(self, target_type: type) -> bool: """Return True if a handler is registered for *target_type* or any MRO ancestor. diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index fe711e737..3b50948df 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -8,14 +8,17 @@ import logging import re -from typing import TYPE_CHECKING, Iterable +import threading +from typing import TYPE_CHECKING, Any, Iterable +from orcapod.errors import MissingOptionalDependencyError from orcapod.logical_types.identifier import LogicalTypeIdentifierError from orcapod.logical_types.identifier import LogicalTypeIdentifier from orcapod.logical_types.protocols import ( LogicalTypeProtocol, LogicalTypeFactoryProtocol, ) +from orcapod.utils.deferred_types import load_object, module_available, qualified_name from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -258,6 +261,15 @@ class LogicalTypeRegistry: the same pattern as the ``logical_types`` constructor argument used by other registries in this package. + An optional ``deferred_logical_types`` list names logical types that are + loaded on first use, so an optional library is never imported up front + (ITL-871). Each entry is a dict: ``python_type`` (qualified class name, e.g. + ``"torch.Tensor"``), ``extension_name``, ``logical_type`` (dotted path of the + logical type class), ``requires`` (the module that must be importable) and, + optionally, ``_config`` (keyword arguments the class is instantiated with, + for example the ignore list of its value hash). See + ``orcapod.utils.deferred_types``. + An optional ``factories`` list can also be passed to pre-register ``LogicalTypeFactoryProtocol`` instances at construction time. Each entry is a dict with keys ``factory`` (the factory instance), ``category`` (optional str), @@ -276,7 +288,15 @@ def __init__( self, logical_types: list[LogicalTypeProtocol] | None = None, factories: list[dict] | None = None, + deferred_logical_types: list[dict[str, Any]] | None = None, ) -> None: + # Serialises loading a deferred type, so concurrent first lookups load it once. + self._deferred_lock = threading.Lock() + self._deferred_by_python_name: dict[str, dict[str, Any]] = {} + self._deferred_by_arrow_name: dict[str, dict[str, Any]] = {} + for entry in deferred_logical_types or []: + self._deferred_by_python_name[entry["python_type"]] = entry + self._deferred_by_arrow_name[entry["extension_name"]] = entry self._by_logical_name: dict[str, LogicalTypeProtocol] = {} self._by_arrow_name: dict[str, LogicalTypeProtocol] = {} self._by_python_type: dict[type, LogicalTypeProtocol] = {} @@ -438,8 +458,9 @@ def get_by_logical_name(self, name: str) -> LogicalTypeProtocol | None: def get_by_python_type(self, python_type: type) -> LogicalTypeProtocol | None: """Return the logical type for *python_type*, or ``None``. - The most specific registered type wins: *python_type* itself, then each - class of its MRO in order, so a subclass registered with its own logical + The most specific type wins: *python_type* itself, then each class of its + MRO in order, and for each class a registered type before a deferred one + (which is loaded), so a subclass registered with its own logical type is used for itself and its subclasses whatever the registration order (an ``np.ndarray`` subclass without one is an array). A logical type with ``exact_python_type_only`` (a dataclass or pydantic model) is never @@ -447,18 +468,34 @@ class of its MRO in order, so a subclass registered with its own logical subclasses only through ``__subclasshook__``/ABC registration fall back to an ``issubclass`` scan, where the first registered match wins. """ - result = self._by_python_type.get(python_type) - if result is not None: - return result - for base in getattr(python_type, "__mro__", ())[1:]: + if not isinstance(python_type, type): + # An annotation that is not a class (a union, a generic alias) is + # found only as itself. + result = self._by_python_type.get(python_type) + if result is not None: + return result + for base in getattr(python_type, "__mro__", ()): result = self._by_python_type.get(base) # A user class matches exactly: its subclass is a different class with # its own fields and identity, never stored as the base (ITL-881). - if result is not None and not getattr( - result, "exact_python_type_only", False + if result is not None and ( + base is python_type + or not getattr(result, "exact_python_type_only", False) ): return result - for registered_type, registered_lt in self._by_python_type.items(): + # A deferred type named for this class comes before any type of a base + # class, registered or deferred. + entry = self._deferred_by_python_name.get(qualified_name(base)) + if entry is not None: + loaded = self._load_deferred(entry) + # The same exact-type rule as for a registered type: such a type + # of a base class is not a match, and the walk goes on. + if base is python_type or not getattr( + loaded, "exact_python_type_only", False + ): + return loaded + # A snapshot: a deferred type loaded by another thread adds an entry. + for registered_type, registered_lt in tuple(self._by_python_type.items()): if getattr(registered_lt, "exact_python_type_only", False): continue try: @@ -466,13 +503,67 @@ class of its MRO in order, so a subclass registered with its own logical return registered_lt except TypeError: continue + # Another thread may have loaded a deferred type between the two reads of the + # walk above: it registers the type before it removes the deferred entry. + for base in getattr(python_type, "__mro__", ()): + loaded = self._by_python_type.get(base) + if loaded is not None and not getattr( + loaded, "exact_python_type_only", False + ): + return loaded return None def get_by_arrow_extension_name( self, arrow_name: str ) -> LogicalTypeProtocol | None: - """Return the logical type registered under *arrow_name*, or ``None``.""" - return self._by_arrow_name.get(arrow_name) + """Return the logical type registered under *arrow_name*, or ``None``. + + A deferred logical type (see the class docstring) is loaded on the first + lookup of its extension name. + + Raises: + MissingOptionalDependencyError: If *arrow_name* belongs to a deferred + logical type whose required module is not installed. + """ + result = self._by_arrow_name.get(arrow_name) + if result is not None: + return result + entry = self._deferred_by_arrow_name.get(arrow_name) + if entry is None: + # Another thread may have loaded the deferred type between the two reads: + # it registers the type before it removes the deferred entry. + return self._by_arrow_name.get(arrow_name) + if not module_available(entry["requires"]): + raise MissingOptionalDependencyError( + f"Arrow extension type {arrow_name!r} needs the optional dependency " + f"{entry['requires']!r}, which is not installed. Install it to read " + "or write values of this type." + ) + return self._load_deferred(entry) + + def _load_deferred(self, entry: dict[str, Any]) -> LogicalTypeProtocol: + """Instantiate and register a deferred logical type. + + Holds a lock and checks again whether another thread has already loaded + the type, so concurrent first lookups instantiate and register it once. + + Args: + entry: The deferred entry (see the class docstring). + + Returns: + The registered logical type. + """ + with self._deferred_lock: + existing = self._by_arrow_name.get(entry["extension_name"]) + if existing is not None: + return existing + logical_type = load_object(entry["logical_type"])( + **entry.get("_config", {}) + ) + self.register_logical_type(logical_type) + self._deferred_by_python_name.pop(entry["python_type"], None) + self._deferred_by_arrow_name.pop(entry["extension_name"], None) + return logical_type def register_logical_type_factory( self, diff --git a/src/orcapod/logical_types/torch_type.py b/src/orcapod/logical_types/torch_type.py new file mode 100644 index 000000000..c1aa644e9 --- /dev/null +++ b/src/orcapod/logical_types/torch_type.py @@ -0,0 +1,526 @@ +"""``torch.Tensor`` logical type: stored with ``torch.save``, hashed by content (ITL-871). + +Storage and hash are decided separately (developer, November 12, 2026). + +**Storage.** One cell is the bytes ``torch.save`` writes for one tensor, in a +``large_binary`` column under the extension name ``orcapod.torch.tensor``. A cell +is read with ``torch.load(map_location="cpu", weights_only=True)``, always with +both arguments explicit: PyTorch's restricted loader, which runs no arbitrary +pickle. There is no unrestricted fallback. What is written is the *stored form* +of the value (``stored_form``): on the CPU, without autograd history, and, when +the tensor does not cover its underlying storage exactly (a slice, a step slice, +an expanded view), a ``clone(memory_format=torch.preserve_format)`` of it, so only +the elements that were passed are stored. The caller's tensor is never modified. + +**Hash.** The value hash ``D`` (``tensor_digest``) is defined by Orcapod and never +reads the stored bytes. It is strict by default:: + + E = array_format.array_digest_v1(dtype tag, shape, element bytes) + D = SHA-256(lp("orcapod.torch.tensor") || E || u8(requires_grad) || u8(conj_bit) + || u8(neg_bit) || u64le(ndim) || i64le(each stride) || lp(class name)) + lp(x) = u64le(len(utf8(x))) || utf8(x) + +``E`` covers what can never be ignored: dtype, shape and every element bit (the +logical values, lazy conjugate and negative bits applied, C order, little-endian). +The five optional properties (``OPTIONS``: ``requires_grad``, ``memory_layout``, +``conj_bit``, ``neg_bit``, ``subclass``) are read from the stored form, so +``hash(read(write(v))) == hash(v)``. An ignored property's slot is written with its +canonical value (0, the C-contiguous strides of the shape, ``torch.Tensor``). Which +properties are ignored is the type's ignore list (``logical_types/ignore_list.py``, +ITL-981): configured per data context in ``contexts/logical_types.json`` and +written in the extension metadata, ``{"ignore":[...],"version":1}``. + +A property storage cannot restore never counts in the hash and has no option +name: device, autograd history (``grad_fn``, ``.grad``, hooks), the storage offset +and hidden elements of a larger buffer, pinned or shared memory. + +Supported dtypes: ``bool``, ``int8`` to ``int64``, ``uint8`` to ``uint64``, +``float16``, ``float32``, ``float64``, ``complex64`` and ``complex128`` (element +bytes as NumPy's), and the torch-only dtypes ``bfloat16``, ``complex32`` and the +float8 family, hashed as their raw bits under the torch dtype name. Refused at +write time with ``UnsupportedValueError`` naming the reason (the type converter +adds the column): sparse and mkldnn layouts, quantized, nested and meta tensors, +named tensors, other dtypes, and subclasses other than ``torch.nn.Parameter`` (the +restricted loader cannot read them back). + +This module imports torch; it is loaded on first use through the deferred +registration in ``contexts/logical_types.json`` (``orcapod.utils.deferred_types``), so +``import orcapod`` never imports torch. +""" + +from __future__ import annotations + +import hashlib +import io +import pickle +import struct +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +import pyarrow as pa +import torch + +from orcapod.errors import UnsupportedValueError +from orcapod.logical_types.array_format import array_digest_v1 +from orcapod.logical_types.ignore_list import IgnoreListLogicalType +from orcapod.logical_types.numpy_type import array_bytes, canonical_array + +if TYPE_CHECKING: + from orcapod.logical_types.protocols import TypeConverterProtocol + +TORCH_TENSOR_EXTENSION_NAME = "orcapod.torch.tensor" +"""Arrow extension name (and logical type name) of a stored tensor. Also the +leading type text of the value hash, which keeps a tensor apart from every other +array type.""" + +TORCH_TENSOR_HASH_LABEL = "torch_tensor_v1" +"""``ContentHash.method`` of a tensor hash.""" + +TORCH_TENSOR_VERSION = 1 +"""Version of the stored layout and of the hash, written in the extension metadata.""" + +OPTIONS = frozenset( + {"requires_grad", "memory_layout", "conj_bit", "neg_bit", "subclass"} +) +"""The properties the value hash includes by default and can be told to ignore.""" + +_CLASS_NAMES: dict[type, str] = { + torch.Tensor: "torch.Tensor", + torch.nn.Parameter: "torch.nn.parameter.Parameter", +} +"""The classes that can be stored, with the name each is hashed under. The names +are part of the hash definition and stay fixed if PyTorch moves a class.""" + +_PICKLE_PROTOCOL = 2 +"""``torch.save``'s default pickle protocol, passed explicitly.""" + +_SHARED_TAGS = ( + "bool", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "float16", + "float32", + "float64", + "complex64", + "complex128", +) +"""Tags whose element bytes are the same as NumPy's (and whose name is the torch dtype name).""" + +_RAW_BIT_TAGS: dict[str, tuple[str, int]] = { + "bfloat16": ("int16", 1), + "complex32": ("int16", 2), + "float8_e4m3fn": ("uint8", 1), + "float8_e5m2": ("uint8", 1), + "float8_e4m3fnuz": ("uint8", 1), + "float8_e5m2fnuz": ("uint8", 1), + "float8_e8m0fnu": ("uint8", 1), +} +"""Tags without a NumPy dtype: the NumPy tag their bits are hashed as, and how many +of those make one element (``complex32`` is a real then an imaginary binary16).""" + +_SUPPORTED_TAGS = frozenset(_SHARED_TAGS) | frozenset(_RAW_BIT_TAGS) + + +def _tag_of(dtype: Any) -> str | None: + """Return the tag of a torch dtype (its name), or ``None`` if it is not supported.""" + tag = str(dtype).removeprefix("torch.") + return tag if tag in _SUPPORTED_TAGS else None + + +def _refuse(what: str, reason: str) -> UnsupportedValueError: + """Build the refusal for a tensor that cannot be stored. + + Args: + what: The offending property, e.g. ``"layout torch.sparse_coo"``. + reason: Why it cannot be stored and, where possible, what to do. + + Returns: + The error to raise; the type converter prefixes the column name. + """ + return UnsupportedValueError( + f"torch tensors with {what} cannot be stored: {reason}" + ) + + +def _check_storable(value: Any) -> str: + """Refuse what cannot be stored, and return the dtype tag of ``value``. + + Raises: + TypeError: If ``value`` is not a ``torch.Tensor``. + UnsupportedValueError: For extended or unsupported tensors. + """ + if not isinstance(value, torch.Tensor): + raise TypeError(f"expected a torch.Tensor, got {type(value).__name__}") + if value.is_nested: + raise _refuse( + "nested (jagged) layout", + "nested tensors are extended data and are not supported", + ) + if value.layout != torch.strided: + raise _refuse( + f"layout {value.layout}", + "sparse and other non-dense layouts are extended data and are not " + "supported (convert with .to_dense())", + ) + if value.is_quantized: + raise _refuse( + f"quantized dtype {value.dtype}", + "quantized tensors are extended data and are not supported " + "(convert with .dequantize())", + ) + if value.device.type == "meta": + raise _refuse("device 'meta'", "a meta tensor holds no data") + names = getattr(value, "names", None) + if names and any(name is not None for name in names): + raise _refuse( + f"dimension names {tuple(names)}", + "named tensors are not supported (drop the names with .rename(None))", + ) + tag = _tag_of(value.dtype) + if tag is None: + raise _refuse( + f"dtype {value.dtype}", + "this dtype is not supported in Orcapod v0.3 (supported: bool, integer, " + "float16/32/64, bfloat16, the float8 types, complex32/64/128)", + ) + return tag + + +def _c_strides(shape: Sequence[int]) -> tuple[int, ...]: + """Return the strides, in elements, of a C-contiguous tensor of ``shape``. + + This is PyTorch's own rule: the last stride is 1 and each earlier stride is the + next one times the next dimension's size, a size of 0 counting as 1. + """ + strides: list[int] = [] + step = 1 + for dim in reversed(shape): + strides.append(step) + step *= max(int(dim), 1) + return tuple(reversed(strides)) + + +def _covers_storage(t: Any) -> bool: + """Whether ``t`` holds exactly the elements of its underlying storage. + + True when the tensor starts at the beginning of its storage, the storage holds + exactly as many elements as the tensor, and no two indices share an element + (the strides are a permutation of a contiguous layout). Such a tensor is + stored as it is; any other is stored as a clone. + """ + if t.storage_offset() != 0: + return False + if t.untyped_storage().nbytes() != t.numel() * t.element_size(): + return False + if t.numel() == 0: + return True + expected = 1 + # Dimensions from the innermost (smallest stride) outwards; a dimension of + # size 1 takes no room, whatever its stride. + for stride, size in sorted(zip(t.stride(), t.shape, strict=True)): + if size == 1: + continue + if stride != expected: + return False + expected *= size + return True + + +def stored_form(value: Any) -> Any: + """Return the tensor that is written for ``value`` and that it is hashed as. + + The stored form is on the CPU and has no autograd history; ``requires_grad``, + the class (``torch.Tensor`` or ``torch.nn.Parameter``), dtype, shape and + elements are those of ``value``. A tensor made under ``torch.inference_mode()`` + that requires grad is cloned, because only an ordinary tensor can carry the + flag outside that mode. When ``value`` covers its underlying storage + exactly, its strides and lazy conjugate and negative bits are kept. Otherwise + the stored form is ``clone(memory_format=torch.preserve_format)``: only the + elements of ``value``, with PyTorch's strides for that clone and with the lazy + bits resolved into the elements. ``value`` itself is never modified; the + stored form may share memory with it. + + Args: + value: A ``torch.Tensor`` or ``torch.nn.Parameter``. + + Returns: + The stored form, a new tensor object. + + Raises: + TypeError: If ``value`` is not a ``torch.Tensor``. + UnsupportedValueError: For tensors that cannot be stored. + """ + _check_storable(value) + if type(value) not in _CLASS_NAMES: + raise _refuse( + f"class {type(value).__module__}.{type(value).__qualname__}", + "only torch.Tensor and torch.nn.Parameter can be read back by PyTorch's " + "restricted loader (convert with .as_subclass(torch.Tensor), or register " + "a logical type for the class)", + ) + t = value.detach() + if t.device.type != "cpu": + t = t.to("cpu") + if not _covers_storage(t): + t = t.clone(memory_format=torch.preserve_format) + if value.requires_grad and t.is_inference(): + # A tensor made under torch.inference_mode() cannot be given requires_grad + # outside it; its clone is an ordinary tensor. + t = t.clone(memory_format=torch.preserve_format) + if isinstance(value, torch.nn.Parameter): + return torch.nn.Parameter(t, requires_grad=value.requires_grad) + if value.requires_grad: + # ``t`` is a tensor object of this function's own: the caller's is untouched. + t.requires_grad_(True) + return t + + +def canonical_tensor(value: Any) -> tuple[str, tuple[int, ...], memoryview]: + """Return the dtype tag, shape and canonical element bytes of a tensor. + + The bytes are the elements in C order, little-endian, packed, read from a + detached CPU copy with lazy conjugate and negative bits resolved. They are + the input of the element digest ``E``. ``value`` itself is never modified, and + its memory is never handed to NumPy (that would make its storage + non-resizable). + + Args: + value: A ``torch.Tensor``. + + Returns: + ``(tag, shape, data)``, the arguments of ``array_format.array_digest_v1``. + + Raises: + TypeError: If ``value`` is not a ``torch.Tensor``. + UnsupportedValueError: For tensors that cannot be stored. + """ + tag = _check_storable(value) + t = value.detach() + if t.device.type != "cpu": + t = t.to("cpu") + t = t.resolve_conj().resolve_neg().contiguous() + if t.untyped_storage().data_ptr() == value.untyped_storage().data_ptr(): + # Still the caller's memory. ``Tensor.numpy()`` below makes the storage it + # exports non-resizable, so it must not be the caller's: take a copy. + t = t.clone() + if tag == "complex32": + # NumPy has no complex32: each element's real then imaginary binary16. + t = torch.view_as_real(t) + raw = _RAW_BIT_TAGS.get(tag) + if raw is not None: + t = t.view(getattr(torch, raw[0])) + _, array = canonical_array(t.numpy()) + return tag, tuple(int(d) for d in value.shape), array_bytes(array) + + +def _lp(text: str) -> bytes: + """Return ``text`` as UTF-8 behind its length as an unsigned 64-bit little-endian integer.""" + raw = text.encode("utf-8") + return struct.pack(" bytes: + """Return the value hash ``D`` of a tensor (the module docstring gives the formula). + + Args: + value: A ``torch.Tensor`` or ``torch.nn.Parameter``. + ignore: Names from ``OPTIONS``. The slot of an ignored property is written + with its canonical value, so two tensors that differ only in ignored + properties have equal digests. + + Returns: + The 32-byte SHA-256 digest. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + TypeError: If ``value`` is not a ``torch.Tensor``. + UnsupportedValueError: For tensors that cannot be stored, and for a + subclass other than ``torch.nn.Parameter``: the hash includes the + class, so such a class needs a value handler of its own. + """ + unknown = ignore - OPTIONS + if unknown: + raise ValueError( + f"unknown torch tensor hash options {sorted(unknown)}; " + f"the options are {sorted(OPTIONS)}" + ) + if isinstance(value, torch.Tensor) and type(value) not in _CLASS_NAMES: + raise UnsupportedValueError( + f"a {type(value).__module__}.{type(value).__qualname__} cannot be hashed " + "as a torch tensor: the tensor value hash includes the class and is " + "defined for torch.Tensor and torch.nn.Parameter only. Register a value " + "handler for the class (a subclass with its own logical type needs its " + "own value handler too), or convert with .as_subclass(torch.Tensor)" + ) + stored = stored_form(value) + tag, shape, data = canonical_tensor(stored) + strides = _c_strides(shape) if "memory_layout" in ignore else stored.stride() + stored_class = torch.Tensor if "subclass" in ignore else type(stored) + preimage = b"".join( + ( + _lp(TORCH_TENSOR_EXTENSION_NAME), + array_digest_v1(tag, shape, data), + bytes( + ( + "requires_grad" not in ignore and stored.requires_grad, + "conj_bit" not in ignore and stored.is_conj(), + "neg_bit" not in ignore and stored.is_neg(), + ) + ), + struct.pack(" bytes: + """Return what ``torch.save`` writes for the stored form of ``value``. + + Args: + value: A ``torch.Tensor`` or ``torch.nn.Parameter``. + + Returns: + The archive bytes: ``torch.save``'s zip format and pickle protocol 2, its + defaults, passed explicitly. + + Raises: + TypeError: If ``value`` is not a ``torch.Tensor``. + UnsupportedValueError: For tensors that cannot be stored. + """ + buffer = io.BytesIO() + torch.save( + stored_form(value), + buffer, + pickle_protocol=_PICKLE_PROTOCOL, + _use_new_zipfile_serialization=True, + ) + return buffer.getvalue() + + +def tensor_from_bytes(data: bytes) -> Any: + """Read a tensor from the bytes ``tensor_to_bytes`` wrote. + + Always ``torch.load(map_location="cpu", weights_only=True)``: PyTorch's + restricted loader. There is no fallback to an unrestricted load, and no class + is ever added to the loader's allow list. + + Args: + data: The stored bytes of one cell. + + Returns: + A CPU ``torch.Tensor`` or ``torch.nn.Parameter``. + + Raises: + ValueError: If PyTorch's restricted loader refuses the bytes (not an + archive, a pickle that is not a plain tensor, a dtype the installed + PyTorch does not have), or the archive holds something other than one + tensor. + """ + try: + # Without autograd: PyTorch rebuilds some dtypes (complex32, the float8 + # family) in place, which a leaf that requires grad refuses. requires_grad + # is restored all the same. + with torch.no_grad(): + value = torch.load(io.BytesIO(data), map_location="cpu", weights_only=True) + except Exception as exc: # noqa: BLE001 # torch.load raises many unrelated types + # An UnpicklingError's text advises an unrestricted load: it stays on the + # cause only. Other errors say what is wrong (for example a dtype the + # installed PyTorch does not have). + detail = type(exc).__name__ + if not isinstance(exc, pickle.UnpicklingError) and str(exc): + detail += f": {str(exc).splitlines()[0]}" + raise ValueError( + "stored torch tensor value cannot be read by the restricted loader " + f"(weights_only=True) of PyTorch {torch.__version__} ({detail}). " + "Orcapod never loads a stored tensor without that restriction" + ) from exc + if type(value) not in _CLASS_NAMES: + raise ValueError( + "stored torch tensor value holds a " + f"{type(value).__module__}.{type(value).__qualname__}, not a tensor" + ) + return value + + +class LogicalTorchTensor(IgnoreListLogicalType): + """Logical type for ``torch.Tensor``: one ``torch.save`` archive per cell. + + The column is ``large_binary`` under the extension name + ``orcapod.torch.tensor``; the extension metadata carries the ignore list of + the value hash. A value reads back as a CPU ``torch.Tensor`` (or + ``torch.nn.Parameter``) with the stored dtype, shape, elements, strides, + ``requires_grad`` and lazy conjugate and negative bits. + + Example: + >>> lt = LogicalTorchTensor() + >>> t = torch.arange(6.0).reshape(2, 3) + >>> torch.equal(lt.storage_to_python(lt.python_to_storage(t)), t) + True + """ + + EXTENSION_NAME = TORCH_TENSOR_EXTENSION_NAME + STORAGE_TYPE = pa.large_binary() + VERSION = TORCH_TENSOR_VERSION + OPTIONS = OPTIONS + python_type: type = torch.Tensor + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None = None + ) -> bytes: + """Encode a tensor as its ``torch.save`` archive. + + Args: + value: A ``torch.Tensor`` or ``torch.nn.Parameter``. + converter: Ignored. Present for protocol conformance. + + Returns: + The bytes of the cell. + + Raises: + UnsupportedValueError: For tensors that cannot be stored. + TypeError: If ``value`` is not a ``torch.Tensor``. + """ + return tensor_to_bytes(value) + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None = None + ) -> Any: + """Decode a cell with PyTorch's restricted loader. + + Args: + storage_value: The stored bytes. + converter: Ignored. Present for protocol conformance. + + Returns: + A CPU ``torch.Tensor`` or ``torch.nn.Parameter``. + + Raises: + ValueError: If the bytes are not a tensor archive the restricted + loader reads. + """ + return tensor_from_bytes(bytes(storage_value)) + + def storage_sort_key(self, storage_value: Any) -> tuple[Any, ...]: + """Ordering key for stored values, used where operators sort values. + + The stored bytes are PyTorch's and give no stable order, so a cell is + ordered by the strict value hash of the tensor it holds; ``None`` sorts + first. + + Args: + storage_value: The stored bytes of a cell, or ``None``. + + Returns: + A tuple that orders stored cells deterministically. + """ + if storage_value is None: + return (0,) + return (1, tensor_digest(tensor_from_bytes(bytes(storage_value)))) diff --git a/src/orcapod/semantic_types/output_conformance.py b/src/orcapod/semantic_types/output_conformance.py index 28a5aa601..4cffaae05 100644 --- a/src/orcapod/semantic_types/output_conformance.py +++ b/src/orcapod/semantic_types/output_conformance.py @@ -1,7 +1,9 @@ """Function-output conformance (ITL-895, ITL-901). the returned value must be an instance of the declared type, or an exact numeric -equivalent; a subclass stands in for its base and is stored as the base. +equivalent; a subclass stands in for its base. What is stored for a subclass instance +is decided by the logical type of the declared class (a user class stores the base's +fields; the tensor type keeps an ``nn.Parameter``). Logical types retain ownership of their compatibility rules. The caller adds function and output context to a mismatch; this module never formats returned values. @@ -232,6 +234,16 @@ def check(self, value: object, declared: object) -> Nonconformance | None: mismatch = self.check(getattr(value, name), hints[name]) if mismatch: return _at(mismatch, f".{name}") + elif ( + logical_type is not None + and isinstance(declared, type) + and not isinstance(value, declared) + ): + # Any other class with a registered logical type (torch.Tensor, + # nn.Parameter, File, ...): the returned value must be an instance of + # the declared class. A subclass instance conforms; what is stored for + # it is decided by the logical type. + return Nonconformance("", declared, value) return None def check_number( diff --git a/src/orcapod/utils/deferred_types.py b/src/orcapod/utils/deferred_types.py new file mode 100644 index 000000000..a59a6e131 --- /dev/null +++ b/src/orcapod/utils/deferred_types.py @@ -0,0 +1,59 @@ +"""Helpers for value types that are registered on first use (ITL-871). + +A registry can name an optional value type (such as ``torch.Tensor``) by the +qualified name of its class and by its Arrow extension name, without importing +the library. The type is loaded when a lookup first needs it: + +* by Python type: a class in the looked-up type's MRO has the deferred + qualified name. Comparing names never imports anything, and such a class can + only exist once its library has been imported. +* by Arrow extension name: reading a stored column. The library is imported if + it is installed; otherwise ``MissingOptionalDependencyError`` names it. +""" + +from __future__ import annotations + +import importlib +import importlib.util +from typing import Any + + +def qualified_name(cls: type) -> str: + """Return ``module.qualname`` of a class. + + Args: + cls: Any class. + + Returns: + The class's module and qualified name joined by a dot, for example + ``"torch.Tensor"``. + """ + return f"{cls.__module__}.{cls.__qualname__}" + + +def load_object(dotted: str) -> Any: + """Import and return the object named by a dotted path. + + Args: + dotted: ``"package.module.Name"``. + + Returns: + The attribute ``Name`` of the imported module. + """ + module_name, _, attr = dotted.rpartition(".") + return getattr(importlib.import_module(module_name), attr) + + +def module_available(module_name: str) -> bool: + """Return whether a top-level module can be imported, without importing it. + + Args: + module_name: A module name such as ``"torch"``. + + Returns: + True when the import system can find the module. + """ + try: + return importlib.util.find_spec(module_name) is not None + except (ImportError, ValueError): + return False diff --git a/src/orcapod/utils/tensor_conversions.py b/src/orcapod/utils/tensor_conversions.py new file mode 100644 index 000000000..fee4efaf6 --- /dev/null +++ b/src/orcapod/utils/tensor_conversions.py @@ -0,0 +1,170 @@ +"""What storing a torch tensor converts, and the warnings that report it (ITL-871). + +Storage does not keep a tensor's device or its autograd history: a tensor on a +GPU (or another non-CPU device) is stored as a CPU tensor, and one computed from +tensors that track gradients (it has a ``grad_fn``) is stored as a leaf. +``requires_grad`` itself is stored, so a leaf tensor that tracks gradients is not +converted. The caller's tensor is never modified. The code +that knows where a value came from (a function pod's output, a source's column) +reports the conversion with ``orcapod.errors.ValueConversionWarning``, once per +(function or source, output or column, conversion). + +This module never imports torch: a value is a tensor when a class in its MRO is +``torch.Tensor`` by qualified name, and such a value can only exist once torch has +been imported. +""" + +from __future__ import annotations + +import sys +import warnings +from collections.abc import Iterable, Mapping +from typing import Any + +from orcapod.errors import ValueConversionWarning +from orcapod.logical_types.user_class_identity import class_kind, stored_field_names +from orcapod.utils.deferred_types import qualified_name + +_TENSOR_CLASS = "torch.Tensor" + +_HAS_HISTORY = "with autograd history (a grad_fn)" + + +def _is_tensor_type(cls: type) -> bool: + """Whether ``cls`` is ``torch.Tensor`` or a subclass, by qualified name. + + Not cached: a cache would keep every class it sees alive, and the MRO walk + is short. + """ + return any(qualified_name(base) == _TENSOR_CLASS for base in cls.__mro__) + + +def tensor_conversions(value: Any) -> tuple[str, ...]: + """Describe what storing ``value`` converts in the torch tensors it holds. + + Looks at ``value`` itself and, recursively, at the items of lists and tuples, + the values of dicts and the stored fields of dataclass and pydantic instances. + + Args: + value: Any value about to be stored. + + Returns: + The conversions, in first-seen order without repeats: + ``"on device 'cuda:0'"`` for a tensor that is not on the CPU, and + ``"with autograd history (a grad_fn)"`` for one that is not a leaf. + Empty when storing converts nothing. A ``meta`` tensor is not + listed: it holds no data and storing it is refused. + """ + found: dict[str, None] = {} + _collect(value, found) + return tuple(found) + + +def _collect(value: Any, found: dict[str, None]) -> None: + """Add the conversions of ``value`` and the values it contains to ``found``.""" + if isinstance(value, (list, tuple)): + for item in value: + _collect(item, found) + elif isinstance(value, dict): + for item in value.values(): + _collect(item, found) + elif _is_tensor_type(type(value)): + device = value.device + if device.type not in ("cpu", "meta"): + found[f"on device '{device}'"] = None + if value.grad_fn is not None: + found[_HAS_HISTORY] = None + elif type(value).__module__ != "builtins" and class_kind(type(value)) is not None: + for name in stored_field_names(type(value)): + _collect(getattr(value, name), found) + + +def describe(conversions: Iterable[str]) -> str: + """Join conversions into one phrase (``"on device 'cuda:0' with autograd ..."``).""" + return " ".join(conversions) + + +def _error_filter_active(message: str, module: str, lineno: int) -> bool: + """Whether the active warning filters turn this warning into an error. + + Reads ``warnings.filters`` as Python does: the first filter whose category, + message pattern, module pattern and line number match decides. + + Args: + message: The warning text. + module: Name of the module the warning is attributed to. + lineno: Line number the warning is attributed to. + + Returns: + True when that filter's action is ``"error"``. + """ + for action, text, category, module_pattern, line in warnings.filters: + if ( + issubclass(ValueConversionWarning, category) + and (text is None or text.match(message)) + and (module_pattern is None or module_pattern.match(module)) + and line in (0, lineno) + ): + return action == "error" + return False + + +def warn_once( + issued: set[tuple[str, str]], + scope: str, + conversions: tuple[str, ...], + message: str, +) -> None: + """Issue ``ValueConversionWarning`` unless every conversion was already reported. + + The record is updated only after ``warnings.warn`` returns, so when the warning + is escalated to an error every conversion raises, on every row. A conversion + that was already reported still raises once an error filter is in place. The + record is needed besides Python's own filters: the warning is reported once + per owner also under an ``always`` filter. + + Args: + issued: The ``(scope, conversion)`` pairs already reported; updated in place. + Its owner (a function, a source) decides how long it lives. + scope: What the value belongs to within the owner, such as an output key. + conversions: The conversions of the value (``tensor_conversions``). + message: The warning text. + """ + if all((scope, c) in issued for c in conversions): + # The frame the warning is attributed to (``stacklevel=3`` below). + frame = sys._getframe(2) # pyright: ignore[reportPrivateUsage] # as the warnings module does + module = frame.f_globals.get("__name__", "") + if not _error_filter_active(message, module, frame.f_lineno): + return + warnings.warn(message, ValueConversionWarning, stacklevel=3) + issued.update((scope, c) for c in conversions) + + +def warn_source_tensor_conversions( + source_id: str, rows: Iterable[Mapping[str, Any]] +) -> None: + """Warn once per column of a Python-value source whose tensors are converted. + + Args: + source_id: The source's id, named in the warning. + rows: The rows the source was built from. + """ + total = 0 + counts: dict[str, int] = {} + kinds: dict[str, dict[str, None]] = {} + for row in rows: + total += 1 + for column, value in row.items(): + conversions = tensor_conversions(value) + if conversions: + counts[column] = counts.get(column, 0) + 1 + kinds.setdefault(column, {}).update(dict.fromkeys(conversions)) + for column, count in counts.items(): + warnings.warn( + f"Source {source_id!r} column {column!r}: {count} of {total} values hold " + f"a torch tensor {describe(kinds[column])}; Orcapod stored CPU tensors " + "without autograd history, and the given tensors are unchanged. Pass " + "value.detach().cpu() to avoid this warning.", + ValueConversionWarning, + stacklevel=3, + ) diff --git a/superpowers/plans/2026-11-11-itl-871-torch-tensor.md b/superpowers/plans/2026-11-11-itl-871-torch-tensor.md new file mode 100644 index 000000000..9da1b8e4b --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-871-torch-tensor.md @@ -0,0 +1,311 @@ +# torch.Tensor value type (ITL-871) Implementation Plan + +> **SUPERSEDED (November 12, 2026).** Kept as the record of the first implementation. Do not +> follow it: the storage, the hash and the extension name changed +> (`superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md`). In particular Task 7 +> below tells you to edit `CHANGELOG.md` and `DESIGN_ISSUES.md`; a pull request never does +> that. The entries are `changelog.d/ITL-871.md` and `design_issues/ITL-871-TT1.md`. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `torch.Tensor` is an Orcapod value type (`orcapod.torch_tensor`, hash label `torch_tensor_v1`) stored in the shared v0.3 array struct, registered lazily so `import orcapod` never imports torch. + +**Architecture:** A new `logical_types/torch_type.py` (imports torch) holds the dtype<->tag table, canonicalisation and `LogicalTorchTensor`; `hashing/semantic_hashing/torch_handler.py` holds `TorchTensorHandler`. Both registries (`LogicalTypeRegistry`, `PythonTypeHandlerRegistry`) gain a "deferred types" table, filled from the context JSON, consulted on a lookup miss: by class qualified name (MRO walk, never imports torch) or by extension name (imports torch if installed, else raises an install error). + +**Tech Stack:** PyArrow, Polars, NumPy, PyTorch (optional peer dependency, tests via `uv run --with torch`). + +**Spec:** `superpowers/specs/2026-11-11-torch-tensor-v0.3.md` (provisional picks at the top override the body). +**Branch:** `eywalker/itl-871-add-pytorch-tensor-support-to-logical-types`, stacked on PR #289 (ITL-888). Draft PR against `main`. + +--- + +## Resume notes (keep updated) + +- 2026-11-11: spec approved provisionally by the coordinator (name `orcapod.torch_tensor`, label + `torch_tensor_v1`, NumPy-shared dtypes + bfloat16, GPU/grad accepted, P1 peer dependency, + no new CI workflow, draft PR). Plan written. Next: Task 1. +- Tasks 1-7 done (deferred registration, sort key, tests, torch_type + handler, declared-output + check, golden `torch` group, docs/CHANGELOG/DESIGN_ISSUES). The delta-async store test is + xfail(strict=False) for ITL-864 / PR #293 (verified it passes with #293 merged locally); remove + the xfail once #293 is on main. Findings reported: ITL-896 (terminal operator node empty after + async run), in-memory pass-through of the returned object (raised with the developer). + Next: Task 8 (full suite, ruff, basedpyright, torch on 3.11-3.13, merge main, draft PR). +- Task 8 done: draft PR #300 (https://github.com/metamorphic-labs/orcapod-python/pull/300), + CI green, CodeRabbit rounds 1 and 2 answered (round 1 fixes in e118c40c; #289-owned findings + relayed). Open follow-ups: (1) when #289 gains the recursive MergeJoin ordering key for lists + of arrays, merge #289's tip and add a colliding list[torch.Tensor] MergeJoin test; (2) remove + the delta-async xfail once #293 (ITL-864) is on main; (3) merge main after #289 lands; + (4) apply the developer's decisions on items (a)-(d) if they differ from the provisional picks. + +## Commands + +- Default suite (no torch): `uv run pytest tests/` +- Torch suite: `uv run --with torch pytest tests/test_torch tests/test_hashing/golden -q` +- Torch on each Python: `uv run --python 3.11 --with torch pytest tests/test_torch -q` (and 3.12, 3.13) +- Lint/type: `uv run --with ruff ruff check .`, `uv run --with ruff ruff format --check .`, + `uv run --with basedpyright basedpyright` (exit code). basedpyright runs WITHOUT torch: the + `import torch` lines carry `# pyright: ignore[reportMissingImports] # optional peer dependency` + and torch values are typed `Any`. + +## File map + +| File | Role | +|---|---| +| `src/orcapod/utils/deferred_types.py` (new) | `qualified_name(cls)`, `load_object(dotted)`, `MissingOptionalDependencyError` | +| `src/orcapod/logical_types/registry.py` | `deferred_logical_types` ctor arg; miss hooks in `get_by_python_type`, `get_by_arrow_extension_name` | +| `src/orcapod/hashing/semantic_hashing/type_handler_registry.py` | `deferred_handlers` ctor arg; miss hooks in `get_handler`, `get_handler_for_type` | +| `src/orcapod/contexts/data/v0.1.json` | deferred entries for torch + changelog line | +| `src/orcapod/logical_types/array_format.py` | `array_storage_sort_key` (moved from `LogicalNumpyArray.storage_sort_key`, shared) | +| `src/orcapod/logical_types/torch_type.py` (new) | tags, `canonical_tensor`, refusals, `LogicalTorchTensor` | +| `src/orcapod/hashing/semantic_hashing/torch_handler.py` (new) | `TorchTensorHandler` (`torch_tensor_v1`) | +| `src/orcapod/core/data_function.py` | declared-output check covers `torch.Tensor` | +| `src/orcapod/errors.py` | `MissingOptionalDependencyError(ImportError)` if no suitable error exists | +| `tests/test_torch/` (new) | all torch tests, `pytest.importorskip("torch")` per module | +| `tests/test_torch_absent/` or `tests/test_logical_types/test_deferred_types.py` | no-torch-import + install error (runs in default suite) | +| `tests/test_hashing/golden/{cases.py,generate_golden_hashes.py,test_golden_hashes.py,data/torch.json}` | new `torch` group, additions only | +| `docs/concepts/arrays.md`, `docs/reference/hashing.md` | Tensors section; label note | +| `CHANGELOG.md`, `DESIGN_ISSUES.md` | `[0.3.0]` entry; one appended entry | + +## Task 1: deferred-type lookup (no torch involved) + +Tests first, `tests/test_logical_types/test_deferred_types.py`, using a fake class defined in a +test helper module `tests/test_logical_types/deferred_fixture.py` with a module-level counter +proving it is imported only on first use: + +- `LogicalTypeRegistry(deferred_logical_types=[{"python_type": ".Thing", + "extension_name": "test.thing", "logical_type": ".LogicalThing", + "requires": ""}])`: `get_by_python_type(Thing)` returns a `LogicalThing` and registers it; + a subclass of `Thing` resolves too; `get_by_arrow_extension_name("test.thing")` loads it; + an unrelated type returns `None` without importing anything. +- `requires` names a module that does not exist -> `get_by_arrow_extension_name` raises + `MissingOptionalDependencyError` whose message names the extension and the module. +- `PythonTypeHandlerRegistry(deferred_handlers=[{"python_type": ..., "handler": ...}])`: + `get_handler(Thing())` loads and registers; unrelated returns `None`. + +Implementation: + +```python +# src/orcapod/utils/deferred_types.py +def qualified_name(cls: type) -> str: + return f"{cls.__module__}.{cls.__qualname__}" + +def load_object(dotted: str) -> Any: + module_name, _, attr = dotted.rpartition(".") + return getattr(importlib.import_module(module_name), attr) +``` + +In `LogicalTypeRegistry.__init__`: `self._deferred_by_python_name` and +`self._deferred_by_arrow_name` dicts of entries. `get_by_python_type`: after the existing +misses, walk `__mro__`; on a name hit call `self._load_deferred(entry)` (instantiate +`load_object(entry["logical_type"])()`, `register_logical_type`, drop the entry from both +dicts) and return the fresh lookup. `get_by_arrow_extension_name`: on a miss with a deferred +entry, `importlib.util.find_spec(entry["requires"])` is `None` -> raise +`MissingOptionalDependencyError(f"Arrow extension type {name!r} needs the optional dependency +{requires!r}, which is not installed: install it (for example `pip install torch`) to read +this column")`; else load. Same pattern for the handler registry (instantiate +`load_object(entry["handler"])()` and `register(load_object(entry["python_type"]), handler)`). + +Commit: `feat(logical_types): deferred registration of optional value types (ITL-871)`. + +## Task 2: shared sort key + +Move the body of `LogicalNumpyArray.storage_sort_key` to +`array_format.array_storage_sort_key(storage_value)`; `LogicalNumpyArray.storage_sort_key` +calls it. Existing `tests/test_arrays` MergeJoin tests cover it (run them). Commit +`refactor(array_format): share the stored-row ordering key (ITL-871)`. + +## Task 3: tests for the tensor type (failing first) + +`tests/test_torch/` modules, each starting `torch = pytest.importorskip("torch")`; mirror +`tests/test_arrays` (`array_cases.py` -> `tensor_cases.py`): + +- `test_tensor_value_format.py` (logical type + hasher through the public context API: + `get_default_context().semantic_hasher.hash_object`, `type_converter`): + stored row per dtype (all shared tags + bfloat16) equals the independent reference + (`hashlib` + `struct`) digest; label `torch_tensor_v1`; tensor vs equal ndarray: hashes differ, + digest bodies equal; invariances (`t.T.contiguous()` vs `t.T`, channels-last, `t.conj()` vs + resolved, `requires_grad`, `nn.Parameter`, MPS when available, CUDA when available); + differences (dtype width, shape `(0,3)`/`(3,0)`, `()`/`(1,)`, `-0.0`/`0.0`); refusals with + the column named (via a pod or `DictSource`): sparse, quantized, nested, meta, + `complex32`, `float8_e4m3fn`, `int4`; no mutation (`requires_grad`, device, strides, + conj bit, `data_ptr`, values of the input unchanged after storing/hashing). +- `test_tensor_round_trip.py`: pod `x: torch.Tensor -> torch.Tensor` in/out, chained pods, + read-back is CPU, contiguous, `requires_grad=False`, exact dtype and shape, plain + `torch.Tensor`, writeable, does not share memory; 0-d and empty; bfloat16 exact bits; + declared `-> torch.Tensor` returning an ndarray or float raises `TypeError` naming the key; + an `np.ndarray` column into a `torch.Tensor` parameter is a type error at graph build. +- `test_tensor_operators.py`: Batch, GroupBy (`list[torch.Tensor]`), MergeJoin with a colliding + tensor column (order by stored row), SemiJoin, Join, PolarsFilter, Select/Drop, MapData; + each sync and async (`tests/test_core/operators/test_async_sync_equivalence.py` pattern). +- `test_tensor_pipeline_stores.py`: in-memory and Delta databases, sync and async. +- `test_tensor_cached_rerun.py`: Delta store, run, then a fresh `subprocess` run hits the cache + (no recompute) and reads equal tensors; cross-process and `PYTHONHASHSEED` stability of hashes. +- `tests/test_logical_types/test_torch_optional.py` (default suite): subprocess + `import orcapod`, default context, run a small numpy pipeline, assert `"torch" not in + sys.modules`; with torch absent (skip if present) reading a hand-built `orcapod.torch_tensor` + column through `ArrowTableSource`/datagram raises `MissingOptionalDependencyError`. + +Run `uv run --with torch pytest tests/test_torch -q`: expect failures (`Unsupported Python type`). +Commit (WIP tests). + +## Task 4: `torch_type.py` and `TorchTensorHandler` + +```python +TORCH_TENSOR_EXTENSION_NAME = "orcapod.torch_tensor" +TORCH_TENSOR_HASH_LABEL = "torch_tensor_v1" +# dtype -> (tag, raw-bit view dtype or None); built at import from getattr(torch, name, None) +_SHARED = ["bool", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", + "float16", "float32", "float64", "complex64", "complex128"] +_RAW_BITS = {"bfloat16": "int16"} +``` + +`canonical_tensor(value) -> tuple[str, tuple[int, ...], memoryview]`: +1. not `torch.Tensor` -> `TypeError`; `layout != torch.strided` -> refuse (sparse/mkldnn, + "convert with .to_dense()"); `is_quantized`, `is_nested`, `device.type == "meta"`, + named (`any(n is not None for n in getattr(value, "names", ()) or ())`) -> refuse; + dtype not in the table -> refuse naming dtype ("not supported in v0.3"). +2. `t = value.detach()`; `if t.device.type != "cpu": t = t.to("cpu")`; + `t = t.resolve_conj().resolve_neg().contiguous()`; raw-bit dtypes `t = t.view(int dtype)`. +3. `arr = t.numpy()`; `_, canon = numpy_type.canonical_array(arr)`; return + `(tag, tuple(int(d) for d in value.shape), numpy_type.array_bytes(canon))`. + +Refusal message: `UnsupportedValueError(f"torch tensors with {what} cannot be stored: {reason}")` +(the converter prefixes the column, as for arrays). + +`LogicalTorchTensor(BaseLogicalType)`: same structure as `LogicalNumpyArray` +(`make_arrow_extension_type(TORCH_TENSOR_EXTENSION_NAME, ARRAY_STORAGE_TYPE, +ARRAY_FORMAT_METADATA)`, polars likewise, `python_type = torch.Tensor`); +`python_to_storage` -> `{"dtype", "shape", "data": bytes}`; `storage_to_python`: validate +fields and length like ndarray, `np_dtype = numpy_type.tag_to_dtype(tag or raw int tag)`, +`arr = np.frombuffer(...).reshape(dims).copy()`, native byte order, `torch.from_numpy(arr)`, +`.view(torch dtype)` for raw-bit tags; unknown tag for tensors -> `ValueError`; +`storage_sort_key = array_storage_sort_key`. + +`TorchTensorHandler.handle(obj, hasher)` -> `ContentHash("torch_tensor_v1", +array_digest_v1(*canonical_tensor(obj)))`. + +Context JSON: `"deferred_logical_types": [{"python_type": "torch.Tensor", "extension_name": +"orcapod.torch_tensor", "logical_type": "orcapod.logical_types.torch_type.LogicalTorchTensor", +"requires": "torch"}]` in the registry `_config`; `"deferred_handlers": [{"python_type": +"torch.Tensor", "handler": "orcapod.hashing.semantic_hashing.torch_handler.TorchTensorHandler"}]` +in the handler registry `_config`; one changelog line. Verify the objectspec parser passes plain +string dicts through unchanged. + +Run torch tests until green; run default suite. Commit +`feat(logical_types): torch.Tensor value type orcapod.torch_tensor (ITL-871)`. + +## Task 5: declared-output check + +Generalise `_declared_ndarray_type` to match `numpy.ndarray` or `torch.Tensor` by MRO names; +message for tensors suggests `torch.as_tensor(...)`. Test in Task 3 goes green. Commit. + +## Task 6: golden `torch` group (additions only) + +`cases.py`: `GROUPS` gains `"torch"`; torch cases registered only when torch imports (the +values of `ndarray.*` cases as tensors plus bool, bfloat16, zero_dim, non_contiguous; +`type.torch_tensor`; `single_field.torch_tensor.{arrow_type, schema_hash}`; +`torch_tensor.{content_hash, pipeline_hash}` from a datagram source). Test module: entries and +file-present checks for the `torch` group skip without torch. Generator: without torch, keep +`torch.json` untouched (never write it empty). Generate with torch; `git diff` must show only +the new file. Commit `test(golden): pin torch tensor hashes (ITL-871)`. + +## Task 7: docs, CHANGELOG, DESIGN_ISSUES + +- `docs/concepts/arrays.md`: "PyTorch tensors" section (install as peer dependency, identity + table, what is not stored, GPU/grad accepted, read-back, dtypes and refusals, distinct from + ndarray). Code samples are executed by `tests/test_docs` without torch: mark them + `` (needs torch, infrastructure the test run lacks) or keep prose. +- `docs/reference/hashing.md`: `torch_tensor_v1` label next to `ndarray_v1`. +- `CHANGELOG.md` `[0.3.0] — unreleased` -> `### Added`: tensor entry. +- `DESIGN_ISSUES.md`: one appended entry for ITL-871 (resolved with Fix note). + +## Task 8: finish + +Full default suite, torch suite on 3.11/3.12/3.13, ruff, basedpyright; merge `origin/main` +(no rebase); push; draft PR (`Fixes ITL-871`, stacked on #289, items a-d await the developer, +torch results pasted); `orca linear attach`, `orca worktree set --pr`; CodeRabbit rounds; +worker_done. + +--- + +# Phase 2: developer decisions of 2026-11-11 (dispatch task_3ef74b556455) + +Decisions: 2a name/label confirmed; 2b add complex32 + float8 family (raw bits, torch names +as tags; a tag the installed torch lacks fails clearly at use, never at import; quantized, +sparse, nested, sub-byte still refused); 2c GPU/grad accepted with a warning (plan below, +needs coordinator OK); 2d `orcapod[torch]` extra, support driven by importability, lazy +registration kept; 2e torch tests run in the REQUIRED CI on every PR (plan below, needs OK). +Also: no-debug-only-parameters over the whole diff; list[torch.Tensor] MergeJoin test after +#289 merges into main; ITL-864 xfail removal after #293 is on main. + +## CI facts (checked 2026-11-11) + +- Org-required `python-required-ci.yml` -> `ci-pytest.yml`: shared `setup-uv` action (installs + uv, Python, uv cache keyed on uv.lock), then `uv run pytest tests/`. No `uv sync`, no + `--all-extras`: `uv run` syncs the project + the default `dev` group only. **An extra alone + is NOT installed there.** +- `ci-typecheck.yml`: `uv run --with basedpyright basedpyright` -> same default sync (dev group). +- `ci-license-check.yml`: `uv run --no-dev --with pip-licenses` -> project deps only, no extras. +- Repo `run-tests.yml` / `release.yml`: `uv sync --locked --all-extras --dev` -> extras installed. +- Repo `_license-check.yml`: `uv sync --locked --dev` -> dev group (so whatever dev pulls). + +## 2e plan + +1. `[project.optional-dependencies] torch = ["torch>=2.3"]`; `all` gains `orcapod[torch]`. +2. `dev` group gains `"orcapod[torch]"` (uv self-reference with an extra): the default + `uv run` sync, i.e. the org-required pytest and typecheck jobs, installs torch; one source of + truth for the version bound. Users never get torch unless they ask for the extra. +3. CPU wheels: `[[tool.uv.index]] name = "pytorch-cpu", url = "https://download.pytorch.org/whl/cpu", + explicit = true` and `[tool.uv.sources] torch = [{ index = "pytorch-cpu", marker = + "sys_platform == 'linux'" }]`; macOS/Windows use PyPI. `tool.uv.sources` is not in the + published metadata, so `pip install orcapod[torch]` is unaffected. +4. `uv lock`; check no `nvidia-*`/`triton` in the Linux resolution. +5. CI guard: `tests/test_torch/test_ci_has_torch.py` (no importorskip): fails when + `CI` env var is set (GitHub sets `CI=true`) and torch is not importable. +6. No-torch path keeps running although dev envs now have torch: subprocess tests with a + `sys.meta_path` blocker for `torch` (not a sys.modules hack): `import orcapod` + run never + imports torch; reading an `orcapod.torch_tensor` column raises + `MissingOptionalDependencyError`. The current skipif-torch-installed test is converted. +7. basedpyright now sees torch: fix new errors; drop the now-unneeded + `reportMissingImports` ignores. +8. Measure: wheel download size and `uv sync` time added per job (cold cache), licenses of + everything the extra pulls (`pip-licenses` on the torch closure). + +## 2c plan (warning) + +- Category `orcapod.errors.ValueConversionWarning(UserWarning)` (name to confirm; #299 has no + warning category as of 2026-11-11). +- Emitted by the callers that have the context (no-debug-only-parameters): a pure helper in + `torch_type` returns what a store converts (`device 'cuda:0'`, `requires_grad=True`), and + (a) `PythonDataFunction._build_output_data` warns naming the function and the output key; + (b) `DictSource`/`ListSource` (Python-value sources) warn naming the source and column. + Tensors nested in lists/dicts of an output are included. +- Once per (function instance, output key, conversion kinds); one per (source, column) with the + row count. A 10,000-row run where every row returns a CUDA grad tensor: 1 warning. +- Never for plain CPU tensors without grad. Tests with `pytest.warns` / `warnings.catch_warnings`. + +## Phase 2 resume notes + +- 2c and 2e plans approved by the coordinator. Shared category `ValueConversionWarning` in + `orcapod.errors` (exact docstring sent to the coordinator for #299; placed just before + `FieldNotResolvableError`, where #299 puts its class, so the later merge conflicts visibly). +- Done: packaging (extra, dev group, CPU index, lock), CI guard, no-torch subprocess tests, + complex32/float8 (+ golden additions), conversion warning (pods + DictSource/ListSource), + basedpyright with torch (exit 0), docs/CHANGELOG/DESIGN_ISSUES/spec, merged origin/main + (#287) and #289's tip (recursive list ordering), list[torch.Tensor] MergeJoin tests (fail + before #289's fix with TypeError dict < dict). +- Measured: Linux CPU torch wheel 209.5 MB (cp311-313), sympy 6.3 MB, mpmath 0.5 MB; macOS + wheel 138.5 MB. Licenses: torch "Apache-2.0 AND ... BSD-3-Clause ... MIT AND Zlib" (passes + the repo's pip-licenses check), sympy/mpmath/networkx/jinja2/markupsafe BSD, fsspec + BSD-3-Clause, filelock Unlicense, typing_extensions PSF-2.0. CI time added: read from the + CI logs of the next run. +- Left: ITL-864 xfail removal once #293 is on main; merge main again when #289 lands. + Next: full suite, push, PR description, mark ready, CI, CodeRabbit (read review bodies). +- 2026-11-11 later: merged main (#296, #297). PR #300 marked ready; description rewritten with + final decisions. CI measured: torch CPU wheel 199.8 MiB, 19-24 s download per required pytest + job, install of all 199 packages 5.3-6.4 s, torch tests ~35-40 s; required pytest jobs went + from 2m29-2m50 to 3m23-4m00 (cold cache for the new lock). CodeRabbit on 7384e3e5: 2 actionable + (torch 2.3-2.6 lack float8_e8m0fnu) fixed in 94564573 (cases.unavailable_entries, generator + keeps pins); 3 nitpicks on #289 code relayed; review of 94564573 had nothing actionable; + docstring-coverage pre-merge warning answered (ITL-871 functions all documented). + Left: remove ITL-864 xfail once #293 is on main; merge main after #289 lands. diff --git a/superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md b/superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md new file mode 100644 index 000000000..32b6e1a95 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md @@ -0,0 +1,196 @@ +# ITL-871 torch.Tensor: native storage and strict hash Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rebuild the `torch.Tensor` value type of PR #300 on the design the developer signed +off on November 12, 2026 (storage `torch.save`, strict two-step hash with an ignore list), and +bring the PR to merge readiness. + +**Architecture:** `logical_types/torch_type.py` owns three functions that do all the work: +`stored_form(value)` (the tensor that is written: CPU, no autograd history, cloned when it does +not cover its storage), `tensor_to_bytes` / `tensor_from_bytes` (`torch.save` / `torch.load`) +and `tensor_digest(value, ignore)` (the value hash `D`). The logical type and the value handler +are thin: after ITL-981 merges they subclass its `IgnoreListLogicalType` / `IgnoreListHandler`. +Until then they are wired with fixed metadata `{"ignore":[],"version":1}`. + +**Tech Stack:** Python 3.11 to 3.13, uv, PyArrow, PyTorch (optional), pytest, ruff, basedpyright. + +--- + +## Decisions + +Developer sign-off (November 12, 2026): + +1. A property storage cannot restore never counts in the hash and has no option name. +2. Storage: `torch.save` with its defaults written explicitly; read with + `torch.load(buffer, map_location="cpu", weights_only=True)`, always explicit, never an + unrestricted fallback. A tensor that does not cover its underlying storage exactly is saved + as `value.clone(memory_format=torch.preserve_format)`, and `memory_layout` is hashed after + that same clone. +3. Hash, strict by default. `E = array_format.array_digest_v1(tag, shape, bytes)` (unchanged). + `D = SHA-256(lp(extension name) || E || u8(requires_grad) || u8(conj_bit) || u8(neg_bit) + || u64le(ndim) || i64le(each stride) || lp(class name))`, + `lp(x) = u64le(len(utf8(x))) || utf8(x)`. Options: `requires_grad`, `memory_layout`, + `conj_bit`, `neg_bit`, `subclass`. An ignored property's slot holds its canonical value: + 0, C-contiguous strides of the shape, 0, 0, `torch.Tensor`. +4. Extension name `orcapod.torch.tensor`; it is the leading text in `D`. Label `torch_tensor_v1`. +5. Metadata `{"ignore":[...],"version":1}`; the list is configured per type in the data + context; a stored column with another list is refused on read. +6. torch stays optional; `import orcapod` must not import torch. + +Coordinator (ask 1, November 12, 2026): + +- Merge conflicts: `errors.py` both sides; `data_function.py` drops the PR's helper for #299's + `check_conformance`; `test_array_value_format.py` is `main`'s. Done in `5d6a3aa3`. +- Output rule gap (a fix to ITL-895's rule): #299 checks `isinstance` only for ndarray, so + `-> nn.Parameter` returning a plain tensor is accepted. Fix generically in + `semantic_types/output_conformance.py`: when the declared type is a class with a registered + logical type and no more specific rule applies, require `isinstance(value, declared)`. + Run the full suite; if any other type's outcome changes beyond "conversion failure becomes + the standard `OutputTypeError`", stop and send the list. +- An `nn.Parameter` returned for a declared `torch.Tensor` is accepted, stored, read back and + hashed as a Parameter. Correct `docs/concepts/function-pods.md` ("stored as the base"): the + logical type decides; the tensor is the example. +- Minimum torch version: the developer decides (measured: `uint16/32/64` cannot be loaded with + `weights_only=True` before torch 2.6). Do not change `pyproject.toml`; tests of those dtypes + must hold under either answer. + +Measured (torch 2.3.0, 2.4.1, 2.5.1, 2.6.0, 2.15.0): + +- `TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1` does not override an explicit `weights_only=True`. +- `requires_grad`, `nn.Parameter`, strides, conj and neg bits round-trip. +- `clone()` resolves the conj and neg bits (2.15.0). So for a tensor that is cloned for + storage those bits are hashed as 0, like its strides: every optional property is read from + the stored form. + +## File structure + +- `src/orcapod/logical_types/torch_type.py`: `stored_form`, `tensor_to_bytes`, + `tensor_from_bytes`, `tensor_digest`, `OPTIONS`, `canonical_tensor` (kept: source of `E`), + `LogicalTorchTensor`. +- `src/orcapod/hashing/semantic_hashing/torch_handler.py`: `TorchTensorHandler`. +- `src/orcapod/utils/tensor_conversions.py`: what storing changes is now device and autograd + history only. +- `src/orcapod/semantic_types/output_conformance.py`: the generic `isinstance` rule. +- `src/orcapod/logical_types/array_format.py`, `numpy_type.py`: back to `main` (the shared + `array_storage_sort_key` is no longer used by the tensor type). +- Tests: `tests/test_torch/` (rewritten where they assert the struct row or the shared digest), + new `tests/test_torch/test_tensor_strict_hash.py`, `tests/test_torch/test_tensor_storage.py`, + `tests/test_torch/test_tensor_job_load_back.py`. + +## Task 1: Output rule (ITL-895 gap) + +- [ ] Failing tests (already in `tests/test_torch/test_tensor_value_format.py`): + `test_declared_tensor_subclass_refuses_a_plain_tensor`, + `test_declared_subclass_with_its_own_logical_type_requires_that_subclass`, + `test_non_tensor_returned_for_declared_tensor_is_refused_at_the_pod` (expect + `OutputTypeError` naming the function, output `'y'`, declared and returned types). +- [ ] In `_Checker.check`, after the `UserClassLogicalType` branch: a declared class with a + registered logical type requires `isinstance(value, declared)`, else + `Nonconformance("", declared, value)`. +- [ ] `uv run pytest tests/`; compare failures with the pre-change run; report any change for + File, Directory, Path, pandas, polars, user classes. +- [ ] Commit `fix(output_conformance): ...`. + +## Task 2: Storage + +- [ ] Tests first (`tests/test_torch/test_tensor_storage.py`, through `DictSource(...).as_table()` + and `iter_data()`): the column type is `orcapod.torch.tensor` over `large_binary`; each + supported dtype reads back with equal dtype, shape, bits; `requires_grad`, `nn.Parameter`, + transposed and channels-last strides, conj and neg bits read back; a 3-element slice of a + 1,000,000-element tensor stores under 10 KB and reads back equal and contiguous; the + user's tensor is unchanged (same `requires_grad`, `grad_fn`, strides, device, version); + another subclass is refused with `UnsupportedValueError`; a cell holding an arbitrary + pickle raises on read and runs nothing (also with + `TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1`); garbage bytes raise `ValueError`. +- [ ] `stored_form(value)`: `_check_storable`; class must be exactly `torch.Tensor` or + `torch.nn.Parameter`; `t = value.detach()`; `.to("cpu")` if needed; clone with + `preserve_format` unless `t.storage_offset() == 0`, the strides are dense and + non-overlapping, and `t.untyped_storage().nbytes() == t.numel() * t.element_size()`; + restore `requires_grad` on the new tensor object; wrap in `nn.Parameter` when the value + is one. +- [ ] `tensor_to_bytes(value)`: `torch.save(stored_form(value), buffer, pickle_protocol=2, + _use_new_zipfile_serialization=True)`. +- [ ] `tensor_from_bytes(data)`: `torch.load(io.BytesIO(data), map_location="cpu", + weights_only=True)`; the result must be exactly a `torch.Tensor` or `nn.Parameter` + (`ValueError` otherwise); torch's own errors are raised as `ValueError` from them. +- [ ] Commit. + +## Task 3: Hash + +- [ ] Tests first (`tests/test_torch/test_tensor_strict_hash.py`): pinned `D` vectors computed + in the test from the formula with `hashlib` (independent of the implementation); each of + the five properties changes the default hash; with its name passed to + `tensor_digest(value, ignore)` the two hashes are equal; all five ignored equals the + strict hash of the plain canonical tensor; dtype, shape and one bit always change the + hash; device (MPS/CUDA when present), autograd history and `.grad` never do; a tensor + never hashes like an equal ndarray, alone or inside `hash_object({"x": value})`; + `hash(read(write(v))) == hash(v)` for every case in `tensor_cases.py`, including the + step slice, the expanded view and a sliced conj view. +- [ ] `tensor_digest(value, ignore: frozenset[str]) -> bytes` reading every optional property + from `stored_form(value)`; `OPTIONS = frozenset({"requires_grad", "memory_layout", + "conj_bit", "neg_bit", "subclass"})`; class name through + `canonical_type_names.canonical_type_name`. +- [ ] `TorchTensorHandler.handle` returns `ContentHash(TORCH_TENSOR_HASH_LABEL, + tensor_digest(obj, frozenset()))` until Task 5. +- [ ] `storage_sort_key`: `None` first, then the strict `D` of the decoded value (stored bytes + are not a stable order). +- [ ] Commit. + +## Task 4: Conversion warning, existing tests, load-back test, docs + +- [ ] `tensor_conversions`: reports a non-CPU device and autograd history (`grad_fn` is not + `None`); `requires_grad` alone no longer warns. Update the messages in + `core/data_function.py`, `tensor_conversions.warn_source_tensor_conversions` and + `tests/test_torch/test_tensor_conversion_warning.py`. +- [ ] Rewrite the assertions of the struct row and the shared digest in + `test_tensor_value_format.py`, `test_tensor_type_identity.py`, + `test_tensor_round_trip.py`, `test_golden_hashes.py` (`TORCH_PINNED_ROWS`). +- [ ] `tests/test_torch/test_tensor_job_load_back.py`: save a `PipelineJob` with a tensor + output on a Delta store, load it in a new job object, read the stored results (#303). +- [ ] Docs: `docs/concepts/arrays.md`, `docs/reference/hashing.md` section 8, + `docs/concepts/logical-type-identifiers.md`, `docs/concepts/function-pods.md` (the + "stored as the base" sentence), `changelog.d/ITL-871.md`, `design_issues/ITL-871-TT1.md`, + `superpowers/specs/2026-11-11-torch-tensor-v0.3.md` (superseded note). +- [ ] Commit. + +## Task 5: Wire to ITL-981 (after the coordinator sends its sha) + +- [ ] `git fetch`, merge the sha locally. +- [ ] `LogicalTorchTensor(IgnoreListLogicalType)`: `EXTENSION_NAME`, `STORAGE_TYPE = + pa.large_binary()`, `VERSION = 1`, `OPTIONS`, `python_type`, the two storage methods. +- [ ] `TorchTensorHandler(IgnoreListHandler)`: `OPTIONS`, `HASH_LABEL`, `digest(obj, ignore)` + returning `tensor_digest(obj, ignore)`. +- [ ] `contexts/logical_types.json`: `"hash_ignore": {"orcapod.torch.tensor": []}` and + `{"ignore": {"_ref": "hash_ignore:orcapod.torch.tensor"}}` in both deferred entries. +- [ ] Tests: a custom `logical_types.json` with `["memory_layout"]` gives equal cell hashes for + a transposed tensor and its contiguous copy, and metadata + `{"ignore":["memory_layout"],"version":1}`; a column stored under another list is refused + on read (`LogicalTypeIdentifierError`); an unknown option name fails at context build; + `torch` is absent from `sys.modules` after `import orcapod` and after building the default + context. + +## Task 6: Finish (only after ITL-981's PR is on `main`) + +- [ ] Merge `origin/main`; regenerate `--group torch` only; `git diff --stat origin/main -- + tests/test_hashing/golden/data` shows `torch.json` only. +- [ ] `uv run --python 3.12 pytest tests/`, then 3.11, 3.13; ruff check, ruff format --check, + basedpyright exit 0; `scripts/assemble_changelog.py --check`, + `scripts/design_issues.py --check`. +- [ ] PR body; ONE push; CodeRabbit on the new head; every review thread answered, including + `PRRT_kwDOOf7mTs62H5kR` and `PRRT_kwDOOf7mTs62MzFD`; `worker_done`. + +## Resume notes + +- `5d6a3aa3`: merge of `origin/main` at `25650c42`. Nothing is pushed; remote head `5f98283f`. +- Tasks 1 to 5 done. ITL-984 was fixed on `main` (#330): the xfail is removed. The extra is + `torch>=2.6` (developer). +- Final merge of `origin/main` at `859106f7` (holds ITL-981, #328): every ITL-981 file is + `main`'s; `contexts/logical_types.json` has one `hash_ignore` object with both types (git + had merged it into two). Torch golden regenerated with `--group torch`: unchanged, 30 + entries; against `main` only `torch.json` is added. +- Verified on that merge: `pytest tests/` on 3.12, 3.11 and 3.13 each 10970 passed, 119 + skipped, 22 xfailed, 0 failed; ruff check, ruff format --check, basedpyright exit 0; + changelog and design-issue checks pass. +- Later type PRs (Polars #286, pandas) touch `logical_types.json`, `docs/reference/hashing.md` + and the golden files: expect one more mechanical merge of `main`. diff --git a/superpowers/plans/2026-11-12-itl-871-refresh.md b/superpowers/plans/2026-11-12-itl-871-refresh.md new file mode 100644 index 000000000..2302f105b --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-871-refresh.md @@ -0,0 +1,125 @@ +# ITL-871 Refresh of PR #300 onto main Implementation Plan + +> **SUPERSEDED (November 12, 2026).** Kept as the record of the first refresh. Do not follow +> its hash, storage or documentation instructions: the tensor type is now +> `orcapod.torch.tensor`, stored with `torch.save` and hashed strictly, not with the digest +> shared with ndarray that decision 3 below describes. The current plan is +> `superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md`. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bring PR #300 (`torch.Tensor` value type `orcapod.torch_tensor`) up to `main` and to the +v0.3 logical-type rules, so it can merge before the v0.3 tag. + +**Architecture:** Merge `origin/main` with a merge commit. The branch holds an old tip of #289 +(`68b5d164`); `main` holds the squashed #289 plus the v0.3 hash step. Every file the PR never +changed takes `main`'s version; the PR's own files are re-merged 3-way with `68b5d164` as base. +The deferred torch registration moves from the deleted `contexts/data/v0.1.json` to +`contexts/logical_types.json`. + +**Tech Stack:** Python 3.11 to 3.13, uv, PyArrow, PyTorch (optional), pytest, ruff, basedpyright. + +--- + +## Decisions (coordinator, November 12, 2026, ask 1) + +1. Deferred registration lives in `src/orcapod/contexts/logical_types.json`: + `deferred_logical_types` in the `logical_type_registry` `_config`, and a new top-level + `deferred_value_handlers`, read by `contexts/registry.py` through + `PythonTypeHandlerRegistry.register_deferred(python_type, handler)`. `v1.json` is untouched. + Keep the test that `torch` is absent from `sys.modules` after `import orcapod` and after + building `std:v1:default`. +2. Function outputs follow `main`'s rule: a returned value is an instance of the declared class + or of a subclass. The PR's relaxation (a plain base class for a declared unregistered + subclass) is dropped for ndarray and tensor. PR #299 replaces the helper + (`_check_declared_array_outputs`); whichever merges second deletes it. +3. Hash: unchanged. A tensor and an equal ndarray share the digest (`array_digest_v1`) under + labels `torch_tensor_v1` and `ndarray_v1`. Type identity comes from the identifier + (`orcapod.torch_tensor;{"version":1}`): type, schema and table hashes differ, including + zero-row and all-null columns. Inside an untyped Python structure the two hash the same + (label rule 6). State this in `docs/reference/hashing.md` and the PR body. +4. Golden: only `tests/test_hashing/golden/data/torch.json` is added against `main` (26 + entries). Regenerate with `--group torch` only. +5. `pyproject.toml` stays as approved on November 11 (extra `orcapod[torch]`, dev group, uv CPU + index). Known gaps: CUDA/MPS, torch 2.3 to 2.6. +6. Saved `PipelineJob` load-back: add the tensor test only if #303 is on `main` at merge time; + otherwise a known gap naming #303. + +## Task 1: Merge origin/main + +**Files:** all conflicted files (34); resolution rules above. + +- [x] Dry run on `scratch/itl-871-merge-dryrun`. +- [ ] Commit the merge there (parents: PR tip `5f98283f`, `origin/main`), fast-forward the PR + branch to it, delete the scratch branch. +- [ ] `git diff --stat origin/main` lists only the PR's own files. + +Resolution per file: + +- Take `main`: every conflicted file whose PR version equals its `68b5d164` version. +- `errors.py`, `logical_types/registry.py`: keep both sides. +- `array_format.py`, `numpy_type.py`: `main` plus `array_storage_sort_key`. +- `contexts/data/v0.1.json`: deleted. `DESIGN_ISSUES.md`: deleted; entry TT1 is + `design_issues/ITL-871-TT1.md`. `CHANGELOG.md`: `main`'s; text in `changelog.d/ITL-871.md`. +- `uv.lock`: `main`'s, then `uv lock`. +- `tests/test_hashing/golden/cases.py`: `main`'s cases, then the torch group. + +## Task 2: Conformance to main + +- [ ] `core/data_function.py`: `_declared_array_type(declared) -> tuple[type, str] | None` + returns the declared class and a hint; `_check_declared_array_outputs(output_data, + output_schema)` raises `TypeError` unless `isinstance(value, declared_class)`. +- [ ] `tests/test_torch/test_tensor_value_format.py`: token is + `b"orcapod.torch_tensor;7b2276657273696f6e223a317d::" + digest`; `-> nn.Parameter` + returning a plain tensor raises; `-> torch.Tensor` returning an `nn.Parameter` is stored. +- [ ] `tests/test_torch/test_tensor_operators.py::test_group_by`: non-key tag `k` stays a tag. +- [ ] `tests/test_torch/test_tensor_pipeline_stores.py`: `count(x)` only; no ITL-864 xfail; the + `merge_join` operator node is read directly. +- [ ] New `tests/test_torch/test_tensor_type_identity.py`: identifier, zero-row and all-null + columns, hashed schema with and without data, tensor against ndarray. +- [ ] Regenerate `torch.json`: + `uv run python tests/test_hashing/golden/generate_golden_hashes.py --group torch`; + `git diff --stat origin/main -- tests/test_hashing/golden/data` shows `torch.json` only. + +## Task 3: Docs + +- [ ] `docs/reference/hashing.md` section 8: tensor token with the identifier; the + nested-structure consequence; label table row for `torch_tensor_v1`. +- [ ] `docs/concepts/arrays.md`, `docs/concepts/logical-type-identifiers.md` (if it lists the + built-in types), `changelog.d/ITL-871.md`, `design_issues/ITL-871-TT1.md`, module + docstrings: `logical_types.json` instead of `v0.1.json`; `register_deferred`. +- [ ] `tests/test_logical_types/test_torch_optional.py` checks `torch` not in `sys.modules` + after building `std:v1:default`. + +## Task 4: Verify and push once + +- [ ] `uv run --python 3.12 pytest tests/`, then 3.11, then 3.13. +- [ ] `uv run --with ruff ruff check .`, `uv run --with ruff ruff format --check .`, + `uv run --with basedpyright basedpyright` (exit 0). +- [ ] `uv run python scripts/assemble_changelog.py --check`, + `uv run python scripts/design_issues.py --check`. +- [ ] Merge `origin/main` again if it moved; update the PR body; one push. + +## Task 5: Readiness + +- [ ] CI green; CodeRabbit review on the pushed head; every review thread answered (GraphQL + `reviewThreads`); review bodies read for outside-diff and nitpick findings. +- [ ] `worker_done` with the reviewed sha, hand-resolved files, golden report, known gaps. + +## Resume notes + +- Golden report (against the PR's old `torch.json`): 15 unchanged (14 `value.*` digests, the + arrow type text); 7 values moved (`datagram.content_hash`, `datagram.pipeline_hash`, two + `data_hash`, `datagram.system_tag_columns`, `single_field.torch_tensor.schema_hash`, + `type.torch_tensor`); 4 keys renamed (the `_tag_record_id::` / `_tag_source_id::` rows, whose + names hold the pipeline hash). Cause of all 11: `main`'s v0.3 hash step. +- ITL-958 and ITL-966 need no change here. +- **HOLD (coordinator, November 12, 2026, 20:11 UTC):** do not push, do not regenerate golden + files, do not post on GitHub. The developer is reconsidering how numpy, torch, pandas and + polars values are stored (likely the library's own serialization; the hash stays + content-based; what the hash ignores may become a per-type option in the extension + metadata). State when held: Tasks 1 to 3 done and committed locally (merge `22d4b717`, + then `073618ed`), nothing pushed, remote head `5f98283f`. Full suite on 3.12, 3.11 and 3.13: + 9666 passed, 103 skipped, 23 xfailed; ruff, format and basedpyright clean. Task 4's push + and PR body update, and Task 5, wait for the coordinator's decisions. The PR body draft + must be rewritten from the plan if the design changes. diff --git a/superpowers/specs/2026-11-11-torch-tensor-v0.3.md b/superpowers/specs/2026-11-11-torch-tensor-v0.3.md new file mode 100644 index 000000000..a0eebc3b7 --- /dev/null +++ b/superpowers/specs/2026-11-11-torch-tensor-v0.3.md @@ -0,0 +1,274 @@ +# `torch.Tensor` as a value type in v0.3 (ITL-871) + +> **SUPERSEDED (November 12, 2026).** The developer replaced the storage and the hash of this +> spec: a tensor is stored with `torch.save` (read with +> `torch.load(map_location="cpu", weights_only=True)`) under the extension name +> `orcapod.torch.tensor`, and hashed with a strict, Orcapod-defined hash with five optional +> properties. The current design is the module docstring of +> `src/orcapod/logical_types/torch_type.py`, `docs/concepts/arrays.md` and +> `docs/reference/hashing.md` section 8; the plan is +> `superpowers/plans/2026-11-12-itl-871-native-storage-strict-hash.md`. What still holds +> from this spec: the dtype list and the CI setup. Also changed: the extra is `torch>=2.6` +> (not 2.3), and `ValueConversionWarning` reports only a non-CPU device or autograd history +> (a `grad_fn`); `requires_grad` alone does not warn, because it is stored. + +**Date:** 2026-11-11 +**Issue:** ITL-871 (refs ITL-888 / PR #289, ITL-874, ITL-869, ITL-847 / PR #291) +**Basis:** the v0.3 array format (`superpowers/specs/2026-11-10-array-format-v0.3.md`, +`src/orcapod/logical_types/array_format.py`, `numpy_type.py`) and the ITL-874 storage review +(`superpowers/specs/2026-11-10-array-tensor-storage.md`, section B4, `b6_torch_mapping.py`). +**Status:** Decided (developer, November 11, 2026). The decisions below are final and override +the proposals in the body, which is kept as the design record. + +**Developer decisions (November 11, 2026)** + +- (a) Name `orcapod.torch_tensor`, label `torch_tensor_v1`: CONFIRMED. +- (b) Dtypes: the NumPy-shared ones plus `bfloat16`, `complex32` and the float8 family + (`float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `float8_e8m0fnu`), + stored as raw bits under the torch dtype name (the table in section (b)). A dtype the + installed torch lacks fails when a stored value of it is read, naming the dtype and the + torch version, never at import. Quantized, sparse, nested and sub-byte types stay refused. +- (c) GPU and grad tensors accepted, stored from a detached CPU copy, never modified, with + `orcapod.errors.ValueConversionWarning` (shared with PR #299): issued by the function pod + output path (function, output) and by `DictSource` / `ListSource` (source, column, count), + once per (function or source, output or column, conversion), explicitly deduplicated; a + conversion is recorded only after `warnings.warn` returns, so under an error filter every + row raises. Never raised for plain CPU tensors. +- (d) Packaging: the convenience extra `orcapod[torch] = ["torch>=2.3"]` (also in `all`). + Support is driven by `torch` being importable, however installed; `import orcapod` never + imports torch. +- (e) CI: the dev dependency group includes `orcapod[torch]`, so the org-required + `uv run pytest tests/` (which syncs the dev group, not extras) installs torch. uv resolves + torch from the PyTorch CPU index on Linux (`[tool.uv.sources]`, explicit index) and from + PyPI elsewhere; `uv.lock` holds no CUDA package. `tests/test_torch/test_ci_has_torch.py` + fails when `CI` is set and torch is missing. The no-torch path is tested in subprocesses + with a `sys.meta_path` import blocker. + +## Decided by the developer (CONFIRMED, not re-asked) + +- A tensor is stored in the **same struct** as an array: `dtype` tag, `shape`, `data` + (little-endian, C order, packed), `ARRAY_STORAGE_TYPE` with metadata `{"version":1}`. +- An ndarray and a tensor are **different data types**: their own logical type name and their + own versioned hash label, `tensor_v1`, over the **same digest body** (`array_digest_v1`). +- Identity is dtype, shape and element values (hashed by their bits). Memory layout and byte + order are not identity. +- "Extended" variants are deferred, and refused at write time with an error naming the column + and the reason. +- User objects are never mutated. + +## What today does (verified) + +Nothing tensor-shaped can enter Orcapod yet: the semantic hasher raises (`strict`, no handler +for `torch.Tensor`), the type converter raises `Unsupported Python type`, and a function pod +annotated with `torch.Tensor` cannot be defined. So adding the type **moves no existing hash, +schema or table path**. Every hash below is new. + +--- + +## (a) Name, storage version, hash label (decided: see the top) + +| Item | Value | +|---|---| +| Logical type name = Arrow extension name | **`orcapod.tensor`** | +| Storage type | `ARRAY_STORAGE_TYPE` (shared; nullable children per F1 of the array spec) | +| Extension metadata (storage version) | `{"version":1}` (`ARRAY_FORMAT_METADATA`, shared) | +| Semantic hash | `ContentHash(method="tensor_v1", digest=array_digest_v1(tag, shape, data))` | +| Token in a table hash | `b"orcapod:tensor::tensor_v1:" + digest` (array: `orcapod:ndarray::ndarray_v1:`) | +| Python type | `torch.Tensor` (annotation); read back as a plain `torch.Tensor` | + +`orcapod.tensor` follows `orcapod.ndarray`. Should JAX or TensorFlow arrays come later, they get +their own names (`orcapod.jax_array`, ...); the bare word "tensor" is taken by PyTorch because +`torch.Tensor` is the only type Python users call a tensor without a qualifier. Alternative: +`orcapod.torch_tensor` (more explicit, one-line change). + +Consequences, all by construction of the confirmed decisions: + +- An ndarray and a tensor with equal dtype, shape and values have **different hashes** (label + and extension name differ) and the **same digest body** (tested). +- A pod declared `x: torch.Tensor` fed from an `np.ndarray` column (or the reverse) is a type + mismatch at graph build, like any other pair of distinct types. No implicit conversion. + +## (b) Dtype mapping (decided: see the top) + +The tag vocabulary is the array format's (A2 of the array spec). Tags are the same strings for +the same element type, whichever library wrote them. + +**Same tags as NumPy** (identical bytes and digest body to the equal ndarray): + +| torch | tag | +|---|---| +| `bool` | `bool` (one byte, 0x00/0x01) | +| `int8 int16 int32 int64` | `int8 int16 int32 int64` | +| `uint8 uint16 uint32 uint64` | `uint8 uint16 uint32 uint64` (`uint16/32/64` exist from torch 2.3) | +| `float16 float32 float64` | `float16 float32 float64` | +| `complex64 complex128` | `complex64 complex128` | + +**Torch-only dtypes: supported via raw bits with their own tag (my proposal).** Each is a +fixed-width, byte-aligned element. The bits are read through a same-width integer view +(`t.view(torch.int16)` for bfloat16, verified to round-trip exactly) and stored LE. The tag +spells the torch dtype name, which is the established cross-library name (ml_dtypes uses the +same ones), so a future NumPy reader can map them too. + +| torch | tag | element bytes | +|---|---|---| +| `bfloat16` | `bfloat16` | 2, bfloat16 bits LE | +| `complex32` (`chalf`) | `complex32` | 4: real then imaginary, each IEEE binary16 LE | +| `float8_e4m3fn` | `float8_e4m3fn` | 1 | +| `float8_e5m2` | `float8_e5m2` | 1 | +| `float8_e4m3fnuz` | `float8_e4m3fnuz` | 1 | +| `float8_e5m2fnuz` | `float8_e5m2fnuz` | 1 | +| `float8_e8m0fnu` | `float8_e8m0fnu` | 1 (torch >= 2.7) | + +bfloat16 is the one that matters for the workshop (model weights and activations). The rest +cost one table row each. If you prefer a smaller public commitment, ship **bfloat16 only** and +refuse the others (see g); adding a tag later is additive and changes no stored value or hash. + +**Refused** (`UnsupportedValueError`, a `ValueError`, naming the column and the reason): + +| Input | Why | +|---|---| +| quantized dtypes (`qint8`, `quint8`, `qint32`, `quint4x2`, `quint2x4`), `t.is_quantized` | scale and zero point are extended data | +| sparse layouts (`sparse_coo`, `sparse_csr/csc/bsr/bsc`), `mkldnn` | not a dense value; "convert with `.to_dense()`" | +| nested / jagged tensors (`t.is_nested`) | variable shape per element | +| sub-byte and packed dtypes (`int1`..`int7`, `uint1`..`uint7`, `float4_e2m1fn_x2`, `bits*`) | no defined element layout in core torch (their `itemsize` is a placeholder) | +| `bcomplex32` (new in torch 2.15) | no stable definition yet | +| `meta` device (and fake tensors without storage) | there is no data | +| named tensors (torch < 2.15 only; the API is gone in 2.15) | names would be dropped silently (refused) | + +## (c) What is not identity and not stored; what reads back (decided: see the top) + +| Property | Identity? | Stored? | On write | +|---|---|---|---| +| dtype, shape, element bits | yes | yes | | +| device (cpu, cuda, mps, ...) | no | no | copied to CPU with `.to("cpu")` (a new tensor) | +| `requires_grad`, `grad`, `grad_fn` | no | no | `.detach()` (a new view, no copy) | +| memory format, strides, storage offset, views (`t.T`, channels-last) | no | no | `.contiguous()` (copy only when needed) | +| conjugate and negative bits (`t.conj()` lazy views) | no | no | `.resolve_conj().resolve_neg()`; the value stored is the conjugated value. Needed: `.numpy()` raises on a conjugate-bit tensor (verified) | +| names (torch < 2.15) | no | no | refused (proposal); alternative: accept and drop | +| subclass (`nn.Parameter`, user subclasses) | no | no | treated as a plain tensor, like D7 for arrays | + +Every step returns a new tensor or view; the user's tensor is never modified (in place +operations are never used). The bytes are read through `numpy()` of that (possibly copied) +tensor and copied into the stored row. + +**GPU and grad-requiring tensors are accepted** (proposal): a pod that returns a CUDA or MPS +tensor, or one that still tracks gradients, is the common case in ML code, and the brief and +issue both state device and gradient state are not identity. The user sees no warning; the +docs say so. Alternative: refuse them, with an error telling the user to call +`.detach().cpu()`; that is stricter but makes nearly every training-loop pod fail on first try. + +**Read back:** a CPU, C-contiguous, writeable `torch.Tensor`, `requires_grad=False`, with exactly +the stored dtype and shape (type strict: a bfloat16 tensor reads back bfloat16, never float32), +plain `torch.Tensor` class, not sharing memory with Arrow. A user who wants it on a GPU moves it +in the pod. + +The **declared-output check** (D8 of the array spec) extends to tensors: a pod declared +`-> torch.Tensor` that returns an `np.ndarray`, a Python number or `t.item()` is refused at the +pod with a `TypeError` naming the output key, the declared type and the returned type. Never +wrapped implicitly. + +## (d) Packaging and CI (decided: see the top) + +**Optional, in the core package.** New module `orcapod/logical_types/torch_type.py` (imports +torch) and a `TensorHandler` in the hashing handlers. Not an extension package: the type shares +the struct, the digest and the operator support with `orcapod.ndarray`, and the SpikeInterface +pattern needs an explicit `op.register_extension(...)` call, which the brief rules out. + +**Lazy registration, without importing torch** (proposal). A small "deferred type" table in the +data context names a type by its qualified class name and extension name: +`{"python_type": "torch.Tensor", "extension_name": "orcapod.tensor", "logical_type": +"orcapod.logical_types.torch_type.LogicalTorchTensor", "handler": "...TensorHandler"}`. +The three lookups that can meet a tensor consult it on a miss: + +1. `LogicalTypeRegistry.get_by_python_type(t)`: a class in `t.__mro__` with module `torch` and + qualname `Tensor` (comparing names, so this never imports torch; a tensor class can only + exist if torch is already imported) registers the type into that registry, then retries. +2. `PythonTypeHandlerRegistry.get_handler(obj)`: the same check registers `TensorHandler`. +3. `LogicalTypeRegistry.get_by_arrow_extension_name("orcapod.tensor")` (reading a stored + column, possibly in a fresh process): imports torch if it is installed and registers; if + torch is **not** installed, raises an error naming the column type and the fix ("install + PyTorch to read `orcapod.tensor` columns") instead of returning the raw struct. + +`import orcapod`, building the default context, and hashing or running anything without tensors +never import torch (tested in a subprocess). Thread safety: registration under the registry's +lock. Context JSON: one new entry in `v0.1.json` plus a changelog line; nothing else in the +context moves (golden tests prove it). + +**Install: choose one.** CI runs `uv sync --locked --all-extras --dev` in the required test job, +so **any** extra I add is installed in every required job (on Linux the default PyPI torch +wheel pulls the CUDA stack, about 2 GB). Options: + +- **P1 (my pick): no extra; torch is a peer dependency.** `pip install orcapod torch` (or the + wheel pytorch.org recommends for the user's CUDA version). PyTorch's own install matrix is + the usual reason ML libraries do not pin torch. No `pyproject.toml` or `uv.lock` change, no + required-workflow change. Docs state the tested version (PyTorch 2.15) and that the PyTorch build must support the installed NumPy (CodeRabbit, PR #300: some builds cannot use NumPy 2). +- **P2: extra `orcapod[torch] = ["torch>=2.3"]`**, plus a change to the required workflow: + `uv sync --locked --all-extras --no-extra torch`. Needs your approval (required workflow), and + adds torch for every platform to `uv.lock`. + +**Tests in CI.** Torch tests live in `tests/test_torch/` and start with +`pytest.importorskip("torch")`, so the required `uv run pytest tests/` skips them (reported as +skips, by name). They run in **one new, non-required workflow** `run-torch-tests.yml` (Linux, +Python 3.12, CPU wheel): +`uv run --with torch --index https://download.pytorch.org/whl/cpu pytest tests/test_torch tests/test_hashing/golden`. +Adding a new workflow file touches no required workflow, but it is still CI configuration: +please confirm. Without it, the torch tests run only where someone runs that command (I will +run it locally and paste the result in the PR). + +Two tests run in the required job without torch: `import orcapod` + a pipeline run leaves +`torch` out of `sys.modules` (meaningful in the torch job), and reading a hand-built +`orcapod.tensor` column with torch absent raises the install error (skipped when torch is +present). + +## (e) Operators (decided: see the top) + +Same struct, so the array work applies: + +- `Batch` and `GroupBy` give `list[torch.Tensor]` columns through the list logical type, as + for `list[np.ndarray]`. +- `MergeJoin` with a colliding tensor column orders by `storage_sort_key`, the stored + `(dtype tag, shape, data)` tuple, as for arrays (F3). +- `SemiJoin` (PR #291, Polars semi join), `Join`, `PolarsFilter`, `Select`/`Drop`, `Map*`: + pass the struct through. +- Sync and async paths, in-memory and Delta databases. + +The test file mirrors `tests/test_arrays` for tensors. + +## (f) Golden pins (decided: see the top) + +Additions only, in a **new file** `tests/test_hashing/golden/data/tensor.json`, with cases in +`cases.py` that need torch (the same values as the `ndarray.*` cases, plus torch-only ones). Reason: the existing files must not lose or skip entries when the +generator or the test runs without torch. Entries: `value.tensor.{int32, int64, float64, empty, bool, +bfloat16, zero_dim, non_contiguous}`, `type.tensor`, `single_field.tensor.{arrow_type, +schema_hash}`, `tensor.{content_hash, pipeline_hash}`. The test module skips without torch and +runs in the torch workflow; the generator refuses to write `tensor.json` without torch. A +torch-free test pins the `tensor_v1` digests against an independent reference (`hashlib` + +`struct`) of the stored rows, and checks each shared case's digest body equals the pinned +`value.ndarray.*` body. No existing golden entry moves. + +## (g) Cuts if time is short, in order + +1. The float8 family, `complex32` (refuse them; keep `bfloat16`). +2. The new CI workflow (local torch run reported in the PR instead). +3. GPU tests beyond MPS (CUDA is environment-gated and unavailable here anyway). +4. Docs worked example (keep the identity table and the format note). + +Not cuttable: storage, name, label, refusals, no torch import at `import orcapod`, lazy +registration on all three paths, the declared-output check, operator and database coverage, +fresh-process cached re-run, golden pins, CHANGELOG. + +--- + +## Implementation outline (for the plan after approval) + +1. `torch_type.py`: dtype <-> tag tables, `canonical_tensor(value) -> (tag, shape, bytes view)`, + refusals, `LogicalTorchTensor` (name `orcapod.tensor`, `storage_sort_key`). +2. `TensorHandler` (`tensor_v1`). +3. Deferred-type table: context JSON entry, the three lookup hooks, install error. +4. Declared-output check generalised from ndarray to tensor. +5. Tests first (public interface): pod in/out, chained pods, operators, sync+async, + in-memory+Delta, fresh-process cached re-run, cross-process stability, array vs tensor hash + differ, digest body equal, every dtype, refusals, no-mutation, GPU (MPS here; CUDA gated), + no torch import. +6. Docs (`docs/concepts/arrays.md` gains a Tensors section; hashing reference notes the label), + CHANGELOG `[0.3.0]`, `DESIGN_ISSUES.md` entry. diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 07cb0bddc..89abec8db 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -41,7 +41,35 @@ CaseFn = Callable[[], dict[str, Any]] -GROUPS = ("semantic", "schema", "arrow", "datagram", "pipeline") +GROUPS = ("semantic", "schema", "arrow", "datagram", "pipeline", "torch") + +OPTIONAL_GROUPS: dict[str, str] = {"torch": "torch"} +"""Groups that need an optional dependency, mapped to the module they need (ITL-871). + +Such a group is computed, generated and checked only where the module is installed; its +golden file is never rewritten without it. +""" + + +def group_available(group: str) -> bool: + """Whether ``group`` can be computed in this environment. + + Args: + group: One of ``GROUPS``. + + Returns: + False when the group needs an optional module that is not installed. + """ + import importlib.util + + module = OPTIONAL_GROUPS.get(group) + return module is None or importlib.util.find_spec(module) is not None + + +def available_groups() -> tuple[str, ...]: + """The groups that can be computed in this environment, in ``GROUPS`` order.""" + return tuple(g for g in GROUPS if group_available(g)) + _CASES: dict[str, list[tuple[str, CaseFn]]] = {group: [] for group in GROUPS} @@ -94,7 +122,7 @@ def compute_group(group: str, *, allow_errors: bool = False) -> dict[str, Any]: def compute_all(*, allow_errors: bool = False) -> dict[str, dict[str, Any]]: - """Compute every golden group. + """Compute every golden group this environment can compute (``available_groups``). Args: allow_errors: Passed to ``compute_group``. @@ -102,7 +130,7 @@ def compute_all(*, allow_errors: bool = False) -> dict[str, dict[str, Any]]: Returns: A dict mapping each group name to its computed entries. """ - return {g: compute_group(g, allow_errors=allow_errors) for g in GROUPS} + return {g: compute_group(g, allow_errors=allow_errors) for g in available_groups()} # --------------------------------------------------------------------------- @@ -1718,3 +1746,141 @@ def pipeline_default_source_ids() -> dict[str, Any]: job.nodes["adjust"].as_source().source_id ) return out + + +# --------------------------------------------------------------------------- +# Group: torch — torch.Tensor values (ITL-871); needs the optional torch package +# --------------------------------------------------------------------------- + + +def _torch_values() -> dict[str, Any]: + """Tensors whose hashes are pinned: the ndarray cases' values, torch-only dtypes + and one case per optional property of the value hash.""" + import torch + + base = torch.arange(6, dtype=torch.float32).reshape(2, 3) + return { + # The same values as the ndarray.* semantic cases. + "int32": torch.arange(6, dtype=torch.int32).reshape(2, 3), + "int64": torch.arange(6, dtype=torch.int64).reshape(2, 3), + "float64": torch.tensor([[0.5, 1.5], [2.5, 3.5]], dtype=torch.float64), + "empty": torch.zeros((0,), dtype=torch.float32), + "bool": torch.tensor([True, False, True]), + "bfloat16": torch.tensor([1.5, -2.25, 3.0], dtype=torch.bfloat16), + "complex32": torch.tensor([1 + 2j, -0.5 + 0.25j], dtype=torch.complex32), + "zero_dim": torch.tensor(2.5), + # One case per optional property of the hash (all others canonical). + "non_contiguous": base.T, + "requires_grad": base.clone().requires_grad_(True), + "parameter": torch.nn.Parameter(base.clone(), requires_grad=False), + "conj": torch.tensor([1 + 2j, 3 - 4j], dtype=torch.complex64).conj(), + # A view of a larger buffer is hashed as its clone: contiguous strides. + "step_slice": torch.arange(12, dtype=torch.float32).reshape(2, 6)[:, ::2], + # Powers of two are exact in every float8 format. A dtype this torch lacks is + # left out; its pinned entry is listed by unavailable_entries. + **{ + name: torch.tensor([0.5, 1.0, 2.0]).to(dtype) + for name in TORCH_VERSIONED_DTYPES + if (dtype := installed_torch_dtype(name)) is not None + }, + } + + +TORCH_VERSIONED_DTYPES = ( + "float8_e4m3fn", + "float8_e5m2", + "float8_e4m3fnuz", + "float8_e5m2fnuz", + "float8_e8m0fnu", +) +"""Pinned torch dtypes that a supported torch may lack (``float8_e8m0fnu`` needs 2.7).""" + + +def installed_torch_dtype(name: str) -> Any: + """Return the installed torch's dtype called ``name``, or ``None`` if it has none. + + The one place the torch cases ask whether a dtype exists, so a test can stand + in for an older torch by replacing this function. + + Args: + name: A torch dtype name, for example ``"float8_e8m0fnu"``. + + Returns: + The ``torch.dtype``, or ``None``. + """ + import torch + + dtype = getattr(torch, name, None) + return dtype if isinstance(dtype, torch.dtype) else None + + +def unavailable_entries(group: str) -> set[str]: + """Pinned entries of ``group`` that this environment cannot compute. + + These are the ``value.*`` entries of torch dtypes the installed torch lacks. The + golden test skips them and the generator keeps their pins. + + Args: + group: One of ``GROUPS``. + + Returns: + The entry names; empty for every group but an available ``torch`` group. + """ + if group != "torch" or not group_available(group): + return set() + return { + f"value.{name}" + for name in TORCH_VERSIONED_DTYPES + if installed_torch_dtype(name) is None + } + + +@case("torch", "value") +def torch_values() -> dict[str, Any]: + """Semantic hashes of the pinned tensors.""" + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(v).to_string() for k, v in _torch_values().items()} + + +@case("torch", "type") +def torch_type() -> dict[str, Any]: + """Semantic hash of the ``torch.Tensor`` type.""" + import torch + + return { + "torch_tensor": _ctx().semantic_hasher.hash_object(torch.Tensor).to_string() + } + + +@case("torch", "single_field") +def torch_single_field() -> dict[str, Any]: + """Arrow type and schema hash of a one-field ``torch.Tensor`` schema.""" + import torch + + from orcapod.types import Schema + + ctx = _ctx() + return { + "torch_tensor.arrow_type": _arrow_type_text( + ctx.type_converter.python_type_to_arrow_type(torch.Tensor) + ), + "torch_tensor.schema_hash": ctx.semantic_hasher.hash_object( + Schema({"c": torch.Tensor}) + ).to_string(), + } + + +@case("torch", "datagram") +def torch_datagram() -> dict[str, Any]: + """Identity of a two-row tensor source.""" + import torch + + from orcapod.sources import DictSource + + rows = [ + {"k": "r1", "c": torch.arange(4, dtype=torch.int64)}, + {"k": "r2", "c": torch.tensor([[0.5, 1.5]], dtype=torch.float64)}, + ] + return stream_summary( + DictSource(data=rows, tag_columns=["k"], source_id="golden_torch_tensor") + ) diff --git a/tests/test_hashing/golden/data/torch.json b/tests/test_hashing/golden/data/torch.json new file mode 100644 index 000000000..32f06937b --- /dev/null +++ b/tests/test_hashing/golden/data/torch.json @@ -0,0 +1,35 @@ +{ + "datagram.content_hash": "semantic_v1:21c4034e75fe28dd3910b1cf22e6d054d85490ba3a1a1ceb584c3d9247f3efac", + "datagram.pipeline_hash": "semantic_v1:992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc", + "datagram.row[{\"k\":\"r1\"}]._tag_record_id::992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc": "910ffa080815572cad8bae17e89ad18f", + "datagram.row[{\"k\":\"r1\"}]._tag_source_id::992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc": "golden_torch_tensor", + "datagram.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000025222f740e4ca886c970a6412c8c6d2c2ec59305a2d4133815bc854431c28590e", + "datagram.row[{\"k\":\"r2\"}]._tag_record_id::992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc": "48091abf41f7536d89fec56ef519f09b", + "datagram.row[{\"k\":\"r2\"}]._tag_source_id::992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc": "golden_torch_tensor", + "datagram.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:0000026f627a426ed1df55cc22ee7d981089645020f2d06b77fe6c57421d32d0a574a7", + "datagram.system_tag_columns": [ + "_tag_source_id::992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc", + "_tag_record_id::992d43f8f37c85f4272a37f482edd2d7f48632e92af8a984b1688f8ed243e1cc" + ], + "single_field.torch_tensor.arrow_type": "extension[large_binary]", + "single_field.torch_tensor.schema_hash": "semantic_v1:f7ba4f8cad5d1e8997281e594d00f3f4bd2af0b3b38cb2e94a70f28b494fba26", + "type.torch_tensor": "semantic_v1:f0ca3c55aaa6f3e8566635e7d32425870ff52faa811de28a1034edf143834b6b", + "value.bfloat16": "torch_tensor_v1:c87edfd3c4bb6c0badca59073a741ce4752251b3b3232b94e2b372ec3e9019e1", + "value.bool": "torch_tensor_v1:204ae3923743296d0112c035a62bbc0ff6e0d3941513cb53cc1b9a8bcebfe4a6", + "value.complex32": "torch_tensor_v1:be40d785478a4188be39a70cc7677f8e0b8ee98921b7669600d2d707287d7c16", + "value.conj": "torch_tensor_v1:a8259bb5fae1ac7dd77c7fada523ec47c121ec6426c8ed677c68d42823678468", + "value.empty": "torch_tensor_v1:c6b4b550c3cde5747c25c08fcad7a93f842befbd77bba9d4d69502c06725d8bb", + "value.float64": "torch_tensor_v1:7116fadbe108fc9a6d2519e3c5da9f53da65fb9248c7935ebc87a4b53f18a51c", + "value.float8_e4m3fn": "torch_tensor_v1:6d5725dac3217d06592827bfee12ac8caa9284ec4faa9598038153e4152c8d60", + "value.float8_e4m3fnuz": "torch_tensor_v1:4a225105f7508c03fb419f14ebc37297c60b62bbfeea842b019e7f1d3bfa9bc3", + "value.float8_e5m2": "torch_tensor_v1:3f84967ba8a6b9046ad1181a3369c13ea24579d3f83a9f170ee18e04fa230478", + "value.float8_e5m2fnuz": "torch_tensor_v1:22774bf7f4cbb301aa5d3a7eb9260e98c54e278a8ad3972d17a43f007341f801", + "value.float8_e8m0fnu": "torch_tensor_v1:c72e922887b2260315b085c36c3ab1f56c81649109a6333b49aa3104aa887fe4", + "value.int32": "torch_tensor_v1:ef644e2390862e3a8b2e119bdcbae7760de2de965de79d215d84f83d664e49b4", + "value.int64": "torch_tensor_v1:5f25bb8b0f3ff7be6f0943e9a0e372f182d3021fc0da0ca5b676c895f4967cd9", + "value.non_contiguous": "torch_tensor_v1:1c942233e11cc7336f0a0c37b8c4ec33a640b796ba347f7c8a9daa0765c03621", + "value.parameter": "torch_tensor_v1:63bd8bf6729f9caae5342914286c3b2b7d5c84a2c5fdefed228d0bfc0d1e1161", + "value.requires_grad": "torch_tensor_v1:2d30ceea79c0109cd6200639f466e846db1b17b6b48b4ebeda35e4c75954fbb6", + "value.step_slice": "torch_tensor_v1:e1dd0586c342eb2a495195235bfef1876ce9109b10398fd67be4911e3659deb1", + "value.zero_dim": "torch_tensor_v1:b2f415e6539834843f8d100b2744d7a54a2c18ed1491b8ebdf8b79b00bc183bc" +} diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index f3983066e..fedbfb4f8 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -110,7 +110,21 @@ def main(argv: list[str] | None = None) -> None: ) args = parser.parse_args(argv) - groups = args.group or list(cases.GROUPS) + unavailable = [g for g in args.group or [] if not cases.group_available(g)] + if unavailable: + parser.error( + f"group(s) {unavailable} need an optional dependency that is not installed " + f"({[cases.OPTIONAL_GROUPS[g] for g in unavailable]})" + ) + groups = args.group or list(cases.available_groups()) + skipped = [ + g for g in cases.GROUPS if g not in groups and not cases.group_available(g) + ] + if skipped and not args.stdout: + # Never rewrite an optional group's file without its dependency (ITL-871). + print( + f"skipped {skipped}: optional dependency not installed; files left as they are" + ) computed = { g: cases.compute_group(g, allow_errors=args.allow_errors) for g in groups } @@ -120,6 +134,25 @@ def main(argv: list[str] | None = None) -> None: args.output_dir.mkdir(parents=True, exist_ok=True) for group, values in computed.items(): path = golden_path(group, args.output_dir) + missing = cases.unavailable_entries(group) + if missing: + # Keep the pins this environment cannot compute (an older optional + # dependency), from the target file or else the repository's, so a + # golden file is never written without them. + source = path if path.exists() else golden_path(group) + pinned = ( + json.loads(source.read_text(encoding="utf-8")) + if source.exists() + else {} + ) + lost = sorted(e for e in missing if e not in pinned) + if lost: + raise SystemExit( + f"cannot write {path}: this environment cannot compute {lost} " + "and no existing golden file pins them; install the newer " + "optional dependency" + ) + values = {**values, **{e: pinned[e] for e in missing}} path.write_text(dump(values), encoding="utf-8") print(f"wrote {len(values)} entries to {path}") diff --git a/tests/test_hashing/golden/test_golden_hashes.py b/tests/test_hashing/golden/test_golden_hashes.py index 72386bc9a..7c5f4acbd 100644 --- a/tests/test_hashing/golden/test_golden_hashes.py +++ b/tests/test_hashing/golden/test_golden_hashes.py @@ -42,7 +42,23 @@ def _load_golden() -> dict[str, dict[str, Any]]: GOLDEN = _load_golden() -GOLDEN_ENTRIES = [(group, entry) for group in cases.GROUPS for entry in GOLDEN[group]] + + +def _needs(group: str) -> Any: + """A skip mark for a group whose optional dependency is not installed (ITL-871).""" + module = cases.OPTIONAL_GROUPS.get(group) + return pytest.mark.skipif( + not cases.group_available(group), + reason=f"golden group {group!r} needs the optional package {module!r}", + ) + + +GOLDEN_ENTRIES = [ + pytest.param(group, entry, id=f"{group}:{entry}", marks=_needs(group)) + for group in cases.GROUPS + for entry in GOLDEN[group] +] +GROUP_PARAMS = [pytest.param(group, marks=_needs(group)) for group in cases.GROUPS] def _moved_message(group: str, entry: str, expected: Any, actual: Any) -> str: @@ -69,16 +85,16 @@ def test_golden_file_present(group: str) -> None: ) -@pytest.mark.parametrize( - ("group", "entry"), GOLDEN_ENTRIES, ids=[f"{g}:{e}" for g, e in GOLDEN_ENTRIES] -) +@pytest.mark.parametrize(("group", "entry"), GOLDEN_ENTRIES) def test_golden_entry(live: dict[str, dict[str, Any]], group: str, entry: str) -> None: + if entry in cases.unavailable_entries(group): + pytest.skip(f"{group}:{entry} needs a newer version of an optional dependency") actual = live[group].get(entry, "") expected = GOLDEN[group][entry] assert actual == expected, _moved_message(group, entry, expected, actual) -@pytest.mark.parametrize("group", cases.GROUPS) +@pytest.mark.parametrize("group", GROUP_PARAMS) def test_no_unpinned_entries(live: dict[str, dict[str, Any]], group: str) -> None: unpinned = sorted(set(live[group]) - set(GOLDEN[group])) assert not unpinned, ( @@ -124,9 +140,10 @@ def test_values_independent_of_process_and_hash_seed(seed: str) -> None: computed = json.loads(result.stdout) moved = [ _moved_message(group, entry, value, computed[group].get(entry)) - for group in cases.GROUPS + for group in cases.available_groups() for entry, value in GOLDEN[group].items() - if computed[group].get(entry) != value + if entry not in cases.unavailable_entries(group) + and computed[group].get(entry) != value ] assert not moved, f"PYTHONHASHSEED={seed}:\n" + "\n".join(moved[:5]) @@ -251,3 +268,180 @@ def test_schema_round_trips_through_arrow_with_equal_hash(python_type: Any) -> N assert ctx.semantic_hasher.hash_object( round_tripped ) == ctx.semantic_hasher.hash_object(schema) + + +def _reference_array_digest(tag: str, shape: tuple[int, ...], data: bytes) -> str: + """The version-1 array digest from its specification (``hashlib`` + ``struct`` only).""" + import hashlib + import struct + + t = tag.encode("utf-8") + preimage = struct.pack(" str: + """The tensor value hash ``D`` from its specification (``hashlib`` + ``struct`` only). + + ``D = SHA-256(lp("orcapod.torch.tensor") || E || u8(requires_grad) || u8(conj_bit) + || u8(neg_bit) || u64le(ndim) || i64le(each stride) || lp(class name))``. + """ + import hashlib + import struct + + def lp(text: str) -> bytes: + """Return ``text`` as UTF-8 behind its length (unsigned 64-bit little-endian).""" + raw = text.encode("utf-8") + return struct.pack(" None: + # Runs without torch: the pinned torch_tensor_v1 values are D computed by hand. + """Pinned tensor hashes equal the value hash computed from the specification.""" + tag, shape, data, flags, strides, class_name = TORCH_PINNED[name] + element = _reference_array_digest(tag, shape, data) + assert GOLDEN["torch"][f"value.{name}"] == ( + "torch_tensor_v1:" + + _reference_tensor_digest(element, flags, strides, class_name) + ) + + +@pytest.mark.parametrize("name", ["int32", "int64", "float64", "empty"]) +def test_tensor_and_ndarray_pins_differ(name: str) -> None: + """Pinned tensor and ndarray hashes of equal elements have different digests.""" + tensor_label, tensor_body = GOLDEN["torch"][f"value.{name}"].split(":") + array_label, array_body = GOLDEN["semantic"][f"value.ndarray.{name}"].split(":") + assert (tensor_label, array_label) == ("torch_tensor_v1", "ndarray_v1") + assert tensor_body != array_body + + +OLDER_TORCH_GOLDEN = """ +import json +import pathlib +import shutil +import sys + +from tests.test_hashing.golden import cases +from tests.test_hashing.golden.generate_golden_hashes import golden_path, main + +# Stand in for a supported PyTorch that predates float8_e8m0fnu (added in torch +# 2.7). The attribute itself cannot be removed: torch.save refers to it. +_installed = cases.installed_torch_dtype +cases.installed_torch_dtype = ( + lambda name: None if name == "float8_e8m0fnu" else _installed(name) +) + +live = cases.compute_group("torch") +print("produced:", "value.float8_e8m0fnu" in live) +print("unavailable:", sorted(cases.unavailable_entries("torch"))) +out = pathlib.Path(sys.argv[1]) +shutil.copy(golden_path("torch"), out / "torch.json") +main(["--group", "torch", "--output-dir", str(out)]) +kept = json.loads((out / "torch.json").read_text()) +pinned = json.loads(golden_path("torch").read_text()) +print("pins kept:", kept == pinned) +empty = out / "empty" +main(["--group", "torch", "--output-dir", str(empty)]) +fresh = json.loads((empty / "torch.json").read_text()) +print("pins kept in an empty output dir:", fresh == pinned) +""" + + +@pytest.mark.skipif(not cases.group_available("torch"), reason="needs torch") +def test_torch_group_runs_on_a_torch_without_a_newer_dtype(tmp_path: Path) -> None: + """With a torch lacking a pinned dtype, the group computes, skips that entry, keeps its pin.""" + repo_root = Path(__file__).resolve().parents[3] + proc = subprocess.run( + [sys.executable, "-c", OLDER_TORCH_GOLDEN, str(tmp_path)], + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": str(repo_root)}, + cwd=repo_root, + timeout=GENERATOR_TIMEOUT_S, + ) + assert proc.returncode == 0, proc.stderr[-3000:] + lines = [line for line in proc.stdout.splitlines() if ": " in line] + assert "produced: False" in lines + assert "unavailable: ['value.float8_e8m0fnu']" in lines + assert "pins kept: True" in lines + assert "pins kept in an empty output dir: True" in lines diff --git a/tests/test_logical_types/deferred_fixture.py b/tests/test_logical_types/deferred_fixture.py new file mode 100644 index 000000000..aa5c01390 --- /dev/null +++ b/tests/test_logical_types/deferred_fixture.py @@ -0,0 +1,147 @@ +"""Fixture classes for the deferred-registration tests (ITL-871). + +``Thing`` plays the role of an optional library's class (such as ``torch.Tensor``); +``LogicalThing`` and ``ThingHandler`` play the logical type and the semantic +handler that are only loaded on first use. Their ``instances`` counters show +when the registries instantiate them. +""" + +from __future__ import annotations + +from typing import Any, ClassVar + +import polars as pl +import pyarrow as pa + +from orcapod.logical_types.base_logical_type import BaseLogicalType +from orcapod.logical_types.registry import ( + make_arrow_extension_type, + make_polars_extension_type, +) +from orcapod.types import ContentHash + +THING_EXTENSION_NAME = "orcapod_test.deferred_thing" + + +class Thing: + """A value type registered lazily.""" + + def __init__(self, text: str) -> None: + """Hold the text.""" + self.text = text + + +class SubThing(Thing): + """A subclass, resolved through the MRO.""" + + +class LogicalThing(BaseLogicalType): + """Logical type for ``Thing`` over ``large_string``.""" + + instances: ClassVar[int] = 0 + _arrow_ext_class = make_arrow_extension_type( + THING_EXTENSION_NAME, pa.large_string() + ) + _polars_ext_class = make_polars_extension_type( + THING_EXTENSION_NAME, pa.large_string() + ) + logical_type_name: str = THING_EXTENSION_NAME + python_type: type = Thing + + def __init__(self) -> None: + """Count the instance.""" + LogicalThing.instances += 1 + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the Arrow extension type over ``large_string``.""" + return self._arrow_ext_class() # pyright: ignore[reportCallIssue] # the generated class takes no arguments + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the Polars extension type over ``large_string``.""" + return self._polars_ext_class() # pyright: ignore[reportCallIssue] # the generated class takes no arguments + + def python_to_storage(self, value: Any, converter: Any = None) -> str: + """Store a ``Thing`` as its text.""" + return value.text + + def storage_to_python(self, storage_value: Any, converter: Any = None) -> Thing: + """Rebuild a ``Thing`` from its text.""" + return Thing(storage_value) + + +class ThingHandler: + """Semantic handler for ``Thing``.""" + + instances: ClassVar[int] = 0 + + def __init__(self) -> None: + """Count the instance.""" + ThingHandler.instances += 1 + + def handle(self, obj: Any, hasher: Any) -> ContentHash: + """Hash a ``Thing`` by its text.""" + return ContentHash(method="thing_v1", digest=obj.text.encode()) + + +SLOW_EXTENSION_NAME = "orcapod_test.deferred_slow_thing" + + +class SlowThing: + """A value type whose logical type is slow to construct (widens race windows).""" + + +class LogicalSlowThing(LogicalThing): + """Logical type for ``SlowThing``; its constructor sleeps.""" + + _arrow_ext_class = make_arrow_extension_type(SLOW_EXTENSION_NAME, pa.large_string()) + _polars_ext_class = make_polars_extension_type( + SLOW_EXTENSION_NAME, pa.large_string() + ) + logical_type_name: str = SLOW_EXTENSION_NAME + python_type: type = SlowThing + + def __init__(self) -> None: + """Sleep, so that concurrent first lookups overlap.""" + import time + + time.sleep(0.05) + super().__init__() + + +SUB_EXTENSION_NAME = "orcapod_test.deferred_sub_thing" + + +class LogicalSubThing(LogicalThing): + """Logical type for ``SubThing``, deferred while ``Thing``'s is registered.""" + + _arrow_ext_class = make_arrow_extension_type(SUB_EXTENSION_NAME, pa.large_string()) + _polars_ext_class = make_polars_extension_type( + SUB_EXTENSION_NAME, pa.large_string() + ) + logical_type_name: str = SUB_EXTENSION_NAME + python_type: type = SubThing + + +EXACT_EXTENSION_NAME = "orcapod_test.deferred_exact_thing" + + +class ExactThing: + """A value type whose logical type matches the class exactly, never a subclass.""" + + +class SubExactThing(ExactThing): + """A subclass: it must not resolve to ``LogicalExactThing``.""" + + +class LogicalExactThing(LogicalThing): + """Deferred logical type for ``ExactThing`` only.""" + + _arrow_ext_class = make_arrow_extension_type( + EXACT_EXTENSION_NAME, pa.large_string() + ) + _polars_ext_class = make_polars_extension_type( + EXACT_EXTENSION_NAME, pa.large_string() + ) + logical_type_name: str = EXACT_EXTENSION_NAME + python_type: type = ExactThing + exact_python_type_only: bool = True diff --git a/tests/test_logical_types/test_deferred_types.py b/tests/test_logical_types/test_deferred_types.py new file mode 100644 index 000000000..338f779d7 --- /dev/null +++ b/tests/test_logical_types/test_deferred_types.py @@ -0,0 +1,199 @@ +"""Deferred registration of optional value types (ITL-871). + +A registry can name a value type by its qualified class name and extension name +without importing it; the type is loaded on the first lookup that needs it. +""" + +from __future__ import annotations + +import pytest + +from orcapod.errors import MissingOptionalDependencyError +from orcapod.hashing.semantic_hashing.type_handler_registry import ( + PythonTypeHandlerRegistry, +) +from orcapod.logical_types.registry import LogicalTypeRegistry + +from . import deferred_fixture as fx + +_MOD = fx.__name__ + + +def _registry(requires: str = "polars") -> LogicalTypeRegistry: + """A logical type registry with ``Thing`` deferred; ``requires`` names the needed module.""" + return LogicalTypeRegistry( + deferred_logical_types=[ + { + "python_type": f"{_MOD}.Thing", + "extension_name": fx.THING_EXTENSION_NAME, + "logical_type": f"{_MOD}.LogicalThing", + "requires": requires, + } + ] + ) + + +def _handlers() -> PythonTypeHandlerRegistry: + """A handler registry with ``ThingHandler`` deferred.""" + registry = PythonTypeHandlerRegistry() + registry.register_deferred(f"{_MOD}.Thing", f"{_MOD}.ThingHandler") + return registry + + +def test_python_type_lookup_loads_the_deferred_type_once() -> None: + """Looking up ``Thing`` loads and registers its logical type once.""" + registry = _registry() + before = fx.LogicalThing.instances + first = registry.get_by_python_type(fx.Thing) + second = registry.get_by_python_type(fx.Thing) + assert isinstance(first, fx.LogicalThing) + assert second is first + assert fx.LogicalThing.instances == before + 1 + assert registry.get_by_arrow_extension_name(fx.THING_EXTENSION_NAME) is first + + +def test_subclass_resolves_through_the_mro() -> None: + """A subclass of a deferred class resolves to the deferred logical type.""" + assert isinstance(_registry().get_by_python_type(fx.SubThing), fx.LogicalThing) + + +def test_extension_name_lookup_loads_the_deferred_type() -> None: + """Looking up the extension name loads the type and binds the Python type too.""" + registry = _registry() + lt = registry.get_by_arrow_extension_name(fx.THING_EXTENSION_NAME) + assert isinstance(lt, fx.LogicalThing) + assert registry.get_by_python_type(fx.Thing) is lt + + +def test_unrelated_lookups_load_nothing() -> None: + """Lookups of unrelated types and names return ``None`` without loading anything.""" + registry = _registry() + before = fx.LogicalThing.instances + assert registry.get_by_python_type(complex) is None + assert registry.get_by_arrow_extension_name("orcapod_test.unknown") is None + assert fx.LogicalThing.instances == before + + +def test_missing_dependency_names_extension_and_module() -> None: + """A deferred type whose module is missing raises an error naming both.""" + registry = _registry(requires="orcapod_test_no_such_module") + with pytest.raises(MissingOptionalDependencyError) as exc_info: + registry.get_by_arrow_extension_name(fx.THING_EXTENSION_NAME) + message = str(exc_info.value) + assert fx.THING_EXTENSION_NAME in message + assert "orcapod_test_no_such_module" in message + assert isinstance(exc_info.value, ImportError) + + +def test_handler_lookup_loads_the_deferred_handler_once() -> None: + """Looking up a ``Thing`` instance or type loads its handler once.""" + handlers = _handlers() + before = fx.ThingHandler.instances + first = handlers.get_handler(fx.Thing("a")) + assert isinstance(first, fx.ThingHandler) + assert handlers.get_handler(fx.SubThing("b")) is first + assert handlers.get_handler_for_type(fx.Thing) is first + assert fx.ThingHandler.instances == before + 1 + + +def test_handler_lookup_for_unrelated_type_loads_nothing() -> None: + """Unrelated handler lookups return ``None`` without loading anything.""" + handlers = _handlers() + before = fx.ThingHandler.instances + assert handlers.get_handler(1.5) is None + assert handlers.get_handler_for_type(complex) is None + assert fx.ThingHandler.instances == before + + +def test_concurrent_first_lookups_load_the_type_once() -> None: + """Threads resolving a deferred type at the same time all get one registered instance.""" + from concurrent.futures import ThreadPoolExecutor + + registry = LogicalTypeRegistry( + deferred_logical_types=[ + { + "python_type": f"{_MOD}.SlowThing", + "extension_name": fx.SLOW_EXTENSION_NAME, + "logical_type": f"{_MOD}.LogicalSlowThing", + "requires": "polars", + } + ] + ) + with ThreadPoolExecutor(8) as pool: + found = list( + pool.map(lambda _: registry.get_by_python_type(fx.SlowThing), range(8)) + ) + assert all(isinstance(lt, fx.LogicalSlowThing) for lt in found) + assert len({id(lt) for lt in found}) == 1 + + +class _BaseHandler: + """A handler registered for the base class ``Thing``.""" + + def handle(self, obj: object, hasher: object) -> str: + """Return a marker.""" + return "base" + + +def test_deferred_handler_of_a_subclass_wins_over_a_registered_base_handler() -> None: + """Lookup walks the MRO once: the most-derived handler wins, registered or deferred.""" + registry = PythonTypeHandlerRegistry() + base_handler = _BaseHandler() + registry.register(fx.Thing, base_handler) # pyright: ignore[reportArgumentType] # toy handler implements the protocol structurally + registry.register_deferred(f"{_MOD}.SubThing", f"{_MOD}.ThingHandler") + + assert registry.get_handler(fx.Thing("a")) is base_handler + assert registry.get_handler_for_type(fx.Thing) is base_handler + assert isinstance(registry.get_handler_for_type(fx.SubThing), fx.ThingHandler) + assert isinstance(registry.get_handler(fx.SubThing("a")), fx.ThingHandler) + + +def test_deferred_type_of_a_subclass_wins_over_a_registered_base_type() -> None: + """The logical type of the most-derived class wins, registered or deferred.""" + registry = LogicalTypeRegistry( + logical_types=[fx.LogicalThing()], + deferred_logical_types=[ + { + "python_type": f"{_MOD}.SubThing", + "extension_name": fx.SUB_EXTENSION_NAME, + "logical_type": f"{_MOD}.LogicalSubThing", + "requires": "polars", + } + ], + ) + assert type(registry.get_by_python_type(fx.Thing)) is fx.LogicalThing + assert type(registry.get_by_python_type(fx.SubThing)) is fx.LogicalSubThing + # The answer is the same once the deferred type is loaded. + assert type(registry.get_by_python_type(fx.SubThing)) is fx.LogicalSubThing + + +def test_deferred_exact_type_is_never_returned_for_a_subclass() -> None: + """A deferred type that matches its class exactly is not used for a subclass. + + The answer is the same whether the subclass or the class is looked up first. + """ + + def registry() -> LogicalTypeRegistry: + return LogicalTypeRegistry( + deferred_logical_types=[ + { + "python_type": f"{_MOD}.ExactThing", + "extension_name": fx.EXACT_EXTENSION_NAME, + "logical_type": f"{_MOD}.LogicalExactThing", + "requires": "polars", + } + ] + ) + + subclass_first = registry() + assert subclass_first.get_by_python_type(fx.SubExactThing) is None + assert isinstance( + subclass_first.get_by_python_type(fx.ExactThing), fx.LogicalExactThing + ) + assert subclass_first.get_by_python_type(fx.SubExactThing) is None + + class_first = registry() + assert isinstance( + class_first.get_by_python_type(fx.ExactThing), fx.LogicalExactThing + ) + assert class_first.get_by_python_type(fx.SubExactThing) is None diff --git a/tests/test_logical_types/test_torch_optional.py b/tests/test_logical_types/test_torch_optional.py new file mode 100644 index 000000000..a9fe2c4e5 --- /dev/null +++ b/tests/test_logical_types/test_torch_optional.py @@ -0,0 +1,134 @@ +"""PyTorch stays optional: ``import orcapod`` never imports torch (ITL-871). + +The development environment installs torch (the dev group pulls ``orcapod[torch]``), so +the behaviour without torch is checked in a subprocess whose import system refuses +``torch`` through a ``sys.meta_path`` finder. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +BLOCK_TORCH = textwrap.dedent( + """ + import importlib.abc + import sys + + class _BlockTorch(importlib.abc.MetaPathFinder): + def find_spec(self, name, path=None, target=None): + if name == "torch" or name.startswith("torch."): + raise ModuleNotFoundError(f"No module named {name!r}", name=name) + return None + + sys.meta_path.insert(0, _BlockTorch()) + """ +) +"""Prelude that makes ``torch`` unimportable, as in an environment without it.""" + + +def _run(script: str) -> subprocess.CompletedProcess[str]: + """Run ``script`` in a fresh interpreter from the repository root.""" + return subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}, + cwd=REPO_ROOT, + timeout=300, + ) + + +SCRIPT = textwrap.dedent( + """ + import sys + + import numpy as np + + import orcapod + from orcapod import function_pod + from orcapod.contexts import get_default_context, resolve_context + from orcapod.sources import DictSource + + # Building the v0.3 data context names the tensor type without importing torch. + assert resolve_context("std:v1:default").context_key == "std:v1:default" + assert "torch" not in sys.modules + + @function_pod(output_keys="y") + def double(x: np.ndarray) -> np.ndarray: + return x * 2 + + source = DictSource(data=[{"k": "a", "x": np.arange(3.0)}], tag_columns=["k"]) + for _, data in double.pod(source).iter_data(): + data.content_hash() + get_default_context().semantic_hasher.hash_object([1, "a", np.arange(2)]) + print("torch" in sys.modules) + """ +) + + +def test_orcapod_never_imports_torch() -> None: + """Importing Orcapod and running a pipeline leaves torch unimported.""" + proc = _run(SCRIPT) + assert proc.returncode == 0, proc.stderr[-3000:] + assert proc.stdout.strip().splitlines()[-1] == "False" + + +def test_orcapod_works_without_torch() -> None: + """The same pipeline runs when torch cannot be imported at all.""" + proc = _run(BLOCK_TORCH + SCRIPT) + assert proc.returncode == 0, proc.stderr[-3000:] + assert proc.stdout.strip().splitlines()[-1] == "False" + + +READ_TENSOR_COLUMN = textwrap.dedent( + """ + import pyarrow as pa + + from orcapod.errors import MissingOptionalDependencyError + from orcapod.sources import ArrowTableSource + + class StoredTensorType(pa.ExtensionType): + # Stands in for a column written by Orcapod with PyTorch installed. + def __init__(self): + super().__init__(pa.large_binary(), "orcapod.torch.tensor") + + def __arrow_ext_serialize__(self): + return b'{"ignore":[],"version":1}' + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + return cls() + + storage = pa.array([b"a torch.save archive"], type=pa.large_binary()) + table = pa.table( + { + "k": pa.array(["r"], pa.large_string()), + "x": pa.ExtensionArray.from_storage(StoredTensorType(), storage), + } + ) + try: + source = ArrowTableSource(table, tag_columns=["k"]) + for _, data in source.iter_data(): + data.as_dict() + except MissingOptionalDependencyError as exc: + print("raised:", exc) + else: + print("no error") + """ +) + + +def test_reading_a_tensor_column_without_torch_names_the_missing_dependency() -> None: + """Reading a tensor column without torch raises naming the type and the package.""" + proc = _run(BLOCK_TORCH + READ_TENSOR_COLUMN) + assert proc.returncode == 0, proc.stderr[-3000:] + last = proc.stdout.strip().splitlines()[-1] + assert last.startswith("raised:"), proc.stdout + assert "'orcapod.torch.tensor'" in last + assert "'torch'" in last diff --git a/tests/test_torch/__init__.py b/tests/test_torch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_torch/tensor_cases.py b/tests/test_torch/tensor_cases.py new file mode 100644 index 000000000..755441f37 --- /dev/null +++ b/tests/test_torch/tensor_cases.py @@ -0,0 +1,207 @@ +"""Shared tensors, reference digest and assertions for the ``torch.Tensor`` tests (ITL-871). + +Import this module only after ``pytest.importorskip("torch")``. Every tensor here +is a CPU, contiguous, plain ``torch.Tensor`` without gradients, so it reads back +unchanged. +""" + +from __future__ import annotations + +import hashlib +import struct +from collections.abc import Sequence +from typing import Any + +import numpy as np +import torch + +TENSORS: dict[str, Any] = { + "bool": torch.tensor([[True, False, True], [False, False, True]]), + "uint8": torch.tensor([0, 1, 255], dtype=torch.uint8), + "int8": torch.tensor([-128, -1, 0, 1, 127], dtype=torch.int8), + "int16": torch.tensor([-32768, 0, 32767], dtype=torch.int16), + "int64": torch.tensor([[-(2**63), 0], [1, 2**63 - 1]], dtype=torch.int64), + "uint16": torch.from_numpy(np.array([0, 1, 2**16 - 1], dtype=np.uint16)), + "uint32": torch.from_numpy(np.array([0, 1, 2**32 - 1], dtype=np.uint32)), + "uint64": torch.from_numpy(np.array([0, 1, 2**64 - 1], dtype=np.uint64)), + "float16": torch.tensor([0.5, -2.0, 65504.0], dtype=torch.float16), + "float32": torch.arange(6, dtype=torch.float32).reshape(2, 3), + "float64_special": torch.tensor( + [float("nan"), float("inf"), float("-inf"), -0.0, 0.0, 1e-308], + dtype=torch.float64, + ), + "complex64": torch.tensor([1 + 2j, -3.5 - 0.5j], dtype=torch.complex64), + "complex128": torch.tensor([[1 + 2j], [3 - 4j]], dtype=torch.complex128), + "bfloat16": torch.tensor([1.5, -2.25, 3.0, float("nan")], dtype=torch.bfloat16), + "complex32": torch.tensor([1 + 2j, -0.5 + 0.25j], dtype=torch.complex32), + "zero_d": torch.tensor(2.5), + "empty_0x3": torch.zeros((0, 3), dtype=torch.float32), + "empty_3x0": torch.zeros((3, 0), dtype=torch.float32), + "three_d": torch.arange(24, dtype=torch.int32).reshape(2, 3, 4), +} +"""One tensor per supported dtype and shape class.""" + +FLOAT8_DTYPES = ( + "float8_e4m3fn", + "float8_e5m2", + "float8_e4m3fnuz", + "float8_e5m2fnuz", + "float8_e8m0fnu", +) +"""The float8 dtypes Orcapod stores as raw bits, where the installed torch has them.""" + +for _name in FLOAT8_DTYPES: + _dtype = getattr(torch, _name, None) + if _dtype is not None: + # Powers of two are exact in every float8 format, including e8m0 (exponent only). + TENSORS[_name] = torch.tensor([0.5, 1.0, 2.0, 4.0]).to(_dtype) + +NUMPY_DTYPES = frozenset( + {"bool", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64"} + | {"float16", "float32", "float64", "complex64", "complex128"} +) +"""Tags whose element bytes are NumPy's: such a tensor stores the row of its ndarray.""" + + +def shares_numpy_dtype(t: Any) -> bool: + """Whether ``t``'s dtype is one NumPy has (so ``t.numpy()`` is the equal ndarray).""" + return tag_of(t) in NUMPY_DTYPES + + +NUMERIC = ( + "int8", + "float32", + "complex128", + "bfloat16", + "zero_d", + "empty_0x3", + "three_d", +) +"""Keys of tensors that support ``x * 2`` and ``x.sum()``.""" + +LABEL = "torch_tensor_v1" +EXTENSION_NAME = "orcapod.torch.tensor" + + +def tag_of(t: Any) -> str: + """The expected dtype tag: the torch dtype name (``torch.float32`` -> ``float32``).""" + return str(t.dtype).removeprefix("torch.") + + +def value_bytes(t: Any) -> bytes: + """The element bytes of ``t`` in C order, little-endian (independent of Orcapod).""" + c = t.detach().cpu().resolve_conj().resolve_neg().contiguous() + if c.dtype == torch.bfloat16: + c = c.view(torch.int16) + elif c.dtype == torch.complex32: + # Real then imaginary part of each element, each binary16. + c = torch.view_as_real(c).view(torch.int16) + elif tag_of(c).startswith("float8_"): + c = c.view(torch.uint8) + arr = c.numpy() + return arr.astype(arr.dtype.newbyteorder("<"), copy=False).tobytes() + + +def reference_digest(tag: str, shape: Sequence[int], data: bytes) -> bytes: + """The version-1 array digest, written from the spec with ``hashlib`` and ``struct`` only.""" + tag_bytes = tag.encode("utf-8") + preimage = ( + struct.pack(" bytes: + """The reference digest of a tensor's value.""" + return reference_digest(tag_of(t), list(t.shape), value_bytes(t)) + + +TENSOR_CLASS = "torch.Tensor" +PARAMETER_CLASS = "torch.nn.parameter.Parameter" +"""The class names written in the value hash.""" + + +def c_strides(shape: Sequence[int]) -> tuple[int, ...]: + """The strides, in elements, of a C-contiguous tensor of ``shape`` (as torch gives them).""" + strides: list[int] = [] + step = 1 + for dim in reversed(list(shape)): + strides.append(step) + step *= max(int(dim), 1) + return tuple(reversed(strides)) + + +def reference_value_digest( + element_digest: bytes, + *, + strides: Sequence[int], + requires_grad: bool = False, + conj_bit: bool = False, + neg_bit: bool = False, + class_name: str = TENSOR_CLASS, +) -> bytes: + """The value hash ``D`` of a tensor, written from the spec with ``hashlib`` and ``struct``. + + ``D = SHA-256(lp(extension name) || E || u8(requires_grad) || u8(conj_bit) || + u8(neg_bit) || u64le(ndim) || i64le(each stride) || lp(class name))``, where + ``lp(x) = u64le(len(utf8(x))) || utf8(x)``. + """ + + def lp(text: str) -> bytes: + """Return ``text`` as UTF-8 behind its length (unsigned 64-bit little-endian).""" + raw = text.encode("utf-8") + return struct.pack(" None: + """Assert ``got`` is a plain CPU tensor equal to ``expected`` bit for bit. + + Args: + got: The value read back. + expected: The value written. + """ + assert type(got) is torch.Tensor, f"expected torch.Tensor, got {type(got)!r}" + assert got.dtype == expected.dtype, f"dtype {got.dtype} != {expected.dtype}" + assert tuple(got.shape) == tuple(expected.shape) + assert got.device.type == "cpu" + assert not got.requires_grad + assert got.is_contiguous() + assert value_bytes(got) == value_bytes(expected) + + +def same_tensor(a: Any, b: Any) -> bool: + """Whether ``a`` is a tensor with ``b``'s dtype, shape and bits.""" + return ( + type(a) is torch.Tensor + and a.dtype == b.dtype + and tuple(a.shape) == tuple(b.shape) + and value_bytes(a) == value_bytes(b) + ) + + +def assert_same_tensors_any_order(got: list[Any], expected: list[Any]) -> None: + """Assert two lists hold the same tensors, each bit for bit, in any order.""" + assert len(got) == len(expected) + remaining = list(expected) + for g in got: + for i, e in enumerate(remaining): + if same_tensor(g, e): + del remaining[i] + break + else: + raise AssertionError(f"unexpected tensor {g!r}") diff --git a/tests/test_torch/test_ci_has_torch.py b/tests/test_torch/test_ci_has_torch.py new file mode 100644 index 000000000..a3c35f3cc --- /dev/null +++ b/tests/test_torch/test_ci_has_torch.py @@ -0,0 +1,23 @@ +"""Required CI runs the torch tests and never skips them (ITL-871). + +The torch test modules skip themselves (``pytest.importorskip``) where torch is not +installed, which is right for a user's environment but would hide them in CI. CI +installs torch through the dev group; this test fails the run if it ever does not. +""" + +from __future__ import annotations + +import importlib.util +import os + +import pytest + + +def test_torch_is_installed_in_ci() -> None: + """Under CI (the ``CI`` environment variable is set) torch must be importable.""" + if not os.environ.get("CI"): + pytest.skip("enforced only in CI, where the CI environment variable is set") + assert importlib.util.find_spec("torch") is not None, ( + "torch is not installed in CI, so every torch test would be skipped; the dev " + "dependency group must pull orcapod[torch]" + ) diff --git a/tests/test_torch/test_tensor_cached_rerun.py b/tests/test_torch/test_tensor_cached_rerun.py new file mode 100644 index 000000000..776d1096a --- /dev/null +++ b/tests/test_torch/test_tensor_cached_rerun.py @@ -0,0 +1,123 @@ +"""A cached re-run in a fresh process recomputes nothing and reads tensors back exactly (ITL-871).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("torch") + +SCRIPT = textwrap.dedent( + """ + import json, sys + from pathlib import Path + + import torch + + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.pipeline import AsyncPipelineOrchestrator + from orcapod.sources import DictSource + from tests.test_torch.tensor_cases import NUMERIC, TENSORS, value_bytes + + store, counter, mode = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] + + def bump(name): + with counter.open("a") as f: + f.write(name + "\\n") + + @function_pod(output_keys="y") + def identity(x: torch.Tensor) -> torch.Tensor: + bump("identity") + return x.clone() + + @function_pod(output_keys="total") + def total(y: torch.Tensor) -> float: + bump("total") + return float(y.to(torch.complex128).real.sum()) + + src = DictSource( + data=[{"k": k, "x": v} for k, v in TENSORS.items()], + tag_columns=["k"], + source_id="every", + ) + numeric = DictSource( + data=[{"k": k, "x": TENSORS[k]} for k in NUMERIC], + tag_columns=["k"], + source_id="numeric", + ) + job = PipelineJob(name="rerun", store=DeltaTableDatabase(base_path=store)) + with job: + identity.pod(src, label="identity_all") + total.pod(identity.pod(numeric, label="identity_numeric"), label="total") + if mode == "sync": + job.run() + else: + AsyncPipelineOrchestrator().run(job.dag) + + out = {"tensors": {}, "hashes": {}} + for t, d in job.nodes["identity_all"].iter_data(): + y = d.as_dict()["y"] + out["tensors"][t.as_dict()["k"]] = [ + type(y).__name__, str(y.dtype), list(y.shape), value_bytes(y).hex() + ] + for label in ("identity_all", "identity_numeric", "total"): + node = job.nodes[label] + out["hashes"][label] = [ + node.content_hash().to_string(), + node.pipeline_hash().to_string(), + sorted(d.content_hash().to_string() for _, d in node.iter_data()), + ] + print("JSON:" + json.dumps(out, sort_keys=True)) + """ +) + + +def _run(store: Path, counter: Path, mode: str) -> dict[str, Any]: + """Run the pipeline script in a fresh process and return its JSON report.""" + repo_root = Path(__file__).resolve().parents[2] + env = {**os.environ, "PYTHONPATH": str(repo_root), "RUST_LOG": "error"} + proc = subprocess.run( + [sys.executable, "-c", SCRIPT, str(store), str(counter), mode], + capture_output=True, + text=True, + env=env, + cwd=repo_root, + timeout=600, + ) + assert proc.returncode == 0, proc.stderr[-3000:] + line = next(x for x in proc.stdout.splitlines() if x.startswith("JSON:")) + return json.loads(line[5:]) + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_cached_rerun_in_fresh_process(tmp_path: Path, mode: str) -> None: + """A second run in a new process recomputes nothing and reads every tensor back exactly.""" + from tests.test_torch.tensor_cases import NUMERIC, TENSORS, value_bytes + + store, counter = tmp_path / "store", tmp_path / "calls.txt" + first = _run(store, counter, mode) + calls_first = counter.read_text().splitlines() + second = _run(store, counter, mode) + calls_second = counter.read_text().splitlines()[len(calls_first) :] + + # ``identity_numeric`` may reuse results ``identity_all`` computed for the same + # inputs (the result cache is keyed by function and input hash). + assert len(TENSORS) <= calls_first.count("identity") <= len(TENSORS) + len(NUMERIC) + assert calls_first.count("total") == len(NUMERIC) + assert calls_second == [] + assert second == first + for name, expected in TENSORS.items(): + assert first["tensors"][name] == [ + "Tensor", + str(expected.dtype), + list(expected.shape), + value_bytes(expected).hex(), + ] diff --git a/tests/test_torch/test_tensor_conversion_warning.py b/tests/test_torch/test_tensor_conversion_warning.py new file mode 100644 index 000000000..587cc98b4 --- /dev/null +++ b/tests/test_torch/test_tensor_conversion_warning.py @@ -0,0 +1,296 @@ +"""GPU and gradient-tracking tensors are stored from a CPU copy, with a warning (ITL-871). + +Decision 2c (developer, November 11, 2026): such tensors are accepted, the user's +tensor is never modified, and ``orcapod.errors.ValueConversionWarning`` names who gave +the value (a function pod's output, a source's column), what was converted and how +to avoid it. The warning is issued once per (function or source, output or column, +conversion), however many rows a run has, and every time when it is escalated to an +error. +""" + +from __future__ import annotations + +import dataclasses +import warnings +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import torch # noqa: E402 + +from orcapod import function_pod # noqa: E402 +from orcapod.errors import ValueConversionWarning # noqa: E402 +from orcapod.sources import ArrowTableSource, DictSource, ListSource # noqa: E402 + + +def _accelerator() -> str | None: + """A non-CPU device available here, if any.""" + if torch.cuda.is_available(): + return "cuda" + if torch.backends.mps.is_available(): + return "mps" + return None + + +def conversion_warnings(caught: list[warnings.WarningMessage]) -> list[str]: + """The messages of the ``ValueConversionWarning`` warnings in ``caught``.""" + return [ + str(w.message) for w in caught if issubclass(w.category, ValueConversionWarning) + ] + + +def run(pod: Any, source: Any) -> list[dict[str, Any]]: + """Run ``pod`` over ``source`` and return the stored output values, row by row. + + The values are read back from the stream's table, which holds what is stored. + """ + table = pod.pod(source).as_table() + stored = ArrowTableSource(table, tag_columns=["k"]) + return [data.as_dict() for _, data in stored.iter_data()] + + +def source_of(values: list[Any]) -> DictSource: + """A source with one row ``{"k": ..., "x": value}`` per value.""" + return DictSource( + data=[{"k": f"r{i:05d}", "x": v} for i, v in enumerate(values)], + tag_columns=["k"], + ) + + +def make_with_grad() -> Any: + """A new pod ``with_grad`` returning a tensor with autograd history. + + Each test builds its own: a function reports a conversion once in its lifetime. + """ + + @function_pod(output_keys="y") + def with_grad(x: torch.Tensor) -> torch.Tensor: + """Return a tensor computed from one that tracks gradients (it has a ``grad_fn``).""" + return x.clone().requires_grad_(True) * 2 + + return with_grad + + +@function_pod(output_keys="y") +def plain(x: torch.Tensor) -> torch.Tensor: + """Return a CPU tensor without gradients.""" + return x * 2 + + +def test_the_category_is_a_user_warning_in_orcapod_errors() -> None: + """``ValueConversionWarning`` is a ``UserWarning``.""" + assert issubclass(ValueConversionWarning, UserWarning) + + +def test_a_returned_gradient_tensor_warns_naming_function_output_and_fix() -> None: + """The warning names the function, the output, the conversion and the fix.""" + with pytest.warns(ValueConversionWarning) as caught: + out = run(make_with_grad(), source_of([torch.ones(2)])) + (message,) = conversion_warnings(caught.list) + assert "Function 'with_grad' output 'y'" in message + assert "autograd history" in message + assert "CPU tensor without autograd history" in message + assert ".detach().cpu()" in message + stored = out[0]["y"] + assert stored.requires_grad and stored.is_leaf + assert torch.equal(stored.detach(), torch.ones(2) * 2) + + +def test_a_returned_tensor_is_not_modified() -> None: + """The returned tensor keeps its autograd history and values.""" + kept: list[Any] = [] + + @function_pod(output_keys="y") + def keep(x: torch.Tensor) -> torch.Tensor: + """Return, and remember, a tensor with autograd history.""" + t = x.clone().requires_grad_(True) * 3 + kept.append(t) + return t + + with pytest.warns(ValueConversionWarning): + run(keep, source_of([torch.ones(2)])) + assert kept[0].requires_grad + assert kept[0].grad_fn is not None + assert torch.equal(kept[0].detach(), torch.full((2,), 3.0)) + + +def test_ten_thousand_rows_give_one_warning() -> None: + """A 10,000-row run where every row converts gives exactly one warning.""" + source = source_of([torch.full((2,), float(i)) for i in range(10_000)]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + out = run(make_with_grad(), source) + assert len(out) == 10_000 + assert len(conversion_warnings(caught)) == 1 + + +def test_each_output_and_each_function_warns_once() -> None: + """Two outputs of one function, and a second function, each warn once.""" + + @function_pod(output_keys=["a", "b"]) + def two(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Return two tensors with autograd history.""" + return x.clone().requires_grad_(True) * 1, x.clone().requires_grad_(True) * 1 + + source = source_of([torch.ones(2), torch.zeros(2), torch.ones(3)]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + run(two, source) + run(make_with_grad(), source) + messages = conversion_warnings(caught) + assert len(messages) == 3 + assert sum("'two' output 'a'" in m for m in messages) == 1 + assert sum("'two' output 'b'" in m for m in messages) == 1 + assert sum("'with_grad' output 'y'" in m for m in messages) == 1 + + +def test_tensors_inside_a_list_output_warn() -> None: + """Tensors nested in a returned list are covered.""" + + @function_pod(output_keys="ys") + def several(x: torch.Tensor) -> list[torch.Tensor]: + """Return a list of tensors, one of which has autograd history.""" + return [x.clone().requires_grad_(True) * 1, x.clone()] + + with pytest.warns(ValueConversionWarning, match="'several' output 'ys'"): + out = run(several, source_of([torch.ones(2)])) + first, second = out[0]["ys"] + assert first.requires_grad and first.is_leaf + assert not second.requires_grad + + +def test_plain_cpu_tensors_never_warn() -> None: + """CPU tensors without autograd history raise no conversion warning. + + ``requires_grad`` is stored, so a leaf that tracks gradients (and an + ``nn.Parameter``) is not converted. + """ + with warnings.catch_warnings(): + warnings.simplefilter("error", ValueConversionWarning) + run(plain, source_of([torch.ones(2), torch.zeros(3)])) + ListSource(name="x", data=[torch.ones(2)]).as_table() + stored = [ + d.as_dict()["x"] + for leaf in ( + torch.ones(2, requires_grad=True), + torch.nn.Parameter(torch.ones(2)), + ) + for _, d in source_of([leaf]).iter_data() + ] + assert all(t.requires_grad for t in stored) + + +def test_escalated_to_an_error_every_conversion_raises() -> None: + """Under an ``error`` filter each conversion raises, also after the first one.""" + with_grad = make_with_grad() + with warnings.catch_warnings(): + warnings.simplefilter("error", ValueConversionWarning) + for _ in range(2): + with pytest.raises(ValueConversionWarning, match="'with_grad' output 'y'"): + run(with_grad, source_of([torch.ones(2)])) + + +def test_a_source_column_warns_once_with_the_count() -> None: + """A source warns once per column, naming the source, the column and how many.""" + values = [ + torch.ones(2, requires_grad=True) * 1, + torch.zeros(2), + torch.ones(1, requires_grad=True) * 1, + ] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + source = DictSource( + data=[{"k": f"r{i}", "x": v} for i, v in enumerate(values)], + tag_columns=["k"], + source_id="weights", + ) + (message,) = conversion_warnings(caught) + assert "Source 'weights' column 'x'" in message + assert "2 of 3" in message + assert "autograd history" in message + assert ".detach().cpu()" in message + assert values[0].grad_fn is not None and values[2].grad_fn is not None + stored = [d.as_dict()["x"] for _, d in source.iter_data()] + assert all(t.is_leaf for t in stored) + + +def test_a_list_source_warns() -> None: + """``ListSource`` warns for its element column.""" + with pytest.warns(ValueConversionWarning, match="column 'w'"): + ListSource(name="w", data=[torch.ones(2, requires_grad=True) * 1]) + + +@pytest.mark.skipif(_accelerator() is None, reason="needs a CUDA or MPS device") +def test_a_returned_gpu_tensor_warns_naming_the_device() -> None: + """A tensor on an accelerator is stored from a CPU copy, naming the device.""" + device = _accelerator() + + @function_pod(output_keys="y") + def on_device(x: torch.Tensor) -> torch.Tensor: + """Return the tensor moved to the accelerator.""" + return x.to(device) + + with pytest.warns(ValueConversionWarning, match=f"device '{device}") as caught: + out = run(on_device, source_of([torch.arange(3.0)])) + (message,) = conversion_warnings(caught.list) + assert "grad_fn" not in message + got = out[0]["y"] + assert got.device.type == "cpu" + assert torch.equal(got, torch.arange(3.0)) + + +def test_an_error_filter_set_after_a_reported_conversion_still_raises() -> None: + """Once reported as a warning, the same conversion raises under a later error filter.""" + with_grad = make_with_grad() + with pytest.warns(ValueConversionWarning): + run(with_grad, source_of([torch.ones(2)])) + with warnings.catch_warnings(): + warnings.simplefilter("error", ValueConversionWarning) + with pytest.raises(ValueConversionWarning, match="'with_grad' output 'y'"): + run(with_grad, source_of([torch.ones(2)])) + # Without the error filter the conversion is not reported a second time. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + run(with_grad, source_of([torch.ones(2)])) + assert conversion_warnings(caught) == [] + + +def test_a_module_restricted_error_filter_set_later_still_raises() -> None: + """A later error filter with a module pattern is honoured like any other.""" + with_grad = make_with_grad() + with pytest.warns(ValueConversionWarning): + run(with_grad, source_of([torch.ones(2)])) + with warnings.catch_warnings(): + warnings.filterwarnings("error", category=ValueConversionWarning, module=r".*") + with pytest.raises(ValueConversionWarning, match="'with_grad' output 'y'"): + run(with_grad, source_of([torch.ones(2)])) + # A filter for another module does not apply. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + warnings.filterwarnings( + "error", category=ValueConversionWarning, module=r"no_such_module_xyz" + ) + run(with_grad, source_of([torch.ones(2)])) + assert conversion_warnings(caught) == [] + + +@dataclasses.dataclass +class Fit: + """A user class with a tensor field.""" + + weights: torch.Tensor + steps: int + + +def test_a_tensor_inside_a_user_class_field_warns() -> None: + """A tensor with autograd history in a dataclass field is reported.""" + + @function_pod(output_keys="fit") + def fit(x: torch.Tensor) -> Fit: + """Return a dataclass holding a tensor with a ``grad_fn``.""" + return Fit(weights=x.clone().requires_grad_(True) * 2, steps=3) + + with pytest.warns(ValueConversionWarning, match="'fit' output 'fit'.*autograd"): + run(fit, source_of([torch.ones(2)])) diff --git a/tests/test_torch/test_tensor_hash_stability.py b/tests/test_torch/test_tensor_hash_stability.py new file mode 100644 index 000000000..b52d4c881 --- /dev/null +++ b/tests/test_torch/test_tensor_hash_stability.py @@ -0,0 +1,72 @@ +"""Tensor hashes are identical across processes and ``PYTHONHASHSEED`` values (ITL-871).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +pytest.importorskip("torch") + +SCRIPT = textwrap.dedent( + """ + import json + + from orcapod.contexts import get_default_context + from orcapod.sources import DictSource + from tests.test_torch.tensor_cases import TENSORS + + hasher = get_default_context().semantic_hasher + src = DictSource( + data=[{"k": k, "x": v} for k, v in TENSORS.items()], + tag_columns=["k"], + source_id="every", + ) + out = { + "semantic": {k: hasher.hash_object(v).to_string() for k, v in TENSORS.items()}, + "data": { + t.as_dict()["k"]: d.content_hash().to_string() for t, d in src.iter_data() + }, + "source_content": src.content_hash().to_string(), + "source_pipeline": src.pipeline_hash().to_string(), + } + print(json.dumps(out, sort_keys=True)) + """ +) + + +def _run(seed: str) -> dict[str, object]: + """Hash the shared tensors in a fresh process with ``PYTHONHASHSEED=seed``.""" + repo_root = Path(__file__).resolve().parents[2] + env = {**os.environ, "PYTHONPATH": str(repo_root), "PYTHONHASHSEED": seed} + proc = subprocess.run( + [sys.executable, "-c", SCRIPT], + capture_output=True, + text=True, + env=env, + cwd=repo_root, + timeout=300, + ) + assert proc.returncode == 0, proc.stderr[-3000:] + return json.loads(proc.stdout.strip().splitlines()[-1]) + + +def test_tensor_hashes_stable_across_processes_and_hash_seeds() -> None: + """Fresh processes with different hash seeds compute identical hashes.""" + results = [_run(seed) for seed in ("0", "1", "4242", "random")] + assert all(r == results[0] for r in results[1:]) + + +def test_tensor_hashes_match_this_process() -> None: + """A fresh process computes the same semantic hashes as this one.""" + from orcapod.contexts import get_default_context + from tests.test_torch.tensor_cases import TENSORS + + hasher = get_default_context().semantic_hasher + here = {k: hasher.hash_object(v).to_string() for k, v in TENSORS.items()} + assert _run("7")["semantic"] == here diff --git a/tests/test_torch/test_tensor_ignore_list.py b/tests/test_torch/test_tensor_ignore_list.py new file mode 100644 index 000000000..641620f2a --- /dev/null +++ b/tests/test_torch/test_tensor_ignore_list.py @@ -0,0 +1,258 @@ +"""What the tensor hash ignores is an option of the type, per data context (ITL-871, ITL-981). + +The five options are ``requires_grad``, ``memory_layout``, ``conj_bit``, +``neg_bit`` and ``subclass``. The first part exercises the hash function +(``torch_type.tensor_digest``) with each option; the second part configures a list +in a logical type registration file, the way a data context does. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import pyarrow as pa # noqa: E402 +import torch # noqa: E402 + +import orcapod.contexts as contexts_package # noqa: E402 +from orcapod.contexts.core import DataContext # noqa: E402 +from orcapod.contexts.registry import JSONDataContextRegistry # noqa: E402 +from orcapod.logical_types.identifier import LogicalTypeIdentifierError # noqa: E402 +from orcapod.logical_types.torch_type import OPTIONS, tensor_digest # noqa: E402 +from orcapod.sources import ArrowTableSource, DictSource # noqa: E402 +from tests.test_torch.tensor_cases import ( # noqa: E402 + EXTENSION_NAME, + TENSOR_CLASS, + c_strides, + expected_digest, + reference_value_digest, +) +from tests.test_torch.test_tensor_strict_hash import ( # noqa: E402 + _differ_in_one_property, # pyright: ignore[reportPrivateUsage] # shared cases +) + +PAIRS = [ + (name.removesuffix("_channels_last"), a, b) + for name, a, b in _differ_in_one_property() +] +"""``(option name, a, b)``: ``a`` and ``b`` differ only in that property.""" + + +def test_the_options_are_exactly_the_five_properties() -> None: + """The option names are fixed.""" + assert OPTIONS == { + "requires_grad", + "memory_layout", + "conj_bit", + "neg_bit", + "subclass", + } + assert {name for name, _, _ in PAIRS} == OPTIONS + + +# --------------------------------------------------------------------------- +# The hash function with each option +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("option", "a", "b"), PAIRS, ids=[f"{p[0]}.{i}" for i, p in enumerate(PAIRS)] +) +def test_ignoring_a_property_makes_values_that_differ_only_in_it_equal( + option: str, a: Any, b: Any +) -> None: + """With the option ignored the two hash equal; ignoring any other keeps them apart.""" + assert tensor_digest(a) != tensor_digest(b) + assert tensor_digest(a, frozenset({option})) == tensor_digest( + b, frozenset({option}) + ) + for other in OPTIONS - {option}: + assert tensor_digest(a, frozenset({other})) != tensor_digest( + b, frozenset({other}) + ), other + + +@pytest.mark.parametrize( + "value", + [p[1] for p in PAIRS] + [p[2] for p in PAIRS], + ids=[f"{p[0]}.a" for p in PAIRS] + [f"{p[0]}.b" for p in PAIRS], +) +def test_everything_ignored_is_the_strict_hash_of_the_plain_tensor(value: Any) -> None: + """An ignored slot holds the canonical value: the slots of a plain contiguous tensor.""" + plain = value.detach().resolve_conj().resolve_neg().contiguous().clone() + plain = plain.as_subclass(torch.Tensor) + everything = tensor_digest(value, OPTIONS) + assert everything == tensor_digest(plain) + assert everything == reference_value_digest( + expected_digest(value), + requires_grad=False, + conj_bit=False, + neg_bit=False, + strides=c_strides(value.shape), + class_name=TENSOR_CLASS, + ) + + +def test_dtype_shape_and_elements_cannot_be_ignored() -> None: + """With every option ignored, dtype, shape and one bit still change the hash.""" + base = torch.arange(6, dtype=torch.float32) + digests = { + tensor_digest(value, OPTIONS) + for value in ( + base, + base.to(torch.float64), + base.reshape(2, 3), + torch.cat([base[:5], torch.tensor([-5.0])]), + torch.cat([torch.tensor([-0.0]), base[1:]]), + ) + } + assert len(digests) == 5 + + +def test_an_unknown_option_name_is_an_error() -> None: + """The hash function refuses a name that is not an option, naming it.""" + with pytest.raises(ValueError, match="device"): + tensor_digest(torch.zeros(1), frozenset({"device"})) + + +# --------------------------------------------------------------------------- +# A list configured in the data context +# --------------------------------------------------------------------------- + +STRICT = b'{"ignore":[],"version":1}' +LAYOUT = b'{"ignore":["memory_layout"],"version":1}' + + +def _context(tmp_path: Path, ignore: list[str]) -> DataContext: + """Build a data context whose tensor type ignores ``ignore``.""" + shipped = Path(contexts_package.__file__).parent / "logical_types.json" + registration = json.loads(shipped.read_text()) + assert registration["hash_ignore"][EXTENSION_NAME] == [] + registration["hash_ignore"][EXTENSION_NAME] = ignore + custom = tmp_path / f"logical_types_{'_'.join(ignore) or 'strict'}.json" + custom.write_text(json.dumps(registration)) + return JSONDataContextRegistry(logical_types_file=custom).get_context() + + +def _source(context: DataContext, value: Any) -> DictSource: + """A one-row source of ``context`` with ``value`` in column ``x``.""" + return DictSource( + data=[{"k": "r", "x": value}], + tag_columns=["k"], + source_id="s", + data_context=context, + ) + + +def _cell_hash(context: DataContext, value: Any) -> Any: + """The hash of the ``x`` column of ``_source``, by the context's own Arrow hasher. + + A row's ``content_hash()`` is not used here: the table handler of a custom + context hashes with the default context's Arrow hasher, which refuses a + column stored under another list (reported under ITL-981). + """ + table = _source(context, value).as_table().select(["x"]) + return context.arrow_hasher.hash_table(table) + + +def _transposed() -> Any: + """A transposed view that covers its storage.""" + return torch.arange(24, dtype=torch.float32).reshape(2, 3, 4).transpose(0, 2) + + +def test_the_configured_list_is_the_columns_extension_metadata(tmp_path: Path) -> None: + """The list of the data context is written into the column's extension metadata.""" + for ignore, metadata in (([], STRICT), (["memory_layout"], LAYOUT)): + context = _context(tmp_path, ignore) + column_type = _source(context, _transposed()).as_table().schema.field("x").type + assert isinstance(column_type, pa.ExtensionType) + assert column_type.extension_name == EXTENSION_NAME + assert column_type.__arrow_ext_serialize__() == metadata + + +def test_a_bare_value_and_a_cell_are_hashed_with_the_configured_list( + tmp_path: Path, +) -> None: + """Under ``["memory_layout"]`` a transposed view hashes like its contiguous copy.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["memory_layout"]) + view, copy = _transposed(), _transposed().contiguous() + + assert strict.semantic_hasher.hash_object( + view + ) != strict.semantic_hasher.hash_object(copy) + assert lenient.semantic_hasher.hash_object( + view + ) == lenient.semantic_hasher.hash_object(copy) + assert lenient.semantic_hasher.hash_object(view).digest == tensor_digest( + view, frozenset({"memory_layout"}) + ) + + assert _cell_hash(strict, view) != _cell_hash(strict, copy) + assert _cell_hash(lenient, view) == _cell_hash(lenient, copy) + # The list is in the identifier, so the same value hashes differently in a + # strict column and in a lenient one. + assert _cell_hash(strict, copy) != _cell_hash(lenient, copy) + # Another property still counts under the lenient list. + assert _cell_hash(lenient, copy) != _cell_hash( + lenient, copy.clone().requires_grad_(True) + ) + + +def test_storage_does_not_depend_on_the_list(tmp_path: Path) -> None: + """An ignored property is still stored and read back. + + The context's converter is used directly: a datagram finds its data context by + key, which for a custom registration file is the default context (reported + under ITL-981). + """ + lenient = _context(tmp_path, ["memory_layout", "requires_grad"]) + value = ( + _transposed().clone(memory_format=torch.preserve_format).requires_grad_(True) + ) + converter = lenient.type_converter + table = converter.python_dicts_to_arrow_table( + [{"x": value}], python_schema={"x": torch.Tensor} + ) + assert table.schema.field("x").type.__arrow_ext_serialize__() == ( + b'{"ignore":["memory_layout","requires_grad"],"version":1}' + ) + got = converter.arrow_table_to_python_dicts(table)[0]["x"] + assert got.stride() == value.stride() == (1, 4, 12) + assert got.requires_grad + assert torch.equal(got.detach(), value.detach()) + + +def test_a_column_stored_under_another_list_is_refused_on_read(tmp_path: Path) -> None: + """A data context reads only tensor columns stored under its own list.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["memory_layout"]) + table = _source(lenient, _transposed()).as_table().select(["k", "x"]) + with pytest.raises(LogicalTypeIdentifierError) as raised: + source = ArrowTableSource( + table, tag_columns=["k"], source_id="stored", data_context=strict + ) + for _, data in source.iter_data(): + data.as_dict() + message = str(raised.value) + assert repr(LAYOUT) in message and repr(STRICT) in message + + +def test_an_unknown_option_name_in_the_configuration_is_an_error( + tmp_path: Path, +) -> None: + """A configured name the tensor type does not have raises naming it. + + The tensor type is loaded on first use (building a context must not import + torch), so the error is raised when the type is first needed. + """ + context = _context(tmp_path, ["device"]) + with pytest.raises(ValueError, match="'device'"): + _source(context, torch.zeros(1)).as_table() + with pytest.raises(ValueError, match="'device'"): + context.semantic_hasher.hash_object(torch.zeros(1)) diff --git a/tests/test_torch/test_tensor_job_load_back.py b/tests/test_torch/test_tensor_job_load_back.py new file mode 100644 index 000000000..bf19ddcfb --- /dev/null +++ b/tests/test_torch/test_tensor_job_load_back.py @@ -0,0 +1,95 @@ +"""A saved ``PipelineJob`` with tensor results is loaded back and read (ITL-871, ITL-904).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import torch # noqa: E402 + +import orcapod as op # noqa: E402 +from orcapod.contexts import get_default_context # noqa: E402 +from orcapod.core.operators import Join # noqa: E402 +from orcapod.databases import DeltaTableDatabase # noqa: E402 +from orcapod.logical_types.identifier import logical_type_identifier # noqa: E402 +from orcapod.pipeline.serialization import LoadStatus # noqa: E402 +from orcapod.sources import DictSource # noqa: E402 + + +def weights(n: int) -> torch.Tensor: + """Return a transposed float32 tensor that tracks gradients.""" + base = torch.arange(n * 2, dtype=torch.float32).reshape(2, n) + return base.t().clone(memory_format=torch.preserve_format).requires_grad_(True) + + +def parameter(n: int) -> torch.nn.Parameter: + """Return an ``nn.Parameter`` of ``n`` ones.""" + return torch.nn.Parameter(torch.ones(n)) + + +def _save(tmp_path: Path, function: Any) -> Path: + """Run and save a job whose pod returns a tensor, followed by a ``Join``.""" + job = op.PipelineJob(name="tensors", store=DeltaTableDatabase(tmp_path / "store")) + with job: + source = DictSource( + [{"id": 1, "n": 3}, {"id": 2, "n": 4}], tag_columns=["id"], source_id="n" + ) + result = op.function_pod(output_keys="value")(function).pod( + source, label="value" + ) + other = DictSource( + [{"id": 1, "other": 2}, {"id": 2, "other": 3}], + tag_columns=["id"], + source_id="other", + ) + Join()(result, other, label="joined") + job.run() + path = tmp_path / "job.json" + job.save(path) + return path + + +def _values(node: Any) -> dict[int, Any]: + """Map each row's ``id`` tag to its ``value``.""" + return {t.as_dict()["id"]: d.as_dict()["value"] for t, d in node.iter_data()} + + +@pytest.mark.parametrize( + ("function", "declared"), + [(weights, torch.Tensor), (parameter, torch.nn.Parameter)], + ids=["tensor", "parameter"], +) +def test_a_loaded_job_reads_its_stored_tensors( + tmp_path: Path, function: Any, declared: type +) -> None: + """The loaded job has the tensor schema and reads the stored tensors exactly.""" + path = _save(tmp_path, function) + descriptors = json.loads(path.read_text())["pipeline"]["nodes"].values() + descriptor = next(d for d in descriptors if d["label"] == "value") + logical = get_default_context().type_converter.get_logical_type(torch.Tensor) + assert logical is not None + identifier = logical_type_identifier(logical) + assert identifier.text.startswith("orcapod.torch.tensor;") + assert descriptor["output_schema"]["data"]["value"] == ( + f"extension<{identifier.text}>" + ) + + loaded = op.PipelineJob.load(path) + assert loaded.nodes["value"].output_schema()[1]["value"] is torch.Tensor + assert loaded.nodes["joined"].load_status == LoadStatus.FULL + assert loaded.nodes["joined"].output_schema()[1]["value"] is torch.Tensor + got = _values(loaded.nodes["value"]) + assert sorted(got) == [1, 2] + for row_id, n in ((1, 3), (2, 4)): + expected = function(n) + value = got[row_id] + assert type(value) is type(expected) and isinstance(value, declared) + assert value.dtype == expected.dtype + assert value.stride() == expected.stride() + assert value.requires_grad == expected.requires_grad + assert torch.equal(value.detach(), expected.detach()) diff --git a/tests/test_torch/test_tensor_operators.py b/tests/test_torch/test_tensor_operators.py new file mode 100644 index 000000000..36b4a5f13 --- /dev/null +++ b/tests/test_torch/test_tensor_operators.py @@ -0,0 +1,245 @@ +"""Tensor columns pass through every operator unchanged (ITL-871). + +Each source holds one row per entry of ``TENSORS``, so a single column carries +every dtype and shape class at once. Operators are called on streams and read +back through the stream's public methods. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +pytest.importorskip("torch") + +from orcapod.operators import ( # noqa: E402 + Batch, + DropDataColumns, + GroupBy, + Join, + MapData, + MergeJoin, + PolarsFilter, + SelectDataColumns, + SemiJoin, +) +from orcapod.protocols.core_protocols import StreamProtocol # noqa: E402 +from orcapod.sources import DictSource # noqa: E402 +from tests.test_core.operators.test_async_sync_equivalence import ( # noqa: E402 + async_rows, + sync_rows, +) +from tests.test_torch.tensor_cases import ( # noqa: E402 + TENSORS, + assert_same_tensors_any_order, + assert_tensors_identical, + same_tensor, +) + +NAMES = sorted(TENSORS) +GROUP = {name: f"g{i % 3}" for i, name in enumerate(NAMES)} + + +def tensors_source(source_id: str = "tensors", shift: int = 0) -> DictSource: + """One row per tensor; ``shift`` rotates which tensor each tag gets.""" + return DictSource( + data=[ + { + "k": name, + "g": GROUP[name], + "x": TENSORS[NAMES[(i + shift) % len(NAMES)]], + "n": i, + } + for i, name in enumerate(NAMES) + ], + tag_columns=["k", "g"], + source_id=source_id, + ) + + +def expected_x(name: str, shift: int = 0) -> Any: + """The tensor the row ``name`` holds in a source rotated by ``shift``.""" + return TENSORS[NAMES[(NAMES.index(name) + shift) % len(NAMES)]] + + +def fields(datagram: Any) -> dict[str, Any]: + """A tag's or data's values as a plain dict.""" + return dict(datagram.as_dict()) + + +def rows(stream: Any) -> dict[str, dict[str, Any]]: + """Map each row's ``k`` tag to its tag and data values.""" + out: dict[str, dict[str, Any]] = {} + for tag, data in stream.iter_data(): + out[str(tag.as_dict()["k"])] = {**tag.as_dict(), **data.as_dict()} + return out + + +def other_source() -> DictSource: + """A tensor-free source keyed by the same ``k`` tags.""" + return DictSource( + data=[{"k": name, "y": i} for i, name in enumerate(NAMES)], + tag_columns=["k"], + source_id="other", + ) + + +def test_join() -> None: + """Join keeps every tensor exact.""" + got = rows(Join()(tensors_source(), other_source())) + assert sorted(got) == NAMES + for name in NAMES: + assert_tensors_identical(got[name]["x"], expected_x(name)) + + +def test_semijoin() -> None: + """SemiJoin keeps the matching rows' tensors exact.""" + keep = NAMES[::2] + right = DictSource( + data=[{"k": name, "y": 0} for name in keep], tag_columns=["k"], source_id="r" + ) + got = rows(SemiJoin()(tensors_source(), right)) + assert sorted(got) == keep + for name in keep: + assert_tensors_identical(got[name]["x"], expected_x(name)) + + +def test_map_data() -> None: + """Renaming the tensor column keeps its values.""" + got = rows(MapData({"x": "z"})(tensors_source())) + for name in NAMES: + assert "x" not in got[name] + assert_tensors_identical(got[name]["z"], expected_x(name)) + + +def test_select_and_drop_data_columns() -> None: + """Selecting and dropping columns keeps the tensors exact.""" + selected = rows(SelectDataColumns(["x"])(tensors_source())) + dropped = rows(DropDataColumns(["n"])(tensors_source())) + for got in (selected, dropped): + for name in NAMES: + assert "n" not in got[name] + assert_tensors_identical(got[name]["x"], expected_x(name)) + + +def test_polars_filter_on_tag() -> None: + """Filtering on a tag keeps the remaining tensors exact.""" + got = rows(PolarsFilter(constraints={"g": "g1"})(tensors_source())) + expected = [n for n in NAMES if GROUP[n] == "g1"] + assert sorted(got) == expected + for name in expected: + assert_tensors_identical(got[name]["x"], expected_x(name)) + + +def test_batch() -> None: + """Batch collects tensors into lists, each exact.""" + out = list(Batch(batch_size=4)(tensors_source()).iter_data()) + seen: list[str] = [] + for tag, data in out: + names = list(fields(tag)["k"]) + xs = fields(data)["x"] + assert len(names) == len(xs) + for name, x in zip(names, xs, strict=True): + assert_tensors_identical(x, expected_x(name)) + seen.extend(names) + assert sorted(seen) == NAMES + + +def test_group_by() -> None: + """GroupBy collects each group's tensors into a list, each exact.""" + out = list(GroupBy(by=["g"])(tensors_source()).iter_data()) + assert sorted(fields(t)["g"] for t, _ in out) == ["g0", "g1", "g2"] + for tag, data in out: + # GroupBy keeps the non-key tag ``k`` as a list-valued tag. + names = list(fields(tag)["k"]) + assert sorted(names) == sorted(n for n in NAMES if GROUP[n] == fields(tag)["g"]) + for name, x in zip(names, fields(data)["x"], strict=True): + assert_tensors_identical(x, expected_x(name)) + + +@pytest.mark.parametrize("shift", [0, 1]) +def test_merge_join_colliding_tensor_column(shift: int) -> None: + """MergeJoin merges a colliding tensor column into a list of both tensors.""" + left = tensors_source("left") + right = tensors_source("right", shift=shift) + got = rows(MergeJoin()(left, right)) + assert sorted(got) == NAMES + for name in NAMES: + assert_same_tensors_any_order( + list(got[name]["x"]), [expected_x(name), expected_x(name, shift)] + ) + + +def test_merge_join_order_does_not_depend_on_input_order() -> None: + """MergeJoin orders merged tensors the same whichever input comes first.""" + forward = rows(MergeJoin()(tensors_source("a"), tensors_source("b", shift=1))) + backward = rows(MergeJoin()(tensors_source("b", shift=1), tensors_source("a"))) + for name in NAMES: + f, b = list(forward[name]["x"]), list(backward[name]["x"]) + assert len(f) == len(b) == 2 + for x, y in zip(f, b, strict=True): + assert_tensors_identical(x, y) + + +def grouped_tensors_source(source_id: str, shift: int = 0) -> StreamProtocol: + """``tensors_source`` grouped on ``g``, so ``x`` becomes ``list[torch.Tensor]``.""" + return GroupBy(by=["g"])(tensors_source(source_id, shift)) + + +def assert_merged_groups(got: dict[str, Any], shift: int) -> None: + """Each group's merged ``x`` holds the left and the right member lists.""" + assert sorted(got) == ["g0", "g1", "g2"] + for group, merged in got.items(): + names = [n for n in NAMES if GROUP[n] == group] + assert len(merged) == 2 + expected = [ + [expected_x(n) for n in names], + [expected_x(n, shift) for n in names], + ] + # One merged list from each input: match each list to a distinct expected one. + remaining = list(expected) + for members in merged: + match = next( + ( + i + for i, e in enumerate(remaining) + if len(members) == len(e) + and all(same_tensor(g, x) for g, x in zip(members, e, strict=True)) + ), + None, + ) + assert match is not None, f"group {group}: unexpected member list" + del remaining[match] + assert not remaining + + +@pytest.mark.parametrize("shift", [0, 1]) +def test_merge_join_colliding_grouped_tensor_column(shift: int) -> None: + """MergeJoin merges colliding ``list[torch.Tensor]`` columns, in a stable order.""" + left = grouped_tensors_source("left") + right = grouped_tensors_source("right", shift) + out = MergeJoin()(left, right) + got = {fields(t)["g"]: fields(d)["x"] for t, d in out.iter_data()} + assert_merged_groups(got, shift) + # The swapped call yields the same table. Source info is left out: on tied values + # from inputs with one pipeline hash, its order follows the call. + swapped = MergeJoin()(right, left) + assert swapped.as_table().equals(out.as_table()) + + +def test_merge_join_colliding_grouped_tensor_column_async_equals_sync() -> None: + """The async MergeJoin of ``list[torch.Tensor]`` columns yields the sync data.""" + left = grouped_tensors_source("left") + right = grouped_tensors_source("right", 1) + expected = {fields(t)["g"]: d for t, d in sync_rows(MergeJoin(), left, right)} + # The async path is under test: this helper drives ``async_execute`` over channels. + actual = {fields(t)["g"]: d for t, d in async_rows(MergeJoin(), left, right)} + # Data only: after GroupBy, MergeJoin's async system tag column names differ from + # the sync ones for any column type (reported separately, as for arrays). + assert sorted(actual) == sorted(expected) + for group, data in actual.items(): + assert data.arrow_schema() == expected[group].arrow_schema() + assert data.content_hash() == expected[group].content_hash() + assert data.source_info() == expected[group].source_info() + assert_merged_groups({g: fields(d)["x"] for g, d in actual.items()}, 1) diff --git a/tests/test_torch/test_tensor_pipeline_stores.py b/tests/test_torch/test_tensor_pipeline_stores.py new file mode 100644 index 000000000..bfdf96c48 --- /dev/null +++ b/tests/test_torch/test_tensor_pipeline_stores.py @@ -0,0 +1,187 @@ +"""Tensor values through ``PipelineJob`` on in-memory and Delta stores, sync and async (ITL-871).""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import torch # noqa: E402 + +from orcapod import PipelineJob, function_pod # noqa: E402 +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase # noqa: E402 +from orcapod.operators import GroupBy, MergeJoin # noqa: E402 +from orcapod.pipeline import AsyncPipelineOrchestrator # noqa: E402 +from orcapod.sources import DictSource # noqa: E402 +from tests.test_torch.tensor_cases import ( # noqa: E402 + NUMERIC, + TENSORS, + assert_same_tensors_any_order, + assert_tensors_identical, +) + + +@function_pod(output_keys="y") +def identity(x: torch.Tensor) -> torch.Tensor: + """Return a copy of the input.""" + return x.clone() + + +@function_pod(output_keys="d") +def double(y: torch.Tensor) -> torch.Tensor: + """Return twice the input.""" + return y * 2 + + +@function_pod(output_keys="total") +def total(d: torch.Tensor) -> float: + """Return the sum of the real parts as a float.""" + return float(d.to(torch.complex128).real.sum()) + + +@function_pod(output_keys="count") +def count(x: list[torch.Tensor]) -> int: + """Return the number of tensors in the group.""" + return len(x) + + +@function_pod(output_keys="pair") +def pair(x: list[torch.Tensor]) -> list[torch.Tensor]: + """Return the merged tensors as a list.""" + return list(x) + + +def _numeric_sources() -> tuple[DictSource, DictSource]: + """The numeric tensors, and the same keys with the tensors shifted by one.""" + numeric = DictSource( + data=[{"k": k, "x": TENSORS[k]} for k in NUMERIC], + tag_columns=["k"], + source_id="numeric", + ) + shifted = DictSource( + data=[ + {"k": k, "x": TENSORS[NUMERIC[(i + 1) % len(NUMERIC)]]} + for i, k in enumerate(NUMERIC) + ], + tag_columns=["k"], + source_id="shifted", + ) + return numeric, shifted + + +def _build(store: Any) -> PipelineJob: + """Build a job with pods, GroupBy and MergeJoin over tensor sources on ``store``.""" + numeric, shifted = _numeric_sources() + every = DictSource( + data=[ + {"k": k, "g": f"g{i % 2}", "x": v} + for i, (k, v) in enumerate(TENSORS.items()) + ], + tag_columns=["k", "g"], + source_id="every", + ) + job = PipelineJob(name="tensors", store=store) + with job: + identity.pod(every, label="identity_all") + y = identity.pod(numeric, label="identity_numeric") + d = double.pod(y, label="double") + total.pod(d, label="total") + count.pod(GroupBy(by=["g"])(every, label="grouped"), label="count") + count.pod(MergeJoin()(numeric, shifted, label="merge_join"), label="merged") + return job + + +def _build_list_output(store: Any) -> PipelineJob: + """Build a job whose pod returns ``list[torch.Tensor]`` on ``store``.""" + numeric, shifted = _numeric_sources() + job = PipelineJob(name="tensor_lists", store=store) + with job: + pair.pod(MergeJoin()(numeric, shifted, label="merge_join"), label="pair") + return job + + +def _run_sync(job: PipelineJob) -> None: + """Run the job synchronously.""" + job.run() + + +def _run_async(job: PipelineJob) -> None: + """Run the job with the async orchestrator.""" + AsyncPipelineOrchestrator().run(job.dag) + + +STORES: dict[str, Callable[[Path], Any]] = { + "in_memory": lambda _: InMemoryArrowDatabase(), + "delta": lambda tmp: DeltaTableDatabase(base_path=tmp / "store"), +} +RUNNERS: dict[str, Callable[[PipelineJob], None]] = { + "sync": _run_sync, + "async": _run_async, +} + + +def _node(job: PipelineJob, label: str, column: str, key: str = "k") -> dict[str, Any]: + """Map each row's ``key`` tag to ``column`` of the node labelled ``label``.""" + return { + str(t.as_dict()[key]): d.as_dict()[column] + for t, d in job.nodes[label].iter_data() + } + + +@pytest.mark.parametrize( + ("store", "runner"), + [(store, runner) for store in sorted(STORES) for runner in sorted(RUNNERS)], +) +def test_pipeline_keeps_tensors_exact(tmp_path: Path, store: str, runner: str) -> None: + """Every node of the job keeps tensors exact on each store and runner.""" + job = _build(STORES[store](tmp_path)) + RUNNERS[runner](job) + + every = _node(job, "identity_all", "y") + assert sorted(every) == sorted(TENSORS) + for name, expected in TENSORS.items(): + assert_tensors_identical(every[name], expected) + + doubled = _node(job, "double", "d") + totals = _node(job, "total", "total") + for name in NUMERIC: + expected = TENSORS[name] * 2 + assert_tensors_identical(doubled[name], expected) + assert totals[name] == pytest.approx( + float(expected.to(torch.complex128).real.sum()), nan_ok=True + ) + + counts = _node(job, "count", "count", key="g") + assert sum(counts.values()) == len(TENSORS) + + merged = _node(job, "merge_join", "x") + assert sorted(merged) == sorted(NUMERIC) + for i, name in enumerate(NUMERIC): + assert_same_tensors_any_order( + list(merged[name]), + [TENSORS[name], TENSORS[NUMERIC[(i + 1) % len(NUMERIC)]]], + ) + assert set(_node(job, "merged", "count").values()) == {2} + + +@pytest.mark.parametrize( + ("store", "runner"), + [(store, runner) for store in sorted(STORES) for runner in sorted(RUNNERS)], +) +def test_pod_returning_a_list_of_tensors( + tmp_path: Path, store: str, runner: str +) -> None: + """A pod output ``list[torch.Tensor]`` is stored and read back exactly.""" + job = _build_list_output(STORES[store](tmp_path)) + RUNNERS[runner](job) + pairs = _node(job, "pair", "pair") + assert sorted(pairs) == sorted(NUMERIC) + for i, name in enumerate(NUMERIC): + assert_same_tensors_any_order( + list(pairs[name]), + [TENSORS[name], TENSORS[NUMERIC[(i + 1) % len(NUMERIC)]]], + ) diff --git a/tests/test_torch/test_tensor_round_trip.py b/tests/test_torch/test_tensor_round_trip.py new file mode 100644 index 000000000..4487104f5 --- /dev/null +++ b/tests/test_torch/test_tensor_round_trip.py @@ -0,0 +1,187 @@ +"""Tensor values round trip exactly through sources and function pods (ITL-871).""" + +from __future__ import annotations + +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import numpy as np # noqa: E402 +import torch # noqa: E402 + +from orcapod import function_pod # noqa: E402 +from orcapod.errors import ValueConversionWarning # noqa: E402 +from orcapod.sources import ArrowTableSource, DictSource # noqa: E402 +from tests.test_torch.tensor_cases import ( # noqa: E402 + NUMERIC, + TENSORS, + assert_tensors_identical, +) + + +def _source(tensors: dict[str, Any], source_id: str = "tensors") -> DictSource: + """A source with one row per tensor, tagged by its name.""" + return DictSource( + data=[{"k": k, "x": v} for k, v in tensors.items()], + tag_columns=["k"], + source_id=source_id, + ) + + +def _read(stream: Any, column: str) -> dict[str, Any]: + """Map each row's ``k`` tag to the value of ``column``.""" + return {t.as_dict()["k"]: d.as_dict()[column] for t, d in stream.iter_data()} + + +def test_one_column_holds_every_dtype_and_shape() -> None: + """One column holds every supported dtype and shape, and each reads back exactly.""" + got = _read(_source(TENSORS), "x") + for name, expected in TENSORS.items(): + assert_tensors_identical(got[name], expected) + + +def test_function_pod_input_and_output() -> None: + """A pod receives plain CPU tensors and its outputs read back exactly.""" + seen: list[Any] = [] + + @function_pod(output_keys="y") + def identity(x: torch.Tensor) -> torch.Tensor: + """Record the input and return a copy.""" + seen.append(x) + return x.clone() + + got = _read(identity.pod(_source(TENSORS)), "y") + for name, expected in TENSORS.items(): + assert_tensors_identical(got[name], expected) + for x in seen: + assert type(x) is torch.Tensor and not x.requires_grad + + +def test_chained_pods() -> None: + """Tensors flow through two chained pods with exact values.""" + + @function_pod(output_keys="d") + def double(x: torch.Tensor) -> torch.Tensor: + """Return twice the input.""" + return x * 2 + + @function_pod(output_keys="total") + def total(d: torch.Tensor) -> float: + """Return the sum of the real parts as a float.""" + return float(d.to(torch.complex128).real.sum()) + + numeric = {k: TENSORS[k] for k in NUMERIC} + doubled = double.pod(_source(numeric)) + totals = _read(total.pod(doubled), "total") + got = _read(doubled, "d") + for name in NUMERIC: + expected = TENSORS[name] * 2 + assert_tensors_identical(got[name], expected) + assert totals[name] == pytest.approx( + float(expected.to(torch.complex128).real.sum()), nan_ok=True + ) + + +def _stored_outputs(stream: Any, column: str) -> dict[str, Any]: + """Read a pod's outputs back from the stored rows (``as_table``). + + In memory, a pod's output datagram holds the very object the function + returned; the canonical CPU form is what is stored and read back from a store. + """ + table = stream.as_table().select(["k", column]) + return _read(ArrowTableSource(table, tag_columns=["k"]), column) + + +def test_pod_returning_a_grad_tensor_stores_its_value() -> None: + """An output with autograd history is stored as a leaf; the weight is untouched.""" + weight = torch.ones(3, requires_grad=True) + + @function_pod(output_keys="y") + def scale(x: torch.Tensor) -> torch.Tensor: + """Multiply by a weight that requires gradients.""" + return x * weight # has a grad_fn + + with pytest.warns(ValueConversionWarning, match="autograd history"): + got = _stored_outputs(scale.pod(_source({"a": torch.arange(3.0)})), "y") + assert got["a"].requires_grad and got["a"].is_leaf + assert torch.equal(got["a"].detach(), torch.arange(3.0)) + assert weight.requires_grad and weight.grad is None + + +@pytest.mark.skipif(not torch.backends.mps.is_available(), reason="needs MPS") +def test_pod_returning_an_mps_tensor_reads_back_on_cpu() -> None: + """An MPS output is stored and reads back as an equal CPU tensor.""" + + @function_pod(output_keys="y") + def to_gpu(x: torch.Tensor) -> torch.Tensor: + """Move to MPS and double.""" + return x.to("mps") * 2 + + got = _stored_outputs(to_gpu.pod(_source({"a": torch.arange(3.0)})), "y") + assert_tensors_identical(got["a"], torch.arange(3.0) * 2) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_pod_returning_a_cuda_tensor_reads_back_on_cpu() -> None: + """A CUDA output is stored and reads back as an equal CPU tensor.""" + + @function_pod(output_keys="y") + def to_gpu(x: torch.Tensor) -> torch.Tensor: + """Move to CUDA and double.""" + return x.to("cuda") * 2 + + got = _stored_outputs(to_gpu.pod(_source({"a": torch.arange(3.0)})), "y") + assert_tensors_identical(got["a"], torch.arange(3.0) * 2) + + +def test_read_back_is_writeable() -> None: + """A tensor read from a source can be modified in place.""" + got = _read(_source({"a": torch.arange(3.0)}), "x")["a"] + got[0] = 99.0 + assert got[0] == 99.0 + + +def test_optional_tensor_column_keeps_none() -> None: + """``torch.Tensor | None`` outputs keep ``None`` and the other tensors exactly.""" + + @function_pod(output_keys="y") + def maybe(x: torch.Tensor) -> torch.Tensor | None: + """Return ``None`` for an empty tensor, else the input.""" + return None if x.numel() == 0 else x + + got = _read(maybe.pod(_source(TENSORS)), "y") + for name, expected in TENSORS.items(): + if expected.numel() == 0: + assert got[name] is None + else: + assert_tensors_identical(got[name], expected) + + +def test_list_of_tensors_round_trips() -> None: + """A ``list[torch.Tensor]`` output reads back element by element.""" + + @function_pod(output_keys="parts") + def split(x: torch.Tensor) -> list[torch.Tensor]: + """Return the input and its double as a list.""" + return [x, x * 2] + + got = _read(split.pod(_source({"a": torch.arange(3.0)})), "parts") + assert len(got["a"]) == 2 + assert_tensors_identical(got["a"][0], torch.arange(3.0)) + assert_tensors_identical(got["a"][1], torch.arange(3.0) * 2) + + +def test_ndarray_column_into_tensor_parameter_is_refused() -> None: + """An ndarray column bound to a ``torch.Tensor`` parameter fails when the graph is built.""" + + @function_pod(output_keys="y") + def identity(x: torch.Tensor) -> torch.Tensor: + """Return the input unchanged.""" + return x + + arrays = DictSource(data=[{"k": "a", "x": np.arange(3.0)}], tag_columns=["k"]) + with pytest.raises(ValueError, match="not compatible") as info: + identity.pod(arrays) + assert "ndarray" in str(info.value) and "Tensor" in str(info.value) diff --git a/tests/test_torch/test_tensor_storage.py b/tests/test_torch/test_tensor_storage.py new file mode 100644 index 000000000..4ecb1c10b --- /dev/null +++ b/tests/test_torch/test_tensor_storage.py @@ -0,0 +1,384 @@ +"""How a ``torch.Tensor`` is stored: ``torch.save`` bytes, restricted load (ITL-871). + +Tests go through sources and their streams. A cell is written with ``torch.save`` +and read with ``torch.load(map_location="cpu", weights_only=True)``. +""" + +from __future__ import annotations + +import io +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import pyarrow as pa # noqa: E402 +import torch # noqa: E402 + +from orcapod.contexts import get_default_context # noqa: E402 +from orcapod.errors import UnsupportedValueError # noqa: E402 +from orcapod.sources import ArrowTableSource, DictSource # noqa: E402 +from tests.test_torch.tensor_cases import ( # noqa: E402 + EXTENSION_NAME, + TENSORS, + value_bytes, +) + + +def _source(value: Any) -> DictSource: + """A one-row source with ``value`` in column ``x``.""" + return DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + + +def stored_cell(value: Any) -> bytes: + """The stored bytes of a one-row ``x`` column.""" + chunk = _source(value).as_table().column("x").chunk(0) + assert isinstance(chunk, pa.ExtensionArray) + return chunk.storage[0].as_py() + + +def read_back(value: Any) -> Any: + """The value of a one-row ``x`` column, read back from its source.""" + return next(iter(_source(value).iter_data()))[1].as_dict()["x"] + + +def read_cell(cell: bytes) -> Any: + """Read a tensor column whose one cell holds ``cell``.""" + table = _source(torch.zeros(1)).as_table() + column_type = table.schema.field("x").type + storage = pa.array([cell], type=column_type.storage_type) + forged = pa.Table.from_arrays( + [table.column("k"), pa.ExtensionArray.from_storage(column_type, storage)], + names=["k", "x"], + ) + source = ArrowTableSource(forged, tag_columns=["k"]) + return next(iter(source.iter_data()))[1].as_dict()["x"] + + +def _base() -> Any: + """A 2x3x4 float32 tensor.""" + return torch.arange(24, dtype=torch.float32).reshape(2, 3, 4) + + +# --------------------------------------------------------------------------- +# Column type and cell +# --------------------------------------------------------------------------- + + +def test_column_is_orcapod_torch_tensor_over_large_binary() -> None: + """A tensor column is ``orcapod.torch.tensor`` over ``large_binary`` with an ignore list.""" + ext = _source(torch.arange(3.0)).as_table().schema.field("x").type + assert isinstance(ext, pa.ExtensionType) + assert ext.extension_name == EXTENSION_NAME == "orcapod.torch.tensor" + assert ext.storage_type == pa.large_binary() + assert ext.__arrow_ext_serialize__() == b'{"ignore":[],"version":1}' + + +def test_a_cell_is_what_torch_save_writes() -> None: + """A cell is a ``torch.save`` archive that torch's restricted loader reads.""" + t = _base() + loaded = torch.load( + io.BytesIO(stored_cell(t)), map_location="cpu", weights_only=True + ) + assert type(loaded) is torch.Tensor + assert torch.equal(loaded, t) + + +# --------------------------------------------------------------------------- +# What reads back +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", sorted(TENSORS)) +def test_dtype_shape_and_bits_read_back(name: str) -> None: + """Each supported dtype and shape reads back as a plain CPU tensor, bit for bit.""" + expected = TENSORS[name] + got = read_back(expected) + assert type(got) is torch.Tensor + assert got.dtype == expected.dtype + assert tuple(got.shape) == tuple(expected.shape) + assert got.device.type == "cpu" + assert value_bytes(got) == value_bytes(expected) + + +def test_requires_grad_reads_back() -> None: + """``requires_grad`` is stored; the tensor reads back as a leaf.""" + got = read_back(_base().requires_grad_(True)) + assert got.requires_grad and got.is_leaf and got.grad is None + assert not read_back(_base()).requires_grad + + +@pytest.mark.parametrize("requires_grad", [True, False]) +def test_parameter_reads_back_as_a_parameter(requires_grad: bool) -> None: + """An ``nn.Parameter`` reads back as an ``nn.Parameter`` with its ``requires_grad``.""" + got = read_back(torch.nn.Parameter(_base(), requires_grad=requires_grad)) + assert type(got) is torch.nn.Parameter + assert got.requires_grad is requires_grad + assert torch.equal(got.detach(), _base()) + + +def test_strides_of_a_tensor_that_covers_its_storage_read_back() -> None: + """Transposed and channels-last layouts read back with their strides.""" + transposed = _base().transpose(0, 2) + got = read_back(transposed) + assert got.stride() == transposed.stride() == (1, 4, 12) + assert torch.equal(got, transposed) + + four_d = torch.arange(120, dtype=torch.float32).reshape(2, 3, 4, 5) + channels_last = four_d.to(memory_format=torch.channels_last) + got = read_back(channels_last) + assert got.stride() == channels_last.stride() + assert got.is_contiguous(memory_format=torch.channels_last) + assert torch.equal(got, four_d) + + +def test_conj_bit_reads_back() -> None: + """A lazy conjugate view that covers its storage keeps its bit and its values.""" + conj = torch.tensor([1 + 2j, 3 - 4j]).conj() + got = read_back(conj) + assert got.is_conj() + assert torch.equal(got, conj) + + +def test_neg_bit_reads_back() -> None: + """A lazy negative view that covers its storage keeps its bit and its values.""" + # torch._neg_view: the only way to get a negative view that covers its storage. + neg = torch._neg_view(torch.tensor([1.0, -2.0])) # pyright: ignore[reportPrivateUsage] + got = read_back(neg) + assert got.is_neg() + assert torch.equal(got, neg) + + +# --------------------------------------------------------------------------- +# Only the passed elements are stored +# --------------------------------------------------------------------------- + + +def test_a_slice_stores_only_its_own_elements() -> None: + """A 3-element slice of a 1,000,000-element tensor stores 3 elements, not the buffer.""" + big = torch.arange(1_000_000, dtype=torch.float32) + piece = big[5:8] + cell = stored_cell(piece) + assert len(cell) < 10_000 + got = read_back(piece) + assert torch.equal(got, piece) + assert got.storage_offset() == 0 + assert got.untyped_storage().nbytes() == 3 * 4 + + +@pytest.mark.parametrize( + ("value", "strides"), + [ + (_base()[:, ::2], (8, 4, 1)), + (torch.ones(1).expand(4), (1,)), + (torch.arange(30, dtype=torch.float32)[6:].reshape(2, 3, 4), (12, 4, 1)), + # Dense inside a larger buffer: the clone keeps the transposed strides. + ( + torch.arange(48, dtype=torch.float32)[24:].reshape(2, 3, 4).transpose(0, 2), + (1, 4, 12), + ), + ], + ids=["step_slice", "expanded", "offset", "offset_transposed"], +) +def test_a_view_of_a_larger_buffer_reads_back_as_its_clone( + value: Any, strides: tuple[int, ...] +) -> None: + """A view that does not cover its storage reads back as ``clone(preserve_format)``.""" + got = read_back(value) + assert torch.equal(got, value) + assert got.stride() == strides + assert got.stride() == value.clone(memory_format=torch.preserve_format).stride() + assert got.untyped_storage().nbytes() == value.numel() * value.element_size() + + +# --------------------------------------------------------------------------- +# The user's tensor is not modified +# --------------------------------------------------------------------------- + + +def _snapshot(t: Any) -> tuple[Any, ...]: + """Everything storing could change about ``t``.""" + return ( + t.requires_grad, + t.grad_fn, + t.is_leaf, + t.grad, + t.device.type, + t.dtype, + tuple(t.shape), + t.stride(), + t.storage_offset(), + t.is_conj(), + t.is_neg(), + t.data_ptr(), + t._version, + type(t), + value_bytes(t), + ) + + +def _given() -> list[tuple[str, Any]]: + """Tensors with every property storing handles.""" + leaf = _base().requires_grad_(True) + with_grad = _base().requires_grad_(True) + with_grad.sum().backward() + return [ + ("plain", _base()), + ("requires_grad", leaf), + ("non_leaf", leaf * 2), + ("with_grad", with_grad), + ("parameter", torch.nn.Parameter(_base())), + ("transposed", _base().transpose(0, 2)), + ("step_slice", _base()[:, ::2]), + ("conj", torch.tensor([1 + 2j]).conj()), + ("conj_imag", torch.tensor([1 + 2j]).conj().imag), + ] + + +@pytest.mark.filterwarnings("ignore::orcapod.errors.ValueConversionWarning") +@pytest.mark.parametrize( + "value", [g[1] for g in _given()], ids=[g[0] for g in _given()] +) +def test_storing_never_modifies_the_tensor(value: Any) -> None: + """Storing, reading and hashing leave the caller's tensor exactly as it was.""" + before = _snapshot(value) + source = _source(value) + source.as_table() + for _, data in source.iter_data(): + data.as_dict() + data.content_hash() + after = _snapshot(value) + assert all(a is b or a == b for a, b in zip(before[:4], after[:4], strict=True)) + assert before[4:] == after[4:] + + +@pytest.mark.filterwarnings("ignore::orcapod.errors.ValueConversionWarning") +def test_autograd_history_is_not_stored() -> None: + """A non-leaf tensor reads back as a leaf that requires grad, without ``.grad``.""" + leaf = _base().requires_grad_(True) + got = read_back(leaf * 2) + assert got.requires_grad and got.is_leaf and got.grad_fn is None + assert torch.equal(got.detach(), _base() * 2) + + with_grad = _base().requires_grad_(True) + with_grad.sum().backward() + assert read_back(with_grad).grad is None + + +# --------------------------------------------------------------------------- +# Refusals at write +# --------------------------------------------------------------------------- + + +class _Wrapped(torch.Tensor): + """A tensor subclass without a logical type of its own.""" + + +def test_a_subclass_the_restricted_loader_cannot_read_is_refused_at_write() -> None: + """Only ``torch.Tensor`` and ``nn.Parameter`` are stored; another subclass is refused.""" + value = torch.ones(2).as_subclass(_Wrapped) + with pytest.raises(UnsupportedValueError, match="'payload'.*_Wrapped"): + DictSource(data=[{"k": "r", "payload": value}], tag_columns=["k"]).as_table() + + +# --------------------------------------------------------------------------- +# Reading never runs a pickle +# --------------------------------------------------------------------------- + + +_PICKLE_RAN: list[str] = [] + + +def _record_that_a_pickle_ran() -> None: + """Called only if a stored pickle is executed.""" + _PICKLE_RAN.append("ran") + + +class _Marker: + """Unpickling this object calls ``_record_that_a_pickle_ran``.""" + + def __reduce__(self) -> tuple[Any, ...]: + return (_record_that_a_pickle_ran, ()) + + +def _archive(value: Any) -> bytes: + """``value`` as an unrestricted ``torch.save`` archive.""" + buffer = io.BytesIO() + torch.save(value, buffer) + return buffer.getvalue() + + +@pytest.mark.parametrize("force_unrestricted", [False, True]) +def test_a_cell_holding_an_arbitrary_pickle_is_refused( + force_unrestricted: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """A forged cell raises naming the column; torch's environment override changes nothing. + + The type converter names the column (``RuntimeError``) around the logical + type's ``ValueError``. + """ + if force_unrestricted: + monkeypatch.setenv("TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD", "1") + with pytest.raises(RuntimeError, match="column 'x'.*restricted loader"): + read_cell(_archive({"x": _Marker()})) + assert _PICKLE_RAN == [] + + +def test_a_cell_holding_something_other_than_a_tensor_is_refused() -> None: + """An archive the restricted loader accepts but that is not a tensor is refused.""" + with pytest.raises(RuntimeError, match="column 'x'.*dict, not a tensor"): + read_cell(_archive({"weights": torch.zeros(2)})) + + +@pytest.mark.parametrize("cell", [b"", b"not a torch archive", b"PK\x03\x04truncated"]) +def test_garbage_bytes_are_refused(cell: bytes) -> None: + """Bytes that are not a ``torch.save`` archive raise naming the column.""" + with pytest.raises(RuntimeError, match="column 'x'.*restricted loader"): + read_cell(cell) + + +# --------------------------------------------------------------------------- +# Gradient-bearing tensors of every floating dtype, and inference tensors +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name", ["float32", "bfloat16", "complex32", "complex64", "float8_e4m3fn"] +) +def test_a_tensor_that_requires_grad_reads_back_for_every_dtype(name: str) -> None: + """``requires_grad`` reads back for the dtypes PyTorch rebuilds in place, too.""" + dtype = getattr(torch, name) + value = torch.zeros(3, dtype=dtype).requires_grad_(True) + got = read_back(value) + assert got.dtype == dtype + assert got.requires_grad and got.is_leaf + assert value_bytes(got) == value_bytes(value) + row = next(iter(_source(value).iter_data()))[1] + stored = ArrowTableSource( + _source(value).as_table().select(["k", "x"]), tag_columns=["k"] + ) + assert next(iter(stored.iter_data()))[1].content_hash() == row.content_hash() + + +def test_an_inference_tensor_that_requires_grad_is_stored() -> None: + """A tensor made under ``torch.inference_mode()`` is stored without touching it.""" + with torch.inference_mode(): + value = torch.arange(3.0, requires_grad=True) + assert value.is_inference() and value.requires_grad + got = read_back(value) + assert got.requires_grad and not got.is_inference() + assert torch.equal(got.detach(), torch.arange(3.0)) + assert value.is_inference() and value.requires_grad + + +def test_hashing_leaves_the_callers_storage_resizable() -> None: + """Hashing and storing do not pin the caller's storage (``resize_`` still works).""" + value = torch.arange(3.0) + source = _source(value) + source.as_table() + for _, data in source.iter_data(): + data.content_hash() + get_default_context().semantic_hasher.hash_object(value) + value.resize_(4) + assert tuple(value.shape) == (4,) diff --git a/tests/test_torch/test_tensor_strict_hash.py b/tests/test_torch/test_tensor_strict_hash.py new file mode 100644 index 000000000..e88aee5f4 --- /dev/null +++ b/tests/test_torch/test_tensor_strict_hash.py @@ -0,0 +1,243 @@ +"""The value hash of a ``torch.Tensor``: strict by default, five properties (ITL-871). + +``D = SHA-256(lp(extension name) || E || requires_grad || conj_bit || neg_bit || +ndim || strides || lp(class name))`` with ``E`` the element digest. The expected +values are computed here from that formula with ``hashlib`` and ``struct`` only +(``tensor_cases.reference_value_digest``). Hashes are taken through the default +data context's semantic hasher and through a source's rows. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import torch # noqa: E402 + +from orcapod.contexts import get_default_context # noqa: E402 +from orcapod.sources import DictSource # noqa: E402 +from tests.test_torch.tensor_cases import ( # noqa: E402 + LABEL, + PARAMETER_CLASS, + TENSOR_CLASS, + TENSORS, + c_strides, + expected_digest, + reference_value_digest, +) + + +def semantic_hash(value: Any) -> Any: + """The default context's semantic hash of ``value``.""" + return get_default_context().semantic_hasher.hash_object(value) + + +def _row(value: Any) -> Any: + """The data of a one-row source holding ``value``.""" + source = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + return next(iter(source.iter_data()))[1] + + +def data_hash(value: Any) -> Any: + """The content hash of a one-row data holding ``value``.""" + return _row(value).content_hash() + + +def read_back(value: Any) -> Any: + """``value`` after it was stored in a source and read from it.""" + return _row(value).as_dict()["x"] + + +def _base() -> Any: + """A 2x3x4 float32 tensor.""" + return torch.arange(24, dtype=torch.float32).reshape(2, 3, 4) + + +def _complex() -> Any: + """A complex64 tensor of two elements.""" + return torch.tensor([1 + 2j, 3 - 4j], dtype=torch.complex64) + + +# --------------------------------------------------------------------------- +# The formula +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", sorted(TENSORS)) +def test_plain_tensor_hash_is_the_formula(name: str) -> None: + """A plain contiguous CPU tensor hashes to ``D`` with every flag 0, under the label.""" + t = TENSORS[name] + h = semantic_hash(t) + assert h.method == LABEL + assert h.digest == reference_value_digest( + expected_digest(t), + requires_grad=False, + conj_bit=False, + neg_bit=False, + strides=c_strides(t.shape), + class_name=TENSOR_CLASS, + ) + + +def test_each_property_is_written_in_its_slot() -> None: + """Each optional property is written with its real value in its own slot.""" + base = _base() + element = expected_digest(base) + contiguous = c_strides(base.shape) + + grad = base.clone().requires_grad_(True) + assert semantic_hash(grad).digest == reference_value_digest( + element, strides=contiguous, requires_grad=True + ) + + transposed = base.transpose(0, 2) + assert transposed.stride() == (1, 4, 12) + assert semantic_hash(transposed).digest == reference_value_digest( + expected_digest(transposed), strides=(1, 4, 12) + ) + + parameter = torch.nn.Parameter(base.clone(), requires_grad=False) + assert semantic_hash(parameter).digest == reference_value_digest( + element, strides=contiguous, class_name=PARAMETER_CLASS + ) + + conj = _complex().conj() + assert conj.is_conj() + assert semantic_hash(conj).digest == reference_value_digest( + expected_digest(conj), strides=(1,), conj_bit=True + ) + + # ``.conj().imag`` is a negative view of every second float: it does not cover + # its storage, so it is stored as a clone, which resolves the bit. + neg = _complex().conj().imag + assert neg.is_neg() + assert semantic_hash(neg).digest == reference_value_digest( + expected_digest(neg), strides=(1,) + ) + + +# --------------------------------------------------------------------------- +# Strict by default: each property changes the hash +# --------------------------------------------------------------------------- + + +def _differ_in_one_property() -> list[tuple[str, Any, Any]]: + """Pairs with equal dtype, shape and elements that differ in one optional property.""" + base = _base() + four_d = torch.arange(2 * 3 * 4 * 5, dtype=torch.float32).reshape(2, 3, 4, 5) + return [ + ("requires_grad", base.clone().requires_grad_(True), base), + ("memory_layout", base.transpose(0, 2), base.transpose(0, 2).contiguous()), + ( + "memory_layout_channels_last", + four_d.to(memory_format=torch.channels_last), + four_d, + ), + ("conj_bit", _complex().conj(), _complex().conj().resolve_conj()), + # torch._neg_view is the only way to get a negative view that covers its + # storage; public code makes one only as a slice (``.conj().imag``). + ( + "neg_bit", + torch._neg_view(torch.tensor([1.0, -2.0])), # pyright: ignore[reportPrivateUsage] + torch._neg_view(torch.tensor([1.0, -2.0])).resolve_neg(), # pyright: ignore[reportPrivateUsage] + ), + ("subclass", torch.nn.Parameter(base.clone(), requires_grad=False), base), + ] + + +@pytest.mark.parametrize( + ("a", "b"), + [p[1:] for p in _differ_in_one_property()], + ids=[p[0] for p in _differ_in_one_property()], +) +def test_each_optional_property_changes_the_hash_by_default(a: Any, b: Any) -> None: + """Two tensors that differ in one optional property hash differently, bare and in a row.""" + assert expected_digest(a) == expected_digest(b) + assert semantic_hash(a) != semantic_hash(b) + assert data_hash(a) != data_hash(b) + + +def _never_in_the_hash() -> list[tuple[str, Any, Any]]: + """Pairs that differ only in what storage cannot restore.""" + base = _base() + leaf = base.clone().requires_grad_(True) + with_grad = base.clone().requires_grad_(True) + with_grad.sum().backward() + pairs = [ + ("autograd_history", leaf * 1, leaf), + ("grad_attribute", with_grad, leaf), + ( + "storage_offset", + torch.cat([torch.zeros(6), base.flatten()])[6:].reshape(2, 3, 4), + base, + ), + ("step_slice", base[:, ::2], base[:, ::2].contiguous()), + ("expanded", torch.ones(1).expand(4), torch.ones(4)), + ] + if torch.backends.mps.is_available(): + pairs.append(("mps", base.to("mps"), base)) + if torch.cuda.is_available(): + pairs.append(("cuda", base.to("cuda"), base)) + return pairs + + +@pytest.mark.filterwarnings("ignore::orcapod.errors.ValueConversionWarning") +@pytest.mark.parametrize( + ("a", "b"), + [p[1:] for p in _never_in_the_hash()], + ids=[p[0] for p in _never_in_the_hash()], +) +def test_what_storage_cannot_restore_never_changes_the_hash(a: Any, b: Any) -> None: + """Device, autograd history, ``.grad`` and hidden storage never enter the hash.""" + assert semantic_hash(a) == semantic_hash(b) + assert data_hash(a) == data_hash(b) + + +# --------------------------------------------------------------------------- +# hash(read(write(v))) == hash(v) +# --------------------------------------------------------------------------- + + +def _round_trip_cases() -> list[tuple[str, Any]]: + """Every tensor whose hash must survive storage.""" + cases = [(name, t) for name, t in sorted(TENSORS.items())] + for name, a, b in _differ_in_one_property() + _never_in_the_hash(): + cases.append((f"{name}.a", a)) + cases.append((f"{name}.b", b)) + cases.append(("sliced_conj_view", torch.arange(6).to(torch.complex64).conj()[1:4])) + cases.append(("parameter_with_grad", torch.nn.Parameter(_base().clone()))) + return cases + + +@pytest.mark.filterwarnings("ignore::orcapod.errors.ValueConversionWarning") +@pytest.mark.parametrize( + "value", + [c[1] for c in _round_trip_cases()], + ids=[c[0] for c in _round_trip_cases()], +) +def test_the_hash_survives_storage(value: Any) -> None: + """A value read back from storage hashes like the value that was written.""" + back = read_back(value) + assert semantic_hash(back) == semantic_hash(value) + assert data_hash(back) == data_hash(value) + # A second trip changes nothing either. + assert semantic_hash(read_back(back)) == semantic_hash(value) + + +# --------------------------------------------------------------------------- +# A tensor is never an ndarray +# --------------------------------------------------------------------------- + + +def test_a_tensor_never_hashes_like_the_equal_ndarray() -> None: + """The leading type text keeps a tensor apart from an equal ndarray, also when nested.""" + t = torch.arange(6, dtype=torch.float32).reshape(2, 3) + a = t.numpy() + hasher = get_default_context().semantic_hasher + assert hasher.hash_object(t).digest != hasher.hash_object(a).digest + assert hasher.hash_object({"x": t}) != hasher.hash_object({"x": a}) + assert hasher.hash_object([t]) != hasher.hash_object([a]) + assert data_hash(t) != data_hash(a) diff --git a/tests/test_torch/test_tensor_type_identity.py b/tests/test_torch/test_tensor_type_identity.py new file mode 100644 index 000000000..d8f23c8a1 --- /dev/null +++ b/tests/test_torch/test_tensor_type_identity.py @@ -0,0 +1,95 @@ +"""A tensor column keeps its logical type identity with and without values (ITL-871). + +The Arrow hasher replaces a value-hashed logical type by ``large_binary`` tokens. +The identifier of ``orcapod.torch.tensor`` must stay on the hashed field, so a +column with no value (zero rows, or nulls only) is still hashed as a tensor +column: differently from an ndarray column, which shares its storage struct, and +from plain binary (the regression fixed for the other types in ITL-967). +""" + +from __future__ import annotations + +from typing import Any, Optional + +import pytest + +pytest.importorskip("torch") + +import numpy as np # noqa: E402 +import pyarrow as pa # noqa: E402 +import torch # noqa: E402 + +from orcapod.contexts import get_default_context # noqa: E402 +from orcapod.logical_types.identifier import logical_type_identifier # noqa: E402 +from orcapod.types import Schema # noqa: E402 + + +def _null_column(python_type: Any, rows: int) -> pa.Table: + """A one-column table of ``python_type`` holding ``rows`` nulls.""" + arrow_type = get_default_context().type_converter.python_type_to_arrow_type( + python_type + ) + assert isinstance(arrow_type, pa.ExtensionType) + storage = pa.array([None] * rows, arrow_type.storage_type) + return pa.table({"c": pa.ExtensionArray.from_storage(arrow_type, storage)}) + + +def test_identifier_is_the_extension_name_and_the_ignore_list() -> None: + """The identifier is ``orcapod.torch.tensor`` plus ``{"ignore":[],"version":1}``.""" + logical_type = get_default_context().type_converter.get_logical_type(torch.Tensor) + assert logical_type is not None + identifier = logical_type_identifier(logical_type) + metadata = b'{"ignore":[],"version":1}' + assert identifier.name == "orcapod.torch.tensor" + assert identifier.metadata == metadata + assert tuple(logical_type.readable_extension_metadata()) == (metadata,) + + +@pytest.mark.parametrize("rows", [0, 2], ids=["zero_rows", "all_null"]) +def test_valueless_tensor_column_hashes_as_a_tensor_column(rows: int) -> None: + """A tensor column with no value differs from an ndarray column and plain binary.""" + hasher = get_default_context().arrow_hasher + hashes = { + "tensor": hasher.hash_table(_null_column(torch.Tensor, rows)), + "ndarray": hasher.hash_table(_null_column(np.ndarray, rows)), + "plain_large_binary": hasher.hash_table( + pa.table({"c": pa.array([None] * rows, pa.large_binary())}) + ), + } + assert len(set(hashes.values())) == len(hashes), hashes + assert hashes["tensor"] == hasher.hash_table(_null_column(torch.Tensor, rows)) + + +def test_hashed_type_is_the_same_with_and_without_data() -> None: + """One declared tensor column is hashed under one type, whatever rows it holds.""" + context = get_default_context() + with_data = context.type_converter.python_dicts_to_arrow_table( + [{"c": torch.arange(3.0)}, {"c": None}], + python_schema=Schema({"c": Optional[torch.Tensor]}), + ) + # Non-public helper: the test is about the schema the hasher hashes under. + hashed = context.arrow_hasher._process_table_columns # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] + assert hashed(with_data).schema == hashed(with_data.slice(0, 0)).schema + assert hashed(with_data).schema == hashed(with_data.slice(1, 1)).schema + + +def test_tensor_and_equal_ndarray_are_told_apart_by_type() -> None: + """With equal elements, the value, type, schema and table hashes all differ.""" + context = get_default_context() + hasher = context.semantic_hasher + tensor = torch.arange(6, dtype=torch.float32).reshape(2, 3) + array = tensor.numpy().copy() + assert hasher.hash_object(tensor).digest != hasher.hash_object(array).digest + assert hasher.hash_object(tensor) != hasher.hash_object(array) + assert hasher.hash_object(torch.Tensor) != hasher.hash_object(np.ndarray) + assert hasher.hash_object(Schema({"c": torch.Tensor})) != hasher.hash_object( + Schema({"c": np.ndarray}) + ) + + def table_hash(python_type: Any, value: Any) -> Any: + table = context.type_converter.python_dicts_to_arrow_table( + [{"c": value}], python_schema=Schema({"c": python_type}) + ) + return context.arrow_hasher.hash_table(table) + + assert table_hash(torch.Tensor, tensor) != table_hash(np.ndarray, array) diff --git a/tests/test_torch/test_tensor_value_format.py b/tests/test_torch/test_tensor_value_format.py new file mode 100644 index 000000000..717b3da1c --- /dev/null +++ b/tests/test_torch/test_tensor_value_format.py @@ -0,0 +1,346 @@ +"""``torch.Tensor`` values: cell token, refusals, declared pod outputs (ITL-871). + +Storage is in ``test_tensor_storage.py`` and the value hash in +``test_tensor_strict_hash.py``. Tests go through sources, streams, function pods +and the default data context's semantic hasher. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +pytest.importorskip("torch") + +import numpy as np # noqa: E402 +import polars as pl # noqa: E402 +import pyarrow as pa # noqa: E402 +import torch # noqa: E402 + +from orcapod import function_pod # noqa: E402 +from orcapod.contexts import create_registry, get_default_context # noqa: E402 +from orcapod.errors import OutputTypeError, UnsupportedValueError # noqa: E402 +from orcapod.logical_types.base_logical_type import BaseLogicalType # noqa: E402 +from orcapod.logical_types.registry import ( # noqa: E402 + make_arrow_extension_type, + make_polars_extension_type, +) +from orcapod.semantic_types.universal_converter import ( # noqa: E402 + UniversalTypeConverter, +) +from orcapod.sources import ArrowTableSource, DictSource # noqa: E402 +from tests.test_torch.tensor_cases import ( # noqa: E402 + EXTENSION_NAME, + assert_tensors_identical, +) + + +def semantic_hash(value: Any) -> Any: + """The default context's semantic hash of ``value``.""" + return get_default_context().semantic_hasher.hash_object(value) + + +def data_hash(value: Any) -> Any: + """The content hash of a one-row data holding ``value``.""" + source = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + return next(iter(source.iter_data()))[1].content_hash() + + +# --------------------------------------------------------------------------- +# Hash +# --------------------------------------------------------------------------- + + +def test_table_token_carries_identifier_and_digest() -> None: + """A stored cell hashes as the logical type identifier and the digest, no label.""" + from orcapod.hashing.arrow_hashers import StarfixArrowHasher + + hasher = get_default_context().arrow_hasher + assert isinstance(hasher, StarfixArrowHasher) + t = torch.arange(3.0) + table = DictSource(data=[{"k": "r", "x": t}], tag_columns=["k"]).as_table() + # Non-public helper: this test is about the token the hasher builds for a cell. + processed = hasher._process_table_columns(table.select(["x"])) # pyright: ignore[reportPrivateUsage] + token = processed.column("x")[0].as_py() + metadata = b'{"ignore":[],"version":1}' + assert ( + token + == b"orcapod.torch.tensor;" + + metadata.hex().encode() + + b"::" + + semantic_hash(t).digest + ) + + +DIFFERENT: list[tuple[str, Any, Any]] = [ + ("width", torch.arange(3, dtype=torch.int32), torch.arange(3, dtype=torch.int64)), + ("empty_shapes", torch.zeros((0, 3)), torch.zeros((3, 0))), + ("zero_d_vs_one", torch.tensor(1.0), torch.tensor([1.0])), + ("signed_zero", torch.tensor([0.0]), torch.tensor([-0.0])), + ( + "bfloat16_vs_float16", + torch.ones(2, dtype=torch.bfloat16), + torch.ones(2, dtype=torch.float16), + ), + ("shape", torch.arange(6.0).reshape(2, 3), torch.arange(6.0).reshape(3, 2)), + ( + "complex32_vs_float32", + torch.zeros(2, dtype=torch.complex32), + torch.zeros(2, dtype=torch.float32), + ), + ( + "float8_e4m3fn_vs_e5m2", + torch.zeros(2, dtype=torch.float8_e4m3fn), + torch.zeros(2, dtype=torch.float8_e5m2), + ), + ( + "float8_vs_uint8", + torch.zeros(2, dtype=torch.float8_e4m3fn), + torch.zeros(2, dtype=torch.uint8), + ), +] + + +@pytest.mark.parametrize( + ("a", "b"), [d[1:] for d in DIFFERENT], ids=[d[0] for d in DIFFERENT] +) +def test_dtype_shape_and_bits_change_the_hash(a: Any, b: Any) -> None: + """Dtype, shape and element bits each change the hash.""" + assert semantic_hash(a) != semantic_hash(b) + assert data_hash(a) != data_hash(b) + + +# --------------------------------------------------------------------------- +# Refusals: named column and reason +# --------------------------------------------------------------------------- + + +def _refused() -> list[tuple[str, Any, str]]: + """Tensors that cannot be stored, with a word the refusal must contain.""" + cases: list[tuple[str, Any, str]] = [ + ("sparse_coo", torch.eye(3).to_sparse(), "sparse"), + ("sparse_csr", torch.eye(3).to_sparse_csr(), "sparse"), + ( + "quantized", + torch.quantize_per_tensor(torch.zeros(3), 1.0, 0, torch.qint8), + "quantized", + ), + ( + "nested", + torch.nested.nested_tensor([torch.zeros(2), torch.zeros(3)]), + "nested", + ), + ("meta", torch.empty(3, device="meta"), "meta"), + ] + # Sub-byte, packed and not yet defined dtypes (where this torch has them). + for name in ("int4", "uint3", "bits8", "float4_e2m1fn_x2", "bcomplex32"): + dtype = getattr(torch, name, None) + if dtype is not None: + cases.append((name, torch.empty(2, dtype=dtype), name)) + return cases + + +@pytest.mark.parametrize( + ("value", "reason"), [c[1:] for c in _refused()], ids=[c[0] for c in _refused()] +) +def test_unsupported_tensors_are_refused_naming_column_and_reason( + value: Any, reason: str +) -> None: + """Storing an unsupported tensor raises naming the column and the reason.""" + with pytest.raises(UnsupportedValueError, match="'payload'") as info: + DictSource(data=[{"k": "r", "payload": value}], tag_columns=["k"]).as_table() + assert reason.lower() in str(info.value).lower() + + +@pytest.mark.parametrize( + ("value", "reason"), [c[1:] for c in _refused()], ids=[c[0] for c in _refused()] +) +def test_unsupported_tensors_are_refused_by_the_hasher(value: Any, reason: str) -> None: + """Hashing an unsupported tensor raises naming the reason.""" + with pytest.raises(UnsupportedValueError) as info: + semantic_hash(value) + assert reason.lower() in str(info.value).lower() + + +# --------------------------------------------------------------------------- +# Function pods: a declared torch.Tensor return must be a tensor +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("returned", "type_name"), + [(np.zeros(2, np.float32), "ndarray"), (1.5, "float")], + ids=["ndarray", "float"], +) +def test_non_tensor_returned_for_declared_tensor_is_refused_at_the_pod( + returned: Any, type_name: str +) -> None: + """A non-tensor returned for ``-> torch.Tensor`` raises naming the output.""" + + @function_pod(output_keys="y") + def bad(x: torch.Tensor) -> torch.Tensor: + """Return the value under test instead of a tensor.""" + return returned + + source = DictSource(data=[{"k": "a", "x": torch.zeros(2)}], tag_columns=["k"]) + with pytest.raises( + OutputTypeError, + match=f"'bad' returned .*{type_name} for output 'y'.*declared torch.Tensor", + ): + list(bad.pod(source).iter_data()) + + +def test_none_returned_for_optional_tensor_is_accepted() -> None: + """``None`` is accepted for ``-> torch.Tensor | None``.""" + + @function_pod(output_keys="y") + def maybe(x: torch.Tensor) -> torch.Tensor | None: + """Return ``None`` for an empty tensor, else the input.""" + return None if x.numel() == 0 else x + + source = DictSource( + data=[{"k": "a", "x": torch.zeros(0)}, {"k": "b", "x": torch.ones(2)}], + tag_columns=["k"], + ) + got = {t.as_dict()["k"]: d.as_dict()["y"] for t, d in maybe.pod(source).iter_data()} + assert got["a"] is None + assert_tensors_identical(got["b"], torch.ones(2)) + + +def test_declared_tensor_subclass_refuses_a_plain_tensor() -> None: + """A pod declared ``-> nn.Parameter`` must return an ``nn.Parameter``.""" + + @function_pod(output_keys="y") + def as_parameter(x: torch.Tensor) -> torch.nn.Parameter: + """Return a plain tensor where ``nn.Parameter`` is declared.""" + return x * 2 # pyright: ignore[reportReturnType] # deliberately the base class + + source = DictSource(data=[{"k": "a", "x": torch.ones(2)}], tag_columns=["k"]) + with pytest.raises( + OutputTypeError, + match="returned torch.Tensor for output 'y'.*declared torch.nn.parameter.Parameter", + ): + list(as_parameter.pod(source).iter_data()) + + +def test_subclass_instance_is_accepted_for_declared_tensor() -> None: + """``-> torch.Tensor`` may return an ``nn.Parameter``; it is stored and read as one. + + What is stored for a subclass is decided by the logical type: the tensor + type keeps the class, and hashes it (option ``subclass``). + """ + + @function_pod(output_keys="y") + def as_parameter(x: torch.Tensor) -> torch.Tensor: + """Return an ``nn.Parameter`` where ``torch.Tensor`` is declared.""" + return torch.nn.Parameter(x * 2, requires_grad=False) + + @function_pod(output_keys="y") + def as_tensor(x: torch.Tensor) -> torch.Tensor: + """Return a plain tensor with the same elements.""" + return x * 2 + + source = DictSource(data=[{"k": "a", "x": torch.ones(2)}], tag_columns=["k"]) + stream = as_parameter.pod(source) + y_type = stream.as_table().schema.field("y").type + assert isinstance(y_type, pa.ExtensionType) + assert y_type.extension_name == EXTENSION_NAME + # Decode the stored cell: the stream's own rows hold the returned object. + stored = ArrowTableSource(stream.as_table().select(["k", "y"]), tag_columns=["k"]) + got = [d.as_dict()["y"] for _, d in stored.iter_data()] + assert type(got[0]) is torch.nn.Parameter + assert not got[0].requires_grad + assert torch.equal(got[0].detach(), torch.ones(2) * 2) + + def row_hash(pod_stream: Any) -> Any: + return next(iter(pod_stream.iter_data()))[1].content_hash() + + assert row_hash(stream) != row_hash(as_tensor.pod(source)) + + +# --------------------------------------------------------------------------- +# A declared subclass with its own logical type +# --------------------------------------------------------------------------- + + +class LabeledTensor(torch.Tensor): + """Toy tensor subclass with its own logical type.""" + + +class LogicalLabeledTensor(BaseLogicalType): + """Toy logical type for ``LabeledTensor``: stores the raw float32 bytes.""" + + _arrow_ext_class = make_arrow_extension_type( + "test.labeled_tensor", pa.large_binary() + ) + _polars_ext_class = make_polars_extension_type( + "test.labeled_tensor", pa.large_binary() + ) + logical_type_name: str = "test.labeled_tensor" + python_type: type = LabeledTensor + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the toy extension type.""" + return self._arrow_ext_class() # pyright: ignore[reportCallIssue] # synthesized class takes no arguments + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the toy Polars extension type.""" + return self._polars_ext_class() # pyright: ignore[reportCallIssue] # synthesized class takes no arguments + + def python_to_storage(self, value: Any, converter: Any = None) -> bytes: + """Store the float32 bytes.""" + return value.detach().to(torch.float32).numpy().tobytes() + + def storage_to_python(self, storage_value: Any, converter: Any = None) -> Any: + """Read the bytes back as a ``LabeledTensor``.""" + array = np.frombuffer(storage_value, dtype=np.float32).copy() + return torch.from_numpy(array).as_subclass(LabeledTensor) + + +def test_declared_subclass_with_its_own_logical_type_requires_that_subclass() -> None: + """A pod declared ``-> LabeledTensor`` (registered) must return a ``LabeledTensor``.""" + ctx = create_registry().get_context() + converter = ctx.type_converter + assert isinstance(converter, UniversalTypeConverter) + converter.register_logical_type(LogicalLabeledTensor()) # pyright: ignore[reportArgumentType] # toy logical type implements the protocol structurally + + @function_pod(output_keys="y", data_context=ctx) + def plain(x: torch.Tensor) -> LabeledTensor: + """Return a plain tensor where the registered subclass is declared.""" + return x * 2 # pyright: ignore[reportReturnType] # deliberately the base class + + @function_pod(output_keys="y", data_context=ctx) + def labeled(x: torch.Tensor) -> LabeledTensor: + """Return the registered subclass.""" + return (x * 2).as_subclass(LabeledTensor) + + source = DictSource( + data=[{"k": "a", "x": torch.ones(2)}], tag_columns=["k"], data_context=ctx + ) + with pytest.raises( + OutputTypeError, + match="returned torch.Tensor for output 'y'.*declared .*LabeledTensor", + ): + list(plain.pod(source).iter_data()) + got = [d.as_dict()["y"] for _, d in labeled.pod(source).iter_data()] + assert type(got[0]) is LabeledTensor + + +def test_hashing_a_subclass_without_its_own_value_handler_says_what_is_missing() -> ( + None +): + """A tensor subclass is not hashed as a tensor: the refusal names the value handler. + + The value hash includes the class, so only ``torch.Tensor`` and + ``nn.Parameter`` have one. A subclass registered with its own logical type + needs its own value handler as well. + """ + value = torch.ones(2).as_subclass(LabeledTensor) + with pytest.raises(UnsupportedValueError) as info: + semantic_hash(value) + message = str(info.value) + assert "LabeledTensor" in message + assert "cannot be hashed" in message + assert "value handler" in message + assert "cannot be stored" not in message diff --git a/uv.lock b/uv.lock index e50d3527f..88152f9c4 100644 --- a/uv.lock +++ b/uv.lock @@ -4,16 +4,20 @@ requires-python = ">=3.11.0" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.13.*' and sys_platform == 'darwin'", "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.12.*' and sys_platform == 'darwin'", "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.12' and sys_platform == 'darwin'", "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", ] [[package]] @@ -211,7 +215,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", ] dependencies = [ { name = "cffi" }, @@ -237,13 +242,16 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.13.*' and sys_platform == 'darwin'", "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.12.*' and sys_platform == 'darwin'", "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.12' and sys_platform == 'darwin'", "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", ] dependencies = [ { name = "cffi" }, @@ -1979,6 +1987,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/71/4ad9a42f2772793a03cb698f0fc42499f04e6e8d2560ba2f7da0fb059a8e/mmh3-5.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:b22fe2e54be81f6c07dcb36b96fa250fb72effe08aa52fbb83eade6e1e2d5fd7", size = 38890, upload-time = "2025-01-25T08:39:25.28Z" }, ] +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + [[package]] name = "msgpack" version = "1.1.1" @@ -2318,6 +2335,8 @@ all = [ { name = "pyspiral" }, { name = "ray", extra = ["default"] }, { name = "redis" }, + { name = "torch", version = "2.15.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.15.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, ] postgresql = [ { name = "psycopg", extra = ["binary"] }, @@ -2332,6 +2351,10 @@ redis = [ spiraldb = [ { name = "pyspiral" }, ] +torch = [ + { name = "torch", version = "2.15.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.15.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, +] [package.dev-dependencies] dev = [ @@ -2346,6 +2369,7 @@ dev = [ { name = "minio" }, { name = "mkdocs-material" }, { name = "mkdocstrings", extra = ["python"] }, + { name = "orcapod", extra = ["torch"] }, { name = "pip-licenses" }, { name = "pre-commit" }, { name = "pre-commit-hooks" }, @@ -2389,13 +2413,17 @@ requires-dist = [ { name = "redis", marker = "extra == 'redis'", specifier = ">=6.2.0" }, { name = "s3fs", specifier = ">=2025.12.0" }, { name = "starfix", specifier = ">=0.5.0,<0.6" }, + { name = "torch", marker = "sys_platform == 'linux' and extra == 'all'", specifier = ">=2.6", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torch", marker = "sys_platform == 'linux' and extra == 'torch'", specifier = ">=2.6", index = "https://download.pytorch.org/whl/cpu" }, + { name = "torch", marker = "sys_platform != 'linux' and extra == 'all'", specifier = ">=2.6" }, + { name = "torch", marker = "sys_platform != 'linux' and extra == 'torch'", specifier = ">=2.6" }, { name = "typer", specifier = ">=0.12" }, { name = "tzdata", specifier = ">=2024.1" }, { name = "universal-pathlib", specifier = ">=0.3.8" }, { name = "uuid-utils", specifier = ">=0.11.1" }, { name = "xxhash" }, ] -provides-extras = ["all", "postgresql", "ray", "redis", "spiraldb"] +provides-extras = ["all", "postgresql", "ray", "redis", "spiraldb", "torch"] [package.metadata.requires-dev] dev = [ @@ -2410,6 +2438,7 @@ dev = [ { name = "minio", specifier = ">=7.2.16" }, { name = "mkdocs-material", specifier = ">=9.7.5" }, { name = "mkdocstrings", extras = ["python"], specifier = ">=1.0.3" }, + { name = "orcapod", extras = ["torch"] }, { name = "pip-licenses", specifier = ">=5.0.0" }, { name = "pre-commit", specifier = ">=4.4.0" }, { name = "pre-commit-hooks", specifier = ">=6.0.0" }, @@ -3989,6 +4018,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/21/84474cbe254b10533f0a79c4a20dcd13cf65bb7012f9c85062ab89066eaa/substrait_protobuf-0.99.0-py3-none-any.whl", hash = "sha256:6d00c8a4f6d5d822a2358f870d642f8fda0350e110025c978daf624ae2491f16", size = 70932, upload-time = "2026-07-26T05:28:32.7Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + [[package]] name = "tenacity" version = "9.1.2" @@ -4058,6 +4099,91 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, ] +[[package]] +name = "torch" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version < '3.12' and sys_platform == 'darwin'", + "python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux'", +] +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/a5/fa947e2bbb072933f8c0ad5e02bd11126bcfc2b624fe234ac29a7b63600c/torch-2.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e808b63d616a43b326e1a5677d45a2e5babe7a8ce27edb03ddb48e7dd9b6d663", size = 138527954, upload-time = "2026-10-28T15:16:30.29Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7e/7f1252812e8048bc24f203fc0757a251f26af797b4ff986e19046e2b5f73/torch-2.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:d2c5938f593a47cf6d151a16d130a3cc3f0c105df1b905a571efc00b01e38c1f", size = 126720349, upload-time = "2026-10-28T15:16:25.498Z" }, + { url = "https://files.pythonhosted.org/packages/17/24/7b9b755e8c4dd2395c6a94bab6a4ed8aded1735ff2538bc63cbba8abeddd/torch-2.15.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:593e5fd0c204284890d1c3dff643be34e549e1d24523e13402510f960d4da438", size = 138558517, upload-time = "2026-10-28T15:16:35.918Z" }, + { url = "https://files.pythonhosted.org/packages/74/0b/a57c0b58a64b7a197dc40fc464bb96c0de867f95a2b1cbb3ec9e54414fde/torch-2.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:f2bd0a3709a7493a6ef2ac5a654111242a854439f7558f910b6b653b56f7504d", size = 126738207, upload-time = "2026-10-28T15:16:40.586Z" }, + { url = "https://files.pythonhosted.org/packages/87/fa/efeb85d3ba1d8ef497664db5358ac72a7d1d0f1d8a8ad487cba89753f279/torch-2.15.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:fbefd73b521d3c75c5a1d7c77056b2c6339b81468d14d4e33c55f525418a5855", size = 138556890, upload-time = "2026-10-28T15:16:45.195Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6c/f26ed528d197859cf8a634bbfafc3bc136f5590b6620df179d1858a34fdc/torch-2.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:e079089d36401261ebe145504309d0114fb9a00529eac4b33048221d2376a15c", size = 126736947, upload-time = "2026-10-28T15:17:36.768Z" }, + { url = "https://files.pythonhosted.org/packages/8a/04/ff26bb464e61f83e0176deca01e72fffa779190bbb476011e89c45791589/torch-2.15.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:0d2b2d9674fa0b9b3f50e83231aa6545a0b19e437a52f0af939a224faaa45515", size = 138575002, upload-time = "2026-10-28T15:17:58.526Z" }, + { url = "https://files.pythonhosted.org/packages/40/4e/e31abad54b00c940b33f8c6de724c7b1240fa6ba86acee3cbbf65a17a0f8/torch-2.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:cf02d11e13819807389e172ecb8f400f6135969af860e2a9f677f67a011e6194", size = 126735321, upload-time = "2026-10-28T15:18:03.587Z" }, + { url = "https://files.pythonhosted.org/packages/66/98/cad8d73df09e1517beeebd6090f00a0d1bdf762285a8082cf0f22a432e62/torch-2.15.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:628f54cb55a2363a5f14471a80c1b6efb63724449d2611ad2772fb5d4c42d19d", size = 138900111, upload-time = "2026-10-28T15:18:40.575Z" }, + { url = "https://files.pythonhosted.org/packages/4f/70/31911a8f46608a6c4fdc7744050a69168312984bed5ab7c8e18f4c9ad9d0/torch-2.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4a906d0fa6dc9aa9a9b5f3b96437c8dfe661f855645b3f838c12fc5cf5cce562", size = 127025368, upload-time = "2026-10-28T15:19:00.808Z" }, + { url = "https://files.pythonhosted.org/packages/99/34/c7ff06165cb09238e945aa2ae63b9186583a47a9880aacd349d37e521f08/torch-2.15.0-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:06a686eb717cd3d1662363082820be45497318887c596d691af0af0c7649c8f0", size = 138573101, upload-time = "2026-10-28T15:19:21.942Z" }, + { url = "https://files.pythonhosted.org/packages/14/37/003c96b78f95a0dc25368c371452ed01ea9d9b63d59120b8e970c93381f6/torch-2.15.0-cp315-cp315-win_amd64.whl", hash = "sha256:04597a77bbcc7a1a01e7b9efce78d258b18a7f31bcb190f4ee00360bff75e7f6", size = 126735953, upload-time = "2026-10-28T15:19:41.917Z" }, + { url = "https://files.pythonhosted.org/packages/8b/e6/1df516e9d3c0a551e118d21a2a6cbc7c47cdb4a09b7cc363ad8e696c4744/torch-2.15.0-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:02ac0b22211c1984e5dc002d2e57fdbc13d32116537ea6ae1a0c9d07cafce0c9", size = 138897802, upload-time = "2026-10-28T15:19:46.702Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/1ad0107cec15706e4d972fcbfaa6b3da26734959109861a60902c922b50c/torch-2.15.0-cp315-cp315t-win_amd64.whl", hash = "sha256:c521a00891e92aba2f34717671c268acd0bde26444623efcceaeaf51c87b3b36", size = 127024483, upload-time = "2026-10-28T15:20:23.183Z" }, +] + +[[package]] +name = "torch" +version = "2.15.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp311-cp311-linux_s390x.whl", hash = "sha256:ef0bd617c9439a54d698d95fb7c3f63efcc48c29efdd42b177e87d862cb574fa", upload-time = "2026-10-28T13:17:54Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:97dc6b0a7cbad2109e615da3eac03df626c61e821e9486438b0248ba896178c6", upload-time = "2026-10-28T13:18:01Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cee15855a32ad89a9bb27b8a9d6e07a859dcd2e11ed1b5561f3d6206e31a103b", upload-time = "2026-10-28T13:18:09Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:29c74659a82483d9a9c5d7f9e1112973dfa2dce5b3956dba5d1b97381b94cfae", upload-time = "2026-10-28T13:18:25Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:75941d429aaf4450bcd364345d7677e2d54aad45b38fbfcb1c0552450a6e9c40", upload-time = "2026-10-28T13:18:32Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0955c7e3446326293f48af18fad67872fa31dee5a36a87690eb9e5be8b465be3", upload-time = "2026-10-28T13:18:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:88f5fe2c5d25a809a315a4a3f494e367539a307356f37752d89cd0b9d84f175a", upload-time = "2026-10-28T13:18:52Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ab35875b15ac536a645e165b296300fed71df7b4f2ddbae40c725ab08f0a5e78", upload-time = "2026-10-28T13:18:58Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0f6f69701b2d38ddc412364dfb92e2e13e74960d33fa3573b4930ced626e81c7", upload-time = "2026-10-28T13:19:05Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp314-cp314-linux_s390x.whl", hash = "sha256:ef810702df4dd70fe1b8ebee9e7394c6491f779841571a71433829e44927eaa5", upload-time = "2026-10-28T13:19:19Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:01bdc7d291e3f6b5c03cf72d8a1718a5012d9fad5f380e67b563b0a76592055a", upload-time = "2026-10-28T13:19:25Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:f089c94912c955269a20e5fee6b6a371209b610849c9ece26e908a748d87a723", upload-time = "2026-10-28T13:19:32Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:7f8e2d696d12f965a5967bda4bc903decf6aa714b0b5ffb50e189733112a4dfd", upload-time = "2026-10-28T13:19:42Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:68e4595afffb1af18480b6ecb7217688c02a0f2a7b482c7a95b4896a3b38d49a", upload-time = "2026-10-28T13:19:48Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:17d66ae6f8a49d4dfca368b50b7dc15f7129175e80b4ae2832dfa0cce86d3159", upload-time = "2026-10-28T13:19:55Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:2cec97b29524b3352bb794321976da7cef2ecee171866499c5630c865396c0a2", upload-time = "2026-10-28T13:20:08Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:3552930ba3d358d3ff0b6fee19cb296e199767443c0c86be2a715471036d9233", upload-time = "2026-10-28T13:20:15Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:3fb23f534fdffdd3a178f8ef3327ab5fff833f291e5955be7edcaa7547a3f1fa", upload-time = "2026-10-28T13:20:26Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.15.0%2Bcpu-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:92551daf5873fc6db15be48fe9644398f13143291c151ab6769f19e7c5ebd10c", upload-time = "2026-10-28T13:20:33Z" }, +] + [[package]] name = "tornado" version = "6.5.1"