diff --git a/.zed/rules b/.zed/rules index 4ce3620b9..a9740f662 100644 --- a/.zed/rules +++ b/.zed/rules @@ -273,6 +273,37 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension never fall back to the storage type. List an older value in `also_reads` only when the current code decodes that layout. +## Library value types: storage, strict hash, ignore list + +Developer, November 12, 2026 (ITL-981); see `docs/concepts/arrays.md` and +`docs/reference/hashing.md` section 8. + +- Storage and hashing are decided separately. A value of a library type (`numpy.ndarray`, + later tensors and data frames) is stored with the library's own serialization and reads + back as given. Its hash is defined by Orcapod on the content: never hash the stored bytes. +- The hash is strict by default: every property of a value that storage keeps is part of + its identity. A property storage does not restore never counts and has no option name, + because `hash(read(write(v)))` must equal `hash(v)`. +- What the hash may ignore is a closed set of property names per type (`OPTIONS`). The names + a data context ignores are declared once, in `hash_ignore` of + `contexts/logical_types.json`, and referred to as `hash_ignore:` by the + logical type entry and the value handler entry. +- The list is in the extension metadata, `{"ignore":[...],"version":N}` (sorted names, + sorted keys, `ignore` always present), written only by `ignore_list.ignore_metadata`. Two + lists are two logical types. Never put the list into a hash preimage: an ignored property's + field is always written, with its canonical value. +- Build such a type from `logical_types/ignore_list.py` (`IgnoreListLogicalType`, + `IgnoreListHandler`). A bare value is hashed with the handler's configured list; a cell + with the list of its own column (`SemanticHashingVisitor` calls `hash_ignoring`). A data + context reads only columns stored under its own list; anything else raises + `LogicalTypeIdentifierError`. +- Extension names of library value types are lower case `orcapod..` + (`orcapod.numpy.ndarray`). The leading type text of a value hash is the extension name. +- `numpy.ndarray`: one `.npy` file per cell (`numpy.lib.format`, format 3.0, + `allow_pickle=False` on write and read; never raise `max_header_size`). Options: + `memory_order`, `byte_order`, `struct_layout`. Where stored arrays must be ordered, order + them by value hash (`storage_sort_key`), never by stored bytes. + ## User classes: exact and structural A class a user defines is one of two kinds (developer, November 12, 2026, ITL-972; see diff --git a/CLAUDE.md b/CLAUDE.md index ccca74a3d..c3dc23676 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -284,6 +284,37 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension never fall back to the storage type. List an older value in `also_reads` only when the current code decodes that layout. +## Library value types: storage, strict hash, ignore list + +Developer, November 12, 2026 (ITL-981); see `docs/concepts/arrays.md` and +`docs/reference/hashing.md` section 8. + +- Storage and hashing are decided separately. A value of a library type (`numpy.ndarray`, + later tensors and data frames) is stored with the library's own serialization and reads + back as given. Its hash is defined by Orcapod on the content: never hash the stored bytes. +- The hash is strict by default: every property of a value that storage keeps is part of + its identity. A property storage does not restore never counts and has no option name, + because `hash(read(write(v)))` must equal `hash(v)`. +- What the hash may ignore is a closed set of property names per type (`OPTIONS`). The names + a data context ignores are declared once, in `hash_ignore` of + `contexts/logical_types.json`, and referred to as `hash_ignore:` by the + logical type entry and the value handler entry. +- The list is in the extension metadata, `{"ignore":[...],"version":N}` (sorted names, + sorted keys, `ignore` always present), written only by `ignore_list.ignore_metadata`. Two + lists are two logical types. Never put the list into a hash preimage: an ignored property's + field is always written, with its canonical value. +- Build such a type from `logical_types/ignore_list.py` (`IgnoreListLogicalType`, + `IgnoreListHandler`). A bare value is hashed with the handler's configured list; a cell + with the list of its own column (`SemanticHashingVisitor` calls `hash_ignoring`). A data + context reads only columns stored under its own list; anything else raises + `LogicalTypeIdentifierError`. +- Extension names of library value types are lower case `orcapod..` + (`orcapod.numpy.ndarray`). The leading type text of a value hash is the extension name. +- `numpy.ndarray`: one `.npy` file per cell (`numpy.lib.format`, format 3.0, + `allow_pickle=False` on write and read; never raise `max_header_size`). Options: + `memory_order`, `byte_order`, `struct_layout`. Where stored arrays must be ordered, order + them by value hash (`storage_sort_key`), never by stored bytes. + ## User classes: exact and structural A class a user defines is one of two kinds (developer, November 12, 2026, ITL-972; see diff --git a/changelog.d/ITL-981.md b/changelog.d/ITL-981.md new file mode 100644 index 000000000..6a35ca09c --- /dev/null +++ b/changelog.d/ITL-981.md @@ -0,0 +1,74 @@ +### Breaking changes and migration + +#### NumPy arrays: stored as `.npy`, hashed strictly, with a per-type ignore list (ITL-981) + +**This replaces the array storage and the array hash described under "NumPy arrays: new +storage format and new hashes" (ITL-888).** That format (the extension type +`orcapod.ndarray` over a `{dtype, shape, data}` struct, with a hash that ignored memory +layout and byte order) was never released; v0.3 ships the one described here. What that +entry says about v0.2 stores still holds: they are not migrated and not read. + +The developer decided on November 12, 2026 that storage and hashing are separate, that the +hash is strict by default, and that what the hash ignores is configured per type. + +- **Storage.** An array column is the Arrow extension type `orcapod.numpy.ndarray` over + `large_binary`. Each cell is one `.npy` file written by NumPy + (`numpy.lib.format.write_array`, format version 3.0, `allow_pickle=False`) and read with + `allow_pickle=False`. An array reads back as it was given: dtype including byte order, + shape, element bits, C or Fortran memory order and structured field offsets and item size + are kept. Not kept: strides other than C or Fortran order (a step slice is stored + C-contiguous), the subclass (read back as plain `np.ndarray`), the `writeable` flag, dtype + `metadata` and the `align` flag of a structured dtype. Padding bytes between structured + fields are not part of the value and are written as zero, so residual process memory + never reaches a store. A cell that does not hold exactly the element bytes its `.npy` + header declares is refused on read. +- **Hash.** The hash is defined on the value, never on the stored bytes, and keeps the label + `ndarray_v1`. It is `SHA-256(lp("orcapod.numpy.ndarray") || E || memory order || byte + order || structured layout)`, where `E` is the element digest of dtype, shape and element + bits (`docs/reference/hashing.md`, section 8, with a reference implementation and pinned + vectors). By default a C-ordered and a Fortran-ordered array, a little- and a big-endian + array, and a packed and an aligned structured array with equal values now hash + **differently**. What storage does not keep never changes the hash, so a value read back + from a store hashes like the value written. +- **Ignore list.** The three properties `memory_order`, `byte_order` and `struct_layout` + can be ignored. The list of ignored names is set per type in the data context + (`hash_ignore` in `contexts/logical_types.json`; default: empty) and is written into the + extension metadata, `{"ignore":[...],"version":1}`. Two lists are two logical types: + different schema hashes, no shared type in a join, and a data context refuses to read a + column stored under another list (`LogicalTypeIdentifierError`, naming the column and both + lists). A bare value is hashed with the data context's list, a cell with the list of its + own column. v0.3 has no user API to change the list. +- **Newly refused at write** (`UnsupportedValueError`, naming the column and the dtype): a + structured dtype whose fields overlap or are not in offset order (`.npy` has no form for + it; the previous format accepted it), and a dtype whose `.npy` header exceeds 10,000 bytes + (several hundred fields). Masked arrays, object dtypes, `StringDType`, `longdouble`, + unit-less datetimes and field titles are refused as before. +- **`MergeJoin`** orders the arrays it merges by their value hash (it used the stored dtype + tag, shape and bytes). +- **What moves.** Every array value hash, and with the new extension name and metadata the + type, schema and pipeline hashes of anything with an array column and the system tag + column names built from those schema hashes. Nothing released is affected: v0.3 is not + tagged, and v0.2 stores are not read by v0.3. + +### Added + +- **The ignore-list mechanism for library value types** (`orcapod.logical_types.ignore_list`): + `IgnoreListLogicalType` and `IgnoreListHandler`, the extension metadata shape + `{"ignore":[...],"version":N}` (`ignore_metadata`, `parse_ignore_metadata`), and the + `hash_ignore` object of the logical type registration file. `numpy.ndarray` is the first + type built on it. (ITL-981) + +### Changed + +- **`StarfixArrowHasher.hash_schema` hashes a field under the same rule as `hash_table`.** A + logical type with a value handler is hashed as a `large_binary` field carrying the type's + identifier, so its storage type is no longer part of that hash. No library code calls this + method, and no stored or pinned value moves. (ITL-981) + +### Known limitations + +- **A non-default ignore list is not usable end to end in a pipeline in v0.3.** A data + context built from a custom logical type registration file hashes bare values and Arrow + tables with its list, but streams and datagrams find their data context by key, so their + rows are read by the default context, which refuses a column of another list. (ITL-981, + ITL-965) diff --git a/design_issues/ITL-888-AR1.md b/design_issues/ITL-888-AR1.md index 3c4afc71a..8b0537595 100644 --- a/design_issues/ITL-888-AR1.md +++ b/design_issues/ITL-888-AR1.md @@ -25,3 +25,9 @@ results. Known compromise: the struct's children are nullable at the Arrow level, because Polars drops nested non-null flags on its way back to Arrow (ITL-842). The logical type enforces non-null instead. Tighten under a new storage version once nested nullability survives Polars. + +**Superseded (ITL-981, November 12, 2026):** the struct storage and the layout-independent hash +described above were replaced before any release by `.npy` storage under +`orcapod.numpy.ndarray` and a strict value hash with a per-type ignore list; see entry LV1 +(`design_issues/ITL-981-LV1.md`). The refusals, the declared-output check and the v0.2 +statements of this entry still hold. diff --git a/design_issues/ITL-981-LV1.md b/design_issues/ITL-981-LV1.md new file mode 100644 index 000000000..6f537df0c --- /dev/null +++ b/design_issues/ITL-981-LV1.md @@ -0,0 +1,33 @@ +## `src/orcapod/logical_types/ignore_list.py`, `src/orcapod/logical_types/numpy_type.py` — library value types + +### LV1 — Library value types: storage and hash decided separately, strict hash, per-type ignore list +**Status:** resolved +**Severity:** high +**Issue:** ITL-981 (supersedes the storage and hash of ITL-888, entry AR1) + +The first v0.3 array format (ITL-888, never released) tied storage to the hash: an array was +stored in Orcapod's own canonical struct (C order, little-endian, packed fields) and hashed +as that row. So an array did not read back as it was given, and the hash silently ignored +memory order and byte order, with no way to say otherwise. The same question was open for +torch tensors, Polars and pandas data frames. + +Developer's decisions (November 12, 2026): storage is the library's own serialization and a +value reads back as given; the hash is defined by Orcapod on the content, never on the stored +bytes; the hash is strict by default (anything about a value that can be changed and that +storage keeps is part of its identity); what the hash ignores is a per-type list of property +names saved in the extension metadata, so two lists are two logical types. A property storage +cannot restore never counts, because a value read back must hash like the value written. + +**Fix:** `logical_types/ignore_list.py` holds the shared mechanism: the metadata shape +`{"ignore":[...],"version":N}`, `IgnoreListLogicalType`, `IgnoreListHandler`, one Arrow +extension class per list. The data context declares each type's list once, in `hash_ignore` +of `contexts/logical_types.json`. The Arrow hasher hashes a cell under the list of its own +column. `numpy.ndarray` is stored as a `.npy` file (format 3.0, no pickle) under +`orcapod.numpy.ndarray` and hashed in two steps: the element digest of dtype, shape and bits, +then the type name with memory order, byte order and structured layout, each ignorable. +Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md`. + +Known limits, by decision: a data context has one list per type and refuses a column stored +under another (the Python schema carries only `np.ndarray`, so a per-column list needs a user +API, which v0.3 does not have); a strided view is stored and hashed as its C-contiguous copy; +unit-less datetimes and field titles stay refused (ITL-911) although `.npy` could store them. diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index 7680a828d..ae1ab6187 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -1,9 +1,12 @@ # NumPy Arrays A `numpy.ndarray` can be a data value like any other: a source column, a function pod -parameter or a return value. Orcapod stores each array by its **value**, its dtype, its shape -and its elements. How the array happens to sit in memory is not stored. The hash is defined on -that value, so two arrays that hold the same values hash the same, however each was laid out. +parameter or a return value. Storage and hashing are two separate things: + +- **Storage** is NumPy's own `.npy` format. An array reads back as it was given: dtype, + byte order, shape, C or Fortran memory order and structured field offsets are kept. +- **The hash** is defined by Orcapod on the array's value, never on the stored bytes. It is + strict: everything about the array that storage keeps is part of its identity. ```python import numpy as np @@ -25,13 +28,14 @@ for tag, data in scale.pod(source).iter_data(): # t1 float32 (2, 3) ``` -!!! warning "v0.3 changed the array format" +!!! warning "v0.3 changed how arrays are stored and hashed" Orcapod v0.2 stored arrays as `.npy` files under the extension name `numpy.ndarray` and - hashed the file. v0.3 replaces both, as described below, and does not migrate older - stores. Every node that handles arrays, and every node downstream of an array column, gets - new tables in v0.3. Because v0.3 also changed every hash of Arrow data, v0.3 reuses no - result of a store written by v0.2: it ignores the v0.2 rows and computes every input - again (see [A store written by an older version](function-pods.md#a-store-written-by-an-older-version)). + hashed the file. v0.3 stores them under `orcapod.numpy.ndarray`, hashes the value as + described below, and does not migrate older stores. Every node that handles arrays, and + every node downstream of an array column, gets new tables in v0.3. Because v0.3 also + changed every hash of Arrow data, v0.3 reuses no result of a store written by v0.2: it + ignores the v0.2 rows and computes every input again (see + [A store written by an older version](function-pods.md#a-store-written-by-an-older-version)). Nodes that perform side effects are the exception: a side-effect pod, or a function pod that takes the invocation context, does not run again on such a store but raises `SideEffectLogDataContextError` before it delivers anything. @@ -40,26 +44,43 @@ for tag, data in scale.pod(source).iter_data(): ## What counts as the same array -Two arrays are the same value when they have the same dtype, the same shape and the same -element bits. - -- **Memory layout does not count.** C order, Fortran order, strided views and transposed views - of equal values are the same array. An array always reads back in C order. -- **Byte order does not count.** Arrays are stored little-endian; a big-endian `>f4` array - reads back as `f4"))) -# True +# False print(hasher.hash_object(a) == hasher.hash_object(a.astype(np.float64))) # False -ab = np.zeros(1, dtype=[("a", "i4"), ("b", "f8")]) -ba = np.zeros(1, dtype=[("b", "f8"), ("a", "i4")]) -print(hasher.hash_object(ab) == hasher.hash_object(ba)) +wide = np.arange(12, dtype=np.float32).reshape(2, 6) +view = wide[:, ::2] +print(hasher.hash_object(view) == hasher.hash_object(np.ascontiguousarray(view))) +# True +packed = np.zeros(1, dtype=[("a", "i1"), ("b", "f8")]) +aligned = np.zeros(1, dtype=np.dtype([("a", "i1"), ("b", "f8")], align=True)) +print(hasher.hash_object(packed) == hasher.hash_object(aligned)) # False ``` +## The ignore list + +What the hash ignores is a setting of the array type, not of a single call. The data context +holds one list of ignored property names for the type; the default list is empty. The list +is written into the type's Arrow extension metadata, so it is part of the +[logical type identifier](logical-type-identifiers.md): the array type with an empty list +and the array type that ignores `memory_order` are two different logical types, with +different schema hashes. + +- A bare array (a function default, a value inside a Python structure) is hashed with the + data context's list. +- An array in a table is hashed with the list of its own column. +- A data context reads only array columns stored under its own list. A column stored under + another list raises `LogicalTypeIdentifierError`, naming the column and both lists. + +When a property is ignored, arrays that differ only in it hash equal, and they hash like +the array in canonical form (C order, little-endian, packed fields) does under the empty +list. The list is configured in the `hash_ignore` object of Orcapod's logical type +registration file (`contexts/logical_types.json`); v0.3 has no other way to change it. A +name that is not one of the three above is an error when the data context is built. + +!!! note "Known limitation in v0.3" + A list other than the default cannot be used end to end in a pipeline yet. A data + context built from your own registration file hashes bare values and Arrow tables with + its list, but streams and datagrams find their data context by its key, which is the + key of the default context, so their rows are read by the default context, and it + refuses a column of another list (ITL-965). In v0.3 the default list, empty, is the + one that works everywhere. + +The same mechanism serves the other library value types, each with its own names. + ## What cannot be stored Some arrays are refused when they are written, with a `ValueError` (an @@ -92,6 +150,8 @@ Some arrays are refused when they are written, with a `ValueError` (an | `longdouble`, `clongdouble` | width and format differ by platform | `.astype(np.float64)` or `.astype(np.complex128)` | | `datetime64` or `timedelta64` without a unit | the integer has no defined meaning | give a unit, e.g. `datetime64[ns]` | | Structured dtypes with field titles | titles are aliases of fields | drop the titles | +| Structured dtypes whose fields overlap or are not in offset order | `.npy` has no form for them | declare the fields in offset order | +| Structured dtypes with several hundred fields (a `.npy` header over 10,000 bytes) | NumPy does not read such a header without pickle support | split the array | Extended data may be supported later, stored as several parts rather than as one array. Whether `longdouble` and `clongdouble`, unit-less datetimes and timedeltas, and structured @@ -115,51 +175,63 @@ example `x * 2` on a 0-d array, raises `TypeError` that names the output. Return ## How an array is stored -An array column has the Arrow extension type `orcapod.ndarray`, with extension metadata -`{"version":1}`. Its storage is one struct per row: +An array column has the Arrow extension type `orcapod.numpy.ndarray` over `large_binary`. +Each cell is one `.npy` file, written by NumPy itself +(`numpy.lib.format.write_array`, format version 3.0, `allow_pickle=False`) and read with +`numpy.lib.format.read_array(..., allow_pickle=False)`. Format 3.0 is the one that stores +field names outside Latin-1. Pickle is never written and never read: a cell that holds an +object array, is not a complete `.npy` file of format 3.0, or does not hold exactly the +element bytes its header declares, raises a `ValueError` that names the column. -| Field | Arrow type | Holds | -|---|---|---| -| `dtype` | `large_string` | the Orcapod dtype tag (below) | -| `shape` | `large_list` | the dimensions, outermost first; empty for a 0-d array | -| `data` | `large_binary` | the elements in C order, little-endian, packed | - -The struct's children are nullable in the Arrow schema only because Polars drops nested -non-null flags. A stored array never has a null child. A null array is a null struct, allowed -when the column is annotated `np.ndarray | None`. +The extension metadata is `{"ignore":[],"version":1}`: the ignore list and the version of +the type. A null array is a null cell, allowed when the column is annotated +`np.ndarray | None`. ```python +import io + table = source.as_table() print(table.schema.field("signal").type.extension_name) -# orcapod.ndarray -print(table.column("signal").chunk(0).storage[0].as_py()["dtype"]) +# orcapod.numpy.ndarray +cell = table.column("signal").chunk(0).storage[0].as_py() +print(np.load(io.BytesIO(cell), allow_pickle=False).dtype) # float32 ``` -### Dtype tags +What reads back: -The tag names the element type without a byte order. For a structured dtype, the tag is a JSON -list with one `[name, tag]` entry per field, or `[name, tag, [dims]]` for a subarray field, -in declared field order. It is written with no whitespace and with non-ASCII characters escaped -(`json.dumps(..., separators=(",", ":"), ensure_ascii=True)`). +| Property | Reads back | +|---|---| +| dtype, including byte order, units, string widths, field names, offsets and item size | as given | +| shape and element bits | as given | +| C or Fortran memory order | as given | +| any other strides (step slices, reversed or broadcast views) | C-contiguous | +| subclass | plain `np.ndarray` | +| `writeable` flag | writeable | +| dtype `metadata`, the `align` flag | not kept | +| padding bytes between structured fields | zero | -| NumPy | Tag | Element bytes | -|---|---|---| -| `bool` | `bool` | one byte, 0 or 1 | -| `int8` ... `int64`, `uint8` ... `uint64` | `int8` ... `uint64` | two's complement or unsigned, little-endian | -| `float16`, `float32`, `float64` | same names | IEEE 754, little-endian | -| `complex64`, `complex128` | same names | real part, then imaginary part | -| `datetime64[ns]`, `datetime64[10ms]`, ... | `datetime64[ns]`, `datetime64[10ms]`, ... | `int64`, little-endian (NaT is the minimum) | -| `timedelta64[s]`, ... | `timedelta64[s]`, ... | `int64`, little-endian | -| `U5` | `ucs4[5]` | five UCS-4 code units, each `uint32` little-endian, NUL-padded | -| `S3` | `bytes[3]` | three raw bytes, NUL-padded | -| `V4` | `void[4]` | four raw bytes | -| `[("b", "f8"), ("a", "i4", (2,))]` | `[["b","float64"],["a","int32",[2]]]` | fields packed in tag order | +```python +big = np.asfortranarray(np.arange(6, dtype=">f4").reshape(2, 3)) +kept = DictSource( + data=[{"trial": "t1", "signal": big}], tag_columns=["trial"], source_id="kept" +) +for tag, data in kept.iter_data(): + print(data["signal"].dtype.str, data["signal"].flags.f_contiguous) +# >f4 True +``` + +A stored array costs at least 128 bytes of `.npy` header. The stored bytes are NumPy's and +may differ between NumPy versions; nothing in Orcapod depends on them. Where an operator has +to order arrays (`MergeJoin` sorts the values it merges), it orders them by their value hash. ## How an array is hashed -The hash of an array is the version-1 array digest of its stored row, labelled `ndarray_v1`. -The byte-level definition, and a reference implementation that does not use Orcapod, are in -the [Hashing Reference](../reference/hashing.md#8-array-value-digest). A future PyTorch tensor -type will share the digest under its own label, so an array and a tensor with equal values -have different hashes. +The hash of an array is labelled `ndarray_v1` and is computed in two steps. The first is the +*element digest* of the array's dtype, shape and elements, taken on the array in canonical +form (C order, little-endian, fields packed). The second adds the type's name and the three +properties of the table above. The byte-level definition, the dtype tags and a reference +implementation that does not use Orcapod are in the +[Hashing Reference](../reference/hashing.md#8-array-value-hash). A PyTorch tensor type shares +the element digest and has its own name and properties, so an array and a tensor with equal +values have different hashes. diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index ce05f271a..0cf49171d 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -1,7 +1,7 @@ # Logical Type Identifiers A *logical type* binds a Python class to the way its values are stored in an Arrow column: -`pathlib.Path` is stored as a string, `numpy.ndarray` as a struct, a dataclass as a struct +`pathlib.Path` is stored as a string, `numpy.ndarray` as a `.npy` file, a dataclass as a struct of its fields. Orcapod ships logical types for paths, files, directories, UUIDs, NumPy arrays and pandas objects, and you can register your own. @@ -68,7 +68,7 @@ under one identifier. ## Built-in types Orcapod's own types follow the same contract as yours. Their metadata is the JSON text -`{"version":N}`: +`{"version":N}`, or `{"ignore":[...],"version":N}` for a library value type (below): | Python type | Extension name | Extension metadata | |---|---|---| @@ -77,12 +77,31 @@ Orcapod's own types follow the same contract as yours. Their metadata is the JSO | `uuid.UUID` | `orcapod.uuid` | `{"version":1}` | | `orcapod.File` | `orcapod.file` | `{"version":1}` | | `orcapod.Directory` | `orcapod.directory` | `{"version":1}` | -| `numpy.ndarray` | `orcapod.ndarray` | `{"version":1}` | +| `numpy.ndarray` | `orcapod.numpy.ndarray` | `{"ignore":[],"version":1}` | | `pandas.DataFrame` | `pandas.dataframe` | `{"version":1}` | | `pandas.Series` | `pandas.series` | `{"version":1}` | `{"version":N}` is the convention of the built-in types only. Your types may use any bytes. +### Library value types: the ignore list + +A value from a library such as NumPy has properties beyond its content, for example memory +order and byte order. Its hash counts every property the storage keeps, and the type names +the properties a data context may choose to ignore. The names a data context ignores are the +type's *ignore list*, and the list is part of the identifier: the extension metadata is + +``` +{"ignore":[],"version":1} +{"ignore":["byte_order","memory_order"],"version":1} +``` + +with the names sorted, each once, the keys sorted and no whitespace; `ignore` is present +also when empty. So one type under two lists is two logical types: the schema hashes differ, +a join of two such columns is an error, and a data context reads only columns stored under +its own list (anything else raises `LogicalTypeIdentifierError`, naming the column and both +lists). Storage is the same under every list. `numpy.ndarray` is the first such type; its +property names are in [NumPy Arrays](arrays.md#the-ignore-list). + ### Native containers: `list[T]` and `set[T]` A column typed `list[T]` or `set[T]`, where `T` is a logical type, is stored under a wrapper diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 660dfdb31..8fbb7a3d3 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -56,7 +56,7 @@ The labels of v0.3: | `semantic_v1` | the structure hash: a Python structure expanded to tagged JSON, SHA-256 of its canonical text | `SEMANTIC_HASH_LABEL`, `hashing/semantic_hashing/semantic_hasher.py` | | `arrow_v1` | the Arrow table and schema hash (§9) | `ARROW_HASH_LABEL`, `hashing/arrow_hashers.py` | | `directory_v1` | the directory hash: a SHA-256 Merkle tree over the entries | `DIRECTORY_HASH_LABEL`, `hashing/directory_hashers.py` | -| `ndarray_v1` | the array value digest (§8) | `NumpyArrayHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | +| `ndarray_v1` | the array value hash (§8) | `NumpyArrayHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | | `polars_expr_v1` | the Polars expression hash (§1c) | `POLARS_EXPR_HASH_LABEL`, `hashing/semantic_hashing/polars_expr.py` | | `sha256` | the file content hash: SHA-256 of the file's bytes | the file hasher's `algorithm` | @@ -816,63 +816,149 @@ in any data-level preimage; they serve logging, observability, and result inspec --- -## 8. Array Value Digest +## 8. Array Value Hash **Value hash labels.** A type whose values are hashed by a dedicated handler labels its hash -`_v`, for example `ndarray_v1` (and later `tensor_v1`). The label is the -`ContentHash.method`; the type's implementer chooses it. The bytes produced under an existing -label never change. A new definition gets a new label, so a hash always says which definition -produced it. The label is not part of any hash input (see -[The hash label rule](#the-hash-label-rule)). +`_v`, for example `ndarray_v1`. The label is the `ContentHash.method`; the type's +implementer chooses it. The bytes produced under an existing label never change. A new +definition gets a new label, so a hash always says which definition produced it. The label +is not part of any hash input (see [The hash label rule](#the-hash-label-rule)). -A `numpy.ndarray` value is hashed by value (since v0.3, ITL-888). The semantic hasher's -`NumpyArrayHandler` returns `ContentHash(method="ndarray_v1", digest=D)`, where `D` is the -version-1 array digest of the array's stored row. Inside a table hash, an array cell becomes -the token `b"orcapod.ndarray;7b2276657273696f6e223a317d::" + D`: the logical type's identifier (the extension name, `;`, the extension metadata `{"version":1}` in hex), then the digest, without the label. The stored row, the dtype tags and the canonical form are described in -[NumPy Arrays](../concepts/arrays.md). +A `numpy.ndarray` is stored as a `.npy` file and hashed by value (ITL-981). The hash is +defined on the array, never on the stored bytes. The semantic hasher's `NumpyArrayHandler` +returns `ContentHash(method="ndarray_v1", digest=D)`. What is stored and what reads back is +described in [NumPy Arrays](../concepts/arrays.md). -For a stored row with dtype tag `T`, shape `(d0, ..., dn-1)` and data bytes `B`: +`D` is computed in two steps. Notation: `u8` is one byte, `u64le` and `i64le` are 8-byte +little-endian unsigned and signed integers, and `lp(x) = u64le(len(utf8(x))) || utf8(x)`. + +**Step 1, the element digest `E`.** Take the array in canonical form: C order, +little-endian, structured fields packed in declared order. With dtype tag `T` (below), shape +`(d0, ..., dn-1)` and the canonical element bytes `B`: ``` -P = u64le(len(utf8(T))) || utf8(T) - || u64le(n) || i64le(d0) || ... || i64le(dn-1) - || u64le(len(B)) || B -D = SHA-256(P) +E = SHA-256( lp(T) || u64le(n) || i64le(d0) || ... || i64le(dn-1) || u64le(len(B)) || B ) ``` -`u64le` and `i64le` are 8-byte little-endian unsigned and signed integers. Every field is -length-prefixed, so two different rows never share a preimage. `D` depends only on the stored -row, so it can be recomputed from a stored table by any SHA-256 implementation. The label -`ndarray_v1` names this definition; a different definition gets a different label. A -`torch.Tensor` type is to share the digest under its own label. +`E` covers the dtype, the shape and every element bit, and nothing else. It is shared with +other array types (a tensor type computes the same `E`). + +**Step 2, the value hash `D`.** + +``` +D = SHA-256( lp("orcapod.numpy.ndarray") || E || u8(memory_order) + || lp(byte_order) || lp(struct_layout) ) +``` -| Variation | Same `D`? | +| Field | Value | Canonical value (written when the property is ignored) | +|---|---|---| +| leading text | the type's extension name, `orcapod.numpy.ndarray` | | +| `memory_order` | `1` when the array is Fortran-contiguous and not C-contiguous, else `0` | `0` | +| `byte_order` | one character per scalar field of the dtype, depth first in declared field order, a subarray field counting once: `<` little-endian, `>` big-endian, `\|` for a type without byte order (`bool`, 8-bit integers, `S`, `V`). NumPy's native marker `=` is resolved to the machine's order. `float32` gives `<` | the same text with every `>` replaced by `<` | +| `struct_layout` | JSON without whitespace of `L(dtype)`: `null` for an unstructured dtype, otherwise the list `[itemsize, [offset_1, L(field_1)], [offset_2, L(field_2)], ...]` in declared field order; a subarray field uses its base dtype. `float32` gives `null` | `L` of the packed dtype | + +Every field is always present and length-prefixed or of fixed width, so two different +inputs never share a preimage. + +**The ignore list.** The three properties `memory_order`, `byte_order` and `struct_layout` +count by default. A data context can list any of them as ignored for the type. An ignored +property's field is still written, with its canonical value, so arrays that differ only in +ignored properties have equal `D`, and with all three ignored `D` equals the default `D` of +the canonical array. The list itself is not in the preimage. It is in the type's extension +metadata, `{"ignore":[...],"version":1}` (names sorted, keys sorted, no whitespace), and so +in the logical type identifier: two lists are two logical types. A bare array is hashed with +the data context's list; a cell is hashed with the list of its own column. + +**Cells.** Inside a table hash, an array cell becomes the token +`b"orcapod.numpy.ndarray;7b2269676e6f7265223a5b5d2c2276657273696f6e223a317d::" + D`: the +logical type's identifier (the extension name, `;`, the extension metadata +`{"ignore":[],"version":1}` in hex), then the digest, without the label. The cell is decoded +to an array first; the stored `.npy` bytes are never hashed. + +| Variation | Same `D` by default? | |---|---| -| C or Fortran order, strides, views | yes | -| Little- or big-endian byte order | yes | -| Structured offsets, alignment padding | yes | +| Dtype (including width, units, string width), shape, any element bit, structured field names or field order | no | +| C against Fortran memory order | no (`memory_order`) | +| Little- against big-endian byte order | no (`byte_order`) | +| Structured field offsets or item size (aligned against packed) | no (`struct_layout`) | +| Step slices, reversed or broadcast views against their C-contiguous copy | yes | | `np.matrix`, `np.recarray`, `np.memmap` against the plain array | yes | -| Dtype (including width), shape, any element bit, structured field names or field order | no | +| The `writeable` flag, dtype `metadata`, the `align` flag, padding bytes | yes | + +The last three rows are what the `.npy` storage does not keep: a value read back from a +store must hash like the value that was written. -The reference implementation below uses only the standard library, and the vectors are pinned: +### Dtype tags + +The tag names the element type without a byte order. For a structured dtype, the tag is a +JSON list with one `[name, tag]` entry per field, or `[name, tag, [dims]]` for a subarray +field, in declared field order. It is written with no whitespace and with non-ASCII +characters escaped (`json.dumps(..., separators=(",", ":"), ensure_ascii=True)`). + +| NumPy | Tag | Canonical element bytes | +|---|---|---| +| `bool` | `bool` | one byte, 0 or 1 | +| `int8` ... `int64`, `uint8` ... `uint64` | `int8` ... `uint64` | two's complement or unsigned, little-endian | +| `float16`, `float32`, `float64` | same names | IEEE 754, little-endian | +| `complex64`, `complex128` | same names | real part, then imaginary part | +| `datetime64[ns]`, `datetime64[10ms]`, ... | `datetime64[ns]`, `datetime64[10ms]`, ... | `int64`, little-endian (NaT is the minimum) | +| `timedelta64[s]`, ... | `timedelta64[s]`, ... | `int64`, little-endian | +| `U5` | `ucs4[5]` | five UCS-4 code units, each `uint32` little-endian, NUL-padded | +| `S3` | `bytes[3]` | three raw bytes, NUL-padded | +| `V4` | `void[4]` | four raw bytes | +| `[("b", "f8"), ("a", "i4", (2,))]` | `[["b","float64"],["a","int32",[2]]]` | fields packed in tag order | + +### Reference implementation + +The implementation below uses only the standard library, and the vectors are pinned: ```python import hashlib import struct -def array_digest_v1(tag: str, shape: tuple[int, ...], data: bytes) -> str: - t = tag.encode("utf-8") - p = struct.pack(" bytes: + data = text.encode("utf-8") + return struct.pack(" bytes: + p = lp(tag) + struct.pack(" str: + p = lp("orcapod.numpy.ndarray") + element_digest(tag, shape, data) + p += bytes([memory_order]) + lp(byte_order) + lp(struct_layout) return hashlib.sha256(p).hexdigest() -print(array_digest_v1("float32", (2, 3), struct.pack("<6f", 0, 1, 2, 3, 4, 5))) +six = struct.pack("<6f", 0, 1, 2, 3, 4, 5) +print(element_digest("float32", (2, 3), six).hex()) # 318f6e503e8064e89ce6039453fce709f8e01f60c501af2b63bbbe9cf8fbf90e -print(array_digest_v1("float64", (), struct.pack("f4")).to_string()) +# ndarray_v1:1d5b6c2eb0ceaafd3758356256d651c5edf9ca0efa4deecf12c76c4e15bdb637 print(hasher.hash_object(np.array(2.5)).to_string()) -# ndarray_v1:47f182d862f5dd597d88836deddaed609ed94490827302e75e8364e877633cae +# ndarray_v1:642ad052de6aa53ffd059e5cd9a2eeeb5519caa3d3eb12efcf9e0c3fdbeab17b +aligned = np.dtype([("a", "i1"), ("b", ".", + "hash_ignore": { + "orcapod.numpy.ndarray": [] + }, "logical_type_registry": { "_class": "orcapod.logical_types.registry.LogicalTypeRegistry", "_config": { @@ -26,7 +29,11 @@ }, { "_class": "orcapod.logical_types.numpy_type.LogicalNumpyArray", - "_config": {} + "_config": { + "ignore": { + "_ref": "hash_ignore:orcapod.numpy.ndarray" + } + } }, { "_class": "orcapod.logical_types.pandas_type.LogicalPandasDataFrame", @@ -128,7 +135,11 @@ }, { "_class": "orcapod.hashing.semantic_hashing.builtin_handlers.NumpyArrayHandler", - "_config": {} + "_config": { + "ignore": { + "_ref": "hash_ignore:orcapod.numpy.ndarray" + } + } } ], [ diff --git a/src/orcapod/contexts/registry.py b/src/orcapod/contexts/registry.py index a3b498fb3..ec48bdf9e 100644 --- a/src/orcapod/contexts/registry.py +++ b/src/orcapod/contexts/registry.py @@ -321,6 +321,10 @@ def get_context(self, context_string: str | None = None) -> DataContext: # to them by this name, as ``{"_ref": "logical_type_registry"}``. _LOGICAL_TYPE_REGISTRY_REF = "logical_type_registry" + # The ignore list declared for an extension name in the registration file's + # ``hash_ignore`` object is referred to as ``hash_ignore:``. + _HASH_IGNORE_REF_PREFIX = "hash_ignore:" + def _uses_logical_type_registration(self, spec: dict[str, Any]) -> bool: """Whether ``spec`` takes its logical types from the registration file. @@ -339,8 +343,10 @@ def _load_logical_type_registration(self) -> dict[str, Any]: """Read the logical type registration file. Returns: - Its ``logical_type_registry`` object spec and its ``value_handlers`` - list of ``[type, handler]`` object-spec pairs. + Its ``logical_type_registry`` object spec, its ``value_handlers`` list + of ``[type, handler]`` object-spec pairs and, when present, its + ``hash_ignore`` object: per extension name, the property names the + value hash of that type ignores. """ with open(self.logical_types_file, "r") as f: return json.load(f) @@ -365,6 +371,20 @@ def _create_context_from_spec(self, spec: dict[str, Any]) -> DataContext: registration: dict[str, Any] | None = None if self._uses_logical_type_registration(spec): registration = self._load_logical_type_registration() + # The ignore list of each library value type is declared once, by + # extension name; its logical type and its value handler both take + # it as ``{"_ref": "hash_ignore:"}`` (ITL-981). + for name, ignore in registration.get("hash_ignore", {}).items(): + # A JSON object or string would iterate as names; only a list + # of names is a list of names. + if not isinstance(ignore, list) or not all( + isinstance(item, str) for item in ignore + ): + raise ValueError( + f"hash_ignore[{name!r}] must be a list of property " + f"names, got {ignore!r}" + ) + ref_lut[f"{self._HASH_IGNORE_REF_PREFIX}{name}"] = list(ignore) ref_lut[self._LOGICAL_TYPE_REGISTRY_REF] = parse_objectspec( registration["logical_type_registry"], ref_lut=ref_lut ) diff --git a/src/orcapod/core/operators/merge_join.py b/src/orcapod/core/operators/merge_join.py index 7c5d6b53b..a5fabc65b 100644 --- a/src/orcapod/core/operators/merge_join.py +++ b/src/orcapod/core/operators/merge_join.py @@ -60,8 +60,8 @@ def _storage_sort_key( ) -> Callable[[Any], Any]: """Return the ordering key MergeJoin uses for a column's stored values. - The array logical type defines ``storage_sort_key`` (its stored ``(dtype tag, - shape, data)``). A list of such values, at any depth (an array column after + The array logical type defines ``storage_sort_key`` (the value digest of the + stored array). A list of such values, at any depth (an array column after ``GroupBy`` or ``Batch``), is ordered by the tuple of its elements' keys, with ``None`` first. Every other type is ordered by its stored value as before. General struct ordering is ITL-839. diff --git a/src/orcapod/hashing/arrow_hashers.py b/src/orcapod/hashing/arrow_hashers.py index 89e148f39..382ac8d56 100644 --- a/src/orcapod/hashing/arrow_hashers.py +++ b/src/orcapod/hashing/arrow_hashers.py @@ -155,10 +155,20 @@ def _process_table_columns(self, table: "pa.Table | pa.RecordBatch") -> "pa.Tabl def hash_schema(self, schema: "pa.Schema") -> ContentHash: """Hash an Arrow schema using the starfix canonical algorithm. - Extension-typed columns are normalized to their storage type with the - extension identity in field metadata, as ``hash_table`` does, so the - extension name and metadata are part of the hash. + Each field is hashed under the same rule as in ``hash_table`` + (``SemanticHashingVisitor.hashed_field``). A logical type whose values + are hashed by a value handler is a ``large_binary`` field carrying the + type's identifier (extension name and metadata), whatever its storage + type: how such a type is stored is not part of its identity. Any other + extension type is normalized to its storage type with the extension + identity in field metadata, because its stored values are what is + hashed. """ + visitor = SemanticHashingVisitor(self._type_converter, self._semantic_hasher) + hashed = pa.schema([visitor.hashed_field(field) for field in schema]) + if schema.metadata is not None: + hashed = hashed.with_metadata(dict(schema.metadata)) + schema = hashed schema = normalize_extension_columns(schema.empty_table()).schema include_meta = has_extension_metadata(schema) or has_extension_type(schema) if include_meta: diff --git a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py index 89c8de483..c764bedeb 100644 --- a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py +++ b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py @@ -29,7 +29,9 @@ from orcapod.hashing.semantic_hashing.encoded import Encoded from orcapod.hashing.semantic_hashing.function_info_extractors import ParameterDefault +from orcapod.logical_types.array_format import NDARRAY_OPTIONS from orcapod.logical_types.identifier import logical_type_identifier +from orcapod.logical_types.ignore_list import IgnoreListHandler from orcapod.types import ContentHash, Schema from orcapod.utils.canonical_type_names import canonical_type_name @@ -401,47 +403,45 @@ def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: return self.directory_hasher.hash_directory(wrapped, ignore=ignore) -class NumpyArrayHandler: - """Hasher for ``numpy.ndarray``: the version-1 value digest under ``ndarray_v1``. +class NumpyArrayHandler(IgnoreListHandler): + """Hasher for ``numpy.ndarray``: the strict value digest under ``ndarray_v1``. - The array is put in its canonical form (C order, little-endian, structured - fields packed in declared order; see ``logical_types.numpy_type``), which is - exactly what ``LogicalNumpyArray`` stores. The digest is - ``array_format.array_digest_v1(tag, shape, data)`` of that stored row, so - memory layout, byte order, structured padding and subclass do not change the - hash, while dtype (including structured field names and order), shape and - every element bit do. + The digest is ``numpy_type.array_value_digest``: the element digest of the + array (dtype, shape and element bits) and, on top of it, memory order, byte + order and structured layout, each of which the configured ignore list can + take out. What storage does not keep (strides beyond C or Fortran + contiguity, subclass, flags) never changes the hash. - The label ``ndarray_v1`` names the definition (and binds the digest to the - ndarray type): another array type sharing the digest body uses its own label. + The label ``ndarray_v1`` names this algorithm; the ignore list is part of + the logical type identifier, not of the label or the preimage. + + Args: + ignore: Names from ``array_format.NDARRAY_OPTIONS`` ignored when a bare + array is hashed (the data context's list for the type). """ + OPTIONS = NDARRAY_OPTIONS HASH_LABEL = "ndarray_v1" """``ContentHash.method`` of an ndarray hash.""" - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: - """Return the ``ndarray_v1`` value hash of ``obj``. + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return the value digest of ``obj``, ignoring the named properties. Args: obj: A ``numpy.ndarray`` instance (subclasses are hashed as plain arrays). - hasher: Ignored. Present for protocol conformance. + ignore: Names from ``OPTIONS`` to ignore. Returns: - ``ContentHash("ndarray_v1", array_digest_v1(tag, shape, data))``. + ``numpy_type.array_value_digest(obj, ignore)``. Raises: TypeError: If ``obj`` is not a ``numpy.ndarray``. ValueError: If the array cannot be stored (masked, object, ``StringDType``, ``longdouble``, ...). """ - from orcapod.logical_types.array_format import array_digest_v1 - from orcapod.logical_types.numpy_type import array_bytes, canonical_array + from orcapod.logical_types.numpy_type import array_value_digest - tag, array = canonical_array(obj) - return ContentHash( - method=self.HASH_LABEL, - digest=array_digest_v1(tag, array.shape, array_bytes(array)), - ) + return array_value_digest(obj, ignore) class PandasDataFrameHandler: diff --git a/src/orcapod/hashing/visitors.py b/src/orcapod/hashing/visitors.py index 18ef48372..3ac720279 100644 --- a/src/orcapod/hashing/visitors.py +++ b/src/orcapod/hashing/visitors.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any from orcapod.logical_types.identifier import arrow_type_identifier +from orcapod.logical_types.ignore_list import IgnoreListHandler, parse_ignore_metadata from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -199,8 +200,10 @@ class SemanticHashingVisitor(ArrowTypeDataVisitor): 1. Look up the corresponding Python type via ``type_converter``. 2. If the Python type has a semantic hasher registered in ``python_hasher``, - convert the storage value to a Python object and hash it, replacing the - column with a ``pa.large_binary()`` value of the form:: + convert the storage value to a Python object and hash it (a handler with + an ignore list, ``IgnoreListHandler``, hashes it under the list in the + column's own extension metadata), replacing the column with a + ``pa.large_binary()`` value of the form:: + b"::" + content_hash.digest @@ -422,7 +425,15 @@ def visit_extension( # Convert storage value → Python object and hash it. python_obj = self._type_converter.storage_to_python(storage_value, python_type) - content_hash = self._python_hasher.hash_object(python_obj) + handler = self._python_hasher.type_handler_registry.get_handler(python_obj) + if isinstance(handler, IgnoreListHandler): + # A value type with an ignore list (ITL-981): the cell is hashed under + # the list of its own column, read from the column's extension + # metadata, never under the list the data context configures. + _, ignore = parse_ignore_metadata(extension_type.__arrow_ext_serialize__()) + content_hash = handler.hash_ignoring(python_obj, ignore) + else: + content_hash = self._python_hasher.hash_object(python_obj) # Encode as binary: "::", where the identifier text # is ";" diff --git a/src/orcapod/logical_types/array_format.py b/src/orcapod/logical_types/array_format.py index bc8a04b5f..621de1500 100644 --- a/src/orcapod/logical_types/array_format.py +++ b/src/orcapod/logical_types/array_format.py @@ -1,38 +1,25 @@ -"""Array storage struct and value digest shared by array-like logical types. +"""Element digest shared by array-like value types, and the ndarray constants. -This module defines the parts of Orcapod's array format that do not depend on -any array library: the Arrow struct an array value is stored as, and the -version-1 value digest computed from a stored row. ``numpy.ndarray`` uses them -today (``numpy_type.LogicalNumpyArray``); other array types (such as -``torch.Tensor``) reuse them under their own extension name and hash label. +An array-like value (a ``numpy.ndarray``, a tensor) is hashed in two steps. +This module holds the first, which does not depend on any array library: the +*element digest* of a dtype tag, a shape and the element bytes in a fixed +layout. Each type builds its value hash on top of it (``numpy_type`` for +``numpy.ndarray``), adding its own leading type text and one field per property +the element digest does not cover. -Storage struct (one value per row): - - struct< - dtype: large_string, # Orcapod dtype tag - shape: large_list, # dimensions, outermost first - data: large_binary, # element bytes, C order - > - -The children are nullable at the Arrow level only because Polars drops nested -non-null flags when it converts back to Arrow (ITL-842), and the extension type -would then no longer match. A stored array value never has a null child: the -logical type refuses one on write and on read. A null array is a null struct. - -Value digest, version 1. For a row with dtype tag ``T``, shape -``(d0, ..., d{n-1})`` and data bytes ``D``: +Element digest, version 1. For a dtype tag ``T``, shape ``(d0, ..., d{n-1})`` +and element bytes ``D`` (C order, little-endian, structured fields packed in +declared order): P = u64le(len(utf8(T))) || utf8(T) || u64le(n) || i64le(d0) || ... || i64le(d{n-1}) || u64le(len(D)) || D - digest = SHA-256(P) + E = SHA-256(P) ``u64le`` and ``i64le`` are 8-byte little-endian unsigned and signed integers. -Every field is length-prefixed, so distinct rows never share a preimage. The -digest depends only on the three stored fields, so it can be recomputed from a -stored table with any SHA-256 implementation. +Every field is length-prefixed, so distinct inputs never share a preimage. -This module must not import NumPy. +This module must not import NumPy or Arrow. """ from __future__ import annotations @@ -41,43 +28,43 @@ import struct from collections.abc import Sequence -import pyarrow as pa - -ARRAY_FORMAT_VERSION = 1 -"""Version of the array storage struct; recorded in the extension metadata.""" +NDARRAY_EXTENSION_NAME = "orcapod.numpy.ndarray" +"""Arrow extension name (and logical type name) of ``numpy.ndarray`` values. -ARRAY_FORMAT_METADATA = b'{"version":1}' -"""``ARROW:extension:metadata`` of every array-format extension type (canonical JSON). - -A reader rejects a stored type whose metadata differs, so a later storage version -is never decoded as this one. +It is also the leading type text of the ndarray value hash. """ - -ARRAY_STORAGE_TYPE: pa.StructType = pa.struct( - [ - pa.field("dtype", pa.large_string()), - pa.field("shape", pa.large_list(pa.int64())), - pa.field("data", pa.large_binary()), - ] -) -"""The Arrow struct one array value is stored as (see the module docstring).""" +NDARRAY_OPTIONS = frozenset({"byte_order", "memory_order", "struct_layout"}) +"""The property names the ndarray value hash can ignore (see ``numpy_type``).""" _U64 = struct.Struct(" bytes: + """Return ``u64le(len(utf8(text))) || utf8(text)``. + + Args: + text: The text of one preimage field. + + Returns: + The length-prefixed UTF-8 bytes. + """ + data = text.encode("utf-8") + return _U64.pack(len(data)) + data + + def array_digest_v1( tag: str, shape: Sequence[int], data: bytes | bytearray | memoryview ) -> bytes: - """Compute the version-1 value digest of a stored array row. + """Compute the version-1 element digest of an array. The buffer is fed to SHA-256 directly, so a large array is not copied. Args: - tag: The Orcapod dtype tag, as stored in the ``dtype`` field. - shape: The dimensions, outermost first, as stored in the ``shape`` field. - data: The element bytes, as stored in the ``data`` field. Any object + tag: The Orcapod dtype tag. + shape: The dimensions, outermost first. + data: The element bytes in C order, little-endian, packed. Any object exposing a C-contiguous buffer is accepted. Returns: @@ -86,10 +73,8 @@ def array_digest_v1( Raises: ValueError: If a dimension is negative or not an integer. """ - tag_bytes = tag.encode("utf-8") h = hashlib.sha256() - h.update(_U64.pack(len(tag_bytes))) - h.update(tag_bytes) + h.update(length_prefixed(tag)) h.update(_U64.pack(len(shape))) for dim in shape: if isinstance(dim, bool) or not isinstance(dim, int) or dim < 0: diff --git a/src/orcapod/logical_types/ignore_list.py b/src/orcapod/logical_types/ignore_list.py new file mode 100644 index 000000000..6ead1d736 --- /dev/null +++ b/src/orcapod/logical_types/ignore_list.py @@ -0,0 +1,487 @@ +"""Value types whose hash ignores a configurable list of properties (ITL-981). + +A value from a library (a NumPy array, a tensor, a data frame) has properties +beyond its content: memory order, byte order, column order. By default every +property the storage keeps is part of the value's hash. A type declares the +closed set of property names its hash can ignore (its *options*), and a data +context configures, per type, which of them are ignored (the *ignore list*). + +The ignore list is saved in the Arrow extension metadata, so it is part of the +logical type identifier: two lists are two logical types. The metadata has one +shape for every such type, canonical JSON with sorted keys: + + {"ignore":[],"version":1} + {"ignore":["byte_order","memory_order"],"version":1} + +``ignore`` holds the names sorted by code point, each once; it is present also +when empty. ``version`` is the version of the type's stored layout and hashing. + +The list is not part of the value hash preimage. A handler writes every optional +property into its preimage as a field that is always present, with the real +value, or with a fixed canonical value when the property is ignored. So values +that differ only in ignored properties hash equal. + +A type is built from two base classes here: ``IgnoreListLogicalType`` (the +Arrow and Polars extension types) and ``IgnoreListHandler`` (the value hash). +A data context has exactly one list per type; a column stored under another +list is refused on read with ``LogicalTypeIdentifierError``. +""" + +from __future__ import annotations + +import json +import re +import threading +from collections.abc import Callable, Collection, Iterable +from typing import TYPE_CHECKING, Any, ClassVar, cast + +from orcapod.logical_types.base_logical_type import BaseLogicalType +from orcapod.logical_types.identifier import ( + LogicalTypeIdentifier, + LogicalTypeIdentifierError, +) +from orcapod.types import ContentHash +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import polars as pl + import pyarrow as pa + + from orcapod.protocols.hashing_protocols import SemanticHasherProtocol +else: + pa = LazyModule("pyarrow") + pl = LazyModule("polars") + +_OPTION_NAME = re.compile(r"[a-z][a-z0-9_]*") + + +def _sanitize(name: str) -> str: + """Return ``name`` with every non-alphanumeric character replaced by ``_``.""" + return re.sub(r"[^A-Za-z0-9]", "_", name) + + +def checked_ignore(ignore: Iterable[str], options: Collection[str]) -> tuple[str, ...]: + """Return an ignore list in canonical form, after checking every name. + + Args: + ignore: The property names to ignore, in any order. + options: The closed set of names the type's hash can ignore. + + Returns: + The names sorted by code point, each once. + + Raises: + ValueError: If ``ignore`` is a string, or holds a name that is not a + string or not one of ``options``. + """ + if isinstance(ignore, str): + raise ValueError( + f"an ignore list is a list of property names, got the string {ignore!r}" + ) + names = list(ignore) + for name in names: + if not isinstance(name, str) or name not in options: + allowed = ", ".join(sorted(options)) or "none" + raise ValueError( + f"unknown property name {name!r} in ignore list; allowed names: " + f"{allowed}" + ) + return tuple(sorted(set(names))) + + +def ignore_metadata(version: int, ignore: Iterable[str]) -> bytes: + """Return the extension metadata for a version and an ignore list. + + Args: + version: The version of the type's stored layout and hashing. + ignore: The ignored property names, in any order. + + Returns: + Canonical JSON with sorted keys and no whitespace, for example + ``b'{"ignore":["byte_order"],"version":1}'``. + + Raises: + ValueError: If a name is not lower-case snake case, or ``version`` is not + a positive integer. + """ + if isinstance(version, bool) or not isinstance(version, int) or version < 1: + raise ValueError(f"version must be a positive integer, got {version!r}") + names = sorted(set(ignore)) + for name in names: + if not isinstance(name, str) or _OPTION_NAME.fullmatch(name) is None: + raise ValueError( + f"a property name is lower-case snake case ([a-z][a-z0-9_]*), " + f"got {name!r}" + ) + return json.dumps( + {"ignore": names, "version": version}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("ascii") + + +def parse_ignore_metadata(metadata: bytes) -> tuple[int, tuple[str, ...]]: + """Return the version and the ignore list of stored extension metadata. + + Args: + metadata: The ``ARROW:extension:metadata`` bytes of a column. + + Returns: + ``(version, names)`` with the names sorted. + + Raises: + ValueError: If ``metadata`` is not exactly what ``ignore_metadata`` + writes: other keys, other value types, unsorted or repeated names, + or any other byte sequence. + """ + try: + parsed = json.loads(metadata.decode("ascii")) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise ValueError( + f"extension metadata {metadata!r} is not ignore-list metadata: not JSON" + ) from exc + if ( + not isinstance(parsed, dict) + or set(parsed) != {"ignore", "version"} + or not isinstance(parsed["ignore"], list) + ): + raise ValueError( + f"extension metadata {metadata!r} is not ignore-list metadata: expected " + 'an object with exactly the keys "ignore" (a list) and "version"' + ) + try: + canonical = ignore_metadata(parsed["version"], parsed["ignore"]) + except (TypeError, ValueError) as exc: + raise ValueError( + f"extension metadata {metadata!r} is not ignore-list metadata: {exc}" + ) from exc + if canonical != metadata: + raise ValueError( + f"extension metadata {metadata!r} is not in canonical form; expected " + f"{canonical!r}" + ) + return parsed["version"], tuple(parsed["ignore"]) + + +class _ExtensionFamily: + """The Arrow and Polars extension types of one extension name, one per list. + + Each ignore list has its own ``pa.ExtensionType`` class, so two lists are + unequal Arrow types everywhere (type equality, schema equality, concatenation). + Whichever class is registered with Arrow under the name deserialises a stored + column to the class of the column's own list. + """ + + def __init__( + self, + extension_name: str, + storage_type: "pa.DataType", + version: int, + options: frozenset[str], + ) -> None: + self.extension_name = extension_name + self.storage_type = storage_type + self.version = version + self.options = options + self._arrow_types: dict[tuple[str, ...], pa.ExtensionType] = {} + self._polars_class: type[pl.BaseExtension] | None = None + self._polars_types: dict[tuple[str, ...], pl.BaseExtension] = {} + + def arrow_type(self, ignore: tuple[str, ...]) -> "pa.ExtensionType": + """Return the Arrow extension type of a checked ignore list.""" + with _LOCK: + cached = self._arrow_types.get(ignore) + if cached is None: + cached = self._make_arrow_class(ignore)() + self._arrow_types[ignore] = cached + return cached + + def _deserialize( + self, storage_type: "pa.DataType", serialized: bytes + ) -> "pa.ExtensionType": + """Return the type of a stored column. + + Raises: + TypeIdentityError: If ``storage_type`` is not the family's storage. + ValueError: If ``serialized`` is not the metadata of a valid ignore + list of this family's version. + """ + from orcapod.errors import TypeIdentityError + from orcapod.logical_types.registry import storage_type_matches + + if not storage_type_matches(self.storage_type, storage_type): + raise TypeIdentityError( + f"Arrow extension type '{self.extension_name}': expected storage_type " + f"{self.storage_type!r} but got {storage_type!r}." + ) + version, names = parse_ignore_metadata(serialized) + if version != self.version: + raise ValueError(f"version {version} is not version {self.version}") + return self.arrow_type(checked_ignore(names, self.options)) + + def _make_arrow_class( + self, ignore: tuple[str, ...] + ) -> Callable[[], "pa.ExtensionType"]: + family = self + metadata = ignore_metadata(self.version, ignore) + suffix = "__ignore_" + "_and_".join(ignore) if ignore else "" + + def _init(self: "pa.ExtensionType") -> None: + pa.ExtensionType.__init__(self, family.storage_type, family.extension_name) + + def _serialize(self: "pa.ExtensionType") -> bytes: + return metadata + + def _deserialize( + cls: type["pa.ExtensionType"], + storage_type: "pa.DataType", + serialized: bytes, + ) -> "pa.ExtensionType": + from orcapod.errors import TypeIdentityError + + try: + return family._deserialize(storage_type, serialized) + except TypeIdentityError: + raise + except ValueError as exc: + # The class knows what it reads; the family only what is valid. + raise LogicalTypeIdentifierError( + LogicalTypeIdentifier(family.extension_name, serialized), + (metadata,), + ) from exc + + extension_class = type( + f"_ArrowExt_{_sanitize(self.extension_name)}{suffix}", + (pa.ExtensionType,), + { + "__init__": _init, + "__arrow_ext_serialize__": _serialize, + "__arrow_ext_deserialize__": classmethod(_deserialize), + "readable_metadata": (metadata,), + }, + ) + # The synthesized ``__init__`` takes no argument; the stubs know only the + # base class's. + return cast("Callable[[], pa.ExtensionType]", extension_class) + + def polars_type(self, ignore: tuple[str, ...]) -> "pl.BaseExtension": + """Return the Polars extension type of a checked ignore list.""" + with _LOCK: + cached = self._polars_types.get(ignore) + if cached is None: + if self._polars_class is None: + self._polars_class = self._make_polars_class() + metadata = ignore_metadata(self.version, ignore).decode("ascii") + cached = self._polars_class(metadata) # pyright: ignore[reportCallIssue] # the synthesized class takes the metadata text + self._polars_types[ignore] = cached + return cached + + def _make_polars_class(self) -> type["pl.BaseExtension"]: + name = self.extension_name + polars_dtype = cast( + "pl.Series", pl.from_arrow(pa.array([], type=self.storage_type)) + ).dtype + + def _init(self: "pl.BaseExtension", metadata: str | None = None) -> None: + pl.BaseExtension.__init__(self, name, polars_dtype, metadata) + + def _ext_from_params( + cls: Any, + ext_name: str, + storage_dtype: Any, + metadata_str: str | None, + ) -> "pl.BaseExtension": + # Polars has one class per extension name; the instance keeps the + # metadata text of the column it was built for. + return cls(metadata_str) + + return type( + f"_PolarsExt_{_sanitize(name)}", + (pl.BaseExtension,), + {"__init__": _init, "ext_from_params": classmethod(_ext_from_params)}, + ) + + +_FAMILIES: dict[str, _ExtensionFamily] = {} + +_LOCK = threading.RLock() +"""Guards ``_FAMILIES`` and each family's caches. + +Arrow compares extension types by class, so one ignore list must never get two +classes: every cache lookup that may create a class runs under this lock. +""" + + +def _family( + extension_name: str, + storage_type: "pa.DataType", + version: int, + options: frozenset[str], +) -> _ExtensionFamily: + """Return the one extension family of ``extension_name`` in this process. + + Raises: + ValueError: If the name was already declared with another storage type, + version or set of options. + """ + with _LOCK: + family = _FAMILIES.get(extension_name) + if family is None: + family = _ExtensionFamily(extension_name, storage_type, version, options) + _FAMILIES[extension_name] = family + elif ( + family.storage_type != storage_type + or family.version != version + or family.options != options + ): + raise ValueError( + f"extension name {extension_name!r} is already declared with storage " + f"type {family.storage_type}, version {family.version} and options " + f"{sorted(family.options)}" + ) + return family + + +class IgnoreListLogicalType(BaseLogicalType): + """Base of a logical type whose identifier carries an ignore list. + + A subclass sets the class attributes and implements ``python_to_storage`` + and ``storage_to_python``. Storage is the same under every ignore list; the + list changes only the identifier, and with it what the value hash ignores. + + Attributes: + EXTENSION_NAME: The Arrow extension name, also the logical type name. + STORAGE_TYPE: The Arrow storage type of one value. + VERSION: The version of the stored layout and hashing. + OPTIONS: The closed set of property names the hash can ignore. + python_type: The Python class of the values (a subclass sets it). + + Args: + ignore: The property names this instance's identifier lists as ignored. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + """ + + EXTENSION_NAME: ClassVar[str] + STORAGE_TYPE: ClassVar["pa.DataType"] + VERSION: ClassVar[int] + OPTIONS: ClassVar[frozenset[str]] + python_type: type = object + + def __init__(self, ignore: Iterable[str] = ()) -> None: + self._ignore: tuple[str, ...] = checked_ignore(ignore, self.OPTIONS) + + @property + def logical_type_name(self) -> str: + """The logical type name: the Arrow extension name.""" + return self.EXTENSION_NAME + + @property + def ignore(self) -> tuple[str, ...]: + """The ignored property names, sorted.""" + return self._ignore + + def _extension_family(self) -> _ExtensionFamily: + return _family( + self.EXTENSION_NAME, self.STORAGE_TYPE, self.VERSION, self.OPTIONS + ) + + def get_arrow_extension_type(self) -> "pa.ExtensionType": + """Return the Arrow extension type whose metadata lists ``ignore``. + + Returns: + The extension type; equal only to the type of the same ignore list. + """ + return self._extension_family().arrow_type(self._ignore) + + def get_polars_extension_type(self) -> "pl.BaseExtension": + """Return the Polars extension type whose metadata lists ``ignore``. + + Returns: + The Polars extension registered under the extension name. + """ + return self._extension_family().polars_type(self._ignore) + + +class IgnoreListHandler: + """Base of a value hasher that honours an ignore list. + + A subclass sets ``OPTIONS`` and ``HASH_LABEL`` and implements ``digest``. + A bare value is hashed with the list the handler was configured with (the + data context's). A cell of a typed column is hashed with the list of the + column's own extension metadata, through ``hash_ignoring``. + + Attributes: + OPTIONS: The closed set of property names the hash can ignore; the same + set as the logical type's. + HASH_LABEL: The label of the hash algorithm (``ContentHash.method``). + + Args: + ignore: The property names ignored when a bare value is hashed. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + """ + + OPTIONS: ClassVar[frozenset[str]] + HASH_LABEL: ClassVar[str] + + def __init__(self, ignore: Iterable[str] = ()) -> None: + self._ignore: frozenset[str] = frozenset(checked_ignore(ignore, self.OPTIONS)) + + @property + def ignore(self) -> frozenset[str]: + """The property names ignored when a bare value is hashed.""" + return self._ignore + + def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: + """Return the hash of a bare value under the configured ignore list. + + Args: + obj: The value. + hasher: Ignored. Present for protocol conformance. + + Returns: + ``ContentHash(HASH_LABEL, digest(obj, ignore))``. + """ + return ContentHash( + method=self.HASH_LABEL, digest=self.digest(obj, self._ignore) + ) + + def hash_ignoring(self, obj: Any, ignore: Iterable[str]) -> ContentHash: + """Return the hash of a value under a given ignore list. + + Args: + obj: The value. + ignore: The property names to ignore, for example those of a + column's extension metadata. + + Returns: + ``ContentHash(HASH_LABEL, digest(obj, ignore))``. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + """ + names = frozenset(checked_ignore(ignore, self.OPTIONS)) + return ContentHash(method=self.HASH_LABEL, digest=self.digest(obj, names)) + + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return the digest of a value, ignoring the named properties. + + Every optional property is a field of the preimage that is always + present: its real value, or its canonical value when its name is in + ``ignore``. The list itself is never part of the preimage. + + Args: + obj: The value. + ignore: The property names to ignore; a subset of ``OPTIONS``. + + Returns: + The digest bytes. + + Raises: + NotImplementedError: Always; every handler implements this. + """ + raise NotImplementedError(f"{type(self).__name__} must implement digest.") diff --git a/src/orcapod/logical_types/numpy_type.py b/src/orcapod/logical_types/numpy_type.py index 5844db693..9686eebae 100644 --- a/src/orcapod/logical_types/numpy_type.py +++ b/src/orcapod/logical_types/numpy_type.py @@ -1,54 +1,93 @@ -"""``numpy.ndarray`` logical type: the v0.3 value-based array format. - -An array is stored as one row of the shared array struct -(``array_format.ARRAY_STORAGE_TYPE``): an Orcapod dtype tag, the shape, and the -element bytes in C order, little-endian and packed. The extension name is -``orcapod.ndarray``. The format and the dtype tags are specified in -``superpowers/specs/2026-11-10-array-format-v0.3.md`` and documented in -``docs/concepts/arrays.md``. - -Writing canonicalises what is not part of an array's identity: memory order -and strides (stored C order), byte order (stored little-endian) and structured -offsets, padding and alignment (fields packed). Structured field order is part -of the identity, as NumPy's own dtype equality has it: fields are stored in -declared order, at every nesting depth. So a stored row is the array's -canonical value, and its hash (``NumpyArrayHandler``) is a function of the -stored row alone. - -Refused at write time, with a ``ValueError`` naming the dtype (the type +"""``numpy.ndarray`` logical type: ``.npy`` storage and a strict value hash (ITL-981). + +Storage and hashing are separate. An array is stored as one ``.npy`` file +(NumPy's own format, version 3.0, never pickled) in a ``large_binary`` cell under +the extension name ``orcapod.numpy.ndarray``, and reads back as it was given: +dtype including byte order, shape, element bits, C or Fortran contiguity and +structured field offsets are kept. The hash is defined by Orcapod on the value, +never on the stored bytes. + +What ``.npy`` does not keep is never part of the hash: strides other than C or +Fortran contiguity (a strided view is stored C-contiguous), the subclass, the +``writeable`` flag, dtype metadata, the ``align`` flag of a structured dtype (its +offsets are kept) and padding bytes between structured fields (NumPy does not +define their content; they are written as zero, so memory the array happened +to sit on never reaches a store). + +Value hash ``D`` (label ``ndarray_v1``), in two steps. With +``lp(x) = u64le(len(utf8(x))) || utf8(x)``: + + E = array_format.array_digest_v1(tag, shape, bytes) + D = SHA-256(lp("orcapod.numpy.ndarray") || E || u8(memory_order) + || lp(byte_order_text) || lp(struct_layout_text)) + +``E`` is taken on the canonical form of the array (``canonical_array``: C order, +little-endian, structured fields packed in declared order) and covers dtype, +shape and element bits. The three further fields are the properties the hash +can ignore (``array_format.NDARRAY_OPTIONS``): + +- ``memory_order``: 1 when the array is Fortran-contiguous and not + C-contiguous, else 0 (the flag ``.npy`` keeps). +- ``byte_order``: one character per scalar leaf of the dtype, depth first in + declared field order (a subarray field counts once): ``<``, ``>``, or ``|`` + for a type without byte order. NumPy's native marker is resolved with + ``sys.byteorder``, so the text is the same on every machine. +- ``struct_layout``: compact JSON of ``L(dtype)``: ``null`` for an + unstructured dtype, else ``[itemsize, [offset, L(field)], ...]`` in declared + field order (a subarray field uses its base dtype). + +An ignored property's field is written with its canonical value: 0, the text +with every ``>`` replaced by ``<``, the layout of the packed dtype. The ignore +list is saved in the extension metadata (``ignore_list``), not in the preimage. + +Refused at write time, with ``UnsupportedValueError`` naming the dtype (the type converter adds the column): masked arrays, object dtypes, NumPy 2 -``StringDType`` and other "extended data" arrays, ``longdouble`` and -``clongdouble`` (platform-dependent), datetimes and timedeltas without a unit, -and structured dtypes with field titles. +``StringDType``, ``longdouble`` and ``clongdouble`` (platform-dependent), +datetimes and timedeltas without a unit, structured dtypes with field titles, +structured dtypes with overlapping or out-of-order field offsets (``.npy`` has +no form for them), and dtypes whose ``.npy`` header is longer than NumPy reads +without pickle support. """ from __future__ import annotations +import ast +import hashlib +import io import json +import math import re +import struct +import sys from typing import TYPE_CHECKING, Any import numpy as np import numpy.typing as npt -import polars as pl import pyarrow as pa +from numpy.lib import format as npy_format from orcapod.errors import UnsupportedValueError from orcapod.logical_types.array_format import ( - ARRAY_FORMAT_METADATA, - ARRAY_STORAGE_TYPE, -) -from orcapod.logical_types.base_logical_type import BaseLogicalType -from orcapod.logical_types.registry import ( - make_arrow_extension_type, - make_polars_extension_type, + NDARRAY_EXTENSION_NAME, + NDARRAY_OPTIONS, + array_digest_v1, + length_prefixed, ) +from orcapod.logical_types.ignore_list import IgnoreListLogicalType if TYPE_CHECKING: from orcapod.logical_types.protocols import TypeConverterProtocol -NDARRAY_EXTENSION_NAME = "orcapod.ndarray" -"""Arrow extension name (and logical type name) of the array format.""" +NPY_FORMAT_VERSION = (3, 0) +"""The ``.npy`` format version every array is stored in. + +Version 3.0 is the only one that stores structured field names outside Latin-1. +""" + +_NPY_MAGIC = b"\x93NUMPY" + bytes(NPY_FORMAT_VERSION) + +_MAX_NPY_HEADER_BYTES = 10000 +"""Longest ``.npy`` header ``read_array`` parses when pickle is not allowed.""" _INT_TAGS = {1: "int8", 2: "int16", 4: "int32", 8: "int64"} _UINT_TAGS = {1: "uint8", 2: "uint16", 4: "uint32", 8: "uint64"} @@ -307,128 +346,361 @@ def array_bytes(array: npt.NDArray[Any]) -> memoryview: return array.reshape(-1).view(np.uint8).data -class LogicalNumpyArray(BaseLogicalType): - """Logical type for ``numpy.ndarray`` in the v0.3 value-based array format. +def _plain_array(value: Any) -> npt.NDArray[Any]: + """Return ``value`` as a plain ``np.ndarray`` view, refusing what is not one. + + Raises: + UnsupportedValueError: For masked arrays. + TypeError: If ``value`` is not a ``numpy.ndarray``. + """ + if isinstance(value, np.ma.MaskedArray): + raise UnsupportedValueError( + f"numpy masked arrays (dtype {value.dtype!r}) are extended data and " + "cannot be stored: store the data and the mask as two arrays" + ) + if not isinstance(value, np.ndarray): + raise TypeError(f"expected a numpy.ndarray, got {type(value).__name__}") + # The base implementation, so a subclass's own ``view`` cannot change the values. + return np.ndarray.view(value, np.ndarray) + - Stores each array as ``{"dtype": tag, "shape": [...], "data": bytes}`` in - ``ARRAY_STORAGE_TYPE`` under the extension name ``orcapod.ndarray``. A value - reads back as a writeable, C-ordered, little-endian plain ``np.ndarray``. +def _require_storable_dtype(dtype: np.dtype[Any]) -> None: + """Raise unless arrays of ``dtype`` have a ``.npy`` form Orcapod accepts. - Example: - >>> lt = LogicalNumpyArray() - >>> arr = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float64) - >>> recovered = lt.storage_to_python(lt.python_to_storage(arr)) - >>> np.array_equal(recovered, arr) - True + Raises: + UnsupportedValueError: For the dtypes ``dtype_to_tag`` refuses, and for a + structured dtype whose fields overlap or are not in offset order. """ + dtype_to_tag(dtype) + try: + # NumPy cannot describe such a dtype in a ``.npy`` header. + _ = dtype.descr + except ValueError as exc: + raise _refuse( + dtype, + "structured fields with overlapping or out-of-order offsets have no " + ".npy form (declare the fields in offset order)", + ) from exc - _arrow_ext_class = make_arrow_extension_type( - NDARRAY_EXTENSION_NAME, ARRAY_STORAGE_TYPE, ARRAY_FORMAT_METADATA - ) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type( - NDARRAY_EXTENSION_NAME, ARRAY_STORAGE_TYPE, ARRAY_FORMAT_METADATA.decode() + +def _without_metadata(dtype: np.dtype[Any]) -> np.dtype[Any]: + """Return ``dtype`` with the same layout and no dtype ``metadata``. + + ``.npy`` does not store dtype metadata, and NumPy warns when asked to write + it. Orcapod never counts it, so it is dropped before writing, at every + depth. A dtype without metadata is returned unchanged. + """ + if dtype.fields is not None: + fields = {str(key): value for key, value in dtype.fields.items()} + names = list(dtype.names or ()) + formats = [_without_metadata(fields[name][0]) for name in names] + unchanged = all( + new is fields[name][0] for name, new in zip(names, formats, strict=True) + ) + if unchanged and dtype.metadata is None: + return dtype + return np.dtype( + { + "names": names, + "formats": formats, + "offsets": [fields[name][1] for name in names], + "itemsize": dtype.itemsize, + } + ) + if dtype.subdtype is not None: + base, shape = dtype.subdtype + stripped = _without_metadata(base) + if stripped is base and dtype.metadata is None: + return dtype + return np.dtype((stripped, shape)) + return dtype if dtype.metadata is None else np.dtype(dtype.str) + + +def _has_padding(dtype: np.dtype[Any]) -> bool: + """Return whether a structured dtype has bytes that belong to no field.""" + if dtype.fields is None and dtype.subdtype is None: + return False + return tag_to_dtype(dtype_to_tag(dtype)).itemsize != dtype.itemsize + + +def _with_zeroed_padding(array: npt.NDArray[Any]) -> npt.NDArray[Any]: + """Return a copy of a structured array whose padding bytes are zero. + + Padding between structured fields is not part of the value and NumPy does + not define its content (``np.empty`` leaves whatever the memory held), so it + is never written to a store. The copy has the same dtype, field values and + C or Fortran order; ``array`` is not modified. + """ + fortran = array.flags.f_contiguous and not array.flags.c_contiguous + out = np.zeros(array.shape, dtype=array.dtype, order="F" if fortran else "C") + _copy_by_field_name(out, array) + return out + + +def array_to_npy(value: Any) -> bytes: + """Return the ``.npy`` file an array is stored as. + + The array is written as it is: byte order, C or Fortran contiguity and + structured offsets are kept. A view that is neither C- nor Fortran-contiguous + is written C-contiguous, and padding bytes between structured fields are + written as zero. ``value`` is not modified. + + Args: + value: A ``numpy.ndarray`` (a subclass is stored as a plain array). + + Returns: + The bytes of a ``.npy`` file of format ``NPY_FORMAT_VERSION``. + + Raises: + UnsupportedValueError: For masked arrays and dtypes that cannot be stored. + TypeError: If ``value`` is not a ``numpy.ndarray``. + """ + plain = _plain_array(value) + _require_storable_dtype(plain.dtype) + stored_dtype = _without_metadata(plain.dtype) + if stored_dtype is not plain.dtype: + plain = plain.view(stored_dtype) + if _has_padding(stored_dtype): + plain = _with_zeroed_padding(plain) + buffer = io.BytesIO() + npy_format.write_array( + buffer, plain, version=NPY_FORMAT_VERSION, allow_pickle=False ) - _polars_ext: pl.BaseExtension | None = None + data = buffer.getvalue() + (header_bytes,) = struct.unpack_from(" _MAX_NPY_HEADER_BYTES: + raise _refuse( + plain.dtype, + f"its .npy header is {header_bytes} bytes and NumPy reads at most " + f"{_MAX_NPY_HEADER_BYTES} without pickle support (too many fields)", + ) + return data - logical_type_name: str = NDARRAY_EXTENSION_NAME - python_type: type = np.ndarray - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type ``orcapod.ndarray``. +def _require_declared_payload(data: bytes) -> None: + """Raise unless the bytes after the header are exactly the declared array. - Returns: - The ``pa.ExtensionType`` over ``ARRAY_STORAGE_TYPE``. - """ - if LogicalNumpyArray._arrow_ext is None: - LogicalNumpyArray._arrow_ext = LogicalNumpyArray._arrow_ext_class() - return LogicalNumpyArray._arrow_ext + ``read_array`` allocates the array its header declares before it reads the + elements, so a small cell that declares a huge shape would ask for that much + memory. The header is checked first: its length, and that dtype item size + times the element count equals the number of bytes that follow it. A header + that declares an object array is left for ``read_array`` to refuse. - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type ``orcapod.ndarray``. + Raises: + ValueError: If the header is too long, incomplete or malformed, or the + payload length differs from what the header declares. + """ + prefix = len(_NPY_MAGIC) + 4 + if len(data) < prefix: + raise ValueError("stored numpy array value is truncated: no .npy header") + (header_bytes,) = struct.unpack_from(" _MAX_NPY_HEADER_BYTES: + raise ValueError( + f"stored numpy array value has a .npy header of {header_bytes} bytes; " + f"at most {_MAX_NPY_HEADER_BYTES} are read" + ) + payload_start = prefix + header_bytes + if len(data) < payload_start: + raise ValueError( + "stored numpy array value is truncated: incomplete .npy header" + ) + try: + header = ast.literal_eval(bytes(data[prefix:payload_start]).decode("utf-8")) + shape = header["shape"] + dtype = npy_format.descr_to_dtype(header["descr"]) + if not isinstance(shape, tuple) or not all( + type(dim) is int and dim >= 0 for dim in shape + ): + raise ValueError(f"shape {shape!r} is not a tuple of non-negative integers") + except ( + ValueError, + TypeError, + KeyError, + SyntaxError, + MemoryError, + RecursionError, + ) as exc: + raise ValueError( + f"stored numpy array value has a malformed .npy header: {exc}" + ) from exc + if dtype.hasobject: + return + declared = dtype.itemsize * math.prod(shape) + stored = len(data) - payload_start + if declared != stored: + raise ValueError( + f"stored numpy array value declares {declared} element bytes (shape " + f"{shape}, dtype {dtype}) but holds {stored}" + ) - Returns: - The ``pl.BaseExtension`` registered under ``orcapod.ndarray``. - """ - if LogicalNumpyArray._polars_ext is None: - LogicalNumpyArray._polars_ext = LogicalNumpyArray._polars_ext_class() - return LogicalNumpyArray._polars_ext + +def npy_to_array(data: bytes) -> npt.NDArray[Any]: + """Return the array stored in a ``.npy`` file written by ``array_to_npy``. + + Args: + data: The stored bytes. + + Returns: + A new, writeable plain ``np.ndarray`` with the stored dtype (including + byte order and structured offsets), shape and contiguity; padding + bytes between structured fields are zero. + + Raises: + ValueError: If ``data`` is not a ``.npy`` file of format + ``NPY_FORMAT_VERSION``, is truncated, holds more or fewer element + bytes than its header declares, or holds an object array. + """ + if bytes(data[: len(_NPY_MAGIC)]) != _NPY_MAGIC: + raise ValueError( + "stored numpy array value is not a .npy file of format " + f"{NPY_FORMAT_VERSION[0]}.{NPY_FORMAT_VERSION[1]}" + ) + _require_declared_payload(data) + array = npy_format.read_array(io.BytesIO(data), allow_pickle=False) + if array.dtype.fields is not None and _has_padding(array.dtype): + # NumPy fills a structured array field by field, so the padding of the + # array it returns is whatever the new memory held, not the stored zeros. + array = _with_zeroed_padding(array) + return array + + +def _byte_order_text(dtype: np.dtype[Any]) -> str: + """Return one byte-order character per scalar leaf of ``dtype``.""" + if dtype.subdtype is not None: + return _byte_order_text(dtype.subdtype[0]) + if dtype.fields is not None: + fields = {str(key): value for key, value in dtype.fields.items()} + return "".join(_byte_order_text(fields[name][0]) for name in dtype.names or ()) + order = dtype.byteorder + if order == "=": + return "<" if sys.byteorder == "little" else ">" + return order + + +def _struct_layout(dtype: np.dtype[Any]) -> Any: + """Return ``L(dtype)``: ``None``, or ``[itemsize, [offset, L(field)], ...]``.""" + if dtype.subdtype is not None: + dtype = dtype.subdtype[0] + if dtype.fields is None: + return None + fields = {str(key): value for key, value in dtype.fields.items()} + layout: list[Any] = [int(dtype.itemsize)] + for name in dtype.names or (): + layout.append([int(fields[name][1]), _struct_layout(fields[name][0])]) + return layout + + +def array_value_digest(value: Any, ignore: frozenset[str]) -> bytes: + """Return the value digest ``D`` of an array (see the module docstring). + + Args: + value: A ``numpy.ndarray`` (a subclass is hashed as a plain array). + ignore: Names from ``NDARRAY_OPTIONS`` whose fields are written with + their canonical value. + + Returns: + The 32-byte SHA-256 digest. + + Raises: + UnsupportedValueError: For masked arrays and dtypes that cannot be stored. + TypeError: If ``value`` is not a ``numpy.ndarray``. + """ + plain = _plain_array(value) + _require_storable_dtype(plain.dtype) + tag, canonical = canonical_array(plain) + element_digest = array_digest_v1(tag, canonical.shape, array_bytes(canonical)) + + fortran = plain.flags.f_contiguous and not plain.flags.c_contiguous + memory_order = 1 if fortran and "memory_order" not in ignore else 0 + byte_order = _byte_order_text(plain.dtype) + if "byte_order" in ignore: + byte_order = byte_order.replace(">", "<") + layout_dtype = canonical.dtype if "struct_layout" in ignore else plain.dtype + layout = json.dumps(_struct_layout(layout_dtype), separators=(",", ":")) + + h = hashlib.sha256() + h.update(length_prefixed(NDARRAY_EXTENSION_NAME)) + h.update(element_digest) + h.update(bytes([memory_order])) + h.update(length_prefixed(byte_order)) + h.update(length_prefixed(layout)) + return h.digest() + + +class LogicalNumpyArray(IgnoreListLogicalType): + """Logical type for ``numpy.ndarray``: one ``.npy`` file per value. + + The extension name is ``orcapod.numpy.ndarray`` over ``large_binary``; the + extension metadata lists the properties the value hash ignores. A value reads + back as a writeable plain ``np.ndarray`` with the dtype (including byte + order and structured offsets), shape and C or Fortran contiguity it had. + + Args: + ignore: Names from ``NDARRAY_OPTIONS`` the value hash of this type + ignores. Storage is the same under every list. + + Example: + >>> lt = LogicalNumpyArray() + >>> arr = np.asfortranarray(np.array([[1.0, 2.0], [3.0, 4.0]], dtype=">f8")) + >>> recovered = lt.storage_to_python(lt.python_to_storage(arr)) + >>> recovered.dtype.str, recovered.flags.f_contiguous + ('>f8', True) + """ + + EXTENSION_NAME = NDARRAY_EXTENSION_NAME + STORAGE_TYPE = pa.large_binary() + VERSION = 1 + OPTIONS = NDARRAY_OPTIONS + python_type: type = np.ndarray def python_to_storage( self, value: Any, converter: TypeConverterProtocol | None = None - ) -> dict[str, Any]: - """Encode an array as its canonical stored row. + ) -> bytes: + """Encode an array as its ``.npy`` file. Args: value: A ``numpy.ndarray``. converter: Ignored. Present for protocol conformance. Returns: - ``{"dtype": tag, "shape": list_of_ints, "data": bytes}``. + The ``.npy`` bytes (``array_to_npy``). Raises: UnsupportedValueError: For masked arrays and dtypes that cannot be stored. TypeError: If ``value`` is not a ``numpy.ndarray``. """ - tag, array = canonical_array(value) - return { - "dtype": tag, - "shape": [int(d) for d in array.shape], - "data": array_bytes(array).tobytes(), - } + return array_to_npy(value) def storage_to_python( self, storage_value: Any, converter: TypeConverterProtocol | None = None ) -> npt.NDArray[Any]: - """Decode a stored row into a writeable ``numpy.ndarray``. + """Decode a stored ``.npy`` file into a writeable ``numpy.ndarray``. Args: - storage_value: The stored ``{"dtype", "shape", "data"}`` mapping. + storage_value: The stored bytes. converter: Ignored. Present for protocol conformance. Returns: - A C-ordered plain ``np.ndarray`` of the canonical dtype. + The array as it was given (``npy_to_array``). Raises: - ValueError: If a field is missing or null, or the data length does - not match the dtype and shape. + ValueError: If the bytes are not a ``.npy`` file of the stored format. """ - tag = storage_value.get("dtype") - shape = storage_value.get("shape") - data = storage_value.get("data") - if tag is None or shape is None or data is None: - raise ValueError( - "stored numpy array value has a null field " - f"(dtype={tag!r}, shape={shape!r}, data is None: {data is None})" - ) - dtype = tag_to_dtype(tag) - dims = tuple(int(d) for d in shape) - count = int(np.prod(dims, dtype=np.int64)) if dims else 1 - if len(data) != count * dtype.itemsize: - raise ValueError( - f"stored numpy array value has {len(data)} data bytes; dtype {tag!r} " - f"and shape {list(dims)} need {count * dtype.itemsize}" - ) - if dtype.itemsize == 0: - return np.zeros(dims, dtype=dtype) - return np.frombuffer(data, dtype=dtype, count=count).reshape(dims).copy() + return npy_to_array(storage_value) def storage_sort_key(self, storage_value: Any) -> tuple[Any, ...]: """Ordering key for stored values, used where operators sort values. - Orders by the stored ``(dtype tag, shape, data)``; ``None`` sorts first. + Orders by the strict value digest of the decoded array, never by the + stored bytes (NumPy does not promise them across versions); ``None`` + sorts first. Args: - storage_value: A stored row, or ``None``. + storage_value: Stored ``.npy`` bytes, or ``None``. Returns: - A tuple that orders stored rows deterministically. + A tuple that orders stored values deterministically. """ if storage_value is None: return (0,) - return ( - 1, - storage_value["dtype"], - list(storage_value["shape"]), - bytes(storage_value["data"]), - ) + return (1, array_value_digest(npy_to_array(storage_value), frozenset())) diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index 9233ea36b..fe711e737 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -37,6 +37,29 @@ def _sanitize(name: str) -> str: return re.sub(r"[^A-Za-z0-9]", "_", name) +def storage_type_matches(expected: pa.DataType, stored: pa.DataType) -> bool: + """Return whether ``stored`` is an acceptable storage type for ``expected``. + + When ``expected`` is in its large-normalized form, any physically equivalent + member of the same family is accepted, at any nesting depth (``string`` or + ``string_view`` for ``large_string``, ``binary`` for ``large_binary``): Delta + Lake compaction rewrites large columns as plain ones (ITL-602, ITL-881). + Otherwise the two types must be equal, so a narrowing is never accepted. + + Args: + expected: The storage type the extension type declares. + stored: The storage type found with the stored column. + + Returns: + ``True`` when a column stored as ``stored`` can be read as ``expected``. + """ + from orcapod.utils.arrow_utils import normalize_to_large_types + + if normalize_to_large_types(expected, keep_null=True) == expected: + return normalize_to_large_types(stored, keep_null=True) == expected + return stored == expected + + def make_arrow_extension_type( extension_name: str, storage_type: pa.DataType, @@ -111,21 +134,9 @@ def _deserialize( # # Cross-family mismatches (string where large_binary was registered, or # vice-versa) are genuine semantic errors and always raise. - from orcapod.utils.arrow_utils import normalize_to_large_types - - if normalize_to_large_types(_storage, keep_null=True) == _storage: - # _storage is the canonical large form — accept any same-family variant, - # at any nesting depth (a struct field rewritten from large_string to - # string by Delta compaction, ITL-881). - storage_ok = ( - normalize_to_large_types(storage_type, keep_null=True) == _storage - ) - else: - # Non-canonical _storage — require exact match. - storage_ok = storage_type == _storage from orcapod.errors import TypeIdentityError - if not storage_ok: + if not storage_type_matches(_storage, storage_type): raise TypeIdentityError( f"Arrow extension type '{_name}': expected storage_type " f"{_storage!r} but got {storage_type!r}." diff --git a/superpowers/plans/2026-11-12-itl-981-library-value-types.md b/superpowers/plans/2026-11-12-itl-981-library-value-types.md new file mode 100644 index 000000000..19390c82c --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-981-library-value-types.md @@ -0,0 +1,96 @@ +# ITL-981 plan: library value types (mechanism + NumPy) + +Issue: https://linear.app/metamorphic/issue/ITL-981 +Branch: `eywalker/itl-981-library-value-types-native-storage-strict` (from `origin/main` at `c8e9eb21`) +Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md` + +## State (keep this section current; a successor continues from here) + +- November 12, 2026: design sent to the coordinator (one `ask`). Coordinator's reply: it goes + to the developer with the torch, polars and pandas specs; NO library code until the + coordinator replies again. Allowed meanwhile: this plan and the failing tests (steps 1-2). +- Open decisions (design note, last section): (1) refuse or read a column stored under + another ignore list; (2) label `ndarray_v1` or `ndarray_v2`; (3) keep refusing unit-less + datetimes and field titles; (4) metadata key order. The tests take (2) and (4) from two + constants at the top of `tests/test_arrays/test_array_strict_hash.py`. +- November 12, later: (a) developer decision relayed: extension names are `.`, + so the array type is `numpy.ndarray` (design note vii). The Arrow class registered under + the name must never raise for metadata it does not know (v0.2 stored `numpy.ndarray` with + empty metadata): it returns a "foreign" type keeping name and bytes; Orcapod refuses the + column when it is used. Approved by the coordinator. No `also_reads` for v0.2. + (b) Coordinator: take the 10-line alignment of `StarfixArrowHasher.hash_schema` with + `hash_table` (`SemanticHashingVisitor.hashed_field`) inside ITL-981, with a test, AFTER + sign-off; say in the PR body that no stored or golden value moves (design note viii). + (c) Finding posted as a comment on Linear ITL-980. + (d) FINAL naming (supersedes (a)): `orcapod.numpy.ndarray`. v0.2's `numpy.ndarray` is then + not a registered name, so the "foreign type" is dropped: the registered Arrow class + dispatches valid lists and raises `LogicalTypeIdentifierError` for anything else, as every + built-in type does. The Linear comment on ITL-980 could not be posted from this session + (permission denied); the coordinator posts it. + (e) DEVELOPER SIGN-OFF received November 12, 2026 (through the coordinator): design approved + as recommended. Label stays `ndarray_v1`; a column under another ignore list is refused on + read; unit-less datetimes and field titles stay refused; metadata keys sorted; the leading + type text of the value hash is the extension name; names are `orcapod..`. + (f) Mechanism pushed at `64c63bfe` for the torch, Polars and pandas workers; the public + names of design note (vi) are stable from that commit. + (g) Implemented and pushed: NumPy `.npy` storage and strict hash, tests, golden (22 values + moved, 8 system-tag entries renamed, 4 added, array entries only), docs, changelog + fragment, design-issue entry, CLAUDE.md and `.zed/rules`. Full suite on 3.12: 10092 passed, + 109 skipped, 23 xfailed; array, hashing, logical type and context tests also pass on 3.11 + and 3.13. Left: independent review findings, PR, CodeRabbit rounds. + +## Steps + +1. [x] Plan (this file). +2. [x] Failing tests through the public interface: + `tests/test_arrays/test_array_strict_hash.py`. They fail on `main`'s code (the hash + ignores memory order and byte order, storage is the struct). +3. [x] `src/orcapod/logical_types/ignore_list.py`: `ignore_metadata`, + `parse_ignore_metadata`, `checked_ignore`, `IgnoreListLogicalType`, `IgnoreListHandler`; + one Arrow extension class per (name, list), the registered class deserialising any valid + list. Polars extension keeps `metadata_str`. +4. [x] Context loader (`contexts/registry.py`): `hash_ignore` of `logical_types.json` into + the reference table as `hash_ignore:` before anything is built. + `logical_types.json`: `"hash_ignore": {"orcapod.numpy.ndarray": []}` and the two `_ref`s. +5. [x] `numpy_type.py`: storage `large_binary`, `.npy` format 3.0 write + (`numpy.lib.format.write_array`, `allow_pickle=False`), read after checking the magic; + refusals (masked, object, StringDType, longdouble, unit-less datetimes, titles, + out-of-order or overlapping offsets, header over 10,000 bytes); `storage_sort_key` is the + strict `D` of the decoded value. `array_format.py`: drop the storage struct and its + metadata constant, keep `array_digest_v1`. +6. [x] `NumpyArrayHandler` as an `IgnoreListHandler` computing `D`; + `SemanticHashingVisitor.visit_extension` passes the column's list + (`handler.hash_ignoring`). +6b. [x] `StarfixArrowHasher.hash_schema`: apply `SemanticHashingVisitor.hashed_field` to each + field before hashing (same rule as `hash_table`); test that a value-hashed type's schema + hash does not depend on its storage type and that no golden value moves. +7. [x] Rewrite the ITL-888 tests that assert the struct row and layout independence + (`tests/test_arrays/test_array_value_format.py`, `array_cases.py` + (`assert_arrays_identical` requires C order today), `tests/test_logical_types/test_array_format.py`, + `test_numpy_type.py`, `tests/test_hashing/test_numpy_handler.py`, + `tests/test_hashing/test_hash_label_rule.py` if it names the constant). +8. [x] Golden: add `value.ndarray.{fortran,big_endian,aligned_struct,strided_view}` to + `tests/test_hashing/golden/cases.py`; regenerate; check the diff moves only the 32 values + listed in the design note (iii) and list each by cause in the PR. +9. [x] Docs: `docs/concepts/arrays.md`, `docs/reference/hashing.md` section 8, + `docs/concepts/logical-type-identifiers.md`; "superseded" line in + `superpowers/specs/2026-11-10-array-format-v0.3.md`; CLAUDE.md and `.zed/rules` if a rule + changes (logical type identifiers: the ignore-list metadata shape). +10. [x] `changelog.d/ITL-981.md`, `design_issues/ITL-981-LV1.md`. +11. [x] Checks: `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/`. +12. [ ] Merge `origin/main` if it moved, push, PR against `main` (`Fixes ITL-981`), + `orca linear attach`, `orca worktree set --worktree current --pr `, `worker_done`. + +## Measurements behind the design + +Scripts were run in the session scratchpad (not kept). NumPy 2.2.6, pyarrow 23.0.1. Results +are stated in the design note, section (i). Two facts that shape the code: + +- `pa.ExtensionType` instances of one class compare equal whatever their serialised + metadata, unless the class defines `__eq__`. Two different classes under one extension + name compare unequal everywhere (`==`, schema equality, `concat_tables`, nested lists) and + round-trip through IPC and Parquet when the registered class's + `__arrow_ext_deserialize__` returns an instance of the class for the stored list. +- `read_array(allow_pickle=False)` refuses a header over 10,000 bytes (400 `f8` fields + give 11,584). diff --git a/superpowers/specs/2026-11-10-array-format-v0.3.md b/superpowers/specs/2026-11-10-array-format-v0.3.md index fb1fffefe..c8d565e26 100644 --- a/superpowers/specs/2026-11-10-array-format-v0.3.md +++ b/superpowers/specs/2026-11-10-array-format-v0.3.md @@ -4,7 +4,10 @@ **Issue:** ITL-888 (fixes ITL-883; refs ITL-874, ITL-871, ITL-465, ITL-882, ITL-884) **Design basis:** ITL-874 review, PR #285 (`superpowers/specs/2026-11-10-array-tensor-storage.md`, candidate 4c, sections C4 and C5; experiments `b0`, `c1`, `c2`, `c3`, `b7`). -**Status:** Implemented in PR #289. +**Status:** Implemented in PR #289. SUPERSEDED on November 12, 2026 by ITL-981 +(`superpowers/specs/2026-11-12-itl-981-library-value-types-design.md`): arrays are stored as +`.npy` under `orcapod.numpy.ndarray` and hashed strictly. Of this spec, the dtype tags and the +digest of section C1 remain, as the element digest of the new hash. **Decisions (developer, November 10 and 11, 2026).** Every design choice in this spec is decided: diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md new file mode 100644 index 000000000..2d4508e33 --- /dev/null +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -0,0 +1,355 @@ +# ITL-981: library value types (design note) + +Status: SIGNED OFF by the developer on November 12, 2026 and implemented in PR #328. The +text below is the proposal as it was approved; the four "open decisions" at the end were +decided as proposed (refuse a column of another list; label `ndarray_v1`; keep refusing +unit-less datetimes and field titles; sorted metadata keys), and section (vii) records the +final extension names. Added in review of PR #328: padding bytes between structured fields +are written as zero, and a stored cell must hold exactly the element bytes its header +declares. +Measured with NumPy 2.2.6 and pyarrow 23.0.1 on a little-endian machine, writing with +`numpy.lib.format.write_array(version=(3, 0), allow_pickle=False)` and reading with +`numpy.lib.format.read_array(allow_pickle=False)`. "Measured" below means exactly that. + +Rule applied (proposed by the torch and polars workers, adopted here): a property that +storage does not round-trip can never count in the hash, because `hash(read(write(v)))` must +equal `hash(v)` or every cached re-run recomputes. + +## (i) numpy.ndarray + +### A. Properties + +Each line: option name; in the hash by default; does storage round-trip it. + +Always in the hash, cannot be ignored: + +1. `dtype`: yes / yes. Kind and width (`int32` is not `int64`), `datetime64` / `timedelta64` + unit and multiple (`[ns]`, `[10ms]`), string and void widths (`U5`, `S3`, `V4`), and for a + structured dtype the field names, declared field order, field types and subarray shapes, + at every depth. +2. `shape`: yes / yes. Zero-dim `()` and empty shapes `(0, 3)`, `(3, 0)` are all distinct. +3. `element_bits`: yes / yes. Bit exact: `-0.0`, NaN payloads, NUL padding of strings, every + byte of a void element. + +In the hash by default, can be ignored (these are the three option names): + +4. `memory_order`: yes / yes, as two values. `F` when the array is Fortran-contiguous and + not C-contiguous, otherwise `C`. This is exactly the flag `.npy` keeps (measured: an F + array and a transposed C array read back F-contiguous). +5. `byte_order`: yes / yes. Little or big, per scalar field (measured: `>f4`, `>U3`, + `>M8[ns]` and a structured dtype with mixed orders read back unchanged). It is absolute, + not "native or swapped", so a value hashes the same on every machine. +6. `struct_layout`: yes / yes. Field offsets and itemsize of a structured dtype, at every + depth (measured: an aligned dtype, a nested aligned dtype with a subarray field, and + trailing padding read back with identical offsets and itemsize). + +Never in the hash (no option): + +7. Strides beyond property 4 (step slices `a[:, ::2]`, reversed `a[::-1]`, broadcast views): + never / no. Measured: `.npy` writes them C-contiguous. This is where the round-trip rule + bites: a strided view hashes like its C-contiguous copy. +8. The `align=True` flag of a structured dtype: never / no (measured: `isalignedstruct` + reads back `False`; the offsets it produced are kept, property 6). +9. Padding bytes between structured fields: never / not reliably (corrected after CI: on + Linux a stored aligned array came back with zeroed padding). NumPy does not + define them (`np.empty` leaves garbage, field assignment does not copy them), so they + cannot be hashed deterministically. +10. Subclass (`np.matrix`, `np.recarray`, `np.memmap`): never / no (measured: all read back + as plain `np.ndarray`; a recarray's `np.record` dtype reads back as the plain dtype). +11. `writeable` flag: never / no (measured: a read-only array reads back writeable). +12. dtype `metadata`: never / no (measured: reads back `None`). +13. Ownership (`base`, `OWNDATA`), `ALIGNED` flag, memory address: never / no. Not + properties of the value. + +Refused at write, as on main, with `UnsupportedValueError` naming the dtype: + +14. Masked arrays (the mask would be lost; `.npy` silently drops it, measured), object + dtype and structured dtypes with an object field, NumPy 2 `StringDType` (`.npy` itself + refuses both without pickle, measured), `longdouble` / `clongdouble` (platform width). +15. Unit-less `datetime64` / `timedelta64` and structured field titles: `.npy` round-trips + both (measured), but main refuses them and ITL-911 tracks them. Proposal: keep refusing + in v0.3; accepting them later changes no existing hash. +16. New refusals forced by `.npy`: a structured dtype with overlapping or out-of-order + field offsets (`np.save` raises, measured), and a dtype whose `.npy` header exceeds + 10,000 bytes (roughly 350 or more fields: `read_array(allow_pickle=False)` refuses to + parse a longer header, measured at 400 fields). Checked at write so nothing unreadable + is stored. + +### B. Hash + +Notation: `u8` one byte; `u64le` / `i64le` 8-byte little-endian; `lp(x) = u64le(len(utf8(x))) || utf8(x)`. + +Step 1, element digest `E`. Unchanged from main (`array_format.array_digest_v1`, shared +with torch): take the array as C order, little-endian, structured fields packed in declared +order (`canonical_array`), then +`E = SHA-256(lp(tag) || u64le(ndim) || i64le(each dim) || u64le(len(bytes)) || bytes)`, +where `tag` is main's dtype tag (`float32`, `datetime64[ns]`, `ucs4[5]`, or the JSON field +list for a structured dtype). `E` covers properties 1 to 3 and nothing else. + +Step 2, value hash `D = SHA-256(P)` with +`P = lp("orcapod.numpy.ndarray") || E (32 bytes) || u8(memory_order) || lp(byte_order_text) || lp(struct_layout_text)`. + +- `memory_order`: 1 if `flags.f_contiguous and not flags.c_contiguous`, else 0. +- `byte_order_text`: one character per scalar leaf of the dtype, depth first in tag order (a + subarray field counts once): `<` little-endian, `>` big-endian, `|` for a type without + byte order (bool, 8-bit integers, `S`, `V`). NumPy's native marker `=` is resolved with + `sys.byteorder`. A plain `float32` gives `<`. +- `struct_layout_text`: compact JSON (`separators=(",", ":")`) of `L(dtype)`, where `L` is + `null` for an unstructured dtype and otherwise the list + `[itemsize, [offset_1, L(field_1)], [offset_2, L(field_2)], ...]` in declared field order; + a subarray field uses its base dtype. A plain `float32` gives `null`. + +Ignoring a property: its field is always present and is written with the canonical value +instead of the real one. `memory_order` 0; `byte_order_text` with every `>` replaced by `<`; +`struct_layout_text` computed from the packed dtype `tag_to_dtype(tag)`. So two values that +differ only in an ignored property have equal `D`, and `D` with all three ignored equals the +strict `D` of the canonical array (C order, little-endian, packed). The ignore list is not +in `P`; it is in the extension metadata, so it is in every type identifier and cell token. + +The result is `ContentHash("ndarray_v1", D)`. Proposal: keep the label `ndarray_v1`, because +nothing was released under it (v0.2 hashed the `.npy` file under another name); the +alternative is `ndarray_v2`. + +Changes from main: main's hash is `D = E`: no leading type text, and it ignores all three +options. Every array hash moves. The leading text makes an ndarray never equal a tensor. + +A cell of a stored table is hashed by decoding the cell to an array and computing `D` with +the ignore list of the cell's own column type. The stored bytes are never hashed. + +### C. Storage + +- Column: Arrow extension `orcapod.numpy.ndarray` over `large_binary`; one `.npy` file per cell. + Null array is a null cell. +- Write: `numpy.lib.format.write_array(buffer, array, version=(3, 0), allow_pickle=False)` + on the plain-`ndarray` view of the value (the user's array is not modified). `np.save` + itself cannot pin the version (it picks 1.0 or 3.0 by content); format 3.0 is the only + one that stores non-Latin-1 field names (measured: 1.0 and 2.0 raise on `β`). +- Read: check the magic is format 3.0, then + `numpy.lib.format.read_array(buffer, allow_pickle=False)` (default `max_header_size`). + The result is a fresh, writeable array. +- Round-trips (measured): dtype including byte order, units, widths, field names, offsets + and itemsize; shape; bits; C or Fortran contiguity. Does not: properties 7, 8, 10 to 13. +- Size: 128 bytes of header per value for simple dtypes (a 6-element `float32` array is 152 + bytes), against a few dozen on main. +- Security: `allow_pickle=False` on both sides, so no pickle is ever read; an object array + inside a stored file raises `ValueError` (measured). The header is parsed by NumPy with + `ast.literal_eval` under the 10,000-byte limit. A truncated or garbage cell raises + `ValueError` (measured). Orcapod never passes `allow_pickle=True` and never raises + `max_header_size`. +- The stored bytes are deterministic within one NumPy version (measured) but NumPy does not + promise them across versions. So `storage_sort_key` (MergeJoin sorts stored values) becomes + the strict `D` of the decoded value, not the stored bytes. + +## (ii) Shared mechanism + +### Extension metadata + +`json.dumps({"ignore": names, "version": N}, sort_keys=True, separators=(",", ":"), ensure_ascii=True)` +with `names` sorted by code point and unique. Exact bytes: + +- default: `{"ignore":[],"version":1}` +- non-default: `{"ignore":["byte_order","memory_order"],"version":1}` + +The `ignore` key is always present, also when empty. An option name matches +`[a-z][a-z0-9_]*`. On read the bytes are parsed and must re-serialise to the identical bytes; +keys other than `ignore` and `version` are an error. + +New module `orcapod/logical_types/ignore_list.py` (no heavy imports), beside `identifier.py` +(`version_metadata` stays for types without options): + +- `ignore_metadata(version: int, ignore: Iterable[str]) -> bytes` +- `parse_ignore_metadata(metadata: bytes) -> tuple[int, tuple[str, ...]]` +- `checked_ignore(ignore: Iterable[str], options: Collection[str]) -> tuple[str, ...]`: + sorted and unique; an unknown name raises `ValueError` naming it and the allowed names. +- `class IgnoreListLogicalType(BaseLogicalType)`: class attributes `EXTENSION_NAME`, + `STORAGE_TYPE`, `VERSION`, `OPTIONS`, `python_type`; `__init__(self, ignore=())`; property + `ignore`; implements `get_arrow_extension_type`, `get_polars_extension_type` and + `readable_extension_metadata`. A subclass adds `python_to_storage` / `storage_to_python`. +- `class IgnoreListHandler`: class attributes `OPTIONS`, `HASH_LABEL`; + `__init__(self, ignore=())`; `handle(obj, hasher)` returns + `hash_ignoring(obj, frozenset(self.ignore))`; `hash_ignoring(obj, ignore) -> ContentHash`; + a subclass implements `digest(self, obj, ignore: frozenset[str]) -> bytes`. + +Arrow type: one `pa.ExtensionType` class per (extension name, ignore list), so two lists are +unequal Arrow types everywhere (measured: `==`, schema equality, `concat_tables` and nested +`large_list` all tell them apart; IPC and Parquet round-trip the list). The class registered +with Arrow under the name deserialises any valid list to the class of that list; an unknown +name or non-canonical metadata raises `LogicalTypeIdentifierError`. + +### Configuration + +`contexts/logical_types.json` gets one new top-level object, the single place a list is +declared, keyed by extension name: + + "hash_ignore": {"orcapod.numpy.ndarray": []} + +The context loader puts each entry into the reference table as `hash_ignore:` +before it builds anything. The logical type entry and the value handler entry both take it: +`"_config": {"ignore": {"_ref": "hash_ignore:orcapod.numpy.ndarray"}}`. So: + +- (a) the logical type's Arrow extension metadata is `ignore_metadata(VERSION, self.ignore)`; +- (b) a bare value: the semantic hasher finds the handler by Python type and calls + `handle`, which uses the handler's configured list (the data context's); +- (c) a cell: `SemanticHashingVisitor.visit_extension` already has the cell's extension + type. When the handler is an `IgnoreListHandler` it calls + `handler.hash_ignoring(value, parse_ignore_metadata(column metadata)[1])`. The list comes + from the column, never from the context. The cell token stays + `::`, and the identifier text now contains the list. + +An unknown option name is an error at context build (`ContextValidationError` from the +`ValueError` of `checked_ignore`), for the logical type and for the handler. + +There is no user API in v0.3; a different list needs a custom `logical_types.json` +(`DataContextRegistry(logical_types_file=...)`). + +### Two lists meeting + +- Join / MergeJoin of two columns whose Arrow types carry different lists: + `InputValidationError` naming the column and both types, through the existing + `require_shared_arrow_type` (the two Arrow types are unequal; the class name printed in the + message carries the list). +- DECISION 1. A data context has exactly one list per type. Proposal: a column stored under + another valid list is refused on read with `LogicalTypeIdentifierError` naming the stored + and the readable metadata (the existing identifier rule; `readable_extension_metadata()` + is the configured list only). Reason: the Python schema says only `np.ndarray`, and the + schema hash and type hash take the identifier from the context's logical type, so a column + read under a foreign list would have a schema hash that disagrees with its cells. Reading + such a column needs a per-column Python type (the out-of-scope user API). The visitor + still takes the list from the column, so allowing it later changes no hash. + +## (iii) Golden entries + +Moved (32 values), by cause. Cause H: the hash definition (`D`). Cause M: the identifier +metadata `{"version":1}` to `{"ignore":[],"version":1}`. Cause S: storage type struct to +`large_binary`. + +- `semantic.json`: `value.ndarray.{int32,int64,float64,empty}` (H); `type.ndarray`, + `type.structural_with_array` (M). +- `schema.json`: `single_field.ndarray.{arrow_type,schema_hash}`, + `single_field.structural_with_array.{arrow_type,schema_hash}` (M, S). +- `arrow.json`: `extension_no_values.{all_null,zero_rows}.ndarray` (M). +- `datagram.json`: the 10 `ndarray.*` and 10 `structural_with_array.*` values (H, M; the + system tag column names move because they embed the schema hash). + +Added: `value.ndarray.{fortran,big_endian,aligned_struct,strided_view}` and pinned `D` +vectors in `test_golden_hashes.py`. No other entry moves; the regeneration diff is checked +for that. + +## (iv) Deleted from ITL-888 + +- `array_format.py`: `ARRAY_STORAGE_TYPE`, `ARRAY_FORMAT_METADATA`, `ARRAY_FORMAT_VERSION` + and the storage-struct text. `array_digest_v1` stays (it is `E`). +- `numpy_type.py`: the struct `python_to_storage` / `storage_to_python`, the + `(tag, shape, data)` sort key. `dtype_to_tag`, `tag_to_dtype`, `canonical_array`, + `array_bytes` stay (they produce `E`). +- `NumpyArrayHandler`: rewritten as an `IgnoreListHandler`. +- Docs: `arrays.md` sections "What counts as the same array" and "How an array is stored" + and `hashing.md` section 8 are rewritten; `logical-type-identifiers.md` gains the ignore + list. `superpowers/specs/2026-11-10-array-format-v0.3.md` gets a "superseded" line. +- Tests of the struct row (`test_array_value_format.py`, `test_array_format.py`, + `test_numpy_type.py`, `test_numpy_handler.py`) are rewritten for the new behaviour. + +## (v) Size and commit order + +About 30 files: 8 source, 10 test, 4 golden data plus cases and generator, 4 docs, +changelog fragment, design-issue file, plan. + +1. Plan under `superpowers/plans/`. +2. Failing tests through the public interface (hash distinguishes memory order, byte order, + layout; ignoring makes them equal; exact round-trip; join error; unknown option name). +3. `ignore_list.py` and the context loader (`hash_ignore`). +4. NumPy storage (`.npy`) and logical type. +5. NumPy handler (`D`) and the visitor's cell path. +6. Golden regeneration for array entries, with the moved-entry list. +7. Docs, changelog fragment, design-issue file. + +## (vi) How torch, polars and pandas plug in + +Each branch: + +1. adds its entry to `hash_ignore` in `logical_types.json` (`"orcapod.torch.tensor": []` + ...), and `{"ignore": {"_ref": "hash_ignore:"}}` to its logical type and handler + entries (also inside a deferred registration); +2. subclasses `IgnoreListLogicalType` with `EXTENSION_NAME`, `STORAGE_TYPE`, `VERSION`, + `OPTIONS`, `python_type`, and its two storage methods; +3. subclasses `IgnoreListHandler` with `OPTIONS`, `HASH_LABEL` and + `digest(obj, ignore) -> bytes`, writing each optional property's field with its + canonical value when its name is in `ignore`; +4. for torch, calls `array_format.array_digest_v1(tag, shape, data)` for `E`, unchanged. + +Nothing else is needed: the visitor, the loader, the metadata and the Arrow type classes +are shared. + +## (vii) Extension names are `orcapod..` (developer, November 12, 2026, final) + +First decision of the day: `.` (`numpy.ndarray`). Revised the same day, +because using the libraries' own namespaces is dangerous: prepend `orcapod.`. Final names: +`orcapod.numpy.ndarray`, `orcapod.torch.tensor`, `orcapod.pandas.dataframe`, +`orcapod.pandas.series`, `orcapod.polars.dataframe`, `orcapod.polars.series`; +`orcapod.file` and `orcapod.directory` are unchanged. + +For ITL-981: + +- The array type's extension name and logical type name are `orcapod.numpy.ndarray` (main: + `orcapod.ndarray`). One constant (`NDARRAY_EXTENSION_NAME`) and the `hash_ignore` key; + spelling in 8 test files, `arrays.md`, `hashing.md`, `logical-type-identifiers.md`. +- The leading type text of the value hash is the extension name (proposed by the + coordinator, not yet confirmed): `lp("orcapod.numpy.ndarray")` in `P` of section (i) B, and + `orcapod.torch.tensor` for torch. It is one constant; the tests take it from + `LEADING_TYPE_TEXT`. +- Golden: no further entry. The same 32 values move once; cause M now includes the name. +- The clash with v0.2 does not arise: v0.2 stored arrays as `numpy.ndarray` (empty + metadata), a name v0.3 does not register, so v0.2 tables are read as plain storage, + counted and left unchanged, exactly as on main. (Measured while the name was briefly + `numpy.ndarray`: registering that name made 2 of the 6 tests of + `tests/test_arrays/test_array_v02_store.py` fail, because every pyarrow or Delta read of a + v0.2 table then calls the registered class.) The v0.2 fixture script keeps its guard. +- The "foreign type" is therefore DROPPED. The mechanism does not need it: the class + registered with Arrow deserialises any valid ignore list to the class of that list, and + for anything else (non-canonical bytes, an unknown option name, another version) raises + `LogicalTypeIdentifierError`, which is what every built-in type does on main. +- Note for ITL-897: the pandas rename is different. v0.2 stores used `pandas.dataframe`, + and main reads their empty metadata through `also_reads`. That is ITL-897's question. + +## Open decisions for the developer + +1. Reading a column stored under another list: refuse (proposed) or read. +2. Hash label: keep `ndarray_v1` (proposed) or `ndarray_v2`. +3. Keep refusing unit-less datetimes and field titles in v0.3 (proposed), although `.npy` + stores them. +4. Metadata key order: sorted keys, `{"ignore":[...],"version":1}` (proposed), so that + `json.dumps(sort_keys=True)` reproduces it; the brief sketched version first. + +## (viii) Scoping: storage type inside the Arrow schema hash (not part of ITL-981 unless decided) + +Question from the coordinator: `StarfixArrowHasher.hash_schema` hashes a logical-type field +as its storage type plus the `ARROW:extension:*` keys, so that hash depends on the storage +type. Should a logical-type field be hashed as a fixed placeholder plus name and metadata? + +Findings (read from the code and measured on main): + +- No library code calls `hash_schema` (the only match under `src/` is its definition). It is + reached only by tests and by 47 golden values in `arrow.json` (`schema.*`, 42 plain; + `extension_schema.extension.*`, 5). +- Every schema or type hash Orcapod stores or embeds goes through the semantic hasher over + Python types: the schema hash (Site 3) and the system tag column names built from it, the + type hash, the pipeline hash and the paths built from it, the function signature. There a + logical type contributes its identifier text only (`canonical_annotation_str`, + `TypeObjectHandler`): name and metadata, never the storage type. +- `hash_table` (default source ids, row and datagram hashes) hashes a value-hashed column as + `large_binary` plus the keys, also with zero rows (ITL-967): already independent of + storage. Value-hashed today: UUID, File, Directory, ndarray, pandas DataFrame and Series, + and `list` / `set` of them. +- Types without a value handler (Path, UPath, dataclass, pydantic, structural classes, + lists of them, unregistered extension types) are hashed in `hash_table` as their stored + values under the storage type. For them the stored struct is the value, so the storage + type must stay in the hash. + +So no stored or embedded hash depends on a value-hashed type's storage type today; only the +unused public method does. If it is to be made consistent, the change is to give +`hash_schema` the same field rule as `hash_table` (`SemanticHashingVisitor.hashed_field`): +about 10 lines, one test, and golden moves limited to `arrow.json` +`extension_schema.extension.*` entries of value-hashed types whose storage is not already +`large_binary` (none of the five there today: `uuid` is `large_binary`, `path` has no value +handler). Recommendation: not needed for the tag; leave it to ITL-980, or take the +10-line alignment in ITL-981 if the developer wants the public method consistent. diff --git a/tests/test_arrays/test_array_pipeline_stores.py b/tests/test_arrays/test_array_pipeline_stores.py index 0083c7ef7..229870037 100644 --- a/tests/test_arrays/test_array_pipeline_stores.py +++ b/tests/test_arrays/test_array_pipeline_stores.py @@ -11,6 +11,7 @@ from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.operators import Batch, GroupBy from orcapod.pipeline import AsyncPipelineOrchestrator from orcapod.sources import DictSource from tests.test_arrays.array_cases import ARRAYS, NUMERIC, assert_arrays_identical @@ -94,3 +95,70 @@ def test_pipeline_keeps_arrays_exact(tmp_path: Path, store: str, runner: str) -> expected = np.asarray(ARRAYS[name] * 2) assert_arrays_identical(doubled[name], expected) assert totals[name] == pytest.approx(float(np.real(expected).sum())) + + +@function_pod(output_keys="parts") +def split(x: np.ndarray) -> list[np.ndarray]: # pyright: ignore[reportMissingTypeArgument] # the bare np.ndarray annotation is under test + return [x.copy(), np.asarray(x * 2)] + + +@pytest.mark.parametrize( + ("store", "runner"), + [ + ("in_memory", "sync"), + ("in_memory", "async"), + ("delta", "sync"), + ("delta", "async"), + ], +) +def test_pod_output_list_of_arrays(tmp_path: Path, store: str, runner: str) -> None: + """A pod output ``list[np.ndarray]`` is stored and read back on every store and runner.""" + source = DictSource( + data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], + tag_columns=["k"], + source_id="numeric", + ) + job = PipelineJob(name="array_lists", store=STORES[store](tmp_path)) + with job: + split.pod(source, label="split") + RUNNERS[runner](job) + + parts = _node(job, "split", "parts") + assert sorted(parts) == sorted(NUMERIC) + for name in NUMERIC: + assert_arrays_identical(parts[name][0], ARRAYS[name]) + assert_arrays_identical(parts[name][1], np.asarray(ARRAYS[name] * 2)) + + +@function_pod(output_keys="firsts") +def first_elements(x: list[np.ndarray]) -> list[float]: # pyright: ignore[reportMissingTypeArgument] # the bare np.ndarray annotation is under test + assert all(type(a) is np.ndarray and a.shape == (3,) for a in x) + return sorted(float(a[0]) for a in x) + + +@pytest.mark.parametrize("runner", sorted(RUNNERS)) +@pytest.mark.parametrize("store", sorted(STORES)) +@pytest.mark.parametrize("operator", ["batch", "group_by"]) +def test_batch_and_group_by_over_an_array_column( + tmp_path: Path, operator: str, store: str, runner: str +) -> None: + """``Batch`` and ``GroupBy`` over an array column feed a pod on every store and runner.""" + source = DictSource( + data=[ + {"g": "a" if i < 2 else "b", "k": f"r{i}", "x": np.arange(3.0) + i} + for i in range(4) + ], + tag_columns=["g", "k"], + source_id="grouped", + ) + job = PipelineJob(name="array_groups", store=STORES[store](tmp_path)) + with job: + if operator == "batch": + grouped = Batch(batch_size=2)(source) + else: + grouped = GroupBy(["g"])(source) + first_elements.pod(grouped, label="firsts") + RUNNERS[runner](job) + + firsts = sorted(d.as_dict()["firsts"] for _, d in job.nodes["firsts"].iter_data()) + assert firsts == [[0.0, 1.0], [2.0, 3.0]] diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py new file mode 100644 index 000000000..6827ac181 --- /dev/null +++ b/tests/test_arrays/test_array_strict_hash.py @@ -0,0 +1,379 @@ +"""Strict array hash, exact ``.npy`` storage and the per-type ignore list (ITL-981). + +The hash label and the extension metadata bytes are two constants below: the +developer's decision on each changes only these lines. +""" + +from __future__ import annotations + +import hashlib +import json +import struct +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import numpy.typing as npt +import pyarrow as pa +import pytest + +import orcapod.contexts as contexts_package +from orcapod.contexts import get_default_context +from orcapod.contexts.core import ContextValidationError, DataContext +from orcapod.contexts.registry import JSONDataContextRegistry +from orcapod.core.operators import MergeJoin +from orcapod.logical_types.identifier import LogicalTypeIdentifierError +from orcapod.sources import ArrowTableSource, DictSource +from orcapod.utils.arrow_utils import shared_arrow_type + +HASH_LABEL = "ndarray_v1" +"""Label of an array value hash (open decision 2 of the design note).""" + + +def ignore_metadata(names: list[str]) -> bytes: + """Return the extension metadata for an ignore list (open decision 4: key order).""" + return json.dumps( + {"ignore": sorted(names), "version": 1}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode() + + +EXTENSION_NAME = "orcapod.numpy.ndarray" +"""Arrow extension name of the array type (developer, November 12, 2026).""" + +LEADING_TYPE_TEXT = EXTENSION_NAME +"""Leading text of the value hash preimage: the extension name (proposed).""" + +OPTIONS = ("byte_order", "memory_order", "struct_layout") + +BASE = np.arange(6, dtype=np.float32).reshape(2, 3) +PACKED = np.array([(1, 0.5), (-2, 1.5)], dtype=[("a", "i1"), ("b", "f4")), + "big_endian_field": ( + "byte_order", + PACKED, + PACKED.astype([("a", "i1"), ("b", ">f8")]), + ), + "aligned_struct": ("struct_layout", PACKED, ALIGNED), +} + + +def _lp(text: str) -> bytes: + data = text.encode("utf-8") + return struct.pack(" str: + """Reference implementation of the value hash with the standard library only.""" + p = _lp(tag) + struct.pack(" DataContext: + """Build a data context whose ndarray ignore list is ``names``.""" + shipped = Path(contexts_package.__file__).parent / "logical_types.json" + registration = json.loads(shipped.read_text()) + registration["hash_ignore"][EXTENSION_NAME] = names + custom = tmp_path / f"logical_types_{'_'.join(names) or 'strict'}.json" + custom.write_text(json.dumps(registration)) + return JSONDataContextRegistry(logical_types_file=custom).get_context() + + +def _source(arrays: dict[str, Any], column: str = "x") -> DictSource: + return DictSource( + data=[{"k": k, column: v} for k, v in arrays.items()], + tag_columns=["k"], + source_id="arrays", + ) + + +def _read(stream: Any, column: str = "x") -> dict[str, Any]: + return {t.as_dict()["k"]: d.as_dict()[column] for t, d in stream.iter_data()} + + +def _layout(dtype: np.dtype[Any]) -> Any: + if dtype.subdtype is not None: + dtype = dtype.subdtype[0] + if dtype.names is None: + return None + assert dtype.fields is not None + fields: dict[str, Any] = {str(key): value for key, value in dtype.fields.items()} + return [dtype.itemsize] + [ + [fields[name][1], _layout(fields[name][0])] for name in dtype.names + ] + + +def _value_bytes(array: npt.NDArray[Any]) -> bytes: + """The bytes of every scalar field in memory order, without padding bytes. + + Padding between structured fields is not compared: NumPy does not define its + content and does not always copy it. + """ + if array.dtype.names is None: + return array.tobytes("A") + return b"".join(_value_bytes(array[name]) for name in array.dtype.names) + + +# --- the hash is strict by default ------------------------------------------------ + + +@pytest.mark.parametrize("name", sorted(PAIRS)) +def test_default_hash_distinguishes(name: str) -> None: + """By default memory order, byte order and structured layout are part of the hash.""" + _, one, other = PAIRS[name] + hasher = get_default_context().semantic_hasher + assert hasher.hash_object(one) != hasher.hash_object(other) + + +def test_hash_carries_the_label() -> None: + """An array value hash is labelled with the array algorithm's label.""" + assert get_default_context().semantic_hasher.hash_object(BASE).method == HASH_LABEL + + +def test_strided_view_hashes_like_its_contiguous_copy() -> None: + """A strided view is stored C-contiguous, so it hashes as its C-contiguous copy.""" + hasher = get_default_context().semantic_hasher + wide = np.arange(12, dtype=np.float32).reshape(2, 6) + view = wide[:, ::2] + assert not view.flags.c_contiguous + assert hasher.hash_object(view) == hasher.hash_object(np.ascontiguousarray(view)) + assert hasher.hash_object(BASE[::-1]) == hasher.hash_object( + np.ascontiguousarray(BASE[::-1]) + ) + + +def test_subclass_and_writeable_flag_do_not_count() -> None: + """What storage does not keep (subclass, writeable flag) is never in the hash.""" + hasher = get_default_context().semantic_hasher + ints = np.array([[1, 2], [3, 4]]) + assert hasher.hash_object(np.matrix(ints)) == hasher.hash_object(ints) + frozen = BASE.copy() + frozen.setflags(write=False) + assert hasher.hash_object(frozen) == hasher.hash_object(BASE) + + +def test_reference_vectors() -> None: + """The hash equals the reference implementation, field by field.""" + hasher = get_default_context().semantic_hasher + data = struct.pack("<6f", 0, 1, 2, 3, 4, 5) + assert hasher.hash_object(BASE).to_hex() == reference_digest( + "float32", (2, 3), data, 0, "<", "null" + ) + assert hasher.hash_object(np.asfortranarray(BASE)).to_hex() == reference_digest( + "float32", (2, 3), data, 1, "<", "null" + ) + assert hasher.hash_object(BASE.astype(">f4")).to_hex() == reference_digest( + "float32", (2, 3), data, 0, ">", "null" + ) + packed_bytes = struct.pack("f4"), + "big_endian_unicode": np.array(["ab", "c"], dtype=">U3"), + "aligned_struct": ALIGNED, + "mixed_endian_struct": PACKED.astype([("a", "i1"), ("b", ">f8")]), + "unicode_field_name": np.zeros(2, dtype=[("β", " None: + """Memory order, byte order and structured layout survive a source round trip.""" + expected = EXACT[name] + got = _read(_source({name: expected}))[name] + assert type(got) is np.ndarray + assert got.dtype == expected.dtype + assert got.dtype.str == expected.dtype.str + assert _layout(got.dtype) == _layout(expected.dtype) + assert got.shape == expected.shape + assert got.flags.c_contiguous == expected.flags.c_contiguous + assert got.flags.f_contiguous == expected.flags.f_contiguous + assert _value_bytes(got) == _value_bytes(expected) + assert got.flags.writeable + + +@pytest.mark.parametrize( + "name", sorted(EXACT) + ["strided_view", "matrix", "read_only"] +) +def test_hash_is_unchanged_by_storage(name: str) -> None: + """``hash(read(write(v))) == hash(v)``: a cached re-run never recomputes.""" + frozen = BASE.copy() + frozen.setflags(write=False) + extra = { + "strided_view": np.arange(12, dtype=np.float32).reshape(2, 6)[:, ::2], + "matrix": np.matrix([[1, 2], [3, 4]]), + "read_only": frozen, + } + value = {**EXACT, **extra}[name] + hasher = get_default_context().semantic_hasher + got = _read(_source({name: value}))[name] + assert hasher.hash_object(got) == hasher.hash_object(value) + + +def test_column_is_npy_bytes_under_the_default_identifier() -> None: + """An array column is ``orcapod.numpy.ndarray`` over ``large_binary`` holding ``.npy`` files.""" + field = _source({"a": BASE}).as_table().schema.field("x") + assert field.type.extension_name == EXTENSION_NAME + assert field.type.storage_type == pa.large_binary() + assert field.type.__arrow_ext_serialize__() == ignore_metadata([]) + chunk = _source({"a": BASE}).as_table().column("x").chunk(0) + assert isinstance(chunk, pa.ExtensionArray) + cell = chunk.storage[0].as_py() + assert cell[:8] == b"\x93NUMPY\x03\x00" + + +def test_out_of_order_struct_offsets_are_refused() -> None: + """A structured dtype ``.npy`` cannot store is refused when written.""" + dtype = np.dtype( + { + "names": ["a", "b"], + "formats": ["i4", "f8"], + "offsets": [8, 0], + "itemsize": 16, + } + ) + with pytest.raises(ValueError, match="offset"): + _source({"a": np.zeros(2, dtype=dtype)}) + + +# --- the ignore list ---------------------------------------------------------------- + + +@pytest.mark.parametrize("name", sorted(PAIRS)) +def test_ignoring_a_property_gives_equal_hashes(name: str, tmp_path: Path) -> None: + """With a property in the ignore list, values differing only in it hash equal.""" + option, one, other = PAIRS[name] + hasher = _context_ignoring(tmp_path, [option]).semantic_hasher + assert hasher.hash_object(one) == hasher.hash_object(other) + + +@pytest.mark.parametrize("name", sorted(PAIRS)) +def test_ignoring_another_property_keeps_them_apart(name: str, tmp_path: Path) -> None: + """Ignoring one property leaves the other two in the hash.""" + option, one, other = PAIRS[name] + others = [o for o in OPTIONS if o != option] + hasher = _context_ignoring(tmp_path, others).semantic_hasher + assert hasher.hash_object(one) != hasher.hash_object(other) + + +def test_everything_ignored_equals_strict_hash_of_canonical_array( + tmp_path: Path, +) -> None: + """The ignore list is not a hash input: only the canonical field values are.""" + lenient = _context_ignoring(tmp_path, list(OPTIONS)).semantic_hasher + strict = get_default_context().semantic_hasher + swapped_fortran = np.asfortranarray(BASE.astype(">f4")) + assert lenient.hash_object(swapped_fortran) == strict.hash_object(BASE) + + +def test_ignore_list_is_in_the_extension_metadata(tmp_path: Path) -> None: + """Two ignore lists are two logical types: the list is in the identifier.""" + context = _context_ignoring(tmp_path, ["memory_order", "byte_order"]) + arrow_type = context.type_converter.python_type_to_arrow_type(np.ndarray) + assert isinstance(arrow_type, pa.ExtensionType) + assert arrow_type.__arrow_ext_serialize__() == ignore_metadata( + ["byte_order", "memory_order"] + ) + default_type = get_default_context().type_converter.python_type_to_arrow_type( + np.ndarray + ) + assert arrow_type != default_type + assert context.semantic_hasher.hash_object( + np.ndarray + ) != get_default_context().semantic_hasher.hash_object(np.ndarray) + + +def test_a_cell_is_hashed_with_its_columns_list(tmp_path: Path) -> None: + """A table hash uses the ignore list of the column, so it follows the column type.""" + context = _context_ignoring(tmp_path, ["memory_order"]) + + def table_hash(value: npt.NDArray[Any]) -> Any: + table = context.type_converter.python_dicts_to_arrow_table( + [{"x": value}], python_schema={"x": np.ndarray} + ) + return context.arrow_hasher.hash_table(table) + + assert table_hash(BASE) == table_hash(np.asfortranarray(BASE)) + assert table_hash(BASE) != table_hash(BASE.astype(">f4")) + + +def test_unknown_option_name_is_an_error(tmp_path: Path) -> None: + """A context that names an option the type does not have cannot be built.""" + with pytest.raises(ContextValidationError, match="memory_layout"): + _context_ignoring(tmp_path, ["memory_layout"]) + + +def test_merge_join_of_two_ignore_lists_is_an_error(tmp_path: Path) -> None: + """Joining one column typed with two ignore lists names the column and both lists. + + A data context reads only columns of its own list, so the error is the read + refusal, raised when the join inspects its inputs. + """ + lenient = _context_ignoring(tmp_path, ["memory_order"]) + + def stream(context: DataContext) -> ArrowTableSource: + table = context.type_converter.python_dicts_to_arrow_table( + [{"k": "r1", "x": BASE}], python_schema={"k": str, "x": np.ndarray} + ) + return ArrowTableSource(table, tag_columns=["k"], source_id="s") + + with pytest.raises(LogicalTypeIdentifierError, match="'x'") as raised: + MergeJoin()(stream(get_default_context()), stream(lenient)).as_table() + message = str(raised.value) + assert repr(ignore_metadata(["memory_order"])) in message + assert repr(ignore_metadata([])) in message + + +def test_the_join_type_rule_tells_two_ignore_lists_apart(tmp_path: Path) -> None: + """The rule joins use for a shared column finds no shared type for two lists.""" + lenient = _context_ignoring(tmp_path, ["memory_order"]) + strict_type = get_default_context().type_converter.python_type_to_arrow_type( + np.ndarray + ) + lenient_type = lenient.type_converter.python_type_to_arrow_type(np.ndarray) + assert shared_arrow_type(strict_type, lenient_type) is None + assert shared_arrow_type(strict_type, strict_type) == strict_type + assert "memory_order" in str(lenient_type) + + +def test_native_byte_order_is_absolute() -> None: + """A native array hashes as its explicit byte order, whatever the machine.""" + hasher = get_default_context().semantic_hasher + explicit = "f4" + assert hasher.hash_object(BASE) == hasher.hash_object(BASE.astype(explicit)) diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index e6335eccc..a6f42a2fe 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -1,14 +1,14 @@ -"""The v0.3 value-based array format: storage, canonical form, hash and refusals (ITL-888). +"""Array values: ``.npy`` storage, what reads back, the hash and refusals (ITL-981). -Specification: ``superpowers/specs/2026-11-10-array-format-v0.3.md``. Tests go -through sources, streams, function pods and the default data context's -semantic hasher. +Design: ``superpowers/specs/2026-11-12-itl-981-library-value-types-design.md``. +Tests go through sources, streams, function pods and the default data context's +semantic hasher. The strict hash and the ignore list have their own file, +``test_array_strict_hash.py``. """ from __future__ import annotations -import hashlib -import json +import io import struct from pathlib import Path from typing import Any @@ -20,22 +20,19 @@ from orcapod import function_pod from orcapod.contexts import get_default_context +from orcapod.logical_types.numpy_type import dtype_to_tag from orcapod.operators import MergeJoin from orcapod.sources import ArrowTableSource, DictSource from tests.test_arrays.array_cases import ARRAYS, assert_arrays_identical - -def reference_digest(tag: str, shape: tuple[int, ...], data: bytes) -> bytes: - """Independent implementation of the v1 array digest (spec C1).""" - t = tag.encode("utf-8") - preimage = struct.pack(" dict[str, Any]: - """The stored row of a one-row ``x`` column.""" +def stored(value: Any) -> bytes: + """The stored cell of a one-row ``x`` column.""" table = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]).as_table() chunk = table.column("x").chunk(0) assert isinstance(chunk, pa.ExtensionArray) @@ -61,36 +58,31 @@ def data_hash(value: Any) -> Any: # --------------------------------------------------------------------------- -def test_column_type_is_orcapod_ndarray_over_the_array_struct() -> None: - """An array column is ``orcapod.ndarray`` over the array struct with version metadata.""" +def test_column_type_is_orcapod_numpy_ndarray_over_large_binary() -> None: + """An array column is ``orcapod.numpy.ndarray`` over ``large_binary`` with the ignore list.""" table = DictSource( data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"] ).as_table() ext = table.schema.field("x").type assert isinstance(ext, pa.ExtensionType) - assert ext.extension_name == "orcapod.ndarray" - assert ext.storage_type == pa.struct( - [ - pa.field("dtype", pa.large_string()), - pa.field("shape", pa.large_list(pa.int64())), - pa.field("data", pa.large_binary()), - ] - ) - assert ext.__arrow_ext_serialize__() == b'{"version":1}' + assert ext.extension_name == "orcapod.numpy.ndarray" + assert ext.storage_type == pa.large_binary() + assert ext.__arrow_ext_serialize__() == b'{"ignore":[],"version":1}' -def test_old_extension_name_is_not_registered() -> None: - """Only ``orcapod.ndarray`` is registered; the old ``numpy.ndarray`` name is not.""" +def test_older_extension_names_are_not_registered() -> None: + """Only ``orcapod.numpy.ndarray`` is registered; the names of v0.2 and of the first v0.3 format are not.""" converter: Any = get_default_context().type_converter assert converter.get_logical_type_by_arrow_extension_name("numpy.ndarray") is None + assert converter.get_logical_type_by_arrow_extension_name("orcapod.ndarray") is None assert ( - converter.get_logical_type_by_arrow_extension_name("orcapod.ndarray") + converter.get_logical_type_by_arrow_extension_name("orcapod.numpy.ndarray") is not None ) # --------------------------------------------------------------------------- -# Stored rows: dtype tags, shape, little-endian C-order data +# Stored cells: one .npy file per value; dtype tags # --------------------------------------------------------------------------- TAG_CASES: list[tuple[str, npt.NDArray[Any], str]] = [ @@ -141,54 +133,52 @@ def test_old_extension_name_is_not_registered() -> None: @pytest.mark.parametrize( ("array", "tag"), [c[1:] for c in TAG_CASES], ids=[c[0] for c in TAG_CASES] ) -def test_stored_row_holds_tag_shape_and_c_order_little_endian_data( +def test_stored_cell_is_a_npy_file_numpy_reads( array: npt.NDArray[Any], tag: str ) -> None: - """The stored row holds the dtype tag, the shape and the packed element bytes.""" - row = stored(array) - assert row["dtype"] == tag - assert row["shape"] == list(array.shape) - assert len(row["data"]) == array.size * np.dtype(read_back(array).dtype).itemsize + """The stored cell is a ``.npy`` file of format 3.0 that plain NumPy loads back.""" + cell = stored(array) + assert cell[:8] == b"\x93NUMPY\x03\x00" + loaded = np.load(io.BytesIO(cell), allow_pickle=False) + assert loaded.dtype == array.dtype + assert loaded.shape == array.shape + assert loaded.tobytes() == array.tobytes() @pytest.mark.parametrize( - "name", ["float32", "three_d", "complex128", "unicode", "datetime64_ns"] + ("array", "tag"), [c[1:] for c in TAG_CASES], ids=[c[0] for c in TAG_CASES] ) -def test_data_is_the_c_order_little_endian_buffer(name: str) -> None: - """The stored data is the array's C-order, little-endian buffer.""" - array = ARRAYS[name] - assert stored(array)["data"] == np.ascontiguousarray(array).tobytes() +def test_dtype_tag_of_the_element_digest(array: npt.NDArray[Any], tag: str) -> None: + """The dtype tag that enters the element digest names the dtype without a byte order.""" + assert dtype_to_tag(array.dtype) == tag + assert dtype_to_tag(array.dtype.newbyteorder(">")) == tag def test_zero_d_and_empty_shapes() -> None: - """Zero-dimensional and empty arrays store their exact shape and data.""" - assert stored(np.array(2.5))["shape"] == [] - assert stored(np.array(2.5))["data"] == struct.pack(" None: - """Structured data is stored little-endian and packed in declared field order.""" - array = np.array([(1.5, 7)], dtype=[("b", ">f8"), ("a", " None: + """Within one NumPy version, equal arrays give equal stored bytes.""" + array = np.arange(6, dtype=np.float32).reshape(2, 3) + assert stored(array) == stored(array.copy()) # --------------------------------------------------------------------------- -# Canonical form on read +# What reads back # --------------------------------------------------------------------------- -def test_fortran_order_reads_back_c_order() -> None: - """A Fortran-ordered array reads back C-ordered with the same values.""" +def test_fortran_order_reads_back_fortran_order() -> None: + """A Fortran-ordered array reads back Fortran-ordered with the same values.""" array = np.asfortranarray(np.arange(6, dtype=np.float32).reshape(2, 3)) - assert_arrays_identical(read_back(array), np.ascontiguousarray(array)) + got = read_back(array) + assert got.flags.f_contiguous and not got.flags.c_contiguous + assert (got == array).all() def test_strided_view_reads_back_as_its_values() -> None: @@ -199,35 +189,40 @@ def test_strided_view_reads_back_as_its_values() -> None: @pytest.mark.parametrize("dtype", [">f4", ">i8", ">c16", ">M8[ns]", ">U3"]) -def test_big_endian_reads_back_little_endian_with_equal_values(dtype: str) -> None: - """A big-endian array reads back little-endian with equal values.""" +def test_big_endian_reads_back_big_endian(dtype: str) -> None: + """A big-endian array reads back big-endian with equal values.""" raw = ( ["ab", "c"] if dtype.startswith(">U") else np.arange(2).astype(dtype[0] + "i8") ) big = np.array(raw).astype(dtype) got = read_back(big) - assert got.dtype == big.dtype.newbyteorder("<") - assert (got == big).all() + assert got.dtype.str == big.dtype.str + assert got.tobytes() == big.tobytes() -def test_structured_reads_back_in_stored_field_order_and_packed() -> None: - """An aligned structured array reads back in declared field order, little-endian and packed.""" +def test_aligned_structured_reads_back_with_its_offsets() -> None: + """An aligned structured array keeps field order, byte order, offsets and itemsize.""" aligned = np.dtype([("b", ">f8"), ("a", "f8" + assert got.dtype.fields is not None and got.dtype.fields["a"][1] == 8 assert got["a"].tolist() == [7, 9] assert got["b"].tolist() == [1.5, -2.0] + # The ``align`` flag itself is not kept by ``.npy``; the layout it produced is. + assert not got.dtype.isalignedstruct -def test_nested_structured_reads_back_in_stored_field_order() -> None: - """A nested structured array reads back in declared field order at every depth.""" +def test_nested_structured_reads_back_as_given() -> None: + """A nested structured array keeps field order and byte order at every depth.""" inner = np.dtype([("y", ">i2"), ("x", "i2" assert got["s"]["y"].tolist() == [1] and got["s"]["x"].tolist() == [2] @@ -235,7 +230,6 @@ def test_reordered_fields_are_a_different_stored_value() -> None: """Arrays whose fields differ only in order are different stored values.""" ab = np.array([(1, 2.0)], dtype=[("a", " None: def test_read_back_is_writeable() -> None: - """An array read back from storage is writeable.""" - got = read_back(np.arange(3.0)) + """An array read back from storage is writeable, also when a read-only one was written.""" + frozen = np.arange(3.0) + frozen.setflags(write=False) + got = read_back(frozen) got[0] = 10.0 assert got[0] == 10.0 +def test_dtype_metadata_is_dropped_without_a_warning() -> None: + """Dtype metadata is not stored and not hashed; writing it raises no warning.""" + plain = np.arange(3, dtype=" Non # --------------------------------------------------------------------------- -# Hash: value digest under the ndarray_v1 label +# Hash: the value digest under the ndarray_v1 label # --------------------------------------------------------------------------- @pytest.mark.parametrize("name", sorted(ARRAYS)) -def test_semantic_hash_is_the_v1_digest_of_the_stored_row(name: str) -> None: - """The semantic hash is the ``ndarray_v1`` digest of the stored row.""" - row = stored(ARRAYS[name]) - h = semantic_hash(ARRAYS[name]) - assert h.method == "ndarray_v1" - assert h.digest == reference_digest(row["dtype"], tuple(row["shape"]), row["data"]) - - -def test_pinned_digests() -> None: - """Pinned digests match the spec's byte rules.""" - # Pinned v1 digests; computed by ``reference_digest`` from the spec's byte rules. - cases = { - "float32_2x3": (np.arange(6, dtype=np.float32).reshape(2, 3), "float32"), - "zero_d": (np.array(2.5), "float64"), - "empty": (np.zeros((0, 3), np.int8), "int8"), - } - for array, tag in cases.values(): - expected = reference_digest( - tag, array.shape, np.ascontiguousarray(array).tobytes() - ) - assert semantic_hash(array).digest == expected - assert ( - semantic_hash(cases["zero_d"][0]).digest.hex() - == reference_digest("float64", (), struct.pack(" None: + """The hash is labelled ``ndarray_v1`` and is the same for the value read back.""" + given = semantic_hash(ARRAYS[name]) + assert given.method == "ndarray_v1" + assert semantic_hash(read_back(ARRAYS[name])) == given def _equal_value_variants() -> list[tuple[str, npt.NDArray[Any], npt.NDArray[Any]]]: a = np.arange(24, dtype=np.float32).reshape(4, 6) base = np.zeros((4, 12), dtype=np.float32) base[:, ::2] = a - rec_ab = np.array([(1, 2.0)], dtype=[("a", "i4"), ("b", ">f8")]) - aligned = np.array( - [(1, 2.0)], dtype=np.dtype([("a", "f4")), - ("structured_byte_order", rec_ab, rec_ab_big), - ("aligned", rec_ab, aligned), - ("padded", rec_ab, padded), - ("offsets_swapped", rec_ab, offsets_swapped), + ("read_only", a, frozen), ( "matrix", np.arange(4.0).reshape(2, 2), np.matrix(np.arange(4.0).reshape(2, 2)), ), + ( + "align_flag_only", + np.zeros(2, dtype=[("a", " list[tuple[str, npt.NDArray[Any], npt.NDArray[Any [v[1:] for v in _equal_value_variants()], ids=[v[0] for v in _equal_value_variants()], ) -def test_layout_byte_order_padding_and_subclass_do_not_change_the_hash( +def test_what_storage_does_not_keep_does_not_change_the_hash( left: npt.NDArray[Any], right: npt.NDArray[Any] ) -> None: - """Memory layout, byte order, padding and subclass do not change the hash.""" + """Strides, the writeable flag, the subclass and the ``align`` flag do not change the hash.""" assert semantic_hash(left) == semantic_hash(right) assert data_hash(left) == data_hash(right) @@ -386,6 +352,16 @@ def test_layout_byte_order_padding_and_subclass_do_not_change_the_hash( def _different_value_pairs() -> list[tuple[str, npt.NDArray[Any], npt.NDArray[Any]]]: nan_a = np.array([np.nan]) nan_b = np.frombuffer(struct.pack(" list[tuple[str, npt.NDArray[Any], npt.NDArray[An np.zeros(1, dtype=[("s", [("y", "f4")), + ( + "structured_byte_order", + rec_ab, + rec_ab.astype([("a", ">i4"), ("b", ">f8")]), + ), + ( + "aligned_offsets", + rec_ab, + rec_ab.astype(np.dtype([("a", " list[tuple[str, npt.NDArray[Any], npt.NDArray[An [v[1:] for v in _different_value_pairs()], ids=[v[0] for v in _different_value_pairs()], ) -def test_dtype_shape_and_bits_change_the_hash( +def test_dtype_shape_bits_and_kept_layout_change_the_hash( left: npt.NDArray[Any], right: npt.NDArray[Any] ) -> None: - """Dtype, shape, field order and element bits change the hash.""" + """Dtype, shape, element bits, memory order, byte order and offsets change the hash.""" assert semantic_hash(left) != semantic_hash(right) assert data_hash(left) != data_hash(right) -def test_table_token_carries_extension_name_and_digest() -> None: - """A table token carries the extension name and the value digest, no label.""" - # The Arrow hasher folds the extension name in front of the handler's digest. +def test_table_token_carries_the_identifier_and_the_digest() -> None: + """A table token carries the type identifier (name and ignore list) and the value digest, no label.""" + # The Arrow hasher folds the identifier text in front of the handler's digest. from orcapod.hashing.arrow_hashers import StarfixArrowHasher hasher = get_default_context().arrow_hasher @@ -439,10 +428,7 @@ def test_table_token_carries_extension_name_and_digest() -> None: # Non-public helper: this test is about the token the hasher builds for a cell. processed = hasher._process_table_columns(table.select(["x"])) # pyright: ignore[reportPrivateUsage] token = processed.column("x")[0].as_py() - assert ( - token - == b"orcapod.ndarray;7b2276657273696f6e223a317d::" + semantic_hash(array).digest - ) + assert token == IDENTIFIER_TEXT + b"::" + semantic_hash(array).digest # --------------------------------------------------------------------------- @@ -458,6 +444,41 @@ def test_table_token_carries_extension_name_and_digest() -> None: ("generic_datetime", np.array(["NaT"], dtype="M8"), "unit"), ("generic_timedelta", np.array([0], dtype="m8"), "unit"), ("titles", np.zeros(1, dtype=[(("title", "a"), " None: - """A stored row with a null child raises on read, naming the column.""" +@pytest.mark.parametrize( + "cell", + [ + b"not a npy file", + b"\x93NUMPY\x01\x00" + b"\x00" * 16, + None, + ], + ids=["garbage", "other_format_version", "truncated"], +) +def test_a_cell_that_is_not_a_stored_array_raises_naming_the_column( + cell: bytes | None, +) -> None: + """A cell that is not a complete ``.npy`` file of format 3.0 raises on read, naming the column.""" converter = get_default_context().type_converter ext = converter.python_type_to_arrow_type(np.ndarray) assert isinstance(ext, pa.ExtensionType) - storage = pa.array( - [{"dtype": "float64", "shape": None, "data": b""}], type=ext.storage_type - ) + if cell is None: + cell = stored(np.arange(4.0))[:-5] + storage = pa.array([cell], type=ext.storage_type) table = pa.table( { "k": pa.array(["r"], pa.large_string()), @@ -492,51 +524,96 @@ def test_null_child_on_read_raises_naming_the_column() -> None: } ) # The first read of the value (hashing the source rows) must fail naming the column. - with pytest.raises((ValueError, RuntimeError), match="'x'.*null field"): + with pytest.raises((ValueError, RuntimeError), match="'x'"): source = ArrowTableSource(table, tag_columns=["k"]) for _, data in source.iter_data(): data.as_dict() -@pytest.mark.parametrize( - "entry", - [ - ["a"], - ["a", "int32", [2], "extra"], - [1, "int32"], - ["a", "int32", 2], - ["a", "int32", ["2"]], - ["a", "int32", [True]], - ["a", "int32", [-1]], - ], - ids=[ - "one_part", - "four_parts", - "non_string_name", - "shape_not_a_list", - "shape_of_strings", - "shape_of_bools", - "negative_dimension", - ], -) -def test_malformed_structured_tag_entry_raises_on_read( - entry: list[Any], -) -> None: - """A malformed structured tag entry raises the malformed-tag error on read.""" +def _source_of_cell(cell: bytes) -> ArrowTableSource: converter = get_default_context().type_converter ext = converter.python_type_to_arrow_type(np.ndarray) assert isinstance(ext, pa.ExtensionType) - tag = json.dumps([entry]) - storage = pa.array( - [{"dtype": tag, "shape": [], "data": b"\x00" * 8}], type=ext.storage_type + storage = pa.array([cell], type=ext.storage_type) + table = pa.table( + { + "k": pa.array(["r"], pa.large_string()), + "x": pa.ExtensionArray.from_storage(ext, storage), + } ) + return ArrowTableSource(table, tag_columns=["k"]) + + +def test_a_cell_that_declares_more_than_it_holds_is_refused_before_allocation() -> None: + """A small cell whose header declares a huge array is refused without allocating it.""" + buffer = io.BytesIO() + np.lib.format.write_array_header_2_0( + buffer, {"descr": " None: + """A cell that holds more than the array its header declares is refused.""" + with pytest.raises((ValueError, RuntimeError), match="'x'.*declares"): + for _, data in _source_of_cell(stored(np.arange(4.0)) + b"extra").iter_data(): + data.as_dict() + + +def test_padding_bytes_are_stored_as_zero_and_do_not_change_the_hash() -> None: + """Garbage in structured padding never reaches the store, and is not part of the hash.""" + aligned = np.dtype([("a", "i1"), ("b", " None: + """A ``.npy`` file holding a pickled object array is refused on read.""" + converter = get_default_context().type_converter + ext = converter.python_type_to_arrow_type(np.ndarray) + assert isinstance(ext, pa.ExtensionType) + buffer = io.BytesIO() + np.lib.format.write_array( + buffer, np.array([1, "a"], dtype=object), version=(3, 0), allow_pickle=True + ) + storage = pa.array([buffer.getvalue()], type=ext.storage_type) table = pa.table( { "k": pa.array(["r"], pa.large_string()), "x": pa.ExtensionArray.from_storage(ext, storage), } ) - with pytest.raises((ValueError, RuntimeError), match="malformed array dtype tag"): + with pytest.raises((ValueError, RuntimeError), match="allow_pickle=False"): source = ArrowTableSource(table, tag_columns=["k"]) for _, data in source.iter_data(): data.as_dict() @@ -573,12 +650,16 @@ def nothing(x: np.ndarray) -> np.ndarray | None: # pyright: ignore[reportMissin # --------------------------------------------------------------------------- -# MergeJoin orders merged arrays by the stored (dtype tag, shape, data) +# MergeJoin orders merged arrays by their value hash # --------------------------------------------------------------------------- -def test_merge_join_orders_arrays_by_stored_tag_shape_and_data() -> None: - """``MergeJoin`` orders arrays by their stored dtype tag, shape and data.""" +def _hash_order(arrays: list[npt.NDArray[Any]]) -> list[npt.NDArray[Any]]: + return sorted(arrays, key=lambda a: semantic_hash(a).digest) + + +def test_merge_join_orders_arrays_by_their_value_hash() -> None: + """``MergeJoin`` orders merged arrays by their strict value hash, never by the stored bytes.""" small = np.array([7.5], dtype=np.float64) large = np.array([107.5], dtype=np.float64) other = np.array([1], dtype=np.int8) @@ -592,19 +673,15 @@ def test_merge_join_orders_arrays_by_stored_tag_shape_and_data() -> None: tag_columns=["k"], source_id="r", ) - got: dict[Any, Any] = { - t.as_dict()["k"]: d.as_dict()["x"] - for t, d in MergeJoin()(left, right).iter_data() - } - - def key(a: npt.NDArray[Any]) -> tuple[str, list[int], bytes]: - row = stored(a) - return (row["dtype"], row["shape"], row["data"]) - - for k, pair in (("a", [large, small]), ("b", [other, small])): - expected = sorted(pair, key=key) - assert [x.tobytes() for x in got[k]] == [x.tobytes() for x in expected] - assert [x.dtype for x in got[k]] == [x.dtype for x in expected] + for pair in ((left, right), (right, left)): + got: dict[Any, Any] = { + t.as_dict()["k"]: d.as_dict()["x"] + for t, d in MergeJoin()(*pair).iter_data() + } + for k, values in (("a", [large, small]), ("b", [other, small])): + expected = _hash_order(values) + assert [x.tobytes() for x in got[k]] == [x.tobytes() for x in expected] + assert [x.dtype for x in got[k]] == [x.dtype for x in expected] def test_merge_join_orders_arrays_that_differ_only_in_field_order() -> None: @@ -617,17 +694,24 @@ def test_merge_join_orders_arrays_that_differ_only_in_field_order() -> None: next(d.as_dict()["x"] for _, d in MergeJoin()(*pair).iter_data()) for pair in ((left, right), (right, left)) ] - expected = sorted([ab, ba], key=lambda a: stored(a)["dtype"]) + expected = _hash_order([ab, ba]) for got in outs: assert [x.dtype.names for x in got] == [x.dtype.names for x in expected] for x, e in zip(got, expected, strict=True): assert_arrays_identical(x, e) -def test_stored_tag_is_valid_json_for_structured() -> None: - """A structured dtype tag is valid JSON in declared field order.""" - tag = stored(np.zeros(1, dtype=[("b", " None: + """Merged arrays read back as each side gave them.""" + base = np.arange(6, dtype=np.float32).reshape(2, 3) + fortran_big = np.asfortranarray(base.astype(">f4")) + left = DictSource(data=[{"k": "a", "x": base}], tag_columns=["k"], source_id="l") + right = DictSource( + data=[{"k": "a", "x": fortran_big}], tag_columns=["k"], source_id="r" + ) + got: Any = next(d.as_dict()["x"] for _, d in MergeJoin()(left, right).iter_data()) + assert sorted(x.dtype.str for x in got) == ["f4"] + assert sorted(bool(x.flags.f_contiguous) for x in got) == [False, True] # --------------------------------------------------------------------------- diff --git a/tests/test_contexts/test_data_context_v1.py b/tests/test_contexts/test_data_context_v1.py index 730d171d0..28211bad5 100644 --- a/tests/test_contexts/test_data_context_v1.py +++ b/tests/test_contexts/test_data_context_v1.py @@ -200,7 +200,7 @@ def test_logical_type_registration_is_unchanged_by_the_move() -> None: assert sorted(registry._by_logical_name) == [ "orcapod.directory", "orcapod.file", - "orcapod.ndarray", + "orcapod.numpy.ndarray", "orcapod.path", "orcapod.upath", "orcapod.uuid", diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 200f90b28..866ca986a 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -349,6 +349,13 @@ def register_logical_classes() -> None: "ndarray.int64": np.arange(6, dtype=np.int64).reshape(2, 3), "ndarray.float64": np.array([[0.5, 1.5], [2.5, 3.5]], dtype=np.float64), "ndarray.empty": np.zeros((0,), dtype=np.float32), + # Added by ITL-981: the properties the strict array hash counts or never counts. + "ndarray.fortran": np.asfortranarray(np.arange(6, dtype=np.int32).reshape(2, 3)), + "ndarray.big_endian": np.arange(6, dtype=">i4").reshape(2, 3), + "ndarray.aligned_struct": np.array( + [(1, 0.5), (-2, 1.5)], dtype=np.dtype([("a", "i1"), ("b", "[large_list]", "single_field.list_path.schema_hash": "semantic_v1:a12297fadc22ec8a2ae7612d6a1b594fd9ed2a5188ac088d45fedbaa8d9992d8", - "single_field.ndarray.arrow_type": "extension[struct, data: large_binary>]", - "single_field.ndarray.schema_hash": "semantic_v1:28ef1316c5c6ada4ea3b406ab1ad16b6792b1fb23a823c614767a6227e4aecf5", + "single_field.ndarray.arrow_type": "extension[large_binary]", + "single_field.ndarray.schema_hash": "semantic_v1:1ce515cff952c91067233ab891f9ca53d22ac35f4e8acd02911eceeb697d9a03", "single_field.optional_int.arrow_type": "int64", "single_field.optional_int.schema_hash": "semantic_v1:7613bd137a749aba9e9c6f54091681242a22e408f5e8bf7a5caa47a212cc3141", "single_field.path.arrow_type": "extension[large_string]", @@ -105,8 +105,8 @@ "single_field.structural_twin_ab.schema_hash": "semantic_v1:ae904346e1568bac2921bb6b23be6fd1b53c2dfb267272a25707c13609e96702", "single_field.structural_twin_ba.arrow_type": "extension[struct]", "single_field.structural_twin_ba.schema_hash": "semantic_v1:fec2e905d7d0db153e5b9e5d9d443d698821c51154f304c8dc91fdb9afb6ce4e", - "single_field.structural_with_array.arrow_type": "extension[struct, data: large_binary>>]", - "single_field.structural_with_array.schema_hash": "semantic_v1:ce619b2aeb64419745accb29e02d830666a21b3227ce9da5ff6a98388d332b6a", + "single_field.structural_with_array.arrow_type": "extension[struct]", + "single_field.structural_with_array.schema_hash": "semantic_v1:9474e8c5367f510a353811fb48b9b4b82fd8f883f33fce83f0373d7ee4b4c4f2", "single_field.tuple_int_str.arrow_type": "struct", "single_field.tuple_int_str.schema_hash": "semantic_v1:84de6b5743e160faf5668eb29c7045f4757f150011645c3cec992ecd80d388f0", "single_field.uuid.arrow_type": "extension[large_binary]", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index 4020e77a9..e3caf2370 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -27,7 +27,7 @@ "type.list_int": "semantic_v1:388fcfad62b0e03893e350cabe27e6235aa52e5355df844aefce870cf2bbec75", "type.list_list_int": "semantic_v1:6c345e63e39c385db19c6d4abb56394fb31ea8412024ab8b33b67d3fec3340ef", "type.list_path": "semantic_v1:3f3ce39052a75cde9537faacc70ac47772c4ce7e8a0b94c69b0bd037b123bc36", - "type.ndarray": "semantic_v1:b8eb285c0293ba5612fd8424f14c9cc03b306f9e6bf5bbb36c18d09b5202aaec", + "type.ndarray": "semantic_v1:6b5d34c828127cad41bb98e65450250bc847d3e42da07bba377b49d1bbe2f9f5", "type.optional_int": "semantic_v1:aaf3880090849afb79550bdcd6865c455ded79bf372efb283b6a68b82c627736", "type.path": "semantic_v1:7f0da67eaf9d048c9d1791fbe223333cfcc8018c66878647c8b7ae8571c72bd3", "type.pydantic": "semantic_v1:58c6aad8ca32918a34da1dddb62c72392079b9326b021aa3a4c57de4ae0a4ea8", @@ -39,7 +39,7 @@ "type.structural_same_identity": "semantic_v1:e050877645fb628a5ae83ad6c88398a91f2437a599a65054127056b221077564", "type.structural_twin_ab": "semantic_v1:12ab17179b2f707e3551c93481b7eb7974d79805b2fa6a4e37890014bc8ca62a", "type.structural_twin_ba": "semantic_v1:8f0abe45554d4cd0c68d5c9497a819564f7268cc2631602a2e1db6471f0112c7", - "type.structural_with_array": "semantic_v1:845c46052e7e3ad39f229ac4690663aecf2a24e62983361268c2ec0a44752772", + "type.structural_with_array": "semantic_v1:4bf065b65cffed98adba483cb9f2f0918ed209b19e86abff4b1608decb6043ac", "type.tuple_int_str": "semantic_v1:df199a19b63c33f3b7a8d06276faabbc74082af71710d69223980e84f71434ed", "type.uuid": "semantic_v1:763acbee42cb7bd6c7734ad7ca12ab6cb879ace3991e075366b0536bd67556f0", "value.bool.false": "semantic_v1:fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa", @@ -69,10 +69,14 @@ "value.list.mixed": "semantic_v1:05e6eced3c7100adabbd220cc470b1a97e0e39263653ac62da7c950b86f7526c", "value.namedtuple.xy": "semantic_v1:8573505172fe4b9949938d0aee2358b361e51ef007ea7c7519f0cfdefb2de6b0", "value.namedtuple.yx": "semantic_v1:addb076ec3eee640559a2bdf5d8b901dd1f644f83ada7046d12a7792f7b85443", - "value.ndarray.empty": "ndarray_v1:5cdc53e446e7788fb4c77928e777e78d52c5b0d11af177950e428e49eb68c9d9", - "value.ndarray.float64": "ndarray_v1:4318d0446de3a8ad4b337f1d8582a2a29ffe84fd36cef095e2a9053da04d1aca", - "value.ndarray.int32": "ndarray_v1:c47c7f1dfd1002e9c8901f8c09cbb550cd95a72ad63d08d7b86c836d5bceab56", - "value.ndarray.int64": "ndarray_v1:997e3d95a1ebc06c669978196da71a558c4cac47074f8c5f57067c8b5d6390cf", + "value.ndarray.aligned_struct": "ndarray_v1:91f015765fc0be7a51bee61db5dc110d703016e0c1c285a9aac7bc3117281a00", + "value.ndarray.big_endian": "ndarray_v1:b552ddc9bdabb59ad235eca436d580eadac8f751dfb815c9294bdf760db9cfdf", + "value.ndarray.empty": "ndarray_v1:19f39d097025397bf15c1c486f46ed68c1e8a8c8288479c4cc5221a2b1cc8589", + "value.ndarray.float64": "ndarray_v1:ae046f92a521b1cedcf7f6f8f610c5424428f7fa636a10f7a92474770b749833", + "value.ndarray.fortran": "ndarray_v1:ebb2d9032d4ce00a58768eb0fe14789f31b69b55a985c28fc3eea1939a24fa36", + "value.ndarray.int32": "ndarray_v1:39b7bc559e6e952d7276e100740db800810c28371e3895fe467e4e0351c26f8a", + "value.ndarray.int64": "ndarray_v1:4018e3021187f9efa736a2a969040563083e455c353dfbd27c4ce0e15aad0d8c", + "value.ndarray.strided_view": "ndarray_v1:ed2e1e0dd1a376dec89828113a11c976e74972b54f58e9a28e0065d2ff88b8e2", "value.nested": "semantic_v1:1935b97bf8f80e850dbcab19482cacf2af7bab94d73c2c8e6bce44a0cc71c0ff", "value.none": "semantic_v1:74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b", "value.set.ints": "semantic_v1:d1ff6a34712272cbf01d0bc874fb41db6df312102bd073c95529df6e9ae738bf", diff --git a/tests/test_hashing/test_starfix_arrow_hasher.py b/tests/test_hashing/test_starfix_arrow_hasher.py index c5da1eee4..4395753c3 100644 --- a/tests/test_hashing/test_starfix_arrow_hasher.py +++ b/tests/test_hashing/test_starfix_arrow_hasher.py @@ -496,3 +496,63 @@ def test_table_unrelated_schema_metadata_does_not_affect_hash(self): _make_hasher().hash_table(t_clean).digest == _make_hasher().hash_table(t_noise).digest ) + + +class TestSchemaHashFieldRule: + """``hash_schema`` hashes a field under the same rule as ``hash_table`` (ITL-981).""" + + @staticmethod + def _identified(storage: pa.DataType, extension: pa.ExtensionType) -> pa.Schema: + """A plain ``storage`` field that carries the identifier of ``extension``.""" + return pa.schema( + [ + pa.field( + "c", + storage, + metadata={ + "ARROW:extension:name": extension.extension_name, + "ARROW:extension:metadata": extension.__arrow_ext_serialize__(), + }, + ) + ] + ) + + def test_value_hashed_type_does_not_depend_on_its_storage_type(self): + """A type hashed by a value handler is a ``large_binary`` field with its identifier.""" + import numpy as np + + import orcapod as op + from orcapod.contexts import get_default_context + + context = get_default_context() + hasher = context.arrow_hasher + assert isinstance(hasher, StarfixArrowHasher) + for python_type in (op.File, np.ndarray): + extension = context.type_converter.python_type_to_arrow_type(python_type) + assert isinstance(extension, pa.ExtensionType) + schema_hash = hasher.hash_schema(pa.schema([pa.field("c", extension)])) + assert schema_hash == hasher.hash_schema( + self._identified(pa.large_binary(), extension) + ) + # ``File`` is stored as a string; that storage type is not in its schema hash. + file_type = context.type_converter.python_type_to_arrow_type(op.File) + assert isinstance(file_type, pa.ExtensionType) + assert file_type.storage_type == pa.large_string() + assert hasher.hash_schema( + pa.schema([pa.field("c", file_type)]) + ) != hasher.hash_schema(self._identified(pa.large_string(), file_type)) + + def test_type_without_value_handler_keeps_its_storage_type(self): + """A type whose stored values are hashed keeps its storage type in the schema hash.""" + from pathlib import Path + + from orcapod.contexts import get_default_context + + context = get_default_context() + hasher = context.arrow_hasher + assert isinstance(hasher, StarfixArrowHasher) + extension = context.type_converter.python_type_to_arrow_type(Path) + assert isinstance(extension, pa.ExtensionType) + assert hasher.hash_schema( + pa.schema([pa.field("c", extension)]) + ) == hasher.hash_schema(self._identified(extension.storage_type, extension)) diff --git a/tests/test_logical_types/ignore_list_toy.py b/tests/test_logical_types/ignore_list_toy.py new file mode 100644 index 000000000..c769b53ad --- /dev/null +++ b/tests/test_logical_types/ignore_list_toy.py @@ -0,0 +1,63 @@ +"""A toy value type with an ignore list, for ``test_ignore_list.py`` (ITL-981). + +``Labelled`` has a content (``amount``) and one optional property (``label``). +The classes live in a module of their own so that a logical type registration +file can name them. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any + +import pyarrow as pa + +from orcapod.logical_types.ignore_list import IgnoreListHandler, IgnoreListLogicalType +from orcapod.logical_types.protocols import TypeConverterProtocol + +EXTENSION_NAME = "orcapod.test.labelled" + + +@dataclass(frozen=True) +class Labelled: + """A value with a content and a label that the hash can ignore.""" + + amount: int + label: str + + +class LogicalLabelled(IgnoreListLogicalType): + """Stores a ``Labelled`` as JSON text; the label is always stored.""" + + EXTENSION_NAME = EXTENSION_NAME + STORAGE_TYPE = pa.large_string() + VERSION = 1 + OPTIONS = frozenset({"label"}) + python_type: type = Labelled + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None = None + ) -> str: + """Return the JSON text of a ``Labelled``.""" + return json.dumps([value.amount, value.label]) + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None = None + ) -> Labelled: + """Return the ``Labelled`` of stored JSON text.""" + amount, label = json.loads(storage_value) + return Labelled(amount, label) + + +class LabelledHandler(IgnoreListHandler): + """Hashes the amount and the label; an ignored label is written as empty.""" + + OPTIONS = frozenset({"label"}) + HASH_LABEL = "labelled_v1" + + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return SHA-256 of the amount and the label field.""" + label = "" if "label" in ignore else obj.label + return hashlib.sha256(json.dumps([obj.amount, label]).encode()).digest() diff --git a/tests/test_logical_types/test_array_format.py b/tests/test_logical_types/test_array_format.py index bdbb24d31..83083dd14 100644 --- a/tests/test_logical_types/test_array_format.py +++ b/tests/test_logical_types/test_array_format.py @@ -1,9 +1,9 @@ -"""Tests for the shared array storage struct and value digest (ITL-888). +"""Tests for the element digest shared by array-like value types (ITL-888, ITL-981). The reference digest below is written independently of Orcapod, with ``struct`` and -``hashlib`` only, from the byte-level definition in -``superpowers/specs/2026-11-10-array-format-v0.3.md`` (section C1). It is the -definition other implementations (starfix, torch) must reproduce. +``hashlib`` only, from the byte-level definition in ``docs/reference/hashing.md`` +(section 8). It is the first step of the ``numpy.ndarray`` value hash, and the +definition other array types (torch) must reproduce. """ from __future__ import annotations @@ -11,14 +11,18 @@ import hashlib import struct -import pyarrow as pa import pytest -from orcapod.logical_types.array_format import ARRAY_STORAGE_TYPE, array_digest_v1 +from orcapod.logical_types.array_format import ( + NDARRAY_EXTENSION_NAME, + NDARRAY_OPTIONS, + array_digest_v1, + length_prefixed, +) def reference_digest(tag: str, shape: tuple[int, ...], data: bytes) -> bytes: - """Independent implementation of the v1 array digest.""" + """Independent implementation of the v1 element digest.""" t = tag.encode("utf-8") preimage = struct.pack(" None: def test_digest_separates_every_field() -> None: - """Every field of the stored row contributes to the digest.""" + """Every field of the preimage contributes to the digest.""" assert array_digest_v1("int8", (0, 3), b"") != array_digest_v1("int8", (3, 0), b"") assert array_digest_v1("int8", (), b"\x01") != array_digest_v1( "int8", (1,), b"\x01" @@ -78,14 +82,13 @@ def test_digest_rejects_negative_dimension() -> None: _ = array_digest_v1("int8", (-1,), b"") -def test_storage_type() -> None: - """The storage type is the dtype, shape and data struct of the spec.""" - # Children are nullable at the Arrow level: Polars drops nested non-null flags - # (ITL-842), so non-null is enforced by the logical type instead. - assert ARRAY_STORAGE_TYPE == pa.struct( - [ - pa.field("dtype", pa.large_string()), - pa.field("shape", pa.large_list(pa.int64())), - pa.field("data", pa.large_binary()), - ] - ) +def test_length_prefixed_text() -> None: + """A text field is its UTF-8 bytes after their 8-byte little-endian length.""" + assert length_prefixed("") == struct.pack(" None: + """The array type's extension name and the properties its hash can ignore.""" + assert NDARRAY_EXTENSION_NAME == "orcapod.numpy.ndarray" + assert NDARRAY_OPTIONS == {"byte_order", "memory_order", "struct_layout"} diff --git a/tests/test_logical_types/test_ignore_list.py b/tests/test_logical_types/test_ignore_list.py new file mode 100644 index 000000000..8c12b797d --- /dev/null +++ b/tests/test_logical_types/test_ignore_list.py @@ -0,0 +1,346 @@ +"""The shared ignore-list mechanism of library value types (ITL-981). + +Uses a toy type (``ignore_list_toy``) registered through a logical type +registration file, the way a data context configures a real type. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod.contexts as contexts_package +from orcapod.contexts.core import ContextValidationError, DataContext +from orcapod.contexts.registry import JSONDataContextRegistry +from orcapod.errors import TypeIdentityError +from orcapod.hashing.arrow_hashers import StarfixArrowHasher +from orcapod.logical_types.identifier import LogicalTypeIdentifierError +from orcapod.logical_types.ignore_list import ( + checked_ignore, + ignore_metadata, + parse_ignore_metadata, +) +from orcapod.semantic_types.universal_converter import UniversalTypeConverter +from tests.test_logical_types.ignore_list_toy import ( + EXTENSION_NAME, + Labelled, + LogicalLabelled, +) + +TOY_MODULE = "tests.test_logical_types.ignore_list_toy" +STRICT = b'{"ignore":[],"version":1}' +LENIENT = b'{"ignore":["label"],"version":1}' + + +def _context(tmp_path: Path, ignore: Any) -> DataContext: + """Build a data context that registers the toy type with ``ignore``.""" + shipped = Path(contexts_package.__file__).parent / "logical_types.json" + registration = json.loads(shipped.read_text()) + registration.setdefault("hash_ignore", {})[EXTENSION_NAME] = ignore + reference = {"_ref": f"hash_ignore:{EXTENSION_NAME}"} + registration["logical_type_registry"]["_config"]["logical_types"].append( + {"_class": f"{TOY_MODULE}.LogicalLabelled", "_config": {"ignore": reference}} + ) + registration["value_handlers"].append( + [ + {"_type": f"{TOY_MODULE}.Labelled"}, + { + "_class": f"{TOY_MODULE}.LabelledHandler", + "_config": {"ignore": reference}, + }, + ] + ) + name = "_".join(str(item) for item in ignore) if isinstance(ignore, list) else "" + custom = tmp_path / f"logical_types_{name or 'other'}.json" + custom.write_text(json.dumps(registration)) + return JSONDataContextRegistry(logical_types_file=custom).get_context() + + +def _table(context: DataContext, value: Labelled) -> pa.Table: + return context.type_converter.python_dicts_to_arrow_table( + [{"x": value}], python_schema={"x": Labelled} + ) + + +def _converter(context: DataContext) -> UniversalTypeConverter: + """Return the context's converter as the concrete class the databases use.""" + converter = context.type_converter + assert isinstance(converter, UniversalTypeConverter) + return converter + + +# --- metadata bytes ------------------------------------------------------------------ + + +def test_metadata_bytes_are_canonical_json_with_sorted_keys() -> None: + """The metadata is exactly the documented bytes.""" + assert ignore_metadata(1, []) == b'{"ignore":[],"version":1}' + assert ( + ignore_metadata(1, ["memory_order", "byte_order", "memory_order"]) + == b'{"ignore":["byte_order","memory_order"],"version":1}' + ) + assert ignore_metadata(2, ("a_1",)) == b'{"ignore":["a_1"],"version":2}' + + +@pytest.mark.parametrize("name", ["Memory", "1st", "a-b", "", "é", "a b"]) +def test_a_property_name_is_lower_case_snake_case(name: str) -> None: + """A name outside ``[a-z][a-z0-9_]*`` cannot be written into metadata.""" + with pytest.raises(ValueError, match="snake case"): + ignore_metadata(1, [name]) + + +@pytest.mark.parametrize("version", [0, -1, True, 1.0, "1"]) +def test_version_is_a_positive_integer(version: Any) -> None: + """The version is a positive ``int``.""" + with pytest.raises(ValueError, match="positive integer"): + ignore_metadata(version, []) + + +def test_parse_returns_what_was_written() -> None: + """Parsing is the inverse of writing.""" + assert parse_ignore_metadata(STRICT) == (1, ()) + assert parse_ignore_metadata(ignore_metadata(3, ["b", "a"])) == (3, ("a", "b")) + + +@pytest.mark.parametrize( + "metadata", + [ + b"", + b'{"version":1}', + b'{"version":1,"ignore":[]}', + b'{"ignore": [], "version": 1}', + b'{"ignore":["b","a"],"version":1}', + b'{"ignore":["a","a"],"version":1}', + b'{"ignore":[],"version":1,"extra":0}', + b'{"ignore":"a","version":1}', + b'{"ignore":[1],"version":1}', + b'{"ignore":[],"version":"1"}', + b'{"ignore":[],"version":0}', + b"not json", + b'{"ignore":' + b"[" * 100000 + b"]" * 100000 + b',"version":1}', + b"\xff\xfe", + b"[]", + ], +) +def test_parse_refuses_anything_but_the_canonical_bytes(metadata: bytes) -> None: + """Only the exact bytes ``ignore_metadata`` writes are accepted.""" + with pytest.raises(ValueError, match="ignore-list metadata|canonical form"): + parse_ignore_metadata(metadata) + + +def test_checked_ignore_sorts_and_refuses_unknown_names() -> None: + """An ignore list is sorted and unique; an unknown name is an error naming it.""" + options = frozenset({"a", "b", "c"}) + assert checked_ignore(["c", "a", "c"], options) == ("a", "c") + assert checked_ignore((), options) == () + with pytest.raises(ValueError, match="'d'.*allowed names: a, b, c"): + checked_ignore(["a", "d"], options) + with pytest.raises(ValueError, match="list of property names"): + checked_ignore("a", options) + + +# --- the list reaches the type, the bare value and the cell ---------------------------- + + +def test_the_configured_list_is_the_types_extension_metadata(tmp_path: Path) -> None: + """The data context's list is written into the Arrow extension metadata.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + strict_type = strict.type_converter.python_type_to_arrow_type(Labelled) + lenient_type = lenient.type_converter.python_type_to_arrow_type(Labelled) + assert isinstance(strict_type, pa.ExtensionType) + assert isinstance(lenient_type, pa.ExtensionType) + assert strict_type.extension_name == lenient_type.extension_name == EXTENSION_NAME + assert strict_type.__arrow_ext_serialize__() == STRICT + assert lenient_type.__arrow_ext_serialize__() == LENIENT + + +def test_two_lists_are_two_types(tmp_path: Path) -> None: + """Types of two lists are unequal, alone, nested, in a schema and when hashed.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + strict_type = strict.type_converter.python_type_to_arrow_type(Labelled) + lenient_type = lenient.type_converter.python_type_to_arrow_type(Labelled) + assert strict_type != lenient_type + assert strict_type == LogicalLabelled().get_arrow_extension_type() + assert pa.large_list(strict_type) != pa.large_list(lenient_type) + assert pa.schema([("x", strict_type)]) != pa.schema([("x", lenient_type)]) + with pytest.raises(pa.ArrowInvalid): + pa.concat_tables( + [_table(strict, Labelled(1, "a")), _table(lenient, Labelled(1, "a"))] + ) + # The type hash and the schema hash take the identifier, which holds the list. + strict_hasher, lenient_hasher = strict.arrow_hasher, lenient.arrow_hasher + assert isinstance(strict_hasher, StarfixArrowHasher) + assert isinstance(lenient_hasher, StarfixArrowHasher) + assert strict_hasher.hash_schema( + pa.schema([("x", strict_type)]) + ) != lenient_hasher.hash_schema(pa.schema([("x", lenient_type)])) + assert strict.semantic_hasher.hash_object( + Labelled + ) != lenient.semantic_hasher.hash_object(Labelled) + assert strict.arrow_hasher.hash_table( + _table(strict, Labelled(1, "a")).slice(0, 0) + ) != lenient.arrow_hasher.hash_table(_table(lenient, Labelled(1, "a")).slice(0, 0)) + + +def test_a_bare_value_is_hashed_with_the_contexts_list(tmp_path: Path) -> None: + """Outside a typed column the data context's list decides.""" + strict = _context(tmp_path, []).semantic_hasher + lenient = _context(tmp_path, ["label"]).semantic_hasher + one, other = Labelled(1, "a"), Labelled(1, "b") + assert strict.hash_object(one) != strict.hash_object(other) + assert lenient.hash_object(one) == lenient.hash_object(other) + assert lenient.hash_object(one) != lenient.hash_object(Labelled(2, "a")) + assert lenient.hash_object(one).method == "labelled_v1" + # The list is not a hash input: only the canonical field value is. + assert lenient.hash_object(one) == strict.hash_object(Labelled(1, "")) + + +def test_a_cell_is_hashed_with_the_list_of_its_column(tmp_path: Path) -> None: + """In a table the column's extension metadata decides, not the hashing context.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + one, other = Labelled(1, "a"), Labelled(1, "b") + assert strict.arrow_hasher.hash_table( + _table(strict, one) + ) != strict.arrow_hasher.hash_table(_table(strict, other)) + assert lenient.arrow_hasher.hash_table( + _table(lenient, one) + ) == lenient.arrow_hasher.hash_table(_table(lenient, other)) + # The same values in a strict column and in a lenient column hash differently: + # the cell token and the field carry the identifier, which holds the list. + assert strict.arrow_hasher.hash_table( + _table(strict, one) + ) != lenient.arrow_hasher.hash_table(_table(lenient, one)) + + +def test_an_unknown_option_name_fails_the_context_build(tmp_path: Path) -> None: + """A context that lists a name the type does not have cannot be built.""" + with pytest.raises(ContextValidationError, match="'colour'.*allowed names: label"): + _context(tmp_path, ["colour"]) + + +@pytest.mark.parametrize("value", [{"label": False}, "label", [1], None]) +def test_hash_ignore_must_be_a_list_of_names(tmp_path: Path, value: Any) -> None: + """Only a JSON list of names is an ignore list; an object or a string is refused.""" + with pytest.raises(ContextValidationError, match="list of property names"): + _context(tmp_path, value) + + +# --- Arrow and stored columns ---------------------------------------------------------- + + +def _ipc_round_trip(table: pa.Table) -> pa.Table: + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + return pa.ipc.open_stream(sink.getvalue()).read_all() + + +def test_arrow_ipc_keeps_the_list_of_each_column(tmp_path: Path) -> None: + """One class is registered with Arrow under the name; it restores any valid list.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + for context in (strict, lenient): + table = _table(context, Labelled(1, "a")) + assert _ipc_round_trip(table).schema == table.schema + + +def test_arrow_refuses_metadata_that_is_not_a_valid_list() -> None: + """Unknown option names, other versions and other bytes raise on deserialisation.""" + extension_class = type(LogicalLabelled().get_arrow_extension_type()) + for metadata in ( + b'{"ignore":["colour"],"version":1}', + b'{"ignore":[],"version":2}', + b'{"version":1}', + b"", + ): + with pytest.raises(LogicalTypeIdentifierError) as raised: + extension_class.__arrow_ext_deserialize__(pa.large_string(), metadata) + assert raised.value.stored.metadata == metadata + with pytest.raises(TypeIdentityError, match="storage_type"): + extension_class.__arrow_ext_deserialize__(pa.int64(), STRICT) + # A plain string column (Delta compaction) is the same storage family. + assert ( + extension_class.__arrow_ext_deserialize__(pa.string(), LENIENT) + == LogicalLabelled(["label"]).get_arrow_extension_type() + ) + + +def test_a_column_stored_under_another_list_is_refused_naming_both( + tmp_path: Path, +) -> None: + """A data context reads only columns stored under its own list.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + stored = _table(lenient, Labelled(1, "a")) + with pytest.raises(LogicalTypeIdentifierError) as raised: + _converter(strict).apply_logical_types(stored) + message = str(raised.value) + assert "'x'" in message + assert repr(LENIENT) in message and repr(STRICT) in message + # The same column as plain storage with the identifier in the field metadata, + # which is how a database returns it. + chunk = stored.column("x").chunk(0) + assert isinstance(chunk, pa.ExtensionArray) + plain = pa.table( + {"x": chunk.storage}, + schema=pa.schema( + [ + pa.field( + "x", + pa.large_string(), + metadata={ + "ARROW:extension:name": EXTENSION_NAME, + "ARROW:extension:metadata": LENIENT, + }, + ) + ] + ), + ) + with pytest.raises(LogicalTypeIdentifierError, match="'x'"): + _converter(strict).apply_logical_types(plain) + assert ( + _converter(lenient).apply_logical_types(plain)["x"].type + == stored.schema.field("x").type + ) + + +def test_concurrent_first_use_gives_one_type_per_list() -> None: + """Threads that first ask for a list at the same time all get the same Arrow type.""" + import sys + import threading + + from orcapod.logical_types.ignore_list import IgnoreListLogicalType + + interval = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + for trial in range(30): + + class Racy(IgnoreListLogicalType): + EXTENSION_NAME = f"orcapod.test.racy{trial}" + STORAGE_TYPE = pa.large_string() + VERSION = 1 + OPTIONS = frozenset({"a"}) + + barrier = threading.Barrier(8) + seen: list[pa.ExtensionType] = [] + + def ask(logical_type: type[IgnoreListLogicalType] = Racy) -> None: + barrier.wait() + seen.append(logical_type(["a"]).get_arrow_extension_type()) + + threads = [threading.Thread(target=ask) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len({id(t) for t in seen}) == 1 + finally: + sys.setswitchinterval(interval) diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index 37ff00b59..397b813b7 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -62,7 +62,10 @@ (pd.DataFrame, "pandas.dataframe"), (pd.Series, "pandas.series"), ] -ALL_BUILTINS = [*BUILTINS_WITH_EMPTY_LEGACY, (np.ndarray, "orcapod.ndarray")] +ALL_BUILTINS = [*BUILTINS_WITH_EMPTY_LEGACY, (np.ndarray, "orcapod.numpy.ndarray")] + +# A library value type writes its ignore list beside the version (ITL-981). +BUILTIN_METADATA = {"orcapod.numpy.ndarray": b'{"ignore":[],"version":1}'} def _converter(context: DataContext | None = None) -> UniversalTypeConverter: @@ -153,10 +156,10 @@ def _stored_column( @pytest.mark.parametrize(("python_type", "name"), ALL_BUILTINS) def test_builtin_type_writes_version_1(python_type: Any, name: str) -> None: - """Each built-in extension type has its expected name and writes ``VERSION_1``.""" + """Each built-in extension type has its expected name and writes version 1.""" arrow_type = _extension_type(python_type) assert arrow_type.extension_name == name - assert arrow_type.__arrow_ext_serialize__() == VERSION_1 + assert arrow_type.__arrow_ext_serialize__() == BUILTIN_METADATA.get(name, VERSION_1) @pytest.mark.parametrize(("python_type", "name"), ALL_BUILTINS) @@ -164,12 +167,12 @@ def test_builtin_identifier_is_declared_by_the_extension_type( python_type: Any, name: str ) -> None: """``logical_type_identifier`` of a built-in type is its extension name with - ``VERSION_1``. + its version-1 metadata. """ logical_type = _converter().get_logical_type(python_type) assert logical_type is not None assert logical_type_identifier(logical_type) == LogicalTypeIdentifier( - name, VERSION_1 + name, BUILTIN_METADATA.get(name, VERSION_1) ) @@ -458,11 +461,11 @@ def test_builtin_data_without_metadata_is_read_as_version_1( def test_array_without_metadata_is_refused() -> None: - """The array format never wrote empty metadata, so it does not read it.""" + """The array type never wrote empty metadata, so it does not read it.""" converter = _converter() array_type = _extension_type(np.ndarray) table = _stored_column( - b"orcapod.ndarray", b"", pa.array([], array_type.storage_type) + b"orcapod.numpy.ndarray", b"", pa.array([], array_type.storage_type) ) with pytest.raises(LogicalTypeIdentifierError): converter.load_logical_types(table) diff --git a/tests/test_logical_types/test_numpy_type.py b/tests/test_logical_types/test_numpy_type.py index 8a034a6f7..42fb9f362 100644 --- a/tests/test_logical_types/test_numpy_type.py +++ b/tests/test_logical_types/test_numpy_type.py @@ -15,10 +15,10 @@ def test_isinstance_logical_type(self): assert isinstance(LogicalNumpyArray(), LogicalTypeProtocol) def test_logical_type_name(self): - """The logical type name is ``orcapod.ndarray``.""" + """The logical type name is ``orcapod.numpy.ndarray``.""" from orcapod.logical_types.numpy_type import LogicalNumpyArray - assert LogicalNumpyArray().logical_type_name == "orcapod.ndarray" + assert LogicalNumpyArray().logical_type_name == "orcapod.numpy.ndarray" def test_python_type(self): from orcapod.logical_types.numpy_type import LogicalNumpyArray @@ -26,25 +26,47 @@ def test_python_type(self): assert LogicalNumpyArray().python_type is np.ndarray def test_arrow_ext_name(self): - """The Arrow extension name is ``orcapod.ndarray``.""" + """The Arrow extension name is ``orcapod.numpy.ndarray``.""" from orcapod.logical_types.numpy_type import LogicalNumpyArray assert ( LogicalNumpyArray().get_arrow_extension_type().extension_name - == "orcapod.ndarray" + == "orcapod.numpy.ndarray" ) def test_arrow_ext_storage_type(self): - """The Arrow storage type is the shared array struct.""" - from orcapod.logical_types.numpy_type import LogicalNumpyArray + """The Arrow storage type is ``large_binary``: one ``.npy`` file per value.""" + import pyarrow as pa - from orcapod.logical_types.array_format import ARRAY_STORAGE_TYPE + from orcapod.logical_types.numpy_type import LogicalNumpyArray assert ( LogicalNumpyArray().get_arrow_extension_type().storage_type - == ARRAY_STORAGE_TYPE + == pa.large_binary() ) + def test_ignore_list_is_the_extension_metadata(self): + """The ignore list is written into the extension metadata and makes another type.""" + from orcapod.logical_types.numpy_type import LogicalNumpyArray + + strict = LogicalNumpyArray().get_arrow_extension_type() + lenient = LogicalNumpyArray( + ["memory_order", "byte_order"] + ).get_arrow_extension_type() + assert strict.__arrow_ext_serialize__() == b'{"ignore":[],"version":1}' + assert ( + lenient.__arrow_ext_serialize__() + == b'{"ignore":["byte_order","memory_order"],"version":1}' + ) + assert strict != lenient + + def test_unknown_ignore_name_raises(self): + """A property name the array hash cannot ignore is refused.""" + from orcapod.logical_types.numpy_type import LogicalNumpyArray + + with pytest.raises(ValueError, match="'strides'"): + LogicalNumpyArray(["strides"]) + def test_arrow_ext_is_cached(self): from orcapod.logical_types.numpy_type import LogicalNumpyArray @@ -59,13 +81,17 @@ def test_polars_ext_is_cached(self): class TestLogicalNumpyArrayStorage: - def test_python_to_storage_returns_the_array_struct_row(self): - """``python_to_storage`` returns the array struct row: dtype tag, shape and data.""" + def test_python_to_storage_returns_a_npy_file(self): + """``python_to_storage`` returns a ``.npy`` file of format 3.0 that NumPy loads.""" + import io + from orcapod.logical_types.numpy_type import LogicalNumpyArray arr = np.array([1.0, 2.0, 3.0]) result = LogicalNumpyArray().python_to_storage(arr) - assert result == {"dtype": "float64", "shape": [3], "data": arr.tobytes()} + assert isinstance(result, bytes) + assert result[:8] == b"\x93NUMPY\x03\x00" + assert np.array_equal(np.load(io.BytesIO(result), allow_pickle=False), arr) def test_object_dtype_raises(self): """``python_to_storage`` refuses object arrays.""" @@ -127,10 +153,9 @@ def test_structured_record_array(self): assert np.array_equal(result["x"], arr["x"]) assert np.array_equal(result["y"], arr["y"]) - def test_fortran_order_reads_back_c_order(self): - """A Fortran-ordered array reads back C-contiguous with the same values.""" - # Memory order is not part of an array's identity: storage is C order. + def test_fortran_order_reads_back_fortran_order(self): + """A Fortran-ordered array reads back Fortran-contiguous with the same values.""" arr = np.asfortranarray(np.array([[1, 2], [3, 4]], dtype=np.float64)) result = self._rt(arr) assert np.array_equal(result, arr) - assert result.flags["C_CONTIGUOUS"] + assert result.flags["F_CONTIGUOUS"] and not result.flags["C_CONTIGUOUS"] diff --git a/tests/test_logical_types/test_registry_most_specific.py b/tests/test_logical_types/test_registry_most_specific.py index 3e9d6b272..85bb220d7 100644 --- a/tests/test_logical_types/test_registry_most_specific.py +++ b/tests/test_logical_types/test_registry_most_specific.py @@ -60,7 +60,7 @@ def test_ndarray_subclasses_resolve_to_the_array_type() -> None: registry = LogicalTypeRegistry(logical_types=[LogicalNumpyArray()]) for sub in (np.matrix, np.recarray, np.memmap): lt = registry.get_by_python_type(sub) - assert lt is not None and lt.logical_type_name == "orcapod.ndarray" + assert lt is not None and lt.logical_type_name == "orcapod.numpy.ndarray" def test_registered_subclass_wins_for_itself_and_its_subclasses() -> None: diff --git a/tests/test_logical_types/test_roundtrips.py b/tests/test_logical_types/test_roundtrips.py index fcf0328c1..d3bc1e6e7 100644 --- a/tests/test_logical_types/test_roundtrips.py +++ b/tests/test_logical_types/test_roundtrips.py @@ -275,11 +275,11 @@ def test_builtin_uuid_round_trip( def test_builtin_ndarray_round_trip( storage_backend: _StorageBackend, tmp_path: Path ) -> None: - """numpy.ndarray round-trips through storage with extension name ``orcapod.ndarray``. + """numpy.ndarray round-trips through storage with extension name ``orcapod.numpy.ndarray``. Tests both a simple 1-D float64 array and a structured (record) array with named fields, since structured arrays are a primary motivation for this type. - The read-side converter already knows about ``orcapod.ndarray`` because it is + The read-side converter already knows about ``orcapod.numpy.ndarray`` because it is registered in the default context (``contexts/logical_types.json``). """ import numpy as np @@ -303,7 +303,7 @@ def test_builtin_ndarray_round_trip( assert hasattr(field.type, "extension_name"), ( f"Expected extension type on field 'col', got plain type {field.type!r}" ) - assert field.type.extension_name == "orcapod.ndarray" + assert field.type.extension_name == "orcapod.numpy.ndarray" rows = read_converter.arrow_table_to_python_dicts(result) assert len(rows) == 1