From 431d17955ece0638c9edddffd34dc122d9e3fe38 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 12:44:54 -0800 Subject: [PATCH 01/22] test(arrays): failing tests, design note and plan for the strict array hash (ITL-981) WIP, local only: 28 of the 48 new tests fail on main's code by design. Co-Authored-By: Claude Fable 5.2 --- .../2026-11-12-itl-981-library-value-types.md | 69 ++++ ...1-12-itl-981-library-value-types-design.md | 283 +++++++++++++++ tests/test_arrays/test_array_strict_hash.py | 343 ++++++++++++++++++ 3 files changed, 695 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-981-library-value-types.md create mode 100644 superpowers/specs/2026-11-12-itl-981-library-value-types-design.md create mode 100644 tests/test_arrays/test_array_strict_hash.py diff --git a/superpowers/plans/2026-11-12-itl-981-library-value-types.md b/superpowers/plans/2026-11-12-itl-981-library-value-types.md new file mode 100644 index 000000000..7e209049f --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-981-library-value-types.md @@ -0,0 +1,69 @@ +# ITL-981 plan: library value types (mechanism + NumPy) + +Issue: https://linear.app/metamorphic/issue/ITL-981 +Branch: `eywalker/itl-981-library-value-types-native-storage-strict` (from `origin/main` at `c8e9eb21`) +Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md` + +## State (keep this section current; a successor continues from here) + +- November 12, 2026: design sent to the coordinator (one `ask`). Coordinator's reply: it goes + to the developer with the torch, polars and pandas specs; NO library code until the + coordinator replies again. Allowed meanwhile: this plan and the failing tests (steps 1-2). +- Open decisions (design note, last section): (1) refuse or read a column stored under + another ignore list; (2) label `ndarray_v1` or `ndarray_v2`; (3) keep refusing unit-less + datetimes and field titles; (4) metadata key order. The tests take (2) and (4) from two + constants at the top of `tests/test_arrays/test_array_strict_hash.py`. +- Nothing is pushed. The design note, this plan and the failing tests are local commits. + +## Steps + +1. [x] Plan (this file). +2. [x] Failing tests through the public interface: + `tests/test_arrays/test_array_strict_hash.py`. They fail on `main`'s code (the hash + ignores memory order and byte order, storage is the struct). +3. [ ] `src/orcapod/logical_types/ignore_list.py`: `ignore_metadata`, + `parse_ignore_metadata`, `checked_ignore`, `IgnoreListLogicalType`, `IgnoreListHandler`; + one Arrow extension class per (name, list), the registered class deserialising any valid + list. Polars extension keeps `metadata_str`. +4. [ ] Context loader (`contexts/registry.py`): `hash_ignore` of `logical_types.json` into + the reference table as `hash_ignore:` before anything is built. + `logical_types.json`: `"hash_ignore": {"orcapod.ndarray": []}` and the two `_ref`s. +5. [ ] `numpy_type.py`: storage `large_binary`, `.npy` format 3.0 write + (`numpy.lib.format.write_array`, `allow_pickle=False`), read after checking the magic; + refusals (masked, object, StringDType, longdouble, unit-less datetimes, titles, + out-of-order or overlapping offsets, header over 10,000 bytes); `storage_sort_key` is the + strict `D` of the decoded value. `array_format.py`: drop the storage struct and its + metadata constant, keep `array_digest_v1`. +6. [ ] `NumpyArrayHandler` as an `IgnoreListHandler` computing `D`; + `SemanticHashingVisitor.visit_extension` passes the column's list + (`handler.hash_ignoring`). +7. [ ] Rewrite the ITL-888 tests that assert the struct row and layout independence + (`tests/test_arrays/test_array_value_format.py`, `array_cases.py` + (`assert_arrays_identical` requires C order today), `tests/test_logical_types/test_array_format.py`, + `test_numpy_type.py`, `tests/test_hashing/test_numpy_handler.py`, + `tests/test_hashing/test_hash_label_rule.py` if it names the constant). +8. [ ] Golden: add `value.ndarray.{fortran,big_endian,aligned_struct,strided_view}` to + `tests/test_hashing/golden/cases.py`; regenerate; check the diff moves only the 32 values + listed in the design note (iii) and list each by cause in the PR. +9. [ ] Docs: `docs/concepts/arrays.md`, `docs/reference/hashing.md` section 8, + `docs/concepts/logical-type-identifiers.md`; "superseded" line in + `superpowers/specs/2026-11-10-array-format-v0.3.md`; CLAUDE.md and `.zed/rules` if a rule + changes (logical type identifiers: the ignore-list metadata shape). +10. [ ] `changelog.d/ITL-981.md`, `design_issues/ITL-981-LV1.md`. +11. [ ] Checks: `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/`. +12. [ ] Merge `origin/main` if it moved, push, PR against `main` (`Fixes ITL-981`), + `orca linear attach`, `orca worktree set --worktree current --pr `, `worker_done`. + +## Measurements behind the design + +Scripts were run in the session scratchpad (not kept). NumPy 2.2.6, pyarrow 23.0.1. Results +are stated in the design note, section (i). Two facts that shape the code: + +- `pa.ExtensionType` instances of one class compare equal whatever their serialised + metadata, unless the class defines `__eq__`. Two different classes under one extension + name compare unequal everywhere (`==`, schema equality, `concat_tables`, nested lists) and + round-trip through IPC and Parquet when the registered class's + `__arrow_ext_deserialize__` returns an instance of the class for the stored list. +- `read_array(allow_pickle=False)` refuses a header over 10,000 bytes (400 `f8` fields + give 11,584). diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md new file mode 100644 index 000000000..625099a7c --- /dev/null +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -0,0 +1,283 @@ +# ITL-981: library value types (design note, awaiting developer sign-off) + +Status: PROPOSAL, November 12, 2026. No library code is changed yet. +Measured with NumPy 2.2.6 and pyarrow 23.0.1 on a little-endian machine, writing with +`numpy.lib.format.write_array(version=(3, 0), allow_pickle=False)` and reading with +`numpy.lib.format.read_array(allow_pickle=False)`. "Measured" below means exactly that. + +Rule applied (proposed by the torch and polars workers, adopted here): a property that +storage does not round-trip can never count in the hash, because `hash(read(write(v)))` must +equal `hash(v)` or every cached re-run recomputes. + +## (i) numpy.ndarray + +### A. Properties + +Each line: option name; in the hash by default; does storage round-trip it. + +Always in the hash, cannot be ignored: + +1. `dtype`: yes / yes. Kind and width (`int32` is not `int64`), `datetime64` / `timedelta64` + unit and multiple (`[ns]`, `[10ms]`), string and void widths (`U5`, `S3`, `V4`), and for a + structured dtype the field names, declared field order, field types and subarray shapes, + at every depth. +2. `shape`: yes / yes. Zero-dim `()` and empty shapes `(0, 3)`, `(3, 0)` are all distinct. +3. `element_bits`: yes / yes. Bit exact: `-0.0`, NaN payloads, NUL padding of strings, every + byte of a void element. + +In the hash by default, can be ignored (these are the three option names): + +4. `memory_order`: yes / yes, as two values. `F` when the array is Fortran-contiguous and + not C-contiguous, otherwise `C`. This is exactly the flag `.npy` keeps (measured: an F + array and a transposed C array read back F-contiguous). +5. `byte_order`: yes / yes. Little or big, per scalar field (measured: `>f4`, `>U3`, + `>M8[ns]` and a structured dtype with mixed orders read back unchanged). It is absolute, + not "native or swapped", so a value hashes the same on every machine. +6. `struct_layout`: yes / yes. Field offsets and itemsize of a structured dtype, at every + depth (measured: an aligned dtype, a nested aligned dtype with a subarray field, and + trailing padding read back with identical offsets and itemsize). + +Never in the hash (no option): + +7. Strides beyond property 4 (step slices `a[:, ::2]`, reversed `a[::-1]`, broadcast views): + never / no. Measured: `.npy` writes them C-contiguous. This is where the round-trip rule + bites: a strided view hashes like its C-contiguous copy. +8. The `align=True` flag of a structured dtype: never / no (measured: `isalignedstruct` + reads back `False`; the offsets it produced are kept, property 6). +9. Padding bytes between structured fields: never / yes. They round-trip but NumPy does not + define them (`np.empty` leaves garbage, field assignment does not copy them), so they + cannot be hashed deterministically. +10. Subclass (`np.matrix`, `np.recarray`, `np.memmap`): never / no (measured: all read back + as plain `np.ndarray`; a recarray's `np.record` dtype reads back as the plain dtype). +11. `writeable` flag: never / no (measured: a read-only array reads back writeable). +12. dtype `metadata`: never / no (measured: reads back `None`). +13. Ownership (`base`, `OWNDATA`), `ALIGNED` flag, memory address: never / no. Not + properties of the value. + +Refused at write, as on main, with `UnsupportedValueError` naming the dtype: + +14. Masked arrays (the mask would be lost; `.npy` silently drops it, measured), object + dtype and structured dtypes with an object field, NumPy 2 `StringDType` (`.npy` itself + refuses both without pickle, measured), `longdouble` / `clongdouble` (platform width). +15. Unit-less `datetime64` / `timedelta64` and structured field titles: `.npy` round-trips + both (measured), but main refuses them and ITL-911 tracks them. Proposal: keep refusing + in v0.3; accepting them later changes no existing hash. +16. New refusals forced by `.npy`: a structured dtype with overlapping or out-of-order + field offsets (`np.save` raises, measured), and a dtype whose `.npy` header exceeds + 10,000 bytes (roughly 350 or more fields: `read_array(allow_pickle=False)` refuses to + parse a longer header, measured at 400 fields). Checked at write so nothing unreadable + is stored. + +### B. Hash + +Notation: `u8` one byte; `u64le` / `i64le` 8-byte little-endian; `lp(x) = u64le(len(utf8(x))) || utf8(x)`. + +Step 1, element digest `E`. Unchanged from main (`array_format.array_digest_v1`, shared +with torch): take the array as C order, little-endian, structured fields packed in declared +order (`canonical_array`), then +`E = SHA-256(lp(tag) || u64le(ndim) || i64le(each dim) || u64le(len(bytes)) || bytes)`, +where `tag` is main's dtype tag (`float32`, `datetime64[ns]`, `ucs4[5]`, or the JSON field +list for a structured dtype). `E` covers properties 1 to 3 and nothing else. + +Step 2, value hash `D = SHA-256(P)` with +`P = lp("numpy.ndarray") || E (32 bytes) || u8(memory_order) || lp(byte_order_text) || lp(struct_layout_text)`. + +- `memory_order`: 1 if `flags.f_contiguous and not flags.c_contiguous`, else 0. +- `byte_order_text`: one character per scalar leaf of the dtype, depth first in tag order (a + subarray field counts once): `<` little-endian, `>` big-endian, `|` for a type without + byte order (bool, 8-bit integers, `S`, `V`). NumPy's native marker `=` is resolved with + `sys.byteorder`. A plain `float32` gives `<`. +- `struct_layout_text`: compact JSON (`separators=(",", ":")`) of `L(dtype)`, where `L` is + `null` for an unstructured dtype and otherwise the list + `[itemsize, [offset_1, L(field_1)], [offset_2, L(field_2)], ...]` in declared field order; + a subarray field uses its base dtype. A plain `float32` gives `null`. + +Ignoring a property: its field is always present and is written with the canonical value +instead of the real one. `memory_order` 0; `byte_order_text` with every `>` replaced by `<`; +`struct_layout_text` computed from the packed dtype `tag_to_dtype(tag)`. So two values that +differ only in an ignored property have equal `D`, and `D` with all three ignored equals the +strict `D` of the canonical array (C order, little-endian, packed). The ignore list is not +in `P`; it is in the extension metadata, so it is in every type identifier and cell token. + +The result is `ContentHash("ndarray_v1", D)`. Proposal: keep the label `ndarray_v1`, because +nothing was released under it (v0.2 hashed the `.npy` file under another name); the +alternative is `ndarray_v2`. + +Changes from main: main's hash is `D = E`: no leading type text, and it ignores all three +options. Every array hash moves. The leading text makes an ndarray never equal a tensor. + +A cell of a stored table is hashed by decoding the cell to an array and computing `D` with +the ignore list of the cell's own column type. The stored bytes are never hashed. + +### C. Storage + +- Column: Arrow extension `orcapod.ndarray` over `large_binary`; one `.npy` file per cell. + Null array is a null cell. +- Write: `numpy.lib.format.write_array(buffer, array, version=(3, 0), allow_pickle=False)` + on the plain-`ndarray` view of the value (the user's array is not modified). `np.save` + itself cannot pin the version (it picks 1.0 or 3.0 by content); format 3.0 is the only + one that stores non-Latin-1 field names (measured: 1.0 and 2.0 raise on `β`). +- Read: check the magic is format 3.0, then + `numpy.lib.format.read_array(buffer, allow_pickle=False)` (default `max_header_size`). + The result is a fresh, writeable array. +- Round-trips (measured): dtype including byte order, units, widths, field names, offsets + and itemsize; shape; bits; C or Fortran contiguity. Does not: properties 7, 8, 10 to 13. +- Size: 128 bytes of header per value for simple dtypes (a 6-element `float32` array is 152 + bytes), against a few dozen on main. +- Security: `allow_pickle=False` on both sides, so no pickle is ever read; an object array + inside a stored file raises `ValueError` (measured). The header is parsed by NumPy with + `ast.literal_eval` under the 10,000-byte limit. A truncated or garbage cell raises + `ValueError` (measured). Orcapod never passes `allow_pickle=True` and never raises + `max_header_size`. +- The stored bytes are deterministic within one NumPy version (measured) but NumPy does not + promise them across versions. So `storage_sort_key` (MergeJoin sorts stored values) becomes + the strict `D` of the decoded value, not the stored bytes. + +## (ii) Shared mechanism + +### Extension metadata + +`json.dumps({"ignore": names, "version": N}, sort_keys=True, separators=(",", ":"), ensure_ascii=True)` +with `names` sorted by code point and unique. Exact bytes: + +- default: `{"ignore":[],"version":1}` +- non-default: `{"ignore":["byte_order","memory_order"],"version":1}` + +The `ignore` key is always present, also when empty. An option name matches +`[a-z][a-z0-9_]*`. On read the bytes are parsed and must re-serialise to the identical bytes; +keys other than `ignore` and `version` are an error. + +New module `orcapod/logical_types/ignore_list.py` (no heavy imports), beside `identifier.py` +(`version_metadata` stays for types without options): + +- `ignore_metadata(version: int, ignore: Iterable[str]) -> bytes` +- `parse_ignore_metadata(metadata: bytes) -> tuple[int, tuple[str, ...]]` +- `checked_ignore(ignore: Iterable[str], options: Collection[str]) -> tuple[str, ...]`: + sorted and unique; an unknown name raises `ValueError` naming it and the allowed names. +- `class IgnoreListLogicalType(BaseLogicalType)`: class attributes `EXTENSION_NAME`, + `STORAGE_TYPE`, `VERSION`, `OPTIONS`, `python_type`; `__init__(self, ignore=())`; property + `ignore`; implements `get_arrow_extension_type`, `get_polars_extension_type` and + `readable_extension_metadata`. A subclass adds `python_to_storage` / `storage_to_python`. +- `class IgnoreListHandler`: class attributes `OPTIONS`, `HASH_LABEL`; + `__init__(self, ignore=())`; `handle(obj, hasher)` returns + `hash_ignoring(obj, frozenset(self.ignore))`; `hash_ignoring(obj, ignore) -> ContentHash`; + a subclass implements `digest(self, obj, ignore: frozenset[str]) -> bytes`. + +Arrow type: one `pa.ExtensionType` class per (extension name, ignore list), so two lists are +unequal Arrow types everywhere (measured: `==`, schema equality, `concat_tables` and nested +`large_list` all tell them apart; IPC and Parquet round-trip the list). The class registered +with Arrow under the name deserialises any valid list to the class of that list; an unknown +name or non-canonical metadata raises `LogicalTypeIdentifierError`. + +### Configuration + +`contexts/logical_types.json` gets one new top-level object, the single place a list is +declared, keyed by extension name: + + "hash_ignore": {"orcapod.ndarray": []} + +The context loader puts each entry into the reference table as `hash_ignore:` +before it builds anything. The logical type entry and the value handler entry both take it: +`"_config": {"ignore": {"_ref": "hash_ignore:orcapod.ndarray"}}`. So: + +- (a) the logical type's Arrow extension metadata is `ignore_metadata(VERSION, self.ignore)`; +- (b) a bare value: the semantic hasher finds the handler by Python type and calls + `handle`, which uses the handler's configured list (the data context's); +- (c) a cell: `SemanticHashingVisitor.visit_extension` already has the cell's extension + type. When the handler is an `IgnoreListHandler` it calls + `handler.hash_ignoring(value, parse_ignore_metadata(column metadata)[1])`. The list comes + from the column, never from the context. The cell token stays + `::`, and the identifier text now contains the list. + +An unknown option name is an error at context build (`ContextValidationError` from the +`ValueError` of `checked_ignore`), for the logical type and for the handler. + +There is no user API in v0.3; a different list needs a custom `logical_types.json` +(`DataContextRegistry(logical_types_file=...)`). + +### Two lists meeting + +- Join / MergeJoin of two columns whose Arrow types carry different lists: + `InputValidationError` naming the column and both types, through the existing + `require_shared_arrow_type` (the two Arrow types are unequal; the class name printed in the + message carries the list). +- DECISION 1. A data context has exactly one list per type. Proposal: a column stored under + another valid list is refused on read with `LogicalTypeIdentifierError` naming the stored + and the readable metadata (the existing identifier rule; `readable_extension_metadata()` + is the configured list only). Reason: the Python schema says only `np.ndarray`, and the + schema hash and type hash take the identifier from the context's logical type, so a column + read under a foreign list would have a schema hash that disagrees with its cells. Reading + such a column needs a per-column Python type (the out-of-scope user API). The visitor + still takes the list from the column, so allowing it later changes no hash. + +## (iii) Golden entries + +Moved (32 values), by cause. Cause H: the hash definition (`D`). Cause M: the identifier +metadata `{"version":1}` to `{"ignore":[],"version":1}`. Cause S: storage type struct to +`large_binary`. + +- `semantic.json`: `value.ndarray.{int32,int64,float64,empty}` (H); `type.ndarray`, + `type.structural_with_array` (M). +- `schema.json`: `single_field.ndarray.{arrow_type,schema_hash}`, + `single_field.structural_with_array.{arrow_type,schema_hash}` (M, S). +- `arrow.json`: `extension_no_values.{all_null,zero_rows}.ndarray` (M). +- `datagram.json`: the 10 `ndarray.*` and 10 `structural_with_array.*` values (H, M; the + system tag column names move because they embed the schema hash). + +Added: `value.ndarray.{fortran,big_endian,aligned_struct,strided_view}` and pinned `D` +vectors in `test_golden_hashes.py`. No other entry moves; the regeneration diff is checked +for that. + +## (iv) Deleted from ITL-888 + +- `array_format.py`: `ARRAY_STORAGE_TYPE`, `ARRAY_FORMAT_METADATA`, `ARRAY_FORMAT_VERSION` + and the storage-struct text. `array_digest_v1` stays (it is `E`). +- `numpy_type.py`: the struct `python_to_storage` / `storage_to_python`, the + `(tag, shape, data)` sort key. `dtype_to_tag`, `tag_to_dtype`, `canonical_array`, + `array_bytes` stay (they produce `E`). +- `NumpyArrayHandler`: rewritten as an `IgnoreListHandler`. +- Docs: `arrays.md` sections "What counts as the same array" and "How an array is stored" + and `hashing.md` section 8 are rewritten; `logical-type-identifiers.md` gains the ignore + list. `superpowers/specs/2026-11-10-array-format-v0.3.md` gets a "superseded" line. +- Tests of the struct row (`test_array_value_format.py`, `test_array_format.py`, + `test_numpy_type.py`, `test_numpy_handler.py`) are rewritten for the new behaviour. + +## (v) Size and commit order + +About 30 files: 8 source, 10 test, 4 golden data plus cases and generator, 4 docs, +changelog fragment, design-issue file, plan. + +1. Plan under `superpowers/plans/`. +2. Failing tests through the public interface (hash distinguishes memory order, byte order, + layout; ignoring makes them equal; exact round-trip; join error; unknown option name). +3. `ignore_list.py` and the context loader (`hash_ignore`). +4. NumPy storage (`.npy`) and logical type. +5. NumPy handler (`D`) and the visitor's cell path. +6. Golden regeneration for array entries, with the moved-entry list. +7. Docs, changelog fragment, design-issue file. + +## (vi) How torch, polars and pandas plug in + +Each branch: + +1. adds its entry to `hash_ignore` in `logical_types.json` (`"orcapod.torch_tensor": []` + ...), and `{"ignore": {"_ref": "hash_ignore:"}}` to its logical type and handler + entries (also inside a deferred registration); +2. subclasses `IgnoreListLogicalType` with `EXTENSION_NAME`, `STORAGE_TYPE`, `VERSION`, + `OPTIONS`, `python_type`, and its two storage methods; +3. subclasses `IgnoreListHandler` with `OPTIONS`, `HASH_LABEL` and + `digest(obj, ignore) -> bytes`, writing each optional property's field with its + canonical value when its name is in `ignore`; +4. for torch, calls `array_format.array_digest_v1(tag, shape, data)` for `E`, unchanged. + +Nothing else is needed: the visitor, the loader, the metadata and the Arrow type classes +are shared. + +## Open decisions for the developer + +1. Reading a column stored under another list: refuse (proposed) or read. +2. Hash label: keep `ndarray_v1` (proposed) or `ndarray_v2`. +3. Keep refusing unit-less datetimes and field titles in v0.3 (proposed), although `.npy` + stores them. +4. Metadata key order: sorted keys, `{"ignore":[...],"version":1}` (proposed), so that + `json.dumps(sort_keys=True)` reproduces it; the brief sketched version first. diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py new file mode 100644 index 000000000..78a72336e --- /dev/null +++ b/tests/test_arrays/test_array_strict_hash.py @@ -0,0 +1,343 @@ +"""Strict array hash, exact ``.npy`` storage and the per-type ignore list (ITL-981). + +The hash label and the extension metadata bytes are two constants below: the +developer's decision on each changes only these lines. +""" + +from __future__ import annotations + +import hashlib +import json +import struct +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import numpy.typing as npt +import pyarrow as pa +import pytest + +import orcapod.contexts as contexts_package +from orcapod.contexts import get_default_context +from orcapod.contexts.core import ContextValidationError, DataContext +from orcapod.contexts.registry import JSONDataContextRegistry +from orcapod.core.operators import MergeJoin +from orcapod.errors import InputValidationError +from orcapod.sources import ArrowTableSource, DictSource + +HASH_LABEL = "ndarray_v1" +"""Label of an array value hash (open decision 2 of the design note).""" + + +def ignore_metadata(names: list[str]) -> bytes: + """Return the extension metadata for an ignore list (open decision 4: key order).""" + return json.dumps( + {"ignore": sorted(names), "version": 1}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode() + + +OPTIONS = ("byte_order", "memory_order", "struct_layout") + +BASE = np.arange(6, dtype=np.float32).reshape(2, 3) +PACKED = np.array([(1, 0.5), (-2, 1.5)], dtype=[("a", "i1"), ("b", "f4")), + "big_endian_field": ( + "byte_order", + PACKED, + PACKED.astype([("a", "i1"), ("b", ">f8")]), + ), + "aligned_struct": ("struct_layout", PACKED, ALIGNED), +} + + +def _lp(text: str) -> bytes: + data = text.encode("utf-8") + return struct.pack(" str: + """Reference implementation of the value hash with the standard library only.""" + p = _lp(tag) + struct.pack(" DataContext: + """Build a data context whose ndarray ignore list is ``names``.""" + shipped = Path(contexts_package.__file__).parent / "logical_types.json" + registration = json.loads(shipped.read_text()) + registration["hash_ignore"]["orcapod.ndarray"] = names + custom = tmp_path / f"logical_types_{'_'.join(names) or 'strict'}.json" + custom.write_text(json.dumps(registration)) + return JSONDataContextRegistry(logical_types_file=custom).get_context() + + +def _source(arrays: dict[str, Any], column: str = "x") -> DictSource: + return DictSource( + data=[{"k": k, column: v} for k, v in arrays.items()], + tag_columns=["k"], + source_id="arrays", + ) + + +def _read(stream: Any, column: str = "x") -> dict[str, Any]: + return {t.as_dict()["k"]: d.as_dict()[column] for t, d in stream.iter_data()} + + +def _layout(dtype: np.dtype[Any]) -> Any: + if dtype.subdtype is not None: + dtype = dtype.subdtype[0] + if dtype.names is None: + return None + fields = dtype.fields + assert fields is not None + return [dtype.itemsize] + [ + [fields[name][1], _layout(fields[name][0])] for name in dtype.names + ] + + +# --- the hash is strict by default ------------------------------------------------ + + +@pytest.mark.parametrize("name", sorted(PAIRS)) +def test_default_hash_distinguishes(name: str) -> None: + """By default memory order, byte order and structured layout are part of the hash.""" + _, one, other = PAIRS[name] + hasher = get_default_context().semantic_hasher + assert hasher.hash_object(one) != hasher.hash_object(other) + + +def test_hash_carries_the_label() -> None: + """An array value hash is labelled with the array algorithm's label.""" + assert get_default_context().semantic_hasher.hash_object(BASE).method == HASH_LABEL + + +def test_strided_view_hashes_like_its_contiguous_copy() -> None: + """A strided view is stored C-contiguous, so it hashes as its C-contiguous copy.""" + hasher = get_default_context().semantic_hasher + wide = np.arange(12, dtype=np.float32).reshape(2, 6) + view = wide[:, ::2] + assert not view.flags.c_contiguous + assert hasher.hash_object(view) == hasher.hash_object(np.ascontiguousarray(view)) + assert hasher.hash_object(BASE[::-1]) == hasher.hash_object( + np.ascontiguousarray(BASE[::-1]) + ) + + +def test_subclass_and_writeable_flag_do_not_count() -> None: + """What storage does not keep (subclass, writeable flag) is never in the hash.""" + hasher = get_default_context().semantic_hasher + ints = np.array([[1, 2], [3, 4]]) + assert hasher.hash_object(np.matrix(ints)) == hasher.hash_object(ints) + frozen = BASE.copy() + frozen.setflags(write=False) + assert hasher.hash_object(frozen) == hasher.hash_object(BASE) + + +def test_reference_vectors() -> None: + """The hash equals the reference implementation, field by field.""" + hasher = get_default_context().semantic_hasher + data = struct.pack("<6f", 0, 1, 2, 3, 4, 5) + assert hasher.hash_object(BASE).to_hex() == reference_digest( + "float32", (2, 3), data, 0, "<", "null" + ) + assert hasher.hash_object(np.asfortranarray(BASE)).to_hex() == reference_digest( + "float32", (2, 3), data, 1, "<", "null" + ) + assert hasher.hash_object(BASE.astype(">f4")).to_hex() == reference_digest( + "float32", (2, 3), data, 0, ">", "null" + ) + packed_bytes = struct.pack("f4"), + "big_endian_unicode": np.array(["ab", "c"], dtype=">U3"), + "aligned_struct": ALIGNED, + "mixed_endian_struct": PACKED.astype([("a", "i1"), ("b", ">f8")]), + "unicode_field_name": np.zeros(2, dtype=[("β", " None: + """Memory order, byte order and structured layout survive a source round trip.""" + expected = EXACT[name] + got = _read(_source({name: expected}))[name] + assert type(got) is np.ndarray + assert got.dtype == expected.dtype + assert got.dtype.str == expected.dtype.str + assert _layout(got.dtype) == _layout(expected.dtype) + assert got.shape == expected.shape + assert got.flags.c_contiguous == expected.flags.c_contiguous + assert got.flags.f_contiguous == expected.flags.f_contiguous + assert got.tobytes("A") == expected.tobytes("A") + assert got.flags.writeable + + +@pytest.mark.parametrize( + "name", sorted(EXACT) + ["strided_view", "matrix", "read_only"] +) +def test_hash_is_unchanged_by_storage(name: str) -> None: + """``hash(read(write(v))) == hash(v)``: a cached re-run never recomputes.""" + frozen = BASE.copy() + frozen.setflags(write=False) + extra = { + "strided_view": np.arange(12, dtype=np.float32).reshape(2, 6)[:, ::2], + "matrix": np.matrix([[1, 2], [3, 4]]), + "read_only": frozen, + } + value = {**EXACT, **extra}[name] + hasher = get_default_context().semantic_hasher + got = _read(_source({name: value}))[name] + assert hasher.hash_object(got) == hasher.hash_object(value) + + +def test_column_is_npy_bytes_under_the_default_identifier() -> None: + """An array column is ``orcapod.ndarray`` over ``large_binary`` holding ``.npy`` files.""" + field = _source({"a": BASE}).as_table().schema.field("x") + assert field.type.extension_name == "orcapod.ndarray" + assert field.type.storage_type == pa.large_binary() + assert field.type.__arrow_ext_serialize__() == ignore_metadata([]) + cell = _source({"a": BASE}).as_table().column("x").chunk(0).storage[0].as_py() + assert cell[:8] == b"\x93NUMPY\x03\x00" + + +def test_out_of_order_struct_offsets_are_refused() -> None: + """A structured dtype ``.npy`` cannot store is refused when written.""" + dtype = np.dtype( + { + "names": ["a", "b"], + "formats": ["i4", "f8"], + "offsets": [8, 0], + "itemsize": 16, + } + ) + with pytest.raises(ValueError, match="offset"): + _source({"a": np.zeros(2, dtype=dtype)}) + + +# --- the ignore list ---------------------------------------------------------------- + + +@pytest.mark.parametrize("name", sorted(PAIRS)) +def test_ignoring_a_property_gives_equal_hashes(name: str, tmp_path: Path) -> None: + """With a property in the ignore list, values differing only in it hash equal.""" + option, one, other = PAIRS[name] + hasher = _context_ignoring(tmp_path, [option]).semantic_hasher + assert hasher.hash_object(one) == hasher.hash_object(other) + + +@pytest.mark.parametrize("name", sorted(PAIRS)) +def test_ignoring_another_property_keeps_them_apart(name: str, tmp_path: Path) -> None: + """Ignoring one property leaves the other two in the hash.""" + option, one, other = PAIRS[name] + others = [o for o in OPTIONS if o != option] + hasher = _context_ignoring(tmp_path, others).semantic_hasher + assert hasher.hash_object(one) != hasher.hash_object(other) + + +def test_everything_ignored_equals_strict_hash_of_canonical_array( + tmp_path: Path, +) -> None: + """The ignore list is not a hash input: only the canonical field values are.""" + lenient = _context_ignoring(tmp_path, list(OPTIONS)).semantic_hasher + strict = get_default_context().semantic_hasher + swapped_fortran = np.asfortranarray(BASE.astype(">f4")) + assert lenient.hash_object(swapped_fortran) == strict.hash_object(BASE) + + +def test_ignore_list_is_in_the_extension_metadata(tmp_path: Path) -> None: + """Two ignore lists are two logical types: the list is in the identifier.""" + context = _context_ignoring(tmp_path, ["memory_order", "byte_order"]) + arrow_type = context.type_converter.python_type_to_arrow_type(np.ndarray) + assert isinstance(arrow_type, pa.ExtensionType) + assert arrow_type.__arrow_ext_serialize__() == ignore_metadata( + ["byte_order", "memory_order"] + ) + default_type = get_default_context().type_converter.python_type_to_arrow_type( + np.ndarray + ) + assert arrow_type != default_type + assert context.semantic_hasher.hash_object( + np.ndarray + ) != get_default_context().semantic_hasher.hash_object(np.ndarray) + + +def test_a_cell_is_hashed_with_its_columns_list(tmp_path: Path) -> None: + """A table hash uses the ignore list of the column, so it follows the column type.""" + context = _context_ignoring(tmp_path, ["memory_order"]) + + def table_hash(value: npt.NDArray[Any]) -> Any: + table = context.type_converter.python_dicts_to_arrow_table( + [{"x": value}], python_schema={"x": np.ndarray} + ) + return context.arrow_hasher.hash_table(table) + + assert table_hash(BASE) == table_hash(np.asfortranarray(BASE)) + assert table_hash(BASE) != table_hash(BASE.astype(">f4")) + + +def test_unknown_option_name_is_an_error(tmp_path: Path) -> None: + """A context that names an option the type does not have cannot be built.""" + with pytest.raises(ContextValidationError, match="memory_layout"): + _context_ignoring(tmp_path, ["memory_layout"]) + + +def test_merge_join_of_two_ignore_lists_is_an_error(tmp_path: Path) -> None: + """Joining one column typed with two ignore lists names the column and both types.""" + lenient = _context_ignoring(tmp_path, ["memory_order"]) + + def stream(context: DataContext) -> ArrowTableSource: + table = context.type_converter.python_dicts_to_arrow_table( + [{"k": "r1", "x": BASE}], python_schema={"k": str, "x": np.ndarray} + ) + return ArrowTableSource( + table, tag_columns=["k"], source_id="s", data_context=context + ) + + with pytest.raises(InputValidationError, match="'x'") as raised: + MergeJoin()(stream(get_default_context()), stream(lenient)).as_table() + assert "memory_order" in str(raised.value) + + +def test_native_byte_order_is_absolute() -> None: + """A native array hashes as its explicit byte order, whatever the machine.""" + hasher = get_default_context().semantic_hasher + explicit = "f4" + assert hasher.hash_object(BASE) == hasher.hash_object(BASE.astype(explicit)) From 3900ca6d0b3e03319a69d59e55f648bb1f087de7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 12:58:41 -0800 Subject: [PATCH 02/22] docs(specs): extension name numpy.ndarray and the v0.2 name collision (ITL-981) Co-Authored-By: Claude Fable 5.2 --- .../2026-11-12-itl-981-library-value-types.md | 2 +- ...1-12-itl-981-library-value-types-design.md | 44 +++++++++++++++++-- tests/test_arrays/test_array_strict_hash.py | 6 +-- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/superpowers/plans/2026-11-12-itl-981-library-value-types.md b/superpowers/plans/2026-11-12-itl-981-library-value-types.md index 7e209049f..879ac0f36 100644 --- a/superpowers/plans/2026-11-12-itl-981-library-value-types.md +++ b/superpowers/plans/2026-11-12-itl-981-library-value-types.md @@ -27,7 +27,7 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md list. Polars extension keeps `metadata_str`. 4. [ ] Context loader (`contexts/registry.py`): `hash_ignore` of `logical_types.json` into the reference table as `hash_ignore:` before anything is built. - `logical_types.json`: `"hash_ignore": {"orcapod.ndarray": []}` and the two `_ref`s. + `logical_types.json`: `"hash_ignore": {"numpy.ndarray": []}` and the two `_ref`s. 5. [ ] `numpy_type.py`: storage `large_binary`, `.npy` format 3.0 write (`numpy.lib.format.write_array`, `allow_pickle=False`), read after checking the magic; refusals (masked, object, StringDType, longdouble, unit-less datetimes, titles, diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md index 625099a7c..86283e7f9 100644 --- a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -111,7 +111,7 @@ the ignore list of the cell's own column type. The stored bytes are never hashed ### C. Storage -- Column: Arrow extension `orcapod.ndarray` over `large_binary`; one `.npy` file per cell. +- Column: Arrow extension `numpy.ndarray` over `large_binary`; one `.npy` file per cell. Null array is a null cell. - Write: `numpy.lib.format.write_array(buffer, array, version=(3, 0), allow_pickle=False)` on the plain-`ndarray` view of the value (the user's array is not modified). `np.save` @@ -174,11 +174,11 @@ name or non-canonical metadata raises `LogicalTypeIdentifierError`. `contexts/logical_types.json` gets one new top-level object, the single place a list is declared, keyed by extension name: - "hash_ignore": {"orcapod.ndarray": []} + "hash_ignore": {"numpy.ndarray": []} The context loader puts each entry into the reference table as `hash_ignore:` before it builds anything. The logical type entry and the value handler entry both take it: -`"_config": {"ignore": {"_ref": "hash_ignore:orcapod.ndarray"}}`. So: +`"_config": {"ignore": {"_ref": "hash_ignore:numpy.ndarray"}}`. So: - (a) the logical type's Arrow extension metadata is `ignore_metadata(VERSION, self.ignore)`; - (b) a bare value: the semantic hasher finds the handler by Python type and calls @@ -260,7 +260,7 @@ changelog fragment, design-issue file, plan. Each branch: -1. adds its entry to `hash_ignore` in `logical_types.json` (`"orcapod.torch_tensor": []` +1. adds its entry to `hash_ignore` in `logical_types.json` (`"torch.tensor": []` ...), and `{"ignore": {"_ref": "hash_ignore:"}}` to its logical type and handler entries (also inside a deferred registration); 2. subclasses `IgnoreListLogicalType` with `EXTENSION_NAME`, `STORAGE_TYPE`, `VERSION`, @@ -273,6 +273,42 @@ Each branch: Nothing else is needed: the visitor, the loader, the metadata and the Arrow type classes are shared. +## (vii) Extension names are `.` (developer, November 12, 2026) + +The array type's extension name (and logical type name) is `numpy.ndarray`, not +`orcapod.ndarray`; torch uses `torch.tensor`; `pandas.dataframe`, `pandas.series`, +`polars.dataframe`, `polars.series` are unchanged; Orcapod's own types keep `orcapod.*`. +The leading type text of the value hash is the extension name (`numpy.ndarray`, +`torch.tensor`); for NumPy that is the text section (i) B already has. + +What the rename touches: + +- One constant (`NDARRAY_EXTENSION_NAME`) and the `hash_ignore` key. Tests and docs that + spell the name: 8 test files, `arrays.md`, `hashing.md`, `logical-type-identifiers.md`. +- Golden: no further entry. The same 32 values move; cause M now includes the name. +- Neither pyarrow nor NumPy registers `numpy.ndarray` (measured, pyarrow 23.0.1: only + `arrow.*` names are registered). +- v0.2 stored arrays as `numpy.ndarray` with EMPTY metadata over `large_binary`, which is + also the new storage type. Measured by setting the name on main and running + `tests/test_arrays/test_array_v02_store.py`: 2 of 6 tests fail, because every pyarrow or + Delta read of a v0.2 table (also the read that only counts its rows) now calls the + registered class's `__arrow_ext_deserialize__`, which raises. With the new storage the + storage check would pass and the empty metadata would raise + `LogicalTypeIdentifierError` instead: a v0.2 store could then not even be counted. +- So the registered Arrow class must not raise for metadata it does not know. It returns a + "foreign" extension type that keeps the stored name and metadata bytes unchanged and is + equal to no current type. Orcapod's own read (`require_readable`) refuses such a column + with `LogicalTypeIdentifierError` naming the column, as today, when the column is to be + used. A v0.2 table is never read as current, so it is counted and left unchanged as on + main. This is the mechanism of section (ii) (a class per list, dispatch on the stored + metadata) with one more case, so it costs no extra file. +- The empty v0.2 metadata is NOT listed in `also_reads`: v0.3 does not read v0.2 arrays + (other format version, other hash). +- `tests/test_arrays/data/make_v02_array_store.py` recognises a v0.2 installation by + `logical_type_name == "numpy.ndarray"`; that guard becomes a version check. +- `LogicalTypeIdentifierError` messages for a v0.2 array column now name `numpy.ndarray` + with empty metadata, which reads correctly ("written by a different version"). + ## Open decisions for the developer 1. Reading a column stored under another list: refuse (proposed) or read. diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py index 78a72336e..d2e160119 100644 --- a/tests/test_arrays/test_array_strict_hash.py +++ b/tests/test_arrays/test_array_strict_hash.py @@ -87,7 +87,7 @@ def _context_ignoring(tmp_path: Path, names: list[str]) -> DataContext: """Build a data context whose ndarray ignore list is ``names``.""" shipped = Path(contexts_package.__file__).parent / "logical_types.json" registration = json.loads(shipped.read_text()) - registration["hash_ignore"]["orcapod.ndarray"] = names + registration["hash_ignore"]["numpy.ndarray"] = names custom = tmp_path / f"logical_types_{'_'.join(names) or 'strict'}.json" custom.write_text(json.dumps(registration)) return JSONDataContextRegistry(logical_types_file=custom).get_context() @@ -229,9 +229,9 @@ def test_hash_is_unchanged_by_storage(name: str) -> None: def test_column_is_npy_bytes_under_the_default_identifier() -> None: - """An array column is ``orcapod.ndarray`` over ``large_binary`` holding ``.npy`` files.""" + """An array column is ``numpy.ndarray`` over ``large_binary`` holding ``.npy`` files.""" field = _source({"a": BASE}).as_table().schema.field("x") - assert field.type.extension_name == "orcapod.ndarray" + assert field.type.extension_name == "numpy.ndarray" assert field.type.storage_type == pa.large_binary() assert field.type.__arrow_ext_serialize__() == ignore_metadata([]) cell = _source({"a": BASE}).as_table().column("x").chunk(0).storage[0].as_py() From 962baa636f935691057f6562f92012bfa241b4cd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:00:15 -0800 Subject: [PATCH 03/22] docs(specs): scope the Arrow schema hash question (ITL-981) Co-Authored-By: Claude Fable 5.2 --- ...1-12-itl-981-library-value-types-design.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md index 86283e7f9..d4fcb56d5 100644 --- a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -317,3 +317,37 @@ What the rename touches: stores them. 4. Metadata key order: sorted keys, `{"ignore":[...],"version":1}` (proposed), so that `json.dumps(sort_keys=True)` reproduces it; the brief sketched version first. + +## (viii) Scoping: storage type inside the Arrow schema hash (not part of ITL-981 unless decided) + +Question from the coordinator: `StarfixArrowHasher.hash_schema` hashes a logical-type field +as its storage type plus the `ARROW:extension:*` keys, so that hash depends on the storage +type. Should a logical-type field be hashed as a fixed placeholder plus name and metadata? + +Findings (read from the code and measured on main): + +- No library code calls `hash_schema` (the only match under `src/` is its definition). It is + reached only by tests and by 47 golden values in `arrow.json` (`schema.*`, 42 plain; + `extension_schema.extension.*`, 5). +- Every schema or type hash Orcapod stores or embeds goes through the semantic hasher over + Python types: the schema hash (Site 3) and the system tag column names built from it, the + type hash, the pipeline hash and the paths built from it, the function signature. There a + logical type contributes its identifier text only (`canonical_annotation_str`, + `TypeObjectHandler`): name and metadata, never the storage type. +- `hash_table` (default source ids, row and datagram hashes) hashes a value-hashed column as + `large_binary` plus the keys, also with zero rows (ITL-967): already independent of + storage. Value-hashed today: UUID, File, Directory, ndarray, pandas DataFrame and Series, + and `list` / `set` of them. +- Types without a value handler (Path, UPath, dataclass, pydantic, structural classes, + lists of them, unregistered extension types) are hashed in `hash_table` as their stored + values under the storage type. For them the stored struct is the value, so the storage + type must stay in the hash. + +So no stored or embedded hash depends on a value-hashed type's storage type today; only the +unused public method does. If it is to be made consistent, the change is to give +`hash_schema` the same field rule as `hash_table` (`SemanticHashingVisitor.hashed_field`): +about 10 lines, one test, and golden moves limited to `arrow.json` +`extension_schema.extension.*` entries of value-hashed types whose storage is not already +`large_binary` (none of the five there today: `uuid` is `large_binary`, `path` has no value +handler). Recommendation: not needed for the tag; leave it to ITL-980, or take the +10-line alignment in ITL-981 if the developer wants the public method consistent. From ab2684f5fdb6c51f38e10282bcefa2cd1f8a4df8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:00:49 -0800 Subject: [PATCH 04/22] docs(plans): record coordinator decisions while waiting for sign-off (ITL-981) Co-Authored-By: Claude Fable 5.2 --- .../plans/2026-11-12-itl-981-library-value-types.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/superpowers/plans/2026-11-12-itl-981-library-value-types.md b/superpowers/plans/2026-11-12-itl-981-library-value-types.md index 879ac0f36..4397e93e9 100644 --- a/superpowers/plans/2026-11-12-itl-981-library-value-types.md +++ b/superpowers/plans/2026-11-12-itl-981-library-value-types.md @@ -13,6 +13,16 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md another ignore list; (2) label `ndarray_v1` or `ndarray_v2`; (3) keep refusing unit-less datetimes and field titles; (4) metadata key order. The tests take (2) and (4) from two constants at the top of `tests/test_arrays/test_array_strict_hash.py`. +- November 12, later: (a) developer decision relayed: extension names are `.`, + so the array type is `numpy.ndarray` (design note vii). The Arrow class registered under + the name must never raise for metadata it does not know (v0.2 stored `numpy.ndarray` with + empty metadata): it returns a "foreign" type keeping name and bytes; Orcapod refuses the + column when it is used. Approved by the coordinator. No `also_reads` for v0.2. + (b) Coordinator: take the 10-line alignment of `StarfixArrowHasher.hash_schema` with + `hash_table` (`SemanticHashingVisitor.hashed_field`) inside ITL-981, with a test, AFTER + sign-off; say in the PR body that no stored or golden value moves (design note viii). + (c) Finding posted as a comment on Linear ITL-980. + STILL WAITING for the developer's sign-off of the whole design: no library code, no push. - Nothing is pushed. The design note, this plan and the failing tests are local commits. ## Steps @@ -37,6 +47,9 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md 6. [ ] `NumpyArrayHandler` as an `IgnoreListHandler` computing `D`; `SemanticHashingVisitor.visit_extension` passes the column's list (`handler.hash_ignoring`). +6b. [ ] `StarfixArrowHasher.hash_schema`: apply `SemanticHashingVisitor.hashed_field` to each + field before hashing (same rule as `hash_table`); test that a value-hashed type's schema + hash does not depend on its storage type and that no golden value moves. 7. [ ] Rewrite the ITL-888 tests that assert the struct row and layout independence (`tests/test_arrays/test_array_value_format.py`, `array_cases.py` (`assert_arrays_identical` requires C order today), `tests/test_logical_types/test_array_format.py`, From 592274a81010461801f3a2ca65ac454bc3ed8e6a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:10:12 -0800 Subject: [PATCH 05/22] docs(specs): final extension name orcapod.numpy.ndarray; drop the foreign type (ITL-981) Co-Authored-By: Claude Fable 5.2 --- .../2026-11-12-itl-981-library-value-types.md | 7 +- ...1-12-itl-981-library-value-types-design.md | 75 +++++++++---------- tests/test_arrays/test_array_strict_hash.py | 14 +++- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/superpowers/plans/2026-11-12-itl-981-library-value-types.md b/superpowers/plans/2026-11-12-itl-981-library-value-types.md index 4397e93e9..f78983314 100644 --- a/superpowers/plans/2026-11-12-itl-981-library-value-types.md +++ b/superpowers/plans/2026-11-12-itl-981-library-value-types.md @@ -22,6 +22,11 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md `hash_table` (`SemanticHashingVisitor.hashed_field`) inside ITL-981, with a test, AFTER sign-off; say in the PR body that no stored or golden value moves (design note viii). (c) Finding posted as a comment on Linear ITL-980. + (d) FINAL naming (supersedes (a)): `orcapod.numpy.ndarray`. v0.2's `numpy.ndarray` is then + not a registered name, so the "foreign type" is dropped: the registered Arrow class + dispatches valid lists and raises `LogicalTypeIdentifierError` for anything else, as every + built-in type does. The Linear comment on ITL-980 could not be posted from this session + (permission denied); the coordinator posts it. STILL WAITING for the developer's sign-off of the whole design: no library code, no push. - Nothing is pushed. The design note, this plan and the failing tests are local commits. @@ -37,7 +42,7 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md list. Polars extension keeps `metadata_str`. 4. [ ] Context loader (`contexts/registry.py`): `hash_ignore` of `logical_types.json` into the reference table as `hash_ignore:` before anything is built. - `logical_types.json`: `"hash_ignore": {"numpy.ndarray": []}` and the two `_ref`s. + `logical_types.json`: `"hash_ignore": {"orcapod.numpy.ndarray": []}` and the two `_ref`s. 5. [ ] `numpy_type.py`: storage `large_binary`, `.npy` format 3.0 write (`numpy.lib.format.write_array`, `allow_pickle=False`), read after checking the magic; refusals (masked, object, StringDType, longdouble, unit-less datetimes, titles, diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md index d4fcb56d5..a50b9c258 100644 --- a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -80,7 +80,7 @@ where `tag` is main's dtype tag (`float32`, `datetime64[ns]`, `ucs4[5]`, or the list for a structured dtype). `E` covers properties 1 to 3 and nothing else. Step 2, value hash `D = SHA-256(P)` with -`P = lp("numpy.ndarray") || E (32 bytes) || u8(memory_order) || lp(byte_order_text) || lp(struct_layout_text)`. +`P = lp("orcapod.numpy.ndarray") || E (32 bytes) || u8(memory_order) || lp(byte_order_text) || lp(struct_layout_text)`. - `memory_order`: 1 if `flags.f_contiguous and not flags.c_contiguous`, else 0. - `byte_order_text`: one character per scalar leaf of the dtype, depth first in tag order (a @@ -111,7 +111,7 @@ the ignore list of the cell's own column type. The stored bytes are never hashed ### C. Storage -- Column: Arrow extension `numpy.ndarray` over `large_binary`; one `.npy` file per cell. +- Column: Arrow extension `orcapod.numpy.ndarray` over `large_binary`; one `.npy` file per cell. Null array is a null cell. - Write: `numpy.lib.format.write_array(buffer, array, version=(3, 0), allow_pickle=False)` on the plain-`ndarray` view of the value (the user's array is not modified). `np.save` @@ -174,11 +174,11 @@ name or non-canonical metadata raises `LogicalTypeIdentifierError`. `contexts/logical_types.json` gets one new top-level object, the single place a list is declared, keyed by extension name: - "hash_ignore": {"numpy.ndarray": []} + "hash_ignore": {"orcapod.numpy.ndarray": []} The context loader puts each entry into the reference table as `hash_ignore:` before it builds anything. The logical type entry and the value handler entry both take it: -`"_config": {"ignore": {"_ref": "hash_ignore:numpy.ndarray"}}`. So: +`"_config": {"ignore": {"_ref": "hash_ignore:orcapod.numpy.ndarray"}}`. So: - (a) the logical type's Arrow extension metadata is `ignore_metadata(VERSION, self.ignore)`; - (b) a bare value: the semantic hasher finds the handler by Python type and calls @@ -260,7 +260,7 @@ changelog fragment, design-issue file, plan. Each branch: -1. adds its entry to `hash_ignore` in `logical_types.json` (`"torch.tensor": []` +1. adds its entry to `hash_ignore` in `logical_types.json` (`"orcapod.torch.tensor": []` ...), and `{"ignore": {"_ref": "hash_ignore:"}}` to its logical type and handler entries (also inside a deferred registration); 2. subclasses `IgnoreListLogicalType` with `EXTENSION_NAME`, `STORAGE_TYPE`, `VERSION`, @@ -273,41 +273,36 @@ Each branch: Nothing else is needed: the visitor, the loader, the metadata and the Arrow type classes are shared. -## (vii) Extension names are `.` (developer, November 12, 2026) - -The array type's extension name (and logical type name) is `numpy.ndarray`, not -`orcapod.ndarray`; torch uses `torch.tensor`; `pandas.dataframe`, `pandas.series`, -`polars.dataframe`, `polars.series` are unchanged; Orcapod's own types keep `orcapod.*`. -The leading type text of the value hash is the extension name (`numpy.ndarray`, -`torch.tensor`); for NumPy that is the text section (i) B already has. - -What the rename touches: - -- One constant (`NDARRAY_EXTENSION_NAME`) and the `hash_ignore` key. Tests and docs that - spell the name: 8 test files, `arrays.md`, `hashing.md`, `logical-type-identifiers.md`. -- Golden: no further entry. The same 32 values move; cause M now includes the name. -- Neither pyarrow nor NumPy registers `numpy.ndarray` (measured, pyarrow 23.0.1: only - `arrow.*` names are registered). -- v0.2 stored arrays as `numpy.ndarray` with EMPTY metadata over `large_binary`, which is - also the new storage type. Measured by setting the name on main and running - `tests/test_arrays/test_array_v02_store.py`: 2 of 6 tests fail, because every pyarrow or - Delta read of a v0.2 table (also the read that only counts its rows) now calls the - registered class's `__arrow_ext_deserialize__`, which raises. With the new storage the - storage check would pass and the empty metadata would raise - `LogicalTypeIdentifierError` instead: a v0.2 store could then not even be counted. -- So the registered Arrow class must not raise for metadata it does not know. It returns a - "foreign" extension type that keeps the stored name and metadata bytes unchanged and is - equal to no current type. Orcapod's own read (`require_readable`) refuses such a column - with `LogicalTypeIdentifierError` naming the column, as today, when the column is to be - used. A v0.2 table is never read as current, so it is counted and left unchanged as on - main. This is the mechanism of section (ii) (a class per list, dispatch on the stored - metadata) with one more case, so it costs no extra file. -- The empty v0.2 metadata is NOT listed in `also_reads`: v0.3 does not read v0.2 arrays - (other format version, other hash). -- `tests/test_arrays/data/make_v02_array_store.py` recognises a v0.2 installation by - `logical_type_name == "numpy.ndarray"`; that guard becomes a version check. -- `LogicalTypeIdentifierError` messages for a v0.2 array column now name `numpy.ndarray` - with empty metadata, which reads correctly ("written by a different version"). +## (vii) Extension names are `orcapod..` (developer, November 12, 2026, final) + +First decision of the day: `.` (`numpy.ndarray`). Revised the same day, +because using the libraries' own namespaces is dangerous: prepend `orcapod.`. Final names: +`orcapod.numpy.ndarray`, `orcapod.torch.tensor`, `orcapod.pandas.dataframe`, +`orcapod.pandas.series`, `orcapod.polars.dataframe`, `orcapod.polars.series`; +`orcapod.file` and `orcapod.directory` are unchanged. + +For ITL-981: + +- The array type's extension name and logical type name are `orcapod.numpy.ndarray` (main: + `orcapod.ndarray`). One constant (`NDARRAY_EXTENSION_NAME`) and the `hash_ignore` key; + spelling in 8 test files, `arrays.md`, `hashing.md`, `logical-type-identifiers.md`. +- The leading type text of the value hash is the extension name (proposed by the + coordinator, not yet confirmed): `lp("orcapod.numpy.ndarray")` in `P` of section (i) B, and + `orcapod.torch.tensor` for torch. It is one constant; the tests take it from + `LEADING_TYPE_TEXT`. +- Golden: no further entry. The same 32 values move once; cause M now includes the name. +- The clash with v0.2 does not arise: v0.2 stored arrays as `numpy.ndarray` (empty + metadata), a name v0.3 does not register, so v0.2 tables are read as plain storage, + counted and left unchanged, exactly as on main. (Measured while the name was briefly + `numpy.ndarray`: registering that name made 2 of the 6 tests of + `tests/test_arrays/test_array_v02_store.py` fail, because every pyarrow or Delta read of a + v0.2 table then calls the registered class.) The v0.2 fixture script keeps its guard. +- The "foreign type" is therefore DROPPED. The mechanism does not need it: the class + registered with Arrow deserialises any valid ignore list to the class of that list, and + for anything else (non-canonical bytes, an unknown option name, another version) raises + `LogicalTypeIdentifierError`, which is what every built-in type does on main. +- Note for ITL-897: the pandas rename is different. v0.2 stores used `pandas.dataframe`, + and main reads their empty metadata through `also_reads`. That is ITL-897's question. ## Open decisions for the developer diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py index d2e160119..0e0fffe97 100644 --- a/tests/test_arrays/test_array_strict_hash.py +++ b/tests/test_arrays/test_array_strict_hash.py @@ -40,6 +40,12 @@ def ignore_metadata(names: list[str]) -> bytes: ).encode() +EXTENSION_NAME = "orcapod.numpy.ndarray" +"""Arrow extension name of the array type (developer, November 12, 2026).""" + +LEADING_TYPE_TEXT = EXTENSION_NAME +"""Leading text of the value hash preimage: the extension name (proposed).""" + OPTIONS = ("byte_order", "memory_order", "struct_layout") BASE = np.arange(6, dtype=np.float32).reshape(2, 3) @@ -78,7 +84,7 @@ def reference_digest( p += b"".join(struct.pack(" DataContext: """Build a data context whose ndarray ignore list is ``names``.""" shipped = Path(contexts_package.__file__).parent / "logical_types.json" registration = json.loads(shipped.read_text()) - registration["hash_ignore"]["numpy.ndarray"] = names + registration["hash_ignore"][EXTENSION_NAME] = names custom = tmp_path / f"logical_types_{'_'.join(names) or 'strict'}.json" custom.write_text(json.dumps(registration)) return JSONDataContextRegistry(logical_types_file=custom).get_context() @@ -229,9 +235,9 @@ def test_hash_is_unchanged_by_storage(name: str) -> None: def test_column_is_npy_bytes_under_the_default_identifier() -> None: - """An array column is ``numpy.ndarray`` over ``large_binary`` holding ``.npy`` files.""" + """An array column is ``orcapod.numpy.ndarray`` over ``large_binary`` holding ``.npy`` files.""" field = _source({"a": BASE}).as_table().schema.field("x") - assert field.type.extension_name == "numpy.ndarray" + assert field.type.extension_name == EXTENSION_NAME assert field.type.storage_type == pa.large_binary() assert field.type.__arrow_ext_serialize__() == ignore_metadata([]) cell = _source({"a": BASE}).as_table().column("x").chunk(0).storage[0].as_py() From 64c63bfe613532d26604cc2ea49881ff064b46a0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:18:24 -0800 Subject: [PATCH 06/22] feat(logical_types): ignore-list mechanism for library value types (ITL-981) Adds logical_types/ignore_list.py: the extension metadata shape {"ignore":[...],"version":N}, IgnoreListLogicalType and IgnoreListHandler. The context loader reads the per-type lists from the hash_ignore object of logical_types.json, and the Arrow hasher hashes a cell under the list of its own column. Co-Authored-By: Claude Fable 5.2 --- src/orcapod/contexts/registry.py | 15 +- src/orcapod/hashing/visitors.py | 17 +- src/orcapod/logical_types/ignore_list.py | 459 +++++++++++++++++++ src/orcapod/logical_types/registry.py | 37 +- tests/test_arrays/test_array_strict_hash.py | 8 +- tests/test_logical_types/ignore_list_toy.py | 63 +++ tests/test_logical_types/test_ignore_list.py | 295 ++++++++++++ 7 files changed, 873 insertions(+), 21 deletions(-) create mode 100644 src/orcapod/logical_types/ignore_list.py create mode 100644 tests/test_logical_types/ignore_list_toy.py create mode 100644 tests/test_logical_types/test_ignore_list.py diff --git a/src/orcapod/contexts/registry.py b/src/orcapod/contexts/registry.py index a3b498fb3..db679ec46 100644 --- a/src/orcapod/contexts/registry.py +++ b/src/orcapod/contexts/registry.py @@ -321,6 +321,10 @@ def get_context(self, context_string: str | None = None) -> DataContext: # to them by this name, as ``{"_ref": "logical_type_registry"}``. _LOGICAL_TYPE_REGISTRY_REF = "logical_type_registry" + # The ignore list declared for an extension name in the registration file's + # ``hash_ignore`` object is referred to as ``hash_ignore:``. + _HASH_IGNORE_REF_PREFIX = "hash_ignore:" + def _uses_logical_type_registration(self, spec: dict[str, Any]) -> bool: """Whether ``spec`` takes its logical types from the registration file. @@ -339,8 +343,10 @@ def _load_logical_type_registration(self) -> dict[str, Any]: """Read the logical type registration file. Returns: - Its ``logical_type_registry`` object spec and its ``value_handlers`` - list of ``[type, handler]`` object-spec pairs. + Its ``logical_type_registry`` object spec, its ``value_handlers`` list + of ``[type, handler]`` object-spec pairs and, when present, its + ``hash_ignore`` object: per extension name, the property names the + value hash of that type ignores. """ with open(self.logical_types_file, "r") as f: return json.load(f) @@ -365,6 +371,11 @@ def _create_context_from_spec(self, spec: dict[str, Any]) -> DataContext: registration: dict[str, Any] | None = None if self._uses_logical_type_registration(spec): registration = self._load_logical_type_registration() + # The ignore list of each library value type is declared once, by + # extension name; its logical type and its value handler both take + # it as ``{"_ref": "hash_ignore:"}`` (ITL-981). + for name, ignore in registration.get("hash_ignore", {}).items(): + ref_lut[f"{self._HASH_IGNORE_REF_PREFIX}{name}"] = list(ignore) ref_lut[self._LOGICAL_TYPE_REGISTRY_REF] = parse_objectspec( registration["logical_type_registry"], ref_lut=ref_lut ) diff --git a/src/orcapod/hashing/visitors.py b/src/orcapod/hashing/visitors.py index 18ef48372..3ac720279 100644 --- a/src/orcapod/hashing/visitors.py +++ b/src/orcapod/hashing/visitors.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any from orcapod.logical_types.identifier import arrow_type_identifier +from orcapod.logical_types.ignore_list import IgnoreListHandler, parse_ignore_metadata from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -199,8 +200,10 @@ class SemanticHashingVisitor(ArrowTypeDataVisitor): 1. Look up the corresponding Python type via ``type_converter``. 2. If the Python type has a semantic hasher registered in ``python_hasher``, - convert the storage value to a Python object and hash it, replacing the - column with a ``pa.large_binary()`` value of the form:: + convert the storage value to a Python object and hash it (a handler with + an ignore list, ``IgnoreListHandler``, hashes it under the list in the + column's own extension metadata), replacing the column with a + ``pa.large_binary()`` value of the form:: + b"::" + content_hash.digest @@ -422,7 +425,15 @@ def visit_extension( # Convert storage value → Python object and hash it. python_obj = self._type_converter.storage_to_python(storage_value, python_type) - content_hash = self._python_hasher.hash_object(python_obj) + handler = self._python_hasher.type_handler_registry.get_handler(python_obj) + if isinstance(handler, IgnoreListHandler): + # A value type with an ignore list (ITL-981): the cell is hashed under + # the list of its own column, read from the column's extension + # metadata, never under the list the data context configures. + _, ignore = parse_ignore_metadata(extension_type.__arrow_ext_serialize__()) + content_hash = handler.hash_ignoring(python_obj, ignore) + else: + content_hash = self._python_hasher.hash_object(python_obj) # Encode as binary: "::", where the identifier text # is ";" diff --git a/src/orcapod/logical_types/ignore_list.py b/src/orcapod/logical_types/ignore_list.py new file mode 100644 index 000000000..5651c2e7d --- /dev/null +++ b/src/orcapod/logical_types/ignore_list.py @@ -0,0 +1,459 @@ +"""Value types whose hash ignores a configurable list of properties (ITL-981). + +A value from a library (a NumPy array, a tensor, a data frame) has properties +beyond its content: memory order, byte order, column order. By default every +property the storage keeps is part of the value's hash. A type declares the +closed set of property names its hash can ignore (its *options*), and a data +context configures, per type, which of them are ignored (the *ignore list*). + +The ignore list is saved in the Arrow extension metadata, so it is part of the +logical type identifier: two lists are two logical types. The metadata has one +shape for every such type, canonical JSON with sorted keys: + + {"ignore":[],"version":1} + {"ignore":["byte_order","memory_order"],"version":1} + +``ignore`` holds the names sorted by code point, each once; it is present also +when empty. ``version`` is the version of the type's stored layout and hashing. + +The list is not part of the value hash preimage. A handler writes every optional +property into its preimage as a field that is always present, with the real +value, or with a fixed canonical value when the property is ignored. So values +that differ only in ignored properties hash equal. + +A type is built from two base classes here: ``IgnoreListLogicalType`` (the +Arrow and Polars extension types) and ``IgnoreListHandler`` (the value hash). +A data context has exactly one list per type; a column stored under another +list is refused on read with ``LogicalTypeIdentifierError``. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Collection, Iterable +from typing import TYPE_CHECKING, Any, ClassVar, cast + +from orcapod.logical_types.base_logical_type import BaseLogicalType +from orcapod.logical_types.identifier import ( + LogicalTypeIdentifier, + LogicalTypeIdentifierError, +) +from orcapod.types import ContentHash +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import polars as pl + import pyarrow as pa + + from orcapod.protocols.hashing_protocols import SemanticHasherProtocol +else: + pa = LazyModule("pyarrow") + pl = LazyModule("polars") + +_OPTION_NAME = re.compile(r"[a-z][a-z0-9_]*") + + +def _sanitize(name: str) -> str: + """Return ``name`` with every non-alphanumeric character replaced by ``_``.""" + return re.sub(r"[^A-Za-z0-9]", "_", name) + + +def checked_ignore(ignore: Iterable[str], options: Collection[str]) -> tuple[str, ...]: + """Return an ignore list in canonical form, after checking every name. + + Args: + ignore: The property names to ignore, in any order. + options: The closed set of names the type's hash can ignore. + + Returns: + The names sorted by code point, each once. + + Raises: + ValueError: If ``ignore`` is a string, or holds a name that is not a + string or not one of ``options``. + """ + if isinstance(ignore, str): + raise ValueError( + f"an ignore list is a list of property names, got the string {ignore!r}" + ) + names = list(ignore) + for name in names: + if not isinstance(name, str) or name not in options: + allowed = ", ".join(sorted(options)) or "none" + raise ValueError( + f"unknown property name {name!r} in ignore list; allowed names: " + f"{allowed}" + ) + return tuple(sorted(set(names))) + + +def ignore_metadata(version: int, ignore: Iterable[str]) -> bytes: + """Return the extension metadata for a version and an ignore list. + + Args: + version: The version of the type's stored layout and hashing. + ignore: The ignored property names, in any order. + + Returns: + Canonical JSON with sorted keys and no whitespace, for example + ``b'{"ignore":["byte_order"],"version":1}'``. + + Raises: + ValueError: If a name is not lower-case snake case, or ``version`` is not + a positive integer. + """ + if isinstance(version, bool) or not isinstance(version, int) or version < 1: + raise ValueError(f"version must be a positive integer, got {version!r}") + names = sorted(set(ignore)) + for name in names: + if not isinstance(name, str) or _OPTION_NAME.fullmatch(name) is None: + raise ValueError( + f"a property name is lower-case snake case ([a-z][a-z0-9_]*), " + f"got {name!r}" + ) + return json.dumps( + {"ignore": names, "version": version}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("ascii") + + +def parse_ignore_metadata(metadata: bytes) -> tuple[int, tuple[str, ...]]: + """Return the version and the ignore list of stored extension metadata. + + Args: + metadata: The ``ARROW:extension:metadata`` bytes of a column. + + Returns: + ``(version, names)`` with the names sorted. + + Raises: + ValueError: If ``metadata`` is not exactly what ``ignore_metadata`` + writes: other keys, other value types, unsorted or repeated names, + or any other byte sequence. + """ + try: + parsed = json.loads(metadata.decode("ascii")) + except (UnicodeDecodeError, ValueError) as exc: + raise ValueError( + f"extension metadata {metadata!r} is not ignore-list metadata: not JSON" + ) from exc + if ( + not isinstance(parsed, dict) + or set(parsed) != {"ignore", "version"} + or not isinstance(parsed["ignore"], list) + ): + raise ValueError( + f"extension metadata {metadata!r} is not ignore-list metadata: expected " + 'an object with exactly the keys "ignore" (a list) and "version"' + ) + try: + canonical = ignore_metadata(parsed["version"], parsed["ignore"]) + except (TypeError, ValueError) as exc: + raise ValueError( + f"extension metadata {metadata!r} is not ignore-list metadata: {exc}" + ) from exc + if canonical != metadata: + raise ValueError( + f"extension metadata {metadata!r} is not in canonical form; expected " + f"{canonical!r}" + ) + return parsed["version"], tuple(parsed["ignore"]) + + +class _ExtensionFamily: + """The Arrow and Polars extension types of one extension name, one per list. + + Each ignore list has its own ``pa.ExtensionType`` class, so two lists are + unequal Arrow types everywhere (type equality, schema equality, concatenation). + Whichever class is registered with Arrow under the name deserialises a stored + column to the class of the column's own list. + """ + + def __init__( + self, + extension_name: str, + storage_type: "pa.DataType", + version: int, + options: frozenset[str], + ) -> None: + self.extension_name = extension_name + self.storage_type = storage_type + self.version = version + self.options = options + self._arrow_types: dict[tuple[str, ...], pa.ExtensionType] = {} + self._polars_class: type[pl.BaseExtension] | None = None + + def arrow_type(self, ignore: tuple[str, ...]) -> "pa.ExtensionType": + """Return the Arrow extension type of a checked ignore list.""" + cached = self._arrow_types.get(ignore) + if cached is None: + cached = self._make_arrow_class(ignore)() + self._arrow_types[ignore] = cached + return cached + + def _deserialize( + self, storage_type: "pa.DataType", serialized: bytes, readable: bytes + ) -> "pa.ExtensionType": + """Return the type of a stored column, or raise if it cannot be one.""" + from orcapod.errors import TypeIdentityError + from orcapod.logical_types.registry import storage_type_matches + + if not storage_type_matches(self.storage_type, storage_type): + raise TypeIdentityError( + f"Arrow extension type '{self.extension_name}': expected storage_type " + f"{self.storage_type!r} but got {storage_type!r}." + ) + try: + version, names = parse_ignore_metadata(serialized) + if version != self.version: + raise ValueError(f"version {version} is not version {self.version}") + ignore = checked_ignore(names, self.options) + except ValueError as exc: + raise LogicalTypeIdentifierError( + LogicalTypeIdentifier(self.extension_name, serialized), (readable,) + ) from exc + return self.arrow_type(ignore) + + def _make_arrow_class( + self, ignore: tuple[str, ...] + ) -> Callable[[], "pa.ExtensionType"]: + family = self + metadata = ignore_metadata(self.version, ignore) + suffix = "__ignore_" + "_and_".join(ignore) if ignore else "" + + def _init(self: "pa.ExtensionType") -> None: + pa.ExtensionType.__init__(self, family.storage_type, family.extension_name) + + def _serialize(self: "pa.ExtensionType") -> bytes: + return metadata + + def _deserialize( + cls: type["pa.ExtensionType"], + storage_type: "pa.DataType", + serialized: bytes, + ) -> "pa.ExtensionType": + return family._deserialize(storage_type, serialized, metadata) + + extension_class = type( + f"_ArrowExt_{_sanitize(self.extension_name)}{suffix}", + (pa.ExtensionType,), + { + "__init__": _init, + "__arrow_ext_serialize__": _serialize, + "__arrow_ext_deserialize__": classmethod(_deserialize), + "readable_metadata": (metadata,), + }, + ) + # The synthesized ``__init__`` takes no argument; the stubs know only the + # base class's. + return cast("Callable[[], pa.ExtensionType]", extension_class) + + def polars_type(self, ignore: tuple[str, ...]) -> "pl.BaseExtension": + """Return the Polars extension type of a checked ignore list.""" + if self._polars_class is None: + self._polars_class = self._make_polars_class() + return self._polars_class(ignore_metadata(self.version, ignore).decode("ascii")) # pyright: ignore[reportCallIssue] # the synthesized class takes the metadata text + + def _make_polars_class(self) -> type["pl.BaseExtension"]: + name = self.extension_name + polars_dtype = cast( + "pl.Series", pl.from_arrow(pa.array([], type=self.storage_type)) + ).dtype + + def _init(self: "pl.BaseExtension", metadata: str | None = None) -> None: + pl.BaseExtension.__init__(self, name, polars_dtype, metadata) + + def _ext_from_params( + cls: Any, + ext_name: str, + storage_dtype: Any, + metadata_str: str | None, + ) -> "pl.BaseExtension": + # Polars has one class per extension name; the instance keeps the + # metadata text of the column it was built for. + return cls(metadata_str) + + return type( + f"_PolarsExt_{_sanitize(name)}", + (pl.BaseExtension,), + {"__init__": _init, "ext_from_params": classmethod(_ext_from_params)}, + ) + + +_FAMILIES: dict[str, _ExtensionFamily] = {} + + +def _family( + extension_name: str, + storage_type: "pa.DataType", + version: int, + options: frozenset[str], +) -> _ExtensionFamily: + """Return the one extension family of ``extension_name`` in this process. + + Raises: + ValueError: If the name was already declared with another storage type, + version or set of options. + """ + family = _FAMILIES.get(extension_name) + if family is None: + family = _ExtensionFamily(extension_name, storage_type, version, options) + _FAMILIES[extension_name] = family + elif ( + family.storage_type != storage_type + or family.version != version + or family.options != options + ): + raise ValueError( + f"extension name {extension_name!r} is already declared with storage " + f"type {family.storage_type}, version {family.version} and options " + f"{sorted(family.options)}" + ) + return family + + +class IgnoreListLogicalType(BaseLogicalType): + """Base of a logical type whose identifier carries an ignore list. + + A subclass sets the class attributes and implements ``python_to_storage`` + and ``storage_to_python``. Storage is the same under every ignore list; the + list changes only the identifier, and with it what the value hash ignores. + + Attributes: + EXTENSION_NAME: The Arrow extension name, also the logical type name. + STORAGE_TYPE: The Arrow storage type of one value. + VERSION: The version of the stored layout and hashing. + OPTIONS: The closed set of property names the hash can ignore. + python_type: The Python class of the values (a subclass sets it). + + Args: + ignore: The property names this instance's identifier lists as ignored. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + """ + + EXTENSION_NAME: ClassVar[str] + STORAGE_TYPE: ClassVar["pa.DataType"] + VERSION: ClassVar[int] + OPTIONS: ClassVar[frozenset[str]] + python_type: type = object + + def __init__(self, ignore: Iterable[str] = ()) -> None: + self._ignore: tuple[str, ...] = checked_ignore(ignore, self.OPTIONS) + + @property + def logical_type_name(self) -> str: + """The logical type name: the Arrow extension name.""" + return self.EXTENSION_NAME + + @property + def ignore(self) -> tuple[str, ...]: + """The ignored property names, sorted.""" + return self._ignore + + def _extension_family(self) -> _ExtensionFamily: + return _family( + self.EXTENSION_NAME, self.STORAGE_TYPE, self.VERSION, self.OPTIONS + ) + + def get_arrow_extension_type(self) -> "pa.ExtensionType": + """Return the Arrow extension type whose metadata lists ``ignore``. + + Returns: + The extension type; equal only to the type of the same ignore list. + """ + return self._extension_family().arrow_type(self._ignore) + + def get_polars_extension_type(self) -> "pl.BaseExtension": + """Return the Polars extension type whose metadata lists ``ignore``. + + Returns: + The Polars extension registered under the extension name. + """ + return self._extension_family().polars_type(self._ignore) + + +class IgnoreListHandler: + """Base of a value hasher that honours an ignore list. + + A subclass sets ``OPTIONS`` and ``HASH_LABEL`` and implements ``digest``. + A bare value is hashed with the list the handler was configured with (the + data context's). A cell of a typed column is hashed with the list of the + column's own extension metadata, through ``hash_ignoring``. + + Attributes: + OPTIONS: The closed set of property names the hash can ignore; the same + set as the logical type's. + HASH_LABEL: The label of the hash algorithm (``ContentHash.method``). + + Args: + ignore: The property names ignored when a bare value is hashed. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + """ + + OPTIONS: ClassVar[frozenset[str]] + HASH_LABEL: ClassVar[str] + + def __init__(self, ignore: Iterable[str] = ()) -> None: + self._ignore: frozenset[str] = frozenset(checked_ignore(ignore, self.OPTIONS)) + + @property + def ignore(self) -> frozenset[str]: + """The property names ignored when a bare value is hashed.""" + return self._ignore + + def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: + """Return the hash of a bare value under the configured ignore list. + + Args: + obj: The value. + hasher: Ignored. Present for protocol conformance. + + Returns: + ``ContentHash(HASH_LABEL, digest(obj, ignore))``. + """ + return ContentHash( + method=self.HASH_LABEL, digest=self.digest(obj, self._ignore) + ) + + def hash_ignoring(self, obj: Any, ignore: Iterable[str]) -> ContentHash: + """Return the hash of a value under a given ignore list. + + Args: + obj: The value. + ignore: The property names to ignore, for example those of a + column's extension metadata. + + Returns: + ``ContentHash(HASH_LABEL, digest(obj, ignore))``. + + Raises: + ValueError: If ``ignore`` holds a name that is not in ``OPTIONS``. + """ + names = frozenset(checked_ignore(ignore, self.OPTIONS)) + return ContentHash(method=self.HASH_LABEL, digest=self.digest(obj, names)) + + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return the digest of a value, ignoring the named properties. + + Every optional property is a field of the preimage that is always + present: its real value, or its canonical value when its name is in + ``ignore``. The list itself is never part of the preimage. + + Args: + obj: The value. + ignore: The property names to ignore; a subset of ``OPTIONS``. + + Returns: + The digest bytes. + + Raises: + NotImplementedError: Always; every handler implements this. + """ + raise NotImplementedError(f"{type(self).__name__} must implement digest.") diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index 9233ea36b..fe711e737 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -37,6 +37,29 @@ def _sanitize(name: str) -> str: return re.sub(r"[^A-Za-z0-9]", "_", name) +def storage_type_matches(expected: pa.DataType, stored: pa.DataType) -> bool: + """Return whether ``stored`` is an acceptable storage type for ``expected``. + + When ``expected`` is in its large-normalized form, any physically equivalent + member of the same family is accepted, at any nesting depth (``string`` or + ``string_view`` for ``large_string``, ``binary`` for ``large_binary``): Delta + Lake compaction rewrites large columns as plain ones (ITL-602, ITL-881). + Otherwise the two types must be equal, so a narrowing is never accepted. + + Args: + expected: The storage type the extension type declares. + stored: The storage type found with the stored column. + + Returns: + ``True`` when a column stored as ``stored`` can be read as ``expected``. + """ + from orcapod.utils.arrow_utils import normalize_to_large_types + + if normalize_to_large_types(expected, keep_null=True) == expected: + return normalize_to_large_types(stored, keep_null=True) == expected + return stored == expected + + def make_arrow_extension_type( extension_name: str, storage_type: pa.DataType, @@ -111,21 +134,9 @@ def _deserialize( # # Cross-family mismatches (string where large_binary was registered, or # vice-versa) are genuine semantic errors and always raise. - from orcapod.utils.arrow_utils import normalize_to_large_types - - if normalize_to_large_types(_storage, keep_null=True) == _storage: - # _storage is the canonical large form — accept any same-family variant, - # at any nesting depth (a struct field rewritten from large_string to - # string by Delta compaction, ITL-881). - storage_ok = ( - normalize_to_large_types(storage_type, keep_null=True) == _storage - ) - else: - # Non-canonical _storage — require exact match. - storage_ok = storage_type == _storage from orcapod.errors import TypeIdentityError - if not storage_ok: + if not storage_type_matches(_storage, storage_type): raise TypeIdentityError( f"Arrow extension type '{_name}': expected storage_type " f"{_storage!r} but got {storage_type!r}." diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py index 0e0fffe97..8f8906e7b 100644 --- a/tests/test_arrays/test_array_strict_hash.py +++ b/tests/test_arrays/test_array_strict_hash.py @@ -116,8 +116,8 @@ def _layout(dtype: np.dtype[Any]) -> Any: dtype = dtype.subdtype[0] if dtype.names is None: return None - fields = dtype.fields - assert fields is not None + assert dtype.fields is not None + fields: dict[str, Any] = {str(key): value for key, value in dtype.fields.items()} return [dtype.itemsize] + [ [fields[name][1], _layout(fields[name][0])] for name in dtype.names ] @@ -240,7 +240,9 @@ def test_column_is_npy_bytes_under_the_default_identifier() -> None: assert field.type.extension_name == EXTENSION_NAME assert field.type.storage_type == pa.large_binary() assert field.type.__arrow_ext_serialize__() == ignore_metadata([]) - cell = _source({"a": BASE}).as_table().column("x").chunk(0).storage[0].as_py() + chunk = _source({"a": BASE}).as_table().column("x").chunk(0) + assert isinstance(chunk, pa.ExtensionArray) + cell = chunk.storage[0].as_py() assert cell[:8] == b"\x93NUMPY\x03\x00" diff --git a/tests/test_logical_types/ignore_list_toy.py b/tests/test_logical_types/ignore_list_toy.py new file mode 100644 index 000000000..c769b53ad --- /dev/null +++ b/tests/test_logical_types/ignore_list_toy.py @@ -0,0 +1,63 @@ +"""A toy value type with an ignore list, for ``test_ignore_list.py`` (ITL-981). + +``Labelled`` has a content (``amount``) and one optional property (``label``). +The classes live in a module of their own so that a logical type registration +file can name them. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any + +import pyarrow as pa + +from orcapod.logical_types.ignore_list import IgnoreListHandler, IgnoreListLogicalType +from orcapod.logical_types.protocols import TypeConverterProtocol + +EXTENSION_NAME = "orcapod.test.labelled" + + +@dataclass(frozen=True) +class Labelled: + """A value with a content and a label that the hash can ignore.""" + + amount: int + label: str + + +class LogicalLabelled(IgnoreListLogicalType): + """Stores a ``Labelled`` as JSON text; the label is always stored.""" + + EXTENSION_NAME = EXTENSION_NAME + STORAGE_TYPE = pa.large_string() + VERSION = 1 + OPTIONS = frozenset({"label"}) + python_type: type = Labelled + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None = None + ) -> str: + """Return the JSON text of a ``Labelled``.""" + return json.dumps([value.amount, value.label]) + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None = None + ) -> Labelled: + """Return the ``Labelled`` of stored JSON text.""" + amount, label = json.loads(storage_value) + return Labelled(amount, label) + + +class LabelledHandler(IgnoreListHandler): + """Hashes the amount and the label; an ignored label is written as empty.""" + + OPTIONS = frozenset({"label"}) + HASH_LABEL = "labelled_v1" + + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return SHA-256 of the amount and the label field.""" + label = "" if "label" in ignore else obj.label + return hashlib.sha256(json.dumps([obj.amount, label]).encode()).digest() diff --git a/tests/test_logical_types/test_ignore_list.py b/tests/test_logical_types/test_ignore_list.py new file mode 100644 index 000000000..51819b118 --- /dev/null +++ b/tests/test_logical_types/test_ignore_list.py @@ -0,0 +1,295 @@ +"""The shared ignore-list mechanism of library value types (ITL-981). + +Uses a toy type (``ignore_list_toy``) registered through a logical type +registration file, the way a data context configures a real type. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod.contexts as contexts_package +from orcapod.contexts.core import ContextValidationError, DataContext +from orcapod.contexts.registry import JSONDataContextRegistry +from orcapod.errors import TypeIdentityError +from orcapod.logical_types.identifier import LogicalTypeIdentifierError +from orcapod.logical_types.ignore_list import ( + checked_ignore, + ignore_metadata, + parse_ignore_metadata, +) +from orcapod.semantic_types.universal_converter import UniversalTypeConverter +from tests.test_logical_types.ignore_list_toy import ( + EXTENSION_NAME, + Labelled, + LogicalLabelled, +) + +TOY_MODULE = "tests.test_logical_types.ignore_list_toy" +STRICT = b'{"ignore":[],"version":1}' +LENIENT = b'{"ignore":["label"],"version":1}' + + +def _context(tmp_path: Path, ignore: list[str]) -> DataContext: + """Build a data context that registers the toy type with ``ignore``.""" + shipped = Path(contexts_package.__file__).parent / "logical_types.json" + registration = json.loads(shipped.read_text()) + registration.setdefault("hash_ignore", {})[EXTENSION_NAME] = ignore + reference = {"_ref": f"hash_ignore:{EXTENSION_NAME}"} + registration["logical_type_registry"]["_config"]["logical_types"].append( + {"_class": f"{TOY_MODULE}.LogicalLabelled", "_config": {"ignore": reference}} + ) + registration["value_handlers"].append( + [ + {"_type": f"{TOY_MODULE}.Labelled"}, + { + "_class": f"{TOY_MODULE}.LabelledHandler", + "_config": {"ignore": reference}, + }, + ] + ) + custom = tmp_path / f"logical_types_{'_'.join(ignore) or 'strict'}.json" + custom.write_text(json.dumps(registration)) + return JSONDataContextRegistry(logical_types_file=custom).get_context() + + +def _table(context: DataContext, value: Labelled) -> pa.Table: + return context.type_converter.python_dicts_to_arrow_table( + [{"x": value}], python_schema={"x": Labelled} + ) + + +def _converter(context: DataContext) -> UniversalTypeConverter: + """Return the context's converter as the concrete class the databases use.""" + converter = context.type_converter + assert isinstance(converter, UniversalTypeConverter) + return converter + + +# --- metadata bytes ------------------------------------------------------------------ + + +def test_metadata_bytes_are_canonical_json_with_sorted_keys() -> None: + """The metadata is exactly the documented bytes.""" + assert ignore_metadata(1, []) == b'{"ignore":[],"version":1}' + assert ( + ignore_metadata(1, ["memory_order", "byte_order", "memory_order"]) + == b'{"ignore":["byte_order","memory_order"],"version":1}' + ) + assert ignore_metadata(2, ("a_1",)) == b'{"ignore":["a_1"],"version":2}' + + +@pytest.mark.parametrize("name", ["Memory", "1st", "a-b", "", "é", "a b"]) +def test_a_property_name_is_lower_case_snake_case(name: str) -> None: + """A name outside ``[a-z][a-z0-9_]*`` cannot be written into metadata.""" + with pytest.raises(ValueError, match="snake case"): + ignore_metadata(1, [name]) + + +@pytest.mark.parametrize("version", [0, -1, True, 1.0, "1"]) +def test_version_is_a_positive_integer(version: Any) -> None: + """The version is a positive ``int``.""" + with pytest.raises(ValueError, match="positive integer"): + ignore_metadata(version, []) + + +def test_parse_returns_what_was_written() -> None: + """Parsing is the inverse of writing.""" + assert parse_ignore_metadata(STRICT) == (1, ()) + assert parse_ignore_metadata(ignore_metadata(3, ["b", "a"])) == (3, ("a", "b")) + + +@pytest.mark.parametrize( + "metadata", + [ + b"", + b'{"version":1}', + b'{"version":1,"ignore":[]}', + b'{"ignore": [], "version": 1}', + b'{"ignore":["b","a"],"version":1}', + b'{"ignore":["a","a"],"version":1}', + b'{"ignore":[],"version":1,"extra":0}', + b'{"ignore":"a","version":1}', + b'{"ignore":[1],"version":1}', + b'{"ignore":[],"version":"1"}', + b'{"ignore":[],"version":0}', + b"not json", + b"\xff\xfe", + b"[]", + ], +) +def test_parse_refuses_anything_but_the_canonical_bytes(metadata: bytes) -> None: + """Only the exact bytes ``ignore_metadata`` writes are accepted.""" + with pytest.raises(ValueError, match="ignore-list metadata|canonical form"): + parse_ignore_metadata(metadata) + + +def test_checked_ignore_sorts_and_refuses_unknown_names() -> None: + """An ignore list is sorted and unique; an unknown name is an error naming it.""" + options = frozenset({"a", "b", "c"}) + assert checked_ignore(["c", "a", "c"], options) == ("a", "c") + assert checked_ignore((), options) == () + with pytest.raises(ValueError, match="'d'.*allowed names: a, b, c"): + checked_ignore(["a", "d"], options) + with pytest.raises(ValueError, match="list of property names"): + checked_ignore("a", options) + + +# --- the list reaches the type, the bare value and the cell ---------------------------- + + +def test_the_configured_list_is_the_types_extension_metadata(tmp_path: Path) -> None: + """The data context's list is written into the Arrow extension metadata.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + strict_type = strict.type_converter.python_type_to_arrow_type(Labelled) + lenient_type = lenient.type_converter.python_type_to_arrow_type(Labelled) + assert isinstance(strict_type, pa.ExtensionType) + assert isinstance(lenient_type, pa.ExtensionType) + assert strict_type.extension_name == lenient_type.extension_name == EXTENSION_NAME + assert strict_type.__arrow_ext_serialize__() == STRICT + assert lenient_type.__arrow_ext_serialize__() == LENIENT + + +def test_two_lists_are_two_types(tmp_path: Path) -> None: + """Types of two lists are unequal, alone, nested, in a schema and when hashed.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + strict_type = strict.type_converter.python_type_to_arrow_type(Labelled) + lenient_type = lenient.type_converter.python_type_to_arrow_type(Labelled) + assert strict_type != lenient_type + assert strict_type == LogicalLabelled().get_arrow_extension_type() + assert pa.large_list(strict_type) != pa.large_list(lenient_type) + assert pa.schema([("x", strict_type)]) != pa.schema([("x", lenient_type)]) + with pytest.raises(pa.ArrowInvalid): + pa.concat_tables( + [_table(strict, Labelled(1, "a")), _table(lenient, Labelled(1, "a"))] + ) + # The type hash and the schema hash take the identifier, which holds the list. + assert strict.semantic_hasher.hash_object( + Labelled + ) != lenient.semantic_hasher.hash_object(Labelled) + assert strict.arrow_hasher.hash_table( + _table(strict, Labelled(1, "a")).slice(0, 0) + ) != lenient.arrow_hasher.hash_table(_table(lenient, Labelled(1, "a")).slice(0, 0)) + + +def test_a_bare_value_is_hashed_with_the_contexts_list(tmp_path: Path) -> None: + """Outside a typed column the data context's list decides.""" + strict = _context(tmp_path, []).semantic_hasher + lenient = _context(tmp_path, ["label"]).semantic_hasher + one, other = Labelled(1, "a"), Labelled(1, "b") + assert strict.hash_object(one) != strict.hash_object(other) + assert lenient.hash_object(one) == lenient.hash_object(other) + assert lenient.hash_object(one) != lenient.hash_object(Labelled(2, "a")) + assert lenient.hash_object(one).method == "labelled_v1" + # The list is not a hash input: only the canonical field value is. + assert lenient.hash_object(one) == strict.hash_object(Labelled(1, "")) + + +def test_a_cell_is_hashed_with_the_list_of_its_column(tmp_path: Path) -> None: + """In a table the column's extension metadata decides, not the hashing context.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + one, other = Labelled(1, "a"), Labelled(1, "b") + assert strict.arrow_hasher.hash_table( + _table(strict, one) + ) != strict.arrow_hasher.hash_table(_table(strict, other)) + assert lenient.arrow_hasher.hash_table( + _table(lenient, one) + ) == lenient.arrow_hasher.hash_table(_table(lenient, other)) + # The same values in a strict column and in a lenient column hash differently: + # the cell token and the field carry the identifier, which holds the list. + assert strict.arrow_hasher.hash_table( + _table(strict, one) + ) != lenient.arrow_hasher.hash_table(_table(lenient, one)) + + +def test_an_unknown_option_name_fails_the_context_build(tmp_path: Path) -> None: + """A context that lists a name the type does not have cannot be built.""" + with pytest.raises(ContextValidationError, match="'colour'.*allowed names: label"): + _context(tmp_path, ["colour"]) + + +# --- Arrow and stored columns ---------------------------------------------------------- + + +def _ipc_round_trip(table: pa.Table) -> pa.Table: + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + return pa.ipc.open_stream(sink.getvalue()).read_all() + + +def test_arrow_ipc_keeps_the_list_of_each_column(tmp_path: Path) -> None: + """One class is registered with Arrow under the name; it restores any valid list.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + for context in (strict, lenient): + table = _table(context, Labelled(1, "a")) + assert _ipc_round_trip(table).schema == table.schema + + +def test_arrow_refuses_metadata_that_is_not_a_valid_list() -> None: + """Unknown option names, other versions and other bytes raise on deserialisation.""" + extension_class = type(LogicalLabelled().get_arrow_extension_type()) + for metadata in ( + b'{"ignore":["colour"],"version":1}', + b'{"ignore":[],"version":2}', + b'{"version":1}', + b"", + ): + with pytest.raises(LogicalTypeIdentifierError) as raised: + extension_class.__arrow_ext_deserialize__(pa.large_string(), metadata) + assert raised.value.stored.metadata == metadata + with pytest.raises(TypeIdentityError, match="storage_type"): + extension_class.__arrow_ext_deserialize__(pa.int64(), STRICT) + # A plain string column (Delta compaction) is the same storage family. + assert ( + extension_class.__arrow_ext_deserialize__(pa.string(), LENIENT) + == LogicalLabelled(["label"]).get_arrow_extension_type() + ) + + +def test_a_column_stored_under_another_list_is_refused_naming_both( + tmp_path: Path, +) -> None: + """A data context reads only columns stored under its own list.""" + strict = _context(tmp_path, []) + lenient = _context(tmp_path, ["label"]) + stored = _table(lenient, Labelled(1, "a")) + with pytest.raises(LogicalTypeIdentifierError) as raised: + _converter(strict).apply_logical_types(stored) + message = str(raised.value) + assert "'x'" in message + assert repr(LENIENT) in message and repr(STRICT) in message + # The same column as plain storage with the identifier in the field metadata, + # which is how a database returns it. + chunk = stored.column("x").chunk(0) + assert isinstance(chunk, pa.ExtensionArray) + plain = pa.table( + {"x": chunk.storage}, + schema=pa.schema( + [ + pa.field( + "x", + pa.large_string(), + metadata={ + "ARROW:extension:name": EXTENSION_NAME, + "ARROW:extension:metadata": LENIENT, + }, + ) + ] + ), + ) + with pytest.raises(LogicalTypeIdentifierError, match="'x'"): + _converter(strict).apply_logical_types(plain) + assert ( + _converter(lenient).apply_logical_types(plain)["x"].type + == stored.schema.field("x").type + ) From e299455d5a4b14abfb1d90c343658099b1a89237 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:21:00 -0800 Subject: [PATCH 07/22] feat(arrays)!: store numpy arrays as .npy and hash them strictly (ITL-981) WIP: the ITL-888 tests, golden values and docs are not updated yet. Co-Authored-By: Claude Fable 5.2 --- src/orcapod/contexts/logical_types.json | 17 +- .../semantic_hashing/builtin_handlers.py | 44 +-- src/orcapod/logical_types/array_format.py | 87 ++--- src/orcapod/logical_types/numpy_type.py | 353 ++++++++++++------ tests/test_arrays/test_array_strict_hash.py | 31 +- 5 files changed, 339 insertions(+), 193 deletions(-) diff --git a/src/orcapod/contexts/logical_types.json b/src/orcapod/contexts/logical_types.json index 4135c1daf..09ff663a6 100644 --- a/src/orcapod/contexts/logical_types.json +++ b/src/orcapod/contexts/logical_types.json @@ -1,5 +1,8 @@ { - "description": "Logical types registered in every data context, with the value handlers that hash their Python values. This file is not frozen: a logical type is versioned by its own identifier (Arrow extension name and metadata), not by the data context version, so adding one edits this file and no released data context manifest.", + "description": "Logical types registered in every data context, with the value handlers that hash their Python values. This file is not frozen: a logical type is versioned by its own identifier (Arrow extension name and metadata), not by the data context version, so adding one edits this file and no released data context manifest. hash_ignore lists, per extension name, the property names the value hash of a library value type ignores (empty: every property the storage keeps is part of the hash); the list is written into the type's extension metadata, so changing it changes the type's identifier. The logical type entry and the value handler entry of such a type both refer to their list as hash_ignore:.", + "hash_ignore": { + "orcapod.numpy.ndarray": [] + }, "logical_type_registry": { "_class": "orcapod.logical_types.registry.LogicalTypeRegistry", "_config": { @@ -26,7 +29,11 @@ }, { "_class": "orcapod.logical_types.numpy_type.LogicalNumpyArray", - "_config": {} + "_config": { + "ignore": { + "_ref": "hash_ignore:orcapod.numpy.ndarray" + } + } }, { "_class": "orcapod.logical_types.pandas_type.LogicalPandasDataFrame", @@ -128,7 +135,11 @@ }, { "_class": "orcapod.hashing.semantic_hashing.builtin_handlers.NumpyArrayHandler", - "_config": {} + "_config": { + "ignore": { + "_ref": "hash_ignore:orcapod.numpy.ndarray" + } + } } ], [ diff --git a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py index 89c8de483..c764bedeb 100644 --- a/src/orcapod/hashing/semantic_hashing/builtin_handlers.py +++ b/src/orcapod/hashing/semantic_hashing/builtin_handlers.py @@ -29,7 +29,9 @@ from orcapod.hashing.semantic_hashing.encoded import Encoded from orcapod.hashing.semantic_hashing.function_info_extractors import ParameterDefault +from orcapod.logical_types.array_format import NDARRAY_OPTIONS from orcapod.logical_types.identifier import logical_type_identifier +from orcapod.logical_types.ignore_list import IgnoreListHandler from orcapod.types import ContentHash, Schema from orcapod.utils.canonical_type_names import canonical_type_name @@ -401,47 +403,45 @@ def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: return self.directory_hasher.hash_directory(wrapped, ignore=ignore) -class NumpyArrayHandler: - """Hasher for ``numpy.ndarray``: the version-1 value digest under ``ndarray_v1``. +class NumpyArrayHandler(IgnoreListHandler): + """Hasher for ``numpy.ndarray``: the strict value digest under ``ndarray_v1``. - The array is put in its canonical form (C order, little-endian, structured - fields packed in declared order; see ``logical_types.numpy_type``), which is - exactly what ``LogicalNumpyArray`` stores. The digest is - ``array_format.array_digest_v1(tag, shape, data)`` of that stored row, so - memory layout, byte order, structured padding and subclass do not change the - hash, while dtype (including structured field names and order), shape and - every element bit do. + The digest is ``numpy_type.array_value_digest``: the element digest of the + array (dtype, shape and element bits) and, on top of it, memory order, byte + order and structured layout, each of which the configured ignore list can + take out. What storage does not keep (strides beyond C or Fortran + contiguity, subclass, flags) never changes the hash. - The label ``ndarray_v1`` names the definition (and binds the digest to the - ndarray type): another array type sharing the digest body uses its own label. + The label ``ndarray_v1`` names this algorithm; the ignore list is part of + the logical type identifier, not of the label or the preimage. + + Args: + ignore: Names from ``array_format.NDARRAY_OPTIONS`` ignored when a bare + array is hashed (the data context's list for the type). """ + OPTIONS = NDARRAY_OPTIONS HASH_LABEL = "ndarray_v1" """``ContentHash.method`` of an ndarray hash.""" - def handle(self, obj: Any, hasher: "SemanticHasherProtocol") -> ContentHash: - """Return the ``ndarray_v1`` value hash of ``obj``. + def digest(self, obj: Any, ignore: frozenset[str]) -> bytes: + """Return the value digest of ``obj``, ignoring the named properties. Args: obj: A ``numpy.ndarray`` instance (subclasses are hashed as plain arrays). - hasher: Ignored. Present for protocol conformance. + ignore: Names from ``OPTIONS`` to ignore. Returns: - ``ContentHash("ndarray_v1", array_digest_v1(tag, shape, data))``. + ``numpy_type.array_value_digest(obj, ignore)``. Raises: TypeError: If ``obj`` is not a ``numpy.ndarray``. ValueError: If the array cannot be stored (masked, object, ``StringDType``, ``longdouble``, ...). """ - from orcapod.logical_types.array_format import array_digest_v1 - from orcapod.logical_types.numpy_type import array_bytes, canonical_array + from orcapod.logical_types.numpy_type import array_value_digest - tag, array = canonical_array(obj) - return ContentHash( - method=self.HASH_LABEL, - digest=array_digest_v1(tag, array.shape, array_bytes(array)), - ) + return array_value_digest(obj, ignore) class PandasDataFrameHandler: diff --git a/src/orcapod/logical_types/array_format.py b/src/orcapod/logical_types/array_format.py index bc8a04b5f..621de1500 100644 --- a/src/orcapod/logical_types/array_format.py +++ b/src/orcapod/logical_types/array_format.py @@ -1,38 +1,25 @@ -"""Array storage struct and value digest shared by array-like logical types. +"""Element digest shared by array-like value types, and the ndarray constants. -This module defines the parts of Orcapod's array format that do not depend on -any array library: the Arrow struct an array value is stored as, and the -version-1 value digest computed from a stored row. ``numpy.ndarray`` uses them -today (``numpy_type.LogicalNumpyArray``); other array types (such as -``torch.Tensor``) reuse them under their own extension name and hash label. +An array-like value (a ``numpy.ndarray``, a tensor) is hashed in two steps. +This module holds the first, which does not depend on any array library: the +*element digest* of a dtype tag, a shape and the element bytes in a fixed +layout. Each type builds its value hash on top of it (``numpy_type`` for +``numpy.ndarray``), adding its own leading type text and one field per property +the element digest does not cover. -Storage struct (one value per row): - - struct< - dtype: large_string, # Orcapod dtype tag - shape: large_list, # dimensions, outermost first - data: large_binary, # element bytes, C order - > - -The children are nullable at the Arrow level only because Polars drops nested -non-null flags when it converts back to Arrow (ITL-842), and the extension type -would then no longer match. A stored array value never has a null child: the -logical type refuses one on write and on read. A null array is a null struct. - -Value digest, version 1. For a row with dtype tag ``T``, shape -``(d0, ..., d{n-1})`` and data bytes ``D``: +Element digest, version 1. For a dtype tag ``T``, shape ``(d0, ..., d{n-1})`` +and element bytes ``D`` (C order, little-endian, structured fields packed in +declared order): P = u64le(len(utf8(T))) || utf8(T) || u64le(n) || i64le(d0) || ... || i64le(d{n-1}) || u64le(len(D)) || D - digest = SHA-256(P) + E = SHA-256(P) ``u64le`` and ``i64le`` are 8-byte little-endian unsigned and signed integers. -Every field is length-prefixed, so distinct rows never share a preimage. The -digest depends only on the three stored fields, so it can be recomputed from a -stored table with any SHA-256 implementation. +Every field is length-prefixed, so distinct inputs never share a preimage. -This module must not import NumPy. +This module must not import NumPy or Arrow. """ from __future__ import annotations @@ -41,43 +28,43 @@ import struct from collections.abc import Sequence -import pyarrow as pa - -ARRAY_FORMAT_VERSION = 1 -"""Version of the array storage struct; recorded in the extension metadata.""" +NDARRAY_EXTENSION_NAME = "orcapod.numpy.ndarray" +"""Arrow extension name (and logical type name) of ``numpy.ndarray`` values. -ARRAY_FORMAT_METADATA = b'{"version":1}' -"""``ARROW:extension:metadata`` of every array-format extension type (canonical JSON). - -A reader rejects a stored type whose metadata differs, so a later storage version -is never decoded as this one. +It is also the leading type text of the ndarray value hash. """ - -ARRAY_STORAGE_TYPE: pa.StructType = pa.struct( - [ - pa.field("dtype", pa.large_string()), - pa.field("shape", pa.large_list(pa.int64())), - pa.field("data", pa.large_binary()), - ] -) -"""The Arrow struct one array value is stored as (see the module docstring).""" +NDARRAY_OPTIONS = frozenset({"byte_order", "memory_order", "struct_layout"}) +"""The property names the ndarray value hash can ignore (see ``numpy_type``).""" _U64 = struct.Struct(" bytes: + """Return ``u64le(len(utf8(text))) || utf8(text)``. + + Args: + text: The text of one preimage field. + + Returns: + The length-prefixed UTF-8 bytes. + """ + data = text.encode("utf-8") + return _U64.pack(len(data)) + data + + def array_digest_v1( tag: str, shape: Sequence[int], data: bytes | bytearray | memoryview ) -> bytes: - """Compute the version-1 value digest of a stored array row. + """Compute the version-1 element digest of an array. The buffer is fed to SHA-256 directly, so a large array is not copied. Args: - tag: The Orcapod dtype tag, as stored in the ``dtype`` field. - shape: The dimensions, outermost first, as stored in the ``shape`` field. - data: The element bytes, as stored in the ``data`` field. Any object + tag: The Orcapod dtype tag. + shape: The dimensions, outermost first. + data: The element bytes in C order, little-endian, packed. Any object exposing a C-contiguous buffer is accepted. Returns: @@ -86,10 +73,8 @@ def array_digest_v1( Raises: ValueError: If a dimension is negative or not an integer. """ - tag_bytes = tag.encode("utf-8") h = hashlib.sha256() - h.update(_U64.pack(len(tag_bytes))) - h.update(tag_bytes) + h.update(length_prefixed(tag)) h.update(_U64.pack(len(shape))) for dim in shape: if isinstance(dim, bool) or not isinstance(dim, int) or dim < 0: diff --git a/src/orcapod/logical_types/numpy_type.py b/src/orcapod/logical_types/numpy_type.py index 5844db693..27bcbb3cf 100644 --- a/src/orcapod/logical_types/numpy_type.py +++ b/src/orcapod/logical_types/numpy_type.py @@ -1,54 +1,89 @@ -"""``numpy.ndarray`` logical type: the v0.3 value-based array format. - -An array is stored as one row of the shared array struct -(``array_format.ARRAY_STORAGE_TYPE``): an Orcapod dtype tag, the shape, and the -element bytes in C order, little-endian and packed. The extension name is -``orcapod.ndarray``. The format and the dtype tags are specified in -``superpowers/specs/2026-11-10-array-format-v0.3.md`` and documented in -``docs/concepts/arrays.md``. - -Writing canonicalises what is not part of an array's identity: memory order -and strides (stored C order), byte order (stored little-endian) and structured -offsets, padding and alignment (fields packed). Structured field order is part -of the identity, as NumPy's own dtype equality has it: fields are stored in -declared order, at every nesting depth. So a stored row is the array's -canonical value, and its hash (``NumpyArrayHandler``) is a function of the -stored row alone. - -Refused at write time, with a ``ValueError`` naming the dtype (the type +"""``numpy.ndarray`` logical type: ``.npy`` storage and a strict value hash (ITL-981). + +Storage and hashing are separate. An array is stored as one ``.npy`` file +(NumPy's own format, version 3.0, never pickled) in a ``large_binary`` cell under +the extension name ``orcapod.numpy.ndarray``, and reads back as it was given: +dtype including byte order, shape, element bits, C or Fortran contiguity and +structured field offsets are kept. The hash is defined by Orcapod on the value, +never on the stored bytes. + +What ``.npy`` does not keep is never part of the hash: strides other than C or +Fortran contiguity (a strided view is stored C-contiguous), the subclass, the +``writeable`` flag, dtype metadata, the ``align`` flag of a structured dtype (its +offsets are kept) and padding bytes between structured fields. + +Value hash ``D`` (label ``ndarray_v1``), in two steps. With +``lp(x) = u64le(len(utf8(x))) || utf8(x)``: + + E = array_format.array_digest_v1(tag, shape, bytes) + D = SHA-256(lp("orcapod.numpy.ndarray") || E || u8(memory_order) + || lp(byte_order_text) || lp(struct_layout_text)) + +``E`` is taken on the canonical form of the array (``canonical_array``: C order, +little-endian, structured fields packed in declared order) and covers dtype, +shape and element bits. The three further fields are the properties the hash +can ignore (``array_format.NDARRAY_OPTIONS``): + +- ``memory_order``: 1 when the array is Fortran-contiguous and not + C-contiguous, else 0 (the flag ``.npy`` keeps). +- ``byte_order``: one character per scalar leaf of the dtype, depth first in + declared field order (a subarray field counts once): ``<``, ``>``, or ``|`` + for a type without byte order. NumPy's native marker is resolved with + ``sys.byteorder``, so the text is the same on every machine. +- ``struct_layout``: compact JSON of ``L(dtype)``: ``null`` for an + unstructured dtype, else ``[itemsize, [offset, L(field)], ...]`` in declared + field order (a subarray field uses its base dtype). + +An ignored property's field is written with its canonical value: 0, the text +with every ``>`` replaced by ``<``, the layout of the packed dtype. The ignore +list is saved in the extension metadata (``ignore_list``), not in the preimage. + +Refused at write time, with ``UnsupportedValueError`` naming the dtype (the type converter adds the column): masked arrays, object dtypes, NumPy 2 -``StringDType`` and other "extended data" arrays, ``longdouble`` and -``clongdouble`` (platform-dependent), datetimes and timedeltas without a unit, -and structured dtypes with field titles. +``StringDType``, ``longdouble`` and ``clongdouble`` (platform-dependent), +datetimes and timedeltas without a unit, structured dtypes with field titles, +structured dtypes with overlapping or out-of-order field offsets (``.npy`` has +no form for them), and dtypes whose ``.npy`` header is longer than NumPy reads +without pickle support. """ from __future__ import annotations +import hashlib +import io import json import re +import struct +import sys from typing import TYPE_CHECKING, Any import numpy as np import numpy.typing as npt -import polars as pl import pyarrow as pa +from numpy.lib import format as npy_format from orcapod.errors import UnsupportedValueError from orcapod.logical_types.array_format import ( - ARRAY_FORMAT_METADATA, - ARRAY_STORAGE_TYPE, -) -from orcapod.logical_types.base_logical_type import BaseLogicalType -from orcapod.logical_types.registry import ( - make_arrow_extension_type, - make_polars_extension_type, + NDARRAY_EXTENSION_NAME, + NDARRAY_OPTIONS, + array_digest_v1, + length_prefixed, ) +from orcapod.logical_types.ignore_list import IgnoreListLogicalType if TYPE_CHECKING: from orcapod.logical_types.protocols import TypeConverterProtocol -NDARRAY_EXTENSION_NAME = "orcapod.ndarray" -"""Arrow extension name (and logical type name) of the array format.""" +NPY_FORMAT_VERSION = (3, 0) +"""The ``.npy`` format version every array is stored in. + +Version 3.0 is the only one that stores structured field names outside Latin-1. +""" + +_NPY_MAGIC = b"\x93NUMPY" + bytes(NPY_FORMAT_VERSION) + +_MAX_NPY_HEADER_BYTES = 10000 +"""Longest ``.npy`` header ``read_array`` parses when pickle is not allowed.""" _INT_TAGS = {1: "int8", 2: "int16", 4: "int32", 8: "int64"} _UINT_TAGS = {1: "uint8", 2: "uint16", 4: "uint32", 8: "uint64"} @@ -307,128 +342,226 @@ def array_bytes(array: npt.NDArray[Any]) -> memoryview: return array.reshape(-1).view(np.uint8).data -class LogicalNumpyArray(BaseLogicalType): - """Logical type for ``numpy.ndarray`` in the v0.3 value-based array format. +def _plain_array(value: Any) -> npt.NDArray[Any]: + """Return ``value`` as a plain ``np.ndarray`` view, refusing what is not one. - Stores each array as ``{"dtype": tag, "shape": [...], "data": bytes}`` in - ``ARRAY_STORAGE_TYPE`` under the extension name ``orcapod.ndarray``. A value - reads back as a writeable, C-ordered, little-endian plain ``np.ndarray``. + Raises: + UnsupportedValueError: For masked arrays. + TypeError: If ``value`` is not a ``numpy.ndarray``. + """ + if isinstance(value, np.ma.MaskedArray): + raise UnsupportedValueError( + f"numpy masked arrays (dtype {value.dtype!r}) are extended data and " + "cannot be stored: store the data and the mask as two arrays" + ) + if not isinstance(value, np.ndarray): + raise TypeError(f"expected a numpy.ndarray, got {type(value).__name__}") + # The base implementation, so a subclass's own ``view`` cannot change the values. + return np.ndarray.view(value, np.ndarray) - Example: - >>> lt = LogicalNumpyArray() - >>> arr = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float64) - >>> recovered = lt.storage_to_python(lt.python_to_storage(arr)) - >>> np.array_equal(recovered, arr) - True + +def array_to_npy(value: Any) -> bytes: + """Return the ``.npy`` file an array is stored as. + + The array is written as it is: byte order, C or Fortran contiguity and + structured offsets are kept. A view that is neither C- nor Fortran-contiguous + is written C-contiguous. ``value`` is not modified. + + Args: + value: A ``numpy.ndarray`` (a subclass is stored as a plain array). + + Returns: + The bytes of a ``.npy`` file of format ``NPY_FORMAT_VERSION``. + + Raises: + UnsupportedValueError: For masked arrays and dtypes that cannot be stored. + TypeError: If ``value`` is not a ``numpy.ndarray``. """ + plain = _plain_array(value) + # The tag is not stored; computing it applies the dtype refusals. + dtype_to_tag(plain.dtype) + buffer = io.BytesIO() + try: + npy_format.write_array( + buffer, plain, version=NPY_FORMAT_VERSION, allow_pickle=False + ) + except ValueError as exc: + # NumPy cannot describe a structured dtype whose fields overlap or are + # not in offset order (``dtype.descr`` is undefined for it). + raise _refuse( + plain.dtype, + "structured fields with overlapping or out-of-order offsets have no " + f".npy form ({exc})", + ) from exc + data = buffer.getvalue() + (header_bytes,) = struct.unpack_from(" _MAX_NPY_HEADER_BYTES: + raise _refuse( + plain.dtype, + f"its .npy header is {header_bytes} bytes and NumPy reads at most " + f"{_MAX_NPY_HEADER_BYTES} without pickle support (too many fields)", + ) + return data - _arrow_ext_class = make_arrow_extension_type( - NDARRAY_EXTENSION_NAME, ARRAY_STORAGE_TYPE, ARRAY_FORMAT_METADATA - ) - _arrow_ext: pa.ExtensionType | None = None - _polars_ext_class = make_polars_extension_type( - NDARRAY_EXTENSION_NAME, ARRAY_STORAGE_TYPE, ARRAY_FORMAT_METADATA.decode() - ) - _polars_ext: pl.BaseExtension | None = None - logical_type_name: str = NDARRAY_EXTENSION_NAME - python_type: type = np.ndarray +def npy_to_array(data: bytes) -> npt.NDArray[Any]: + """Return the array stored in a ``.npy`` file written by ``array_to_npy``. - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the cached Arrow extension type ``orcapod.ndarray``. + Args: + data: The stored bytes. - Returns: - The ``pa.ExtensionType`` over ``ARRAY_STORAGE_TYPE``. - """ - if LogicalNumpyArray._arrow_ext is None: - LogicalNumpyArray._arrow_ext = LogicalNumpyArray._arrow_ext_class() - return LogicalNumpyArray._arrow_ext + Returns: + A new, writeable plain ``np.ndarray`` with the stored dtype (including + byte order and structured offsets), shape and contiguity. - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the cached Polars extension type ``orcapod.ndarray``. + Raises: + ValueError: If ``data`` is not a ``.npy`` file of format + ``NPY_FORMAT_VERSION``, is truncated, or holds an object array. + """ + if bytes(data[: len(_NPY_MAGIC)]) != _NPY_MAGIC: + raise ValueError( + "stored numpy array value is not a .npy file of format " + f"{NPY_FORMAT_VERSION[0]}.{NPY_FORMAT_VERSION[1]}" + ) + return npy_format.read_array(io.BytesIO(data), allow_pickle=False) - Returns: - The ``pl.BaseExtension`` registered under ``orcapod.ndarray``. - """ - if LogicalNumpyArray._polars_ext is None: - LogicalNumpyArray._polars_ext = LogicalNumpyArray._polars_ext_class() - return LogicalNumpyArray._polars_ext + +def _byte_order_text(dtype: np.dtype[Any]) -> str: + """Return one byte-order character per scalar leaf of ``dtype``.""" + if dtype.subdtype is not None: + return _byte_order_text(dtype.subdtype[0]) + if dtype.fields is not None: + fields = {str(key): value for key, value in dtype.fields.items()} + return "".join(_byte_order_text(fields[name][0]) for name in dtype.names or ()) + order = dtype.byteorder + if order == "=": + return "<" if sys.byteorder == "little" else ">" + return order + + +def _struct_layout(dtype: np.dtype[Any]) -> Any: + """Return ``L(dtype)``: ``None``, or ``[itemsize, [offset, L(field)], ...]``.""" + if dtype.subdtype is not None: + dtype = dtype.subdtype[0] + if dtype.fields is None: + return None + fields = {str(key): value for key, value in dtype.fields.items()} + layout: list[Any] = [int(dtype.itemsize)] + for name in dtype.names or (): + layout.append([int(fields[name][1]), _struct_layout(fields[name][0])]) + return layout + + +def array_value_digest(value: Any, ignore: frozenset[str]) -> bytes: + """Return the value digest ``D`` of an array (see the module docstring). + + Args: + value: A ``numpy.ndarray`` (a subclass is hashed as a plain array). + ignore: Names from ``NDARRAY_OPTIONS`` whose fields are written with + their canonical value. + + Returns: + The 32-byte SHA-256 digest. + + Raises: + UnsupportedValueError: For masked arrays and dtypes that cannot be stored. + TypeError: If ``value`` is not a ``numpy.ndarray``. + """ + plain = _plain_array(value) + tag, canonical = canonical_array(plain) + element_digest = array_digest_v1(tag, canonical.shape, array_bytes(canonical)) + + fortran = plain.flags.f_contiguous and not plain.flags.c_contiguous + memory_order = 1 if fortran and "memory_order" not in ignore else 0 + byte_order = _byte_order_text(plain.dtype) + if "byte_order" in ignore: + byte_order = byte_order.replace(">", "<") + layout_dtype = canonical.dtype if "struct_layout" in ignore else plain.dtype + layout = json.dumps(_struct_layout(layout_dtype), separators=(",", ":")) + + h = hashlib.sha256() + h.update(length_prefixed(NDARRAY_EXTENSION_NAME)) + h.update(element_digest) + h.update(bytes([memory_order])) + h.update(length_prefixed(byte_order)) + h.update(length_prefixed(layout)) + return h.digest() + + +class LogicalNumpyArray(IgnoreListLogicalType): + """Logical type for ``numpy.ndarray``: one ``.npy`` file per value. + + The extension name is ``orcapod.numpy.ndarray`` over ``large_binary``; the + extension metadata lists the properties the value hash ignores. A value reads + back as a writeable plain ``np.ndarray`` with the dtype (including byte + order and structured offsets), shape and C or Fortran contiguity it had. + + Args: + ignore: Names from ``NDARRAY_OPTIONS`` the value hash of this type + ignores. Storage is the same under every list. + + Example: + >>> lt = LogicalNumpyArray() + >>> arr = np.asfortranarray(np.array([[1.0, 2.0], [3.0, 4.0]], dtype=">f8")) + >>> recovered = lt.storage_to_python(lt.python_to_storage(arr)) + >>> recovered.dtype.str, recovered.flags.f_contiguous + ('>f8', True) + """ + + EXTENSION_NAME = NDARRAY_EXTENSION_NAME + STORAGE_TYPE = pa.large_binary() + VERSION = 1 + OPTIONS = NDARRAY_OPTIONS + python_type: type = np.ndarray def python_to_storage( self, value: Any, converter: TypeConverterProtocol | None = None - ) -> dict[str, Any]: - """Encode an array as its canonical stored row. + ) -> bytes: + """Encode an array as its ``.npy`` file. Args: value: A ``numpy.ndarray``. converter: Ignored. Present for protocol conformance. Returns: - ``{"dtype": tag, "shape": list_of_ints, "data": bytes}``. + The ``.npy`` bytes (``array_to_npy``). Raises: UnsupportedValueError: For masked arrays and dtypes that cannot be stored. TypeError: If ``value`` is not a ``numpy.ndarray``. """ - tag, array = canonical_array(value) - return { - "dtype": tag, - "shape": [int(d) for d in array.shape], - "data": array_bytes(array).tobytes(), - } + return array_to_npy(value) def storage_to_python( self, storage_value: Any, converter: TypeConverterProtocol | None = None ) -> npt.NDArray[Any]: - """Decode a stored row into a writeable ``numpy.ndarray``. + """Decode a stored ``.npy`` file into a writeable ``numpy.ndarray``. Args: - storage_value: The stored ``{"dtype", "shape", "data"}`` mapping. + storage_value: The stored bytes. converter: Ignored. Present for protocol conformance. Returns: - A C-ordered plain ``np.ndarray`` of the canonical dtype. + The array as it was given (``npy_to_array``). Raises: - ValueError: If a field is missing or null, or the data length does - not match the dtype and shape. + ValueError: If the bytes are not a ``.npy`` file of the stored format. """ - tag = storage_value.get("dtype") - shape = storage_value.get("shape") - data = storage_value.get("data") - if tag is None or shape is None or data is None: - raise ValueError( - "stored numpy array value has a null field " - f"(dtype={tag!r}, shape={shape!r}, data is None: {data is None})" - ) - dtype = tag_to_dtype(tag) - dims = tuple(int(d) for d in shape) - count = int(np.prod(dims, dtype=np.int64)) if dims else 1 - if len(data) != count * dtype.itemsize: - raise ValueError( - f"stored numpy array value has {len(data)} data bytes; dtype {tag!r} " - f"and shape {list(dims)} need {count * dtype.itemsize}" - ) - if dtype.itemsize == 0: - return np.zeros(dims, dtype=dtype) - return np.frombuffer(data, dtype=dtype, count=count).reshape(dims).copy() + return npy_to_array(storage_value) def storage_sort_key(self, storage_value: Any) -> tuple[Any, ...]: """Ordering key for stored values, used where operators sort values. - Orders by the stored ``(dtype tag, shape, data)``; ``None`` sorts first. + Orders by the strict value digest of the decoded array, never by the + stored bytes (NumPy does not promise them across versions); ``None`` + sorts first. Args: - storage_value: A stored row, or ``None``. + storage_value: Stored ``.npy`` bytes, or ``None``. Returns: - A tuple that orders stored rows deterministically. + A tuple that orders stored values deterministically. """ if storage_value is None: return (0,) - return ( - 1, - storage_value["dtype"], - list(storage_value["shape"]), - bytes(storage_value["data"]), - ) + return (1, array_value_digest(npy_to_array(storage_value), frozenset())) diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py index 8f8906e7b..228626e7a 100644 --- a/tests/test_arrays/test_array_strict_hash.py +++ b/tests/test_arrays/test_array_strict_hash.py @@ -23,8 +23,9 @@ from orcapod.contexts.core import ContextValidationError, DataContext from orcapod.contexts.registry import JSONDataContextRegistry from orcapod.core.operators import MergeJoin -from orcapod.errors import InputValidationError +from orcapod.logical_types.identifier import LogicalTypeIdentifierError from orcapod.sources import ArrowTableSource, DictSource +from orcapod.utils.arrow_utils import shared_arrow_type HASH_LABEL = "ndarray_v1" """Label of an array value hash (open decision 2 of the design note).""" @@ -328,20 +329,36 @@ def test_unknown_option_name_is_an_error(tmp_path: Path) -> None: def test_merge_join_of_two_ignore_lists_is_an_error(tmp_path: Path) -> None: - """Joining one column typed with two ignore lists names the column and both types.""" + """Joining one column typed with two ignore lists names the column and both lists. + + A data context reads only columns of its own list, so the error is the read + refusal, raised when the join inspects its inputs. + """ lenient = _context_ignoring(tmp_path, ["memory_order"]) def stream(context: DataContext) -> ArrowTableSource: table = context.type_converter.python_dicts_to_arrow_table( [{"k": "r1", "x": BASE}], python_schema={"k": str, "x": np.ndarray} ) - return ArrowTableSource( - table, tag_columns=["k"], source_id="s", data_context=context - ) + return ArrowTableSource(table, tag_columns=["k"], source_id="s") - with pytest.raises(InputValidationError, match="'x'") as raised: + with pytest.raises(LogicalTypeIdentifierError, match="'x'") as raised: MergeJoin()(stream(get_default_context()), stream(lenient)).as_table() - assert "memory_order" in str(raised.value) + message = str(raised.value) + assert repr(ignore_metadata(["memory_order"])) in message + assert repr(ignore_metadata([])) in message + + +def test_the_join_type_rule_tells_two_ignore_lists_apart(tmp_path: Path) -> None: + """The rule joins use for a shared column finds no shared type for two lists.""" + lenient = _context_ignoring(tmp_path, ["memory_order"]) + strict_type = get_default_context().type_converter.python_type_to_arrow_type( + np.ndarray + ) + lenient_type = lenient.type_converter.python_type_to_arrow_type(np.ndarray) + assert shared_arrow_type(strict_type, lenient_type) is None + assert shared_arrow_type(strict_type, strict_type) == strict_type + assert "memory_order" in str(lenient_type) def test_native_byte_order_is_absolute() -> None: From 2e0109a42e57e4ffac9511af704470ad890790f7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:30:01 -0800 Subject: [PATCH 08/22] test(arrays): rewrite the value-format tests for .npy storage and the strict hash (ITL-981) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/logical_types/numpy_type.py | 74 +++- tests/test_arrays/test_array_value_format.py | 421 ++++++++++--------- 2 files changed, 279 insertions(+), 216 deletions(-) diff --git a/src/orcapod/logical_types/numpy_type.py b/src/orcapod/logical_types/numpy_type.py index 27bcbb3cf..f034f39b4 100644 --- a/src/orcapod/logical_types/numpy_type.py +++ b/src/orcapod/logical_types/numpy_type.py @@ -360,6 +360,58 @@ def _plain_array(value: Any) -> npt.NDArray[Any]: return np.ndarray.view(value, np.ndarray) +def _require_storable_dtype(dtype: np.dtype[Any]) -> None: + """Raise unless arrays of ``dtype`` have a ``.npy`` form Orcapod accepts. + + Raises: + UnsupportedValueError: For the dtypes ``dtype_to_tag`` refuses, and for a + structured dtype whose fields overlap or are not in offset order. + """ + dtype_to_tag(dtype) + try: + # NumPy cannot describe such a dtype in a ``.npy`` header. + _ = dtype.descr + except ValueError as exc: + raise _refuse( + dtype, + "structured fields with overlapping or out-of-order offsets have no " + ".npy form (declare the fields in offset order)", + ) from exc + + +def _without_metadata(dtype: np.dtype[Any]) -> np.dtype[Any]: + """Return ``dtype`` with the same layout and no dtype ``metadata``. + + ``.npy`` does not store dtype metadata, and NumPy warns when asked to write + it. Orcapod never counts it, so it is dropped before writing, at every + depth. A dtype without metadata is returned unchanged. + """ + if dtype.fields is not None: + fields = {str(key): value for key, value in dtype.fields.items()} + names = list(dtype.names or ()) + formats = [_without_metadata(fields[name][0]) for name in names] + unchanged = all( + new is fields[name][0] for name, new in zip(names, formats, strict=True) + ) + if unchanged and dtype.metadata is None: + return dtype + return np.dtype( + { + "names": names, + "formats": formats, + "offsets": [fields[name][1] for name in names], + "itemsize": dtype.itemsize, + } + ) + if dtype.subdtype is not None: + base, shape = dtype.subdtype + stripped = _without_metadata(base) + if stripped is base and dtype.metadata is None: + return dtype + return np.dtype((stripped, shape)) + return dtype if dtype.metadata is None else np.dtype(dtype.str) + + def array_to_npy(value: Any) -> bytes: """Return the ``.npy`` file an array is stored as. @@ -378,21 +430,14 @@ def array_to_npy(value: Any) -> bytes: TypeError: If ``value`` is not a ``numpy.ndarray``. """ plain = _plain_array(value) - # The tag is not stored; computing it applies the dtype refusals. - dtype_to_tag(plain.dtype) + _require_storable_dtype(plain.dtype) + stored_dtype = _without_metadata(plain.dtype) + if stored_dtype is not plain.dtype: + plain = plain.view(stored_dtype) buffer = io.BytesIO() - try: - npy_format.write_array( - buffer, plain, version=NPY_FORMAT_VERSION, allow_pickle=False - ) - except ValueError as exc: - # NumPy cannot describe a structured dtype whose fields overlap or are - # not in offset order (``dtype.descr`` is undefined for it). - raise _refuse( - plain.dtype, - "structured fields with overlapping or out-of-order offsets have no " - f".npy form ({exc})", - ) from exc + npy_format.write_array( + buffer, plain, version=NPY_FORMAT_VERSION, allow_pickle=False + ) data = buffer.getvalue() (header_bytes,) = struct.unpack_from(" _MAX_NPY_HEADER_BYTES: @@ -468,6 +513,7 @@ def array_value_digest(value: Any, ignore: frozenset[str]) -> bytes: TypeError: If ``value`` is not a ``numpy.ndarray``. """ plain = _plain_array(value) + _require_storable_dtype(plain.dtype) tag, canonical = canonical_array(plain) element_digest = array_digest_v1(tag, canonical.shape, array_bytes(canonical)) diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index e6335eccc..3f316bc21 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -1,14 +1,14 @@ -"""The v0.3 value-based array format: storage, canonical form, hash and refusals (ITL-888). +"""Array values: ``.npy`` storage, what reads back, the hash and refusals (ITL-981). -Specification: ``superpowers/specs/2026-11-10-array-format-v0.3.md``. Tests go -through sources, streams, function pods and the default data context's -semantic hasher. +Design: ``superpowers/specs/2026-11-12-itl-981-library-value-types-design.md``. +Tests go through sources, streams, function pods and the default data context's +semantic hasher. The strict hash and the ignore list have their own file, +``test_array_strict_hash.py``. """ from __future__ import annotations -import hashlib -import json +import io import struct from pathlib import Path from typing import Any @@ -20,22 +20,19 @@ from orcapod import function_pod from orcapod.contexts import get_default_context +from orcapod.logical_types.numpy_type import dtype_to_tag from orcapod.operators import MergeJoin from orcapod.sources import ArrowTableSource, DictSource from tests.test_arrays.array_cases import ARRAYS, assert_arrays_identical - -def reference_digest(tag: str, shape: tuple[int, ...], data: bytes) -> bytes: - """Independent implementation of the v1 array digest (spec C1).""" - t = tag.encode("utf-8") - preimage = struct.pack(" dict[str, Any]: - """The stored row of a one-row ``x`` column.""" +def stored(value: Any) -> bytes: + """The stored cell of a one-row ``x`` column.""" table = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]).as_table() chunk = table.column("x").chunk(0) assert isinstance(chunk, pa.ExtensionArray) @@ -61,36 +58,31 @@ def data_hash(value: Any) -> Any: # --------------------------------------------------------------------------- -def test_column_type_is_orcapod_ndarray_over_the_array_struct() -> None: - """An array column is ``orcapod.ndarray`` over the array struct with version metadata.""" +def test_column_type_is_orcapod_numpy_ndarray_over_large_binary() -> None: + """An array column is ``orcapod.numpy.ndarray`` over ``large_binary`` with the ignore list.""" table = DictSource( data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"] ).as_table() ext = table.schema.field("x").type assert isinstance(ext, pa.ExtensionType) - assert ext.extension_name == "orcapod.ndarray" - assert ext.storage_type == pa.struct( - [ - pa.field("dtype", pa.large_string()), - pa.field("shape", pa.large_list(pa.int64())), - pa.field("data", pa.large_binary()), - ] - ) - assert ext.__arrow_ext_serialize__() == b'{"version":1}' + assert ext.extension_name == "orcapod.numpy.ndarray" + assert ext.storage_type == pa.large_binary() + assert ext.__arrow_ext_serialize__() == b'{"ignore":[],"version":1}' -def test_old_extension_name_is_not_registered() -> None: - """Only ``orcapod.ndarray`` is registered; the old ``numpy.ndarray`` name is not.""" +def test_older_extension_names_are_not_registered() -> None: + """Only ``orcapod.numpy.ndarray`` is registered; the names of v0.2 and of the first v0.3 format are not.""" converter: Any = get_default_context().type_converter assert converter.get_logical_type_by_arrow_extension_name("numpy.ndarray") is None + assert converter.get_logical_type_by_arrow_extension_name("orcapod.ndarray") is None assert ( - converter.get_logical_type_by_arrow_extension_name("orcapod.ndarray") + converter.get_logical_type_by_arrow_extension_name("orcapod.numpy.ndarray") is not None ) # --------------------------------------------------------------------------- -# Stored rows: dtype tags, shape, little-endian C-order data +# Stored cells: one .npy file per value; dtype tags # --------------------------------------------------------------------------- TAG_CASES: list[tuple[str, npt.NDArray[Any], str]] = [ @@ -141,54 +133,52 @@ def test_old_extension_name_is_not_registered() -> None: @pytest.mark.parametrize( ("array", "tag"), [c[1:] for c in TAG_CASES], ids=[c[0] for c in TAG_CASES] ) -def test_stored_row_holds_tag_shape_and_c_order_little_endian_data( +def test_stored_cell_is_a_npy_file_numpy_reads( array: npt.NDArray[Any], tag: str ) -> None: - """The stored row holds the dtype tag, the shape and the packed element bytes.""" - row = stored(array) - assert row["dtype"] == tag - assert row["shape"] == list(array.shape) - assert len(row["data"]) == array.size * np.dtype(read_back(array).dtype).itemsize + """The stored cell is a ``.npy`` file of format 3.0 that plain NumPy loads back.""" + cell = stored(array) + assert cell[:8] == b"\x93NUMPY\x03\x00" + loaded = np.load(io.BytesIO(cell), allow_pickle=False) + assert loaded.dtype == array.dtype + assert loaded.shape == array.shape + assert loaded.tobytes() == array.tobytes() @pytest.mark.parametrize( - "name", ["float32", "three_d", "complex128", "unicode", "datetime64_ns"] + ("array", "tag"), [c[1:] for c in TAG_CASES], ids=[c[0] for c in TAG_CASES] ) -def test_data_is_the_c_order_little_endian_buffer(name: str) -> None: - """The stored data is the array's C-order, little-endian buffer.""" - array = ARRAYS[name] - assert stored(array)["data"] == np.ascontiguousarray(array).tobytes() +def test_dtype_tag_of_the_element_digest(array: npt.NDArray[Any], tag: str) -> None: + """The dtype tag that enters the element digest names the dtype without a byte order.""" + assert dtype_to_tag(array.dtype) == tag + assert dtype_to_tag(array.dtype.newbyteorder(">")) == tag def test_zero_d_and_empty_shapes() -> None: - """Zero-dimensional and empty arrays store their exact shape and data.""" - assert stored(np.array(2.5))["shape"] == [] - assert stored(np.array(2.5))["data"] == struct.pack(" None: - """Structured data is stored little-endian and packed in declared field order.""" - array = np.array([(1.5, 7)], dtype=[("b", ">f8"), ("a", " None: + """Within one NumPy version, equal arrays give equal stored bytes.""" + array = np.arange(6, dtype=np.float32).reshape(2, 3) + assert stored(array) == stored(array.copy()) # --------------------------------------------------------------------------- -# Canonical form on read +# What reads back # --------------------------------------------------------------------------- -def test_fortran_order_reads_back_c_order() -> None: - """A Fortran-ordered array reads back C-ordered with the same values.""" +def test_fortran_order_reads_back_fortran_order() -> None: + """A Fortran-ordered array reads back Fortran-ordered with the same values.""" array = np.asfortranarray(np.arange(6, dtype=np.float32).reshape(2, 3)) - assert_arrays_identical(read_back(array), np.ascontiguousarray(array)) + got = read_back(array) + assert got.flags.f_contiguous and not got.flags.c_contiguous + assert (got == array).all() def test_strided_view_reads_back_as_its_values() -> None: @@ -199,35 +189,40 @@ def test_strided_view_reads_back_as_its_values() -> None: @pytest.mark.parametrize("dtype", [">f4", ">i8", ">c16", ">M8[ns]", ">U3"]) -def test_big_endian_reads_back_little_endian_with_equal_values(dtype: str) -> None: - """A big-endian array reads back little-endian with equal values.""" +def test_big_endian_reads_back_big_endian(dtype: str) -> None: + """A big-endian array reads back big-endian with equal values.""" raw = ( ["ab", "c"] if dtype.startswith(">U") else np.arange(2).astype(dtype[0] + "i8") ) big = np.array(raw).astype(dtype) got = read_back(big) - assert got.dtype == big.dtype.newbyteorder("<") - assert (got == big).all() + assert got.dtype.str == big.dtype.str + assert got.tobytes() == big.tobytes() -def test_structured_reads_back_in_stored_field_order_and_packed() -> None: - """An aligned structured array reads back in declared field order, little-endian and packed.""" +def test_aligned_structured_reads_back_with_its_offsets() -> None: + """An aligned structured array keeps field order, byte order, offsets and itemsize.""" aligned = np.dtype([("b", ">f8"), ("a", "f8" + assert got.dtype.fields is not None and got.dtype.fields["a"][1] == 8 assert got["a"].tolist() == [7, 9] assert got["b"].tolist() == [1.5, -2.0] + # The ``align`` flag itself is not kept by ``.npy``; the layout it produced is. + assert not got.dtype.isalignedstruct -def test_nested_structured_reads_back_in_stored_field_order() -> None: - """A nested structured array reads back in declared field order at every depth.""" +def test_nested_structured_reads_back_as_given() -> None: + """A nested structured array keeps field order and byte order at every depth.""" inner = np.dtype([("y", ">i2"), ("x", "i2" assert got["s"]["y"].tolist() == [1] and got["s"]["x"].tolist() == [2] @@ -235,7 +230,6 @@ def test_reordered_fields_are_a_different_stored_value() -> None: """Arrays whose fields differ only in order are different stored values.""" ab = np.array([(1, 2.0)], dtype=[("a", " None: def test_read_back_is_writeable() -> None: - """An array read back from storage is writeable.""" - got = read_back(np.arange(3.0)) + """An array read back from storage is writeable, also when a read-only one was written.""" + frozen = np.arange(3.0) + frozen.setflags(write=False) + got = read_back(frozen) got[0] = 10.0 assert got[0] == 10.0 +def test_dtype_metadata_is_dropped_without_a_warning() -> None: + """Dtype metadata is not stored and not hashed; writing it raises no warning.""" + plain = np.arange(3, dtype=" Non # --------------------------------------------------------------------------- -# Hash: value digest under the ndarray_v1 label +# Hash: the value digest under the ndarray_v1 label # --------------------------------------------------------------------------- @pytest.mark.parametrize("name", sorted(ARRAYS)) -def test_semantic_hash_is_the_v1_digest_of_the_stored_row(name: str) -> None: - """The semantic hash is the ``ndarray_v1`` digest of the stored row.""" - row = stored(ARRAYS[name]) - h = semantic_hash(ARRAYS[name]) - assert h.method == "ndarray_v1" - assert h.digest == reference_digest(row["dtype"], tuple(row["shape"]), row["data"]) - - -def test_pinned_digests() -> None: - """Pinned digests match the spec's byte rules.""" - # Pinned v1 digests; computed by ``reference_digest`` from the spec's byte rules. - cases = { - "float32_2x3": (np.arange(6, dtype=np.float32).reshape(2, 3), "float32"), - "zero_d": (np.array(2.5), "float64"), - "empty": (np.zeros((0, 3), np.int8), "int8"), - } - for array, tag in cases.values(): - expected = reference_digest( - tag, array.shape, np.ascontiguousarray(array).tobytes() - ) - assert semantic_hash(array).digest == expected - assert ( - semantic_hash(cases["zero_d"][0]).digest.hex() - == reference_digest("float64", (), struct.pack(" None: + """The hash is labelled ``ndarray_v1`` and is the same for the value read back.""" + given = semantic_hash(ARRAYS[name]) + assert given.method == "ndarray_v1" + assert semantic_hash(read_back(ARRAYS[name])) == given def _equal_value_variants() -> list[tuple[str, npt.NDArray[Any], npt.NDArray[Any]]]: a = np.arange(24, dtype=np.float32).reshape(4, 6) base = np.zeros((4, 12), dtype=np.float32) base[:, ::2] = a - rec_ab = np.array([(1, 2.0)], dtype=[("a", "i4"), ("b", ">f8")]) - aligned = np.array( - [(1, 2.0)], dtype=np.dtype([("a", "f4")), - ("structured_byte_order", rec_ab, rec_ab_big), - ("aligned", rec_ab, aligned), - ("padded", rec_ab, padded), - ("offsets_swapped", rec_ab, offsets_swapped), + ("read_only", a, frozen), ( "matrix", np.arange(4.0).reshape(2, 2), np.matrix(np.arange(4.0).reshape(2, 2)), ), + ( + "align_flag_only", + np.zeros(2, dtype=[("a", " list[tuple[str, npt.NDArray[Any], npt.NDArray[Any [v[1:] for v in _equal_value_variants()], ids=[v[0] for v in _equal_value_variants()], ) -def test_layout_byte_order_padding_and_subclass_do_not_change_the_hash( +def test_what_storage_does_not_keep_does_not_change_the_hash( left: npt.NDArray[Any], right: npt.NDArray[Any] ) -> None: - """Memory layout, byte order, padding and subclass do not change the hash.""" + """Strides, the writeable flag, the subclass and the ``align`` flag do not change the hash.""" assert semantic_hash(left) == semantic_hash(right) assert data_hash(left) == data_hash(right) @@ -386,6 +352,16 @@ def test_layout_byte_order_padding_and_subclass_do_not_change_the_hash( def _different_value_pairs() -> list[tuple[str, npt.NDArray[Any], npt.NDArray[Any]]]: nan_a = np.array([np.nan]) nan_b = np.frombuffer(struct.pack(" list[tuple[str, npt.NDArray[Any], npt.NDArray[An np.zeros(1, dtype=[("s", [("y", "f4")), + ( + "structured_byte_order", + rec_ab, + rec_ab.astype([("a", ">i4"), ("b", ">f8")]), + ), + ( + "aligned_offsets", + rec_ab, + rec_ab.astype(np.dtype([("a", " list[tuple[str, npt.NDArray[Any], npt.NDArray[An [v[1:] for v in _different_value_pairs()], ids=[v[0] for v in _different_value_pairs()], ) -def test_dtype_shape_and_bits_change_the_hash( +def test_dtype_shape_bits_and_kept_layout_change_the_hash( left: npt.NDArray[Any], right: npt.NDArray[Any] ) -> None: - """Dtype, shape, field order and element bits change the hash.""" + """Dtype, shape, element bits, memory order, byte order and offsets change the hash.""" assert semantic_hash(left) != semantic_hash(right) assert data_hash(left) != data_hash(right) -def test_table_token_carries_extension_name_and_digest() -> None: - """A table token carries the extension name and the value digest, no label.""" - # The Arrow hasher folds the extension name in front of the handler's digest. +def test_table_token_carries_the_identifier_and_the_digest() -> None: + """A table token carries the type identifier (name and ignore list) and the value digest, no label.""" + # The Arrow hasher folds the identifier text in front of the handler's digest. from orcapod.hashing.arrow_hashers import StarfixArrowHasher hasher = get_default_context().arrow_hasher @@ -439,10 +428,7 @@ def test_table_token_carries_extension_name_and_digest() -> None: # Non-public helper: this test is about the token the hasher builds for a cell. processed = hasher._process_table_columns(table.select(["x"])) # pyright: ignore[reportPrivateUsage] token = processed.column("x")[0].as_py() - assert ( - token - == b"orcapod.ndarray;7b2276657273696f6e223a317d::" + semantic_hash(array).digest - ) + assert token == IDENTIFIER_TEXT + b"::" + semantic_hash(array).digest # --------------------------------------------------------------------------- @@ -458,6 +444,41 @@ def test_table_token_carries_extension_name_and_digest() -> None: ("generic_datetime", np.array(["NaT"], dtype="M8"), "unit"), ("generic_timedelta", np.array([0], dtype="m8"), "unit"), ("titles", np.zeros(1, dtype=[(("title", "a"), " None: - """A stored row with a null child raises on read, naming the column.""" +@pytest.mark.parametrize( + "cell", + [ + b"not a npy file", + b"\x93NUMPY\x01\x00" + b"\x00" * 16, + None, + ], + ids=["garbage", "other_format_version", "truncated"], +) +def test_a_cell_that_is_not_a_stored_array_raises_naming_the_column( + cell: bytes | None, +) -> None: + """A cell that is not a complete ``.npy`` file of format 3.0 raises on read, naming the column.""" converter = get_default_context().type_converter ext = converter.python_type_to_arrow_type(np.ndarray) assert isinstance(ext, pa.ExtensionType) - storage = pa.array( - [{"dtype": "float64", "shape": None, "data": b""}], type=ext.storage_type - ) + if cell is None: + cell = stored(np.arange(4.0))[:-5] + storage = pa.array([cell], type=ext.storage_type) table = pa.table( { "k": pa.array(["r"], pa.large_string()), @@ -492,51 +524,29 @@ def test_null_child_on_read_raises_naming_the_column() -> None: } ) # The first read of the value (hashing the source rows) must fail naming the column. - with pytest.raises((ValueError, RuntimeError), match="'x'.*null field"): + with pytest.raises((ValueError, RuntimeError), match="'x'"): source = ArrowTableSource(table, tag_columns=["k"]) for _, data in source.iter_data(): data.as_dict() -@pytest.mark.parametrize( - "entry", - [ - ["a"], - ["a", "int32", [2], "extra"], - [1, "int32"], - ["a", "int32", 2], - ["a", "int32", ["2"]], - ["a", "int32", [True]], - ["a", "int32", [-1]], - ], - ids=[ - "one_part", - "four_parts", - "non_string_name", - "shape_not_a_list", - "shape_of_strings", - "shape_of_bools", - "negative_dimension", - ], -) -def test_malformed_structured_tag_entry_raises_on_read( - entry: list[Any], -) -> None: - """A malformed structured tag entry raises the malformed-tag error on read.""" +def test_a_stored_object_array_is_never_unpickled() -> None: + """A ``.npy`` file holding a pickled object array is refused on read.""" converter = get_default_context().type_converter ext = converter.python_type_to_arrow_type(np.ndarray) assert isinstance(ext, pa.ExtensionType) - tag = json.dumps([entry]) - storage = pa.array( - [{"dtype": tag, "shape": [], "data": b"\x00" * 8}], type=ext.storage_type + buffer = io.BytesIO() + np.lib.format.write_array( + buffer, np.array([1, "a"], dtype=object), version=(3, 0), allow_pickle=True ) + storage = pa.array([buffer.getvalue()], type=ext.storage_type) table = pa.table( { "k": pa.array(["r"], pa.large_string()), "x": pa.ExtensionArray.from_storage(ext, storage), } ) - with pytest.raises((ValueError, RuntimeError), match="malformed array dtype tag"): + with pytest.raises((ValueError, RuntimeError), match="allow_pickle=False"): source = ArrowTableSource(table, tag_columns=["k"]) for _, data in source.iter_data(): data.as_dict() @@ -573,12 +583,16 @@ def nothing(x: np.ndarray) -> np.ndarray | None: # pyright: ignore[reportMissin # --------------------------------------------------------------------------- -# MergeJoin orders merged arrays by the stored (dtype tag, shape, data) +# MergeJoin orders merged arrays by their value hash # --------------------------------------------------------------------------- -def test_merge_join_orders_arrays_by_stored_tag_shape_and_data() -> None: - """``MergeJoin`` orders arrays by their stored dtype tag, shape and data.""" +def _hash_order(arrays: list[npt.NDArray[Any]]) -> list[npt.NDArray[Any]]: + return sorted(arrays, key=lambda a: semantic_hash(a).digest) + + +def test_merge_join_orders_arrays_by_their_value_hash() -> None: + """``MergeJoin`` orders merged arrays by their strict value hash, never by the stored bytes.""" small = np.array([7.5], dtype=np.float64) large = np.array([107.5], dtype=np.float64) other = np.array([1], dtype=np.int8) @@ -592,19 +606,15 @@ def test_merge_join_orders_arrays_by_stored_tag_shape_and_data() -> None: tag_columns=["k"], source_id="r", ) - got: dict[Any, Any] = { - t.as_dict()["k"]: d.as_dict()["x"] - for t, d in MergeJoin()(left, right).iter_data() - } - - def key(a: npt.NDArray[Any]) -> tuple[str, list[int], bytes]: - row = stored(a) - return (row["dtype"], row["shape"], row["data"]) - - for k, pair in (("a", [large, small]), ("b", [other, small])): - expected = sorted(pair, key=key) - assert [x.tobytes() for x in got[k]] == [x.tobytes() for x in expected] - assert [x.dtype for x in got[k]] == [x.dtype for x in expected] + for pair in ((left, right), (right, left)): + got: dict[Any, Any] = { + t.as_dict()["k"]: d.as_dict()["x"] + for t, d in MergeJoin()(*pair).iter_data() + } + for k, values in (("a", [large, small]), ("b", [other, small])): + expected = _hash_order(values) + assert [x.tobytes() for x in got[k]] == [x.tobytes() for x in expected] + assert [x.dtype for x in got[k]] == [x.dtype for x in expected] def test_merge_join_orders_arrays_that_differ_only_in_field_order() -> None: @@ -617,17 +627,24 @@ def test_merge_join_orders_arrays_that_differ_only_in_field_order() -> None: next(d.as_dict()["x"] for _, d in MergeJoin()(*pair).iter_data()) for pair in ((left, right), (right, left)) ] - expected = sorted([ab, ba], key=lambda a: stored(a)["dtype"]) + expected = _hash_order([ab, ba]) for got in outs: assert [x.dtype.names for x in got] == [x.dtype.names for x in expected] for x, e in zip(got, expected, strict=True): assert_arrays_identical(x, e) -def test_stored_tag_is_valid_json_for_structured() -> None: - """A structured dtype tag is valid JSON in declared field order.""" - tag = stored(np.zeros(1, dtype=[("b", " None: + """Merged arrays read back as each side gave them.""" + base = np.arange(6, dtype=np.float32).reshape(2, 3) + fortran_big = np.asfortranarray(base.astype(">f4")) + left = DictSource(data=[{"k": "a", "x": base}], tag_columns=["k"], source_id="l") + right = DictSource( + data=[{"k": "a", "x": fortran_big}], tag_columns=["k"], source_id="r" + ) + got = next(d.as_dict()["x"] for _, d in MergeJoin()(left, right).iter_data()) + assert sorted(x.dtype.str for x in got) == ["f4"] + assert sorted(bool(x.flags.f_contiguous) for x in got) == [False, True] # --------------------------------------------------------------------------- From 36507ef036ed89320c380a0c81de31c78accdc0e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:31:33 -0800 Subject: [PATCH 09/22] test(logical_types): update array type unit tests for orcapod.numpy.ndarray (ITL-981) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/logical_types/ignore_list.py | 12 +++- tests/test_contexts/test_data_context_v1.py | 2 +- tests/test_logical_types/test_array_format.py | 41 +++++++------- .../test_logical_type_identifiers.py | 17 +++--- tests/test_logical_types/test_numpy_type.py | 55 ++++++++++++++----- .../test_registry_most_specific.py | 2 +- tests/test_logical_types/test_roundtrips.py | 6 +- 7 files changed, 86 insertions(+), 49 deletions(-) diff --git a/src/orcapod/logical_types/ignore_list.py b/src/orcapod/logical_types/ignore_list.py index 5651c2e7d..55ba99c8b 100644 --- a/src/orcapod/logical_types/ignore_list.py +++ b/src/orcapod/logical_types/ignore_list.py @@ -185,6 +185,7 @@ def __init__( self.options = options self._arrow_types: dict[tuple[str, ...], pa.ExtensionType] = {} self._polars_class: type[pl.BaseExtension] | None = None + self._polars_types: dict[tuple[str, ...], pl.BaseExtension] = {} def arrow_type(self, ignore: tuple[str, ...]) -> "pa.ExtensionType": """Return the Arrow extension type of a checked ignore list.""" @@ -253,9 +254,14 @@ def _deserialize( def polars_type(self, ignore: tuple[str, ...]) -> "pl.BaseExtension": """Return the Polars extension type of a checked ignore list.""" - if self._polars_class is None: - self._polars_class = self._make_polars_class() - return self._polars_class(ignore_metadata(self.version, ignore).decode("ascii")) # pyright: ignore[reportCallIssue] # the synthesized class takes the metadata text + cached = self._polars_types.get(ignore) + if cached is None: + if self._polars_class is None: + self._polars_class = self._make_polars_class() + metadata = ignore_metadata(self.version, ignore).decode("ascii") + cached = self._polars_class(metadata) # pyright: ignore[reportCallIssue] # the synthesized class takes the metadata text + self._polars_types[ignore] = cached + return cached def _make_polars_class(self) -> type["pl.BaseExtension"]: name = self.extension_name diff --git a/tests/test_contexts/test_data_context_v1.py b/tests/test_contexts/test_data_context_v1.py index 730d171d0..28211bad5 100644 --- a/tests/test_contexts/test_data_context_v1.py +++ b/tests/test_contexts/test_data_context_v1.py @@ -200,7 +200,7 @@ def test_logical_type_registration_is_unchanged_by_the_move() -> None: assert sorted(registry._by_logical_name) == [ "orcapod.directory", "orcapod.file", - "orcapod.ndarray", + "orcapod.numpy.ndarray", "orcapod.path", "orcapod.upath", "orcapod.uuid", diff --git a/tests/test_logical_types/test_array_format.py b/tests/test_logical_types/test_array_format.py index bdbb24d31..83083dd14 100644 --- a/tests/test_logical_types/test_array_format.py +++ b/tests/test_logical_types/test_array_format.py @@ -1,9 +1,9 @@ -"""Tests for the shared array storage struct and value digest (ITL-888). +"""Tests for the element digest shared by array-like value types (ITL-888, ITL-981). The reference digest below is written independently of Orcapod, with ``struct`` and -``hashlib`` only, from the byte-level definition in -``superpowers/specs/2026-11-10-array-format-v0.3.md`` (section C1). It is the -definition other implementations (starfix, torch) must reproduce. +``hashlib`` only, from the byte-level definition in ``docs/reference/hashing.md`` +(section 8). It is the first step of the ``numpy.ndarray`` value hash, and the +definition other array types (torch) must reproduce. """ from __future__ import annotations @@ -11,14 +11,18 @@ import hashlib import struct -import pyarrow as pa import pytest -from orcapod.logical_types.array_format import ARRAY_STORAGE_TYPE, array_digest_v1 +from orcapod.logical_types.array_format import ( + NDARRAY_EXTENSION_NAME, + NDARRAY_OPTIONS, + array_digest_v1, + length_prefixed, +) def reference_digest(tag: str, shape: tuple[int, ...], data: bytes) -> bytes: - """Independent implementation of the v1 array digest.""" + """Independent implementation of the v1 element digest.""" t = tag.encode("utf-8") preimage = struct.pack(" None: def test_digest_separates_every_field() -> None: - """Every field of the stored row contributes to the digest.""" + """Every field of the preimage contributes to the digest.""" assert array_digest_v1("int8", (0, 3), b"") != array_digest_v1("int8", (3, 0), b"") assert array_digest_v1("int8", (), b"\x01") != array_digest_v1( "int8", (1,), b"\x01" @@ -78,14 +82,13 @@ def test_digest_rejects_negative_dimension() -> None: _ = array_digest_v1("int8", (-1,), b"") -def test_storage_type() -> None: - """The storage type is the dtype, shape and data struct of the spec.""" - # Children are nullable at the Arrow level: Polars drops nested non-null flags - # (ITL-842), so non-null is enforced by the logical type instead. - assert ARRAY_STORAGE_TYPE == pa.struct( - [ - pa.field("dtype", pa.large_string()), - pa.field("shape", pa.large_list(pa.int64())), - pa.field("data", pa.large_binary()), - ] - ) +def test_length_prefixed_text() -> None: + """A text field is its UTF-8 bytes after their 8-byte little-endian length.""" + assert length_prefixed("") == struct.pack(" None: + """The array type's extension name and the properties its hash can ignore.""" + assert NDARRAY_EXTENSION_NAME == "orcapod.numpy.ndarray" + assert NDARRAY_OPTIONS == {"byte_order", "memory_order", "struct_layout"} diff --git a/tests/test_logical_types/test_logical_type_identifiers.py b/tests/test_logical_types/test_logical_type_identifiers.py index 37ff00b59..397b813b7 100644 --- a/tests/test_logical_types/test_logical_type_identifiers.py +++ b/tests/test_logical_types/test_logical_type_identifiers.py @@ -62,7 +62,10 @@ (pd.DataFrame, "pandas.dataframe"), (pd.Series, "pandas.series"), ] -ALL_BUILTINS = [*BUILTINS_WITH_EMPTY_LEGACY, (np.ndarray, "orcapod.ndarray")] +ALL_BUILTINS = [*BUILTINS_WITH_EMPTY_LEGACY, (np.ndarray, "orcapod.numpy.ndarray")] + +# A library value type writes its ignore list beside the version (ITL-981). +BUILTIN_METADATA = {"orcapod.numpy.ndarray": b'{"ignore":[],"version":1}'} def _converter(context: DataContext | None = None) -> UniversalTypeConverter: @@ -153,10 +156,10 @@ def _stored_column( @pytest.mark.parametrize(("python_type", "name"), ALL_BUILTINS) def test_builtin_type_writes_version_1(python_type: Any, name: str) -> None: - """Each built-in extension type has its expected name and writes ``VERSION_1``.""" + """Each built-in extension type has its expected name and writes version 1.""" arrow_type = _extension_type(python_type) assert arrow_type.extension_name == name - assert arrow_type.__arrow_ext_serialize__() == VERSION_1 + assert arrow_type.__arrow_ext_serialize__() == BUILTIN_METADATA.get(name, VERSION_1) @pytest.mark.parametrize(("python_type", "name"), ALL_BUILTINS) @@ -164,12 +167,12 @@ def test_builtin_identifier_is_declared_by_the_extension_type( python_type: Any, name: str ) -> None: """``logical_type_identifier`` of a built-in type is its extension name with - ``VERSION_1``. + its version-1 metadata. """ logical_type = _converter().get_logical_type(python_type) assert logical_type is not None assert logical_type_identifier(logical_type) == LogicalTypeIdentifier( - name, VERSION_1 + name, BUILTIN_METADATA.get(name, VERSION_1) ) @@ -458,11 +461,11 @@ def test_builtin_data_without_metadata_is_read_as_version_1( def test_array_without_metadata_is_refused() -> None: - """The array format never wrote empty metadata, so it does not read it.""" + """The array type never wrote empty metadata, so it does not read it.""" converter = _converter() array_type = _extension_type(np.ndarray) table = _stored_column( - b"orcapod.ndarray", b"", pa.array([], array_type.storage_type) + b"orcapod.numpy.ndarray", b"", pa.array([], array_type.storage_type) ) with pytest.raises(LogicalTypeIdentifierError): converter.load_logical_types(table) diff --git a/tests/test_logical_types/test_numpy_type.py b/tests/test_logical_types/test_numpy_type.py index 8a034a6f7..42fb9f362 100644 --- a/tests/test_logical_types/test_numpy_type.py +++ b/tests/test_logical_types/test_numpy_type.py @@ -15,10 +15,10 @@ def test_isinstance_logical_type(self): assert isinstance(LogicalNumpyArray(), LogicalTypeProtocol) def test_logical_type_name(self): - """The logical type name is ``orcapod.ndarray``.""" + """The logical type name is ``orcapod.numpy.ndarray``.""" from orcapod.logical_types.numpy_type import LogicalNumpyArray - assert LogicalNumpyArray().logical_type_name == "orcapod.ndarray" + assert LogicalNumpyArray().logical_type_name == "orcapod.numpy.ndarray" def test_python_type(self): from orcapod.logical_types.numpy_type import LogicalNumpyArray @@ -26,25 +26,47 @@ def test_python_type(self): assert LogicalNumpyArray().python_type is np.ndarray def test_arrow_ext_name(self): - """The Arrow extension name is ``orcapod.ndarray``.""" + """The Arrow extension name is ``orcapod.numpy.ndarray``.""" from orcapod.logical_types.numpy_type import LogicalNumpyArray assert ( LogicalNumpyArray().get_arrow_extension_type().extension_name - == "orcapod.ndarray" + == "orcapod.numpy.ndarray" ) def test_arrow_ext_storage_type(self): - """The Arrow storage type is the shared array struct.""" - from orcapod.logical_types.numpy_type import LogicalNumpyArray + """The Arrow storage type is ``large_binary``: one ``.npy`` file per value.""" + import pyarrow as pa - from orcapod.logical_types.array_format import ARRAY_STORAGE_TYPE + from orcapod.logical_types.numpy_type import LogicalNumpyArray assert ( LogicalNumpyArray().get_arrow_extension_type().storage_type - == ARRAY_STORAGE_TYPE + == pa.large_binary() ) + def test_ignore_list_is_the_extension_metadata(self): + """The ignore list is written into the extension metadata and makes another type.""" + from orcapod.logical_types.numpy_type import LogicalNumpyArray + + strict = LogicalNumpyArray().get_arrow_extension_type() + lenient = LogicalNumpyArray( + ["memory_order", "byte_order"] + ).get_arrow_extension_type() + assert strict.__arrow_ext_serialize__() == b'{"ignore":[],"version":1}' + assert ( + lenient.__arrow_ext_serialize__() + == b'{"ignore":["byte_order","memory_order"],"version":1}' + ) + assert strict != lenient + + def test_unknown_ignore_name_raises(self): + """A property name the array hash cannot ignore is refused.""" + from orcapod.logical_types.numpy_type import LogicalNumpyArray + + with pytest.raises(ValueError, match="'strides'"): + LogicalNumpyArray(["strides"]) + def test_arrow_ext_is_cached(self): from orcapod.logical_types.numpy_type import LogicalNumpyArray @@ -59,13 +81,17 @@ def test_polars_ext_is_cached(self): class TestLogicalNumpyArrayStorage: - def test_python_to_storage_returns_the_array_struct_row(self): - """``python_to_storage`` returns the array struct row: dtype tag, shape and data.""" + def test_python_to_storage_returns_a_npy_file(self): + """``python_to_storage`` returns a ``.npy`` file of format 3.0 that NumPy loads.""" + import io + from orcapod.logical_types.numpy_type import LogicalNumpyArray arr = np.array([1.0, 2.0, 3.0]) result = LogicalNumpyArray().python_to_storage(arr) - assert result == {"dtype": "float64", "shape": [3], "data": arr.tobytes()} + assert isinstance(result, bytes) + assert result[:8] == b"\x93NUMPY\x03\x00" + assert np.array_equal(np.load(io.BytesIO(result), allow_pickle=False), arr) def test_object_dtype_raises(self): """``python_to_storage`` refuses object arrays.""" @@ -127,10 +153,9 @@ def test_structured_record_array(self): assert np.array_equal(result["x"], arr["x"]) assert np.array_equal(result["y"], arr["y"]) - def test_fortran_order_reads_back_c_order(self): - """A Fortran-ordered array reads back C-contiguous with the same values.""" - # Memory order is not part of an array's identity: storage is C order. + def test_fortran_order_reads_back_fortran_order(self): + """A Fortran-ordered array reads back Fortran-contiguous with the same values.""" arr = np.asfortranarray(np.array([[1, 2], [3, 4]], dtype=np.float64)) result = self._rt(arr) assert np.array_equal(result, arr) - assert result.flags["C_CONTIGUOUS"] + assert result.flags["F_CONTIGUOUS"] and not result.flags["C_CONTIGUOUS"] diff --git a/tests/test_logical_types/test_registry_most_specific.py b/tests/test_logical_types/test_registry_most_specific.py index 3e9d6b272..85bb220d7 100644 --- a/tests/test_logical_types/test_registry_most_specific.py +++ b/tests/test_logical_types/test_registry_most_specific.py @@ -60,7 +60,7 @@ def test_ndarray_subclasses_resolve_to_the_array_type() -> None: registry = LogicalTypeRegistry(logical_types=[LogicalNumpyArray()]) for sub in (np.matrix, np.recarray, np.memmap): lt = registry.get_by_python_type(sub) - assert lt is not None and lt.logical_type_name == "orcapod.ndarray" + assert lt is not None and lt.logical_type_name == "orcapod.numpy.ndarray" def test_registered_subclass_wins_for_itself_and_its_subclasses() -> None: diff --git a/tests/test_logical_types/test_roundtrips.py b/tests/test_logical_types/test_roundtrips.py index fcf0328c1..d3bc1e6e7 100644 --- a/tests/test_logical_types/test_roundtrips.py +++ b/tests/test_logical_types/test_roundtrips.py @@ -275,11 +275,11 @@ def test_builtin_uuid_round_trip( def test_builtin_ndarray_round_trip( storage_backend: _StorageBackend, tmp_path: Path ) -> None: - """numpy.ndarray round-trips through storage with extension name ``orcapod.ndarray``. + """numpy.ndarray round-trips through storage with extension name ``orcapod.numpy.ndarray``. Tests both a simple 1-D float64 array and a structured (record) array with named fields, since structured arrays are a primary motivation for this type. - The read-side converter already knows about ``orcapod.ndarray`` because it is + The read-side converter already knows about ``orcapod.numpy.ndarray`` because it is registered in the default context (``contexts/logical_types.json``). """ import numpy as np @@ -303,7 +303,7 @@ def test_builtin_ndarray_round_trip( assert hasattr(field.type, "extension_name"), ( f"Expected extension type on field 'col', got plain type {field.type!r}" ) - assert field.type.extension_name == "orcapod.ndarray" + assert field.type.extension_name == "orcapod.numpy.ndarray" rows = read_converter.arrow_table_to_python_dicts(result) assert len(rows) == 1 From dc911de2b3c9687e261359ca3e1eb8344f17d1ae Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:34:05 -0800 Subject: [PATCH 10/22] feat(hashing)!: array golden values move once; hash_schema uses the field rule of hash_table (ITL-981) Golden: 22 values moved, 8 system-tag entries renamed (their names embed the schema hash), 4 added, all of them array entries; no other entry moved. Co-Authored-By: Claude Fable 5.2 --- src/orcapod/hashing/arrow_hashers.py | 16 ++++- tests/test_hashing/golden/cases.py | 7 +++ tests/test_hashing/golden/data/arrow.json | 4 +- tests/test_hashing/golden/data/datagram.json | 40 ++++++------- tests/test_hashing/golden/data/schema.json | 8 +-- tests/test_hashing/golden/data/semantic.json | 16 +++-- .../test_hashing/test_starfix_arrow_hasher.py | 58 +++++++++++++++++++ tests/test_logical_types/test_ignore_list.py | 3 + 8 files changed, 117 insertions(+), 35 deletions(-) diff --git a/src/orcapod/hashing/arrow_hashers.py b/src/orcapod/hashing/arrow_hashers.py index 89e148f39..13a3cabff 100644 --- a/src/orcapod/hashing/arrow_hashers.py +++ b/src/orcapod/hashing/arrow_hashers.py @@ -155,10 +155,20 @@ def _process_table_columns(self, table: "pa.Table | pa.RecordBatch") -> "pa.Tabl def hash_schema(self, schema: "pa.Schema") -> ContentHash: """Hash an Arrow schema using the starfix canonical algorithm. - Extension-typed columns are normalized to their storage type with the - extension identity in field metadata, as ``hash_table`` does, so the - extension name and metadata are part of the hash. + Each field is hashed under the same rule as in ``hash_table`` + (``SemanticHashingVisitor.hashed_field``). A logical type whose values + are hashed by a value handler is a ``large_binary`` field carrying the + type's identifier (extension name and metadata), whatever its storage + type: how such a type is stored is not part of its identity. Any other + extension type is normalized to its storage type with the extension + identity in field metadata, because its stored values are what is + hashed. """ + visitor = SemanticHashingVisitor(self._type_converter, self._semantic_hasher) + schema = pa.schema( + [visitor.hashed_field(field) for field in schema], + metadata=schema.metadata, + ) schema = normalize_extension_columns(schema.empty_table()).schema include_meta = has_extension_metadata(schema) or has_extension_type(schema) if include_meta: diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 200f90b28..866ca986a 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -349,6 +349,13 @@ def register_logical_classes() -> None: "ndarray.int64": np.arange(6, dtype=np.int64).reshape(2, 3), "ndarray.float64": np.array([[0.5, 1.5], [2.5, 3.5]], dtype=np.float64), "ndarray.empty": np.zeros((0,), dtype=np.float32), + # Added by ITL-981: the properties the strict array hash counts or never counts. + "ndarray.fortran": np.asfortranarray(np.arange(6, dtype=np.int32).reshape(2, 3)), + "ndarray.big_endian": np.arange(6, dtype=">i4").reshape(2, 3), + "ndarray.aligned_struct": np.array( + [(1, 0.5), (-2, 1.5)], dtype=np.dtype([("a", "i1"), ("b", "[large_list]", "single_field.list_path.schema_hash": "semantic_v1:a12297fadc22ec8a2ae7612d6a1b594fd9ed2a5188ac088d45fedbaa8d9992d8", - "single_field.ndarray.arrow_type": "extension[struct, data: large_binary>]", - "single_field.ndarray.schema_hash": "semantic_v1:28ef1316c5c6ada4ea3b406ab1ad16b6792b1fb23a823c614767a6227e4aecf5", + "single_field.ndarray.arrow_type": "extension[large_binary]", + "single_field.ndarray.schema_hash": "semantic_v1:1ce515cff952c91067233ab891f9ca53d22ac35f4e8acd02911eceeb697d9a03", "single_field.optional_int.arrow_type": "int64", "single_field.optional_int.schema_hash": "semantic_v1:7613bd137a749aba9e9c6f54091681242a22e408f5e8bf7a5caa47a212cc3141", "single_field.path.arrow_type": "extension[large_string]", @@ -105,8 +105,8 @@ "single_field.structural_twin_ab.schema_hash": "semantic_v1:ae904346e1568bac2921bb6b23be6fd1b53c2dfb267272a25707c13609e96702", "single_field.structural_twin_ba.arrow_type": "extension[struct]", "single_field.structural_twin_ba.schema_hash": "semantic_v1:fec2e905d7d0db153e5b9e5d9d443d698821c51154f304c8dc91fdb9afb6ce4e", - "single_field.structural_with_array.arrow_type": "extension[struct, data: large_binary>>]", - "single_field.structural_with_array.schema_hash": "semantic_v1:ce619b2aeb64419745accb29e02d830666a21b3227ce9da5ff6a98388d332b6a", + "single_field.structural_with_array.arrow_type": "extension[struct]", + "single_field.structural_with_array.schema_hash": "semantic_v1:9474e8c5367f510a353811fb48b9b4b82fd8f883f33fce83f0373d7ee4b4c4f2", "single_field.tuple_int_str.arrow_type": "struct", "single_field.tuple_int_str.schema_hash": "semantic_v1:84de6b5743e160faf5668eb29c7045f4757f150011645c3cec992ecd80d388f0", "single_field.uuid.arrow_type": "extension[large_binary]", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index 4020e77a9..e3caf2370 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -27,7 +27,7 @@ "type.list_int": "semantic_v1:388fcfad62b0e03893e350cabe27e6235aa52e5355df844aefce870cf2bbec75", "type.list_list_int": "semantic_v1:6c345e63e39c385db19c6d4abb56394fb31ea8412024ab8b33b67d3fec3340ef", "type.list_path": "semantic_v1:3f3ce39052a75cde9537faacc70ac47772c4ce7e8a0b94c69b0bd037b123bc36", - "type.ndarray": "semantic_v1:b8eb285c0293ba5612fd8424f14c9cc03b306f9e6bf5bbb36c18d09b5202aaec", + "type.ndarray": "semantic_v1:6b5d34c828127cad41bb98e65450250bc847d3e42da07bba377b49d1bbe2f9f5", "type.optional_int": "semantic_v1:aaf3880090849afb79550bdcd6865c455ded79bf372efb283b6a68b82c627736", "type.path": "semantic_v1:7f0da67eaf9d048c9d1791fbe223333cfcc8018c66878647c8b7ae8571c72bd3", "type.pydantic": "semantic_v1:58c6aad8ca32918a34da1dddb62c72392079b9326b021aa3a4c57de4ae0a4ea8", @@ -39,7 +39,7 @@ "type.structural_same_identity": "semantic_v1:e050877645fb628a5ae83ad6c88398a91f2437a599a65054127056b221077564", "type.structural_twin_ab": "semantic_v1:12ab17179b2f707e3551c93481b7eb7974d79805b2fa6a4e37890014bc8ca62a", "type.structural_twin_ba": "semantic_v1:8f0abe45554d4cd0c68d5c9497a819564f7268cc2631602a2e1db6471f0112c7", - "type.structural_with_array": "semantic_v1:845c46052e7e3ad39f229ac4690663aecf2a24e62983361268c2ec0a44752772", + "type.structural_with_array": "semantic_v1:4bf065b65cffed98adba483cb9f2f0918ed209b19e86abff4b1608decb6043ac", "type.tuple_int_str": "semantic_v1:df199a19b63c33f3b7a8d06276faabbc74082af71710d69223980e84f71434ed", "type.uuid": "semantic_v1:763acbee42cb7bd6c7734ad7ca12ab6cb879ace3991e075366b0536bd67556f0", "value.bool.false": "semantic_v1:fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa", @@ -69,10 +69,14 @@ "value.list.mixed": "semantic_v1:05e6eced3c7100adabbd220cc470b1a97e0e39263653ac62da7c950b86f7526c", "value.namedtuple.xy": "semantic_v1:8573505172fe4b9949938d0aee2358b361e51ef007ea7c7519f0cfdefb2de6b0", "value.namedtuple.yx": "semantic_v1:addb076ec3eee640559a2bdf5d8b901dd1f644f83ada7046d12a7792f7b85443", - "value.ndarray.empty": "ndarray_v1:5cdc53e446e7788fb4c77928e777e78d52c5b0d11af177950e428e49eb68c9d9", - "value.ndarray.float64": "ndarray_v1:4318d0446de3a8ad4b337f1d8582a2a29ffe84fd36cef095e2a9053da04d1aca", - "value.ndarray.int32": "ndarray_v1:c47c7f1dfd1002e9c8901f8c09cbb550cd95a72ad63d08d7b86c836d5bceab56", - "value.ndarray.int64": "ndarray_v1:997e3d95a1ebc06c669978196da71a558c4cac47074f8c5f57067c8b5d6390cf", + "value.ndarray.aligned_struct": "ndarray_v1:91f015765fc0be7a51bee61db5dc110d703016e0c1c285a9aac7bc3117281a00", + "value.ndarray.big_endian": "ndarray_v1:b552ddc9bdabb59ad235eca436d580eadac8f751dfb815c9294bdf760db9cfdf", + "value.ndarray.empty": "ndarray_v1:19f39d097025397bf15c1c486f46ed68c1e8a8c8288479c4cc5221a2b1cc8589", + "value.ndarray.float64": "ndarray_v1:ae046f92a521b1cedcf7f6f8f610c5424428f7fa636a10f7a92474770b749833", + "value.ndarray.fortran": "ndarray_v1:ebb2d9032d4ce00a58768eb0fe14789f31b69b55a985c28fc3eea1939a24fa36", + "value.ndarray.int32": "ndarray_v1:39b7bc559e6e952d7276e100740db800810c28371e3895fe467e4e0351c26f8a", + "value.ndarray.int64": "ndarray_v1:4018e3021187f9efa736a2a969040563083e455c353dfbd27c4ce0e15aad0d8c", + "value.ndarray.strided_view": "ndarray_v1:ed2e1e0dd1a376dec89828113a11c976e74972b54f58e9a28e0065d2ff88b8e2", "value.nested": "semantic_v1:1935b97bf8f80e850dbcab19482cacf2af7bab94d73c2c8e6bce44a0cc71c0ff", "value.none": "semantic_v1:74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b", "value.set.ints": "semantic_v1:d1ff6a34712272cbf01d0bc874fb41db6df312102bd073c95529df6e9ae738bf", diff --git a/tests/test_hashing/test_starfix_arrow_hasher.py b/tests/test_hashing/test_starfix_arrow_hasher.py index c5da1eee4..3184554d1 100644 --- a/tests/test_hashing/test_starfix_arrow_hasher.py +++ b/tests/test_hashing/test_starfix_arrow_hasher.py @@ -496,3 +496,61 @@ def test_table_unrelated_schema_metadata_does_not_affect_hash(self): _make_hasher().hash_table(t_clean).digest == _make_hasher().hash_table(t_noise).digest ) + + +class TestSchemaHashFieldRule: + """``hash_schema`` hashes a field under the same rule as ``hash_table`` (ITL-981).""" + + @staticmethod + def _identified(storage: pa.DataType, extension: pa.ExtensionType) -> pa.Schema: + """A plain ``storage`` field that carries the identifier of ``extension``.""" + return pa.schema( + [ + pa.field( + "c", + storage, + metadata={ + "ARROW:extension:name": extension.extension_name, + "ARROW:extension:metadata": extension.__arrow_ext_serialize__(), + }, + ) + ] + ) + + def test_value_hashed_type_does_not_depend_on_its_storage_type(self): + """A type hashed by a value handler is a ``large_binary`` field with its identifier.""" + import numpy as np + + import orcapod as op + from orcapod.contexts import get_default_context + + context = get_default_context() + hasher = context.arrow_hasher + for python_type in (op.File, np.ndarray): + extension = context.type_converter.python_type_to_arrow_type(python_type) + assert isinstance(extension, pa.ExtensionType) + schema_hash = hasher.hash_schema(pa.schema([pa.field("c", extension)])) + assert schema_hash == hasher.hash_schema( + self._identified(pa.large_binary(), extension) + ) + # ``File`` is stored as a string; that storage type is not in its schema hash. + file_type = context.type_converter.python_type_to_arrow_type(op.File) + assert isinstance(file_type, pa.ExtensionType) + assert file_type.storage_type == pa.large_string() + assert hasher.hash_schema( + pa.schema([pa.field("c", file_type)]) + ) != hasher.hash_schema(self._identified(pa.large_string(), file_type)) + + def test_type_without_value_handler_keeps_its_storage_type(self): + """A type whose stored values are hashed keeps its storage type in the schema hash.""" + from pathlib import Path + + from orcapod.contexts import get_default_context + + context = get_default_context() + hasher = context.arrow_hasher + extension = context.type_converter.python_type_to_arrow_type(Path) + assert isinstance(extension, pa.ExtensionType) + assert hasher.hash_schema( + pa.schema([pa.field("c", extension)]) + ) == hasher.hash_schema(self._identified(extension.storage_type, extension)) diff --git a/tests/test_logical_types/test_ignore_list.py b/tests/test_logical_types/test_ignore_list.py index 51819b118..f4c80b766 100644 --- a/tests/test_logical_types/test_ignore_list.py +++ b/tests/test_logical_types/test_ignore_list.py @@ -171,6 +171,9 @@ def test_two_lists_are_two_types(tmp_path: Path) -> None: [_table(strict, Labelled(1, "a")), _table(lenient, Labelled(1, "a"))] ) # The type hash and the schema hash take the identifier, which holds the list. + assert strict.arrow_hasher.hash_schema( + pa.schema([("x", strict_type)]) + ) != lenient.arrow_hasher.hash_schema(pa.schema([("x", lenient_type)])) assert strict.semantic_hasher.hash_object( Labelled ) != lenient.semantic_hasher.hash_object(Labelled) From 9ab64d72235f788d2ab6886c8ecc2e45ee943a7f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:36:13 -0800 Subject: [PATCH 11/22] docs: arrays are stored as .npy and hashed strictly; hashing reference section 8 (ITL-981) Co-Authored-By: Claude Fable 5.2 --- docs/concepts/arrays.md | 198 ++++++++++++++-------- docs/concepts/logical-type-identifiers.md | 25 ++- docs/reference/hashing.md | 183 +++++++++++++++----- 3 files changed, 295 insertions(+), 111 deletions(-) diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index 7680a828d..adf838918 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -1,9 +1,12 @@ # NumPy Arrays A `numpy.ndarray` can be a data value like any other: a source column, a function pod -parameter or a return value. Orcapod stores each array by its **value**, its dtype, its shape -and its elements. How the array happens to sit in memory is not stored. The hash is defined on -that value, so two arrays that hold the same values hash the same, however each was laid out. +parameter or a return value. Storage and hashing are two separate things: + +- **Storage** is NumPy's own `.npy` format. An array reads back as it was given: dtype, + byte order, shape, C or Fortran memory order and structured field offsets are kept. +- **The hash** is defined by Orcapod on the array's value, never on the stored bytes. It is + strict: everything about the array that storage keeps is part of its identity. ```python import numpy as np @@ -25,13 +28,14 @@ for tag, data in scale.pod(source).iter_data(): # t1 float32 (2, 3) ``` -!!! warning "v0.3 changed the array format" +!!! warning "v0.3 changed how arrays are stored and hashed" Orcapod v0.2 stored arrays as `.npy` files under the extension name `numpy.ndarray` and - hashed the file. v0.3 replaces both, as described below, and does not migrate older - stores. Every node that handles arrays, and every node downstream of an array column, gets - new tables in v0.3. Because v0.3 also changed every hash of Arrow data, v0.3 reuses no - result of a store written by v0.2: it ignores the v0.2 rows and computes every input - again (see [A store written by an older version](function-pods.md#a-store-written-by-an-older-version)). + hashed the file. v0.3 stores them under `orcapod.numpy.ndarray`, hashes the value as + described below, and does not migrate older stores. Every node that handles arrays, and + every node downstream of an array column, gets new tables in v0.3. Because v0.3 also + changed every hash of Arrow data, v0.3 reuses no result of a store written by v0.2: it + ignores the v0.2 rows and computes every input again (see + [A store written by an older version](function-pods.md#a-store-written-by-an-older-version)). Nodes that perform side effects are the exception: a side-effect pod, or a function pod that takes the invocation context, does not run again on such a store but raises `SideEffectLogDataContextError` before it delivers anything. @@ -40,26 +44,40 @@ for tag, data in scale.pod(source).iter_data(): ## What counts as the same array -Two arrays are the same value when they have the same dtype, the same shape and the same -element bits. - -- **Memory layout does not count.** C order, Fortran order, strided views and transposed views - of equal values are the same array. An array always reads back in C order. -- **Byte order does not count.** Arrays are stored little-endian; a big-endian `>f4` array - reads back as `f4"))) -# True +# False print(hasher.hash_object(a) == hasher.hash_object(a.astype(np.float64))) # False -ab = np.zeros(1, dtype=[("a", "i4"), ("b", "f8")]) -ba = np.zeros(1, dtype=[("b", "f8"), ("a", "i4")]) -print(hasher.hash_object(ab) == hasher.hash_object(ba)) +wide = np.arange(12, dtype=np.float32).reshape(2, 6) +view = wide[:, ::2] +print(hasher.hash_object(view) == hasher.hash_object(np.ascontiguousarray(view))) +# True +packed = np.zeros(1, dtype=[("a", "i1"), ("b", "f8")]) +aligned = np.zeros(1, dtype=np.dtype([("a", "i1"), ("b", "f8")], align=True)) +print(hasher.hash_object(packed) == hasher.hash_object(aligned)) # False ``` +## The ignore list + +What the hash ignores is a setting of the array type, not of a single call. The data context +holds one list of ignored property names for the type; the default list is empty. The list +is written into the type's Arrow extension metadata, so it is part of the +[logical type identifier](logical-type-identifiers.md): the array type with an empty list +and the array type that ignores `memory_order` are two different logical types, with +different schema hashes. + +- A bare array (a function default, a value inside a Python structure) is hashed with the + data context's list. +- An array in a table is hashed with the list of its own column. +- A data context reads only array columns stored under its own list. A column stored under + another list raises `LogicalTypeIdentifierError`, naming the column and both lists. + +When a property is ignored, arrays that differ only in it hash equal, and they hash like +the array in canonical form (C order, little-endian, packed fields) does under the empty +list. The list is configured in the `hash_ignore` object of Orcapod's logical type +registration file (`contexts/logical_types.json`); v0.3 has no other way to change it. A +name that is not one of the three above is an error when the data context is built. + +The same mechanism serves the other library value types, each with its own names. + ## What cannot be stored Some arrays are refused when they are written, with a `ValueError` (an @@ -92,6 +139,8 @@ Some arrays are refused when they are written, with a `ValueError` (an | `longdouble`, `clongdouble` | width and format differ by platform | `.astype(np.float64)` or `.astype(np.complex128)` | | `datetime64` or `timedelta64` without a unit | the integer has no defined meaning | give a unit, e.g. `datetime64[ns]` | | Structured dtypes with field titles | titles are aliases of fields | drop the titles | +| Structured dtypes whose fields overlap or are not in offset order | `.npy` has no form for them | declare the fields in offset order | +| Structured dtypes with several hundred fields (a `.npy` header over 10,000 bytes) | NumPy does not read such a header without pickle support | split the array | Extended data may be supported later, stored as several parts rather than as one array. Whether `longdouble` and `clongdouble`, unit-less datetimes and timedeltas, and structured @@ -115,51 +164,62 @@ example `x * 2` on a 0-d array, raises `TypeError` that names the output. Return ## How an array is stored -An array column has the Arrow extension type `orcapod.ndarray`, with extension metadata -`{"version":1}`. Its storage is one struct per row: +An array column has the Arrow extension type `orcapod.numpy.ndarray` over `large_binary`. +Each cell is one `.npy` file, written by NumPy itself +(`numpy.lib.format.write_array`, format version 3.0, `allow_pickle=False`) and read with +`numpy.lib.format.read_array(..., allow_pickle=False)`. Format 3.0 is the one that stores +field names outside Latin-1. Pickle is never written and never read: a cell that holds an +object array, or is not a complete `.npy` file of format 3.0, raises a `ValueError` that +names the column. -| Field | Arrow type | Holds | -|---|---|---| -| `dtype` | `large_string` | the Orcapod dtype tag (below) | -| `shape` | `large_list` | the dimensions, outermost first; empty for a 0-d array | -| `data` | `large_binary` | the elements in C order, little-endian, packed | - -The struct's children are nullable in the Arrow schema only because Polars drops nested -non-null flags. A stored array never has a null child. A null array is a null struct, allowed -when the column is annotated `np.ndarray | None`. +The extension metadata is `{"ignore":[],"version":1}`: the ignore list and the version of +the type. A null array is a null cell, allowed when the column is annotated +`np.ndarray | None`. ```python +import io + table = source.as_table() print(table.schema.field("signal").type.extension_name) -# orcapod.ndarray -print(table.column("signal").chunk(0).storage[0].as_py()["dtype"]) +# orcapod.numpy.ndarray +cell = table.column("signal").chunk(0).storage[0].as_py() +print(np.load(io.BytesIO(cell), allow_pickle=False).dtype) # float32 ``` -### Dtype tags +What reads back: -The tag names the element type without a byte order. For a structured dtype, the tag is a JSON -list with one `[name, tag]` entry per field, or `[name, tag, [dims]]` for a subarray field, -in declared field order. It is written with no whitespace and with non-ASCII characters escaped -(`json.dumps(..., separators=(",", ":"), ensure_ascii=True)`). +| Property | Reads back | +|---|---| +| dtype, including byte order, units, string widths, field names, offsets and item size | as given | +| shape and element bits | as given | +| C or Fortran memory order | as given | +| any other strides (step slices, reversed or broadcast views) | C-contiguous | +| subclass | plain `np.ndarray` | +| `writeable` flag | writeable | +| dtype `metadata`, the `align` flag | not kept | -| NumPy | Tag | Element bytes | -|---|---|---| -| `bool` | `bool` | one byte, 0 or 1 | -| `int8` ... `int64`, `uint8` ... `uint64` | `int8` ... `uint64` | two's complement or unsigned, little-endian | -| `float16`, `float32`, `float64` | same names | IEEE 754, little-endian | -| `complex64`, `complex128` | same names | real part, then imaginary part | -| `datetime64[ns]`, `datetime64[10ms]`, ... | `datetime64[ns]`, `datetime64[10ms]`, ... | `int64`, little-endian (NaT is the minimum) | -| `timedelta64[s]`, ... | `timedelta64[s]`, ... | `int64`, little-endian | -| `U5` | `ucs4[5]` | five UCS-4 code units, each `uint32` little-endian, NUL-padded | -| `S3` | `bytes[3]` | three raw bytes, NUL-padded | -| `V4` | `void[4]` | four raw bytes | -| `[("b", "f8"), ("a", "i4", (2,))]` | `[["b","float64"],["a","int32",[2]]]` | fields packed in tag order | +```python +big = np.asfortranarray(np.arange(6, dtype=">f4").reshape(2, 3)) +kept = DictSource( + data=[{"trial": "t1", "signal": big}], tag_columns=["trial"], source_id="kept" +) +for tag, data in kept.iter_data(): + print(data["signal"].dtype.str, data["signal"].flags.f_contiguous) +# >f4 True +``` + +A stored array costs at least 128 bytes of `.npy` header. The stored bytes are NumPy's and +may differ between NumPy versions; nothing in Orcapod depends on them. Where an operator has +to order arrays (`MergeJoin` sorts the values it merges), it orders them by their value hash. ## How an array is hashed -The hash of an array is the version-1 array digest of its stored row, labelled `ndarray_v1`. -The byte-level definition, and a reference implementation that does not use Orcapod, are in -the [Hashing Reference](../reference/hashing.md#8-array-value-digest). A future PyTorch tensor -type will share the digest under its own label, so an array and a tensor with equal values -have different hashes. +The hash of an array is labelled `ndarray_v1` and is computed in two steps. The first is the +*element digest* of the array's dtype, shape and elements, taken on the array in canonical +form (C order, little-endian, fields packed). The second adds the type's name and the three +properties of the table above. The byte-level definition, the dtype tags and a reference +implementation that does not use Orcapod are in the +[Hashing Reference](../reference/hashing.md#8-array-value-hash). A PyTorch tensor type shares +the element digest and has its own name and properties, so an array and a tensor with equal +values have different hashes. diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index ce05f271a..0cf49171d 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -1,7 +1,7 @@ # Logical Type Identifiers A *logical type* binds a Python class to the way its values are stored in an Arrow column: -`pathlib.Path` is stored as a string, `numpy.ndarray` as a struct, a dataclass as a struct +`pathlib.Path` is stored as a string, `numpy.ndarray` as a `.npy` file, a dataclass as a struct of its fields. Orcapod ships logical types for paths, files, directories, UUIDs, NumPy arrays and pandas objects, and you can register your own. @@ -68,7 +68,7 @@ under one identifier. ## Built-in types Orcapod's own types follow the same contract as yours. Their metadata is the JSON text -`{"version":N}`: +`{"version":N}`, or `{"ignore":[...],"version":N}` for a library value type (below): | Python type | Extension name | Extension metadata | |---|---|---| @@ -77,12 +77,31 @@ Orcapod's own types follow the same contract as yours. Their metadata is the JSO | `uuid.UUID` | `orcapod.uuid` | `{"version":1}` | | `orcapod.File` | `orcapod.file` | `{"version":1}` | | `orcapod.Directory` | `orcapod.directory` | `{"version":1}` | -| `numpy.ndarray` | `orcapod.ndarray` | `{"version":1}` | +| `numpy.ndarray` | `orcapod.numpy.ndarray` | `{"ignore":[],"version":1}` | | `pandas.DataFrame` | `pandas.dataframe` | `{"version":1}` | | `pandas.Series` | `pandas.series` | `{"version":1}` | `{"version":N}` is the convention of the built-in types only. Your types may use any bytes. +### Library value types: the ignore list + +A value from a library such as NumPy has properties beyond its content, for example memory +order and byte order. Its hash counts every property the storage keeps, and the type names +the properties a data context may choose to ignore. The names a data context ignores are the +type's *ignore list*, and the list is part of the identifier: the extension metadata is + +``` +{"ignore":[],"version":1} +{"ignore":["byte_order","memory_order"],"version":1} +``` + +with the names sorted, each once, the keys sorted and no whitespace; `ignore` is present +also when empty. So one type under two lists is two logical types: the schema hashes differ, +a join of two such columns is an error, and a data context reads only columns stored under +its own list (anything else raises `LogicalTypeIdentifierError`, naming the column and both +lists). Storage is the same under every list. `numpy.ndarray` is the first such type; its +property names are in [NumPy Arrays](arrays.md#the-ignore-list). + ### Native containers: `list[T]` and `set[T]` A column typed `list[T]` or `set[T]`, where `T` is a logical type, is stored under a wrapper diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 660dfdb31..8fbb7a3d3 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -56,7 +56,7 @@ The labels of v0.3: | `semantic_v1` | the structure hash: a Python structure expanded to tagged JSON, SHA-256 of its canonical text | `SEMANTIC_HASH_LABEL`, `hashing/semantic_hashing/semantic_hasher.py` | | `arrow_v1` | the Arrow table and schema hash (§9) | `ARROW_HASH_LABEL`, `hashing/arrow_hashers.py` | | `directory_v1` | the directory hash: a SHA-256 Merkle tree over the entries | `DIRECTORY_HASH_LABEL`, `hashing/directory_hashers.py` | -| `ndarray_v1` | the array value digest (§8) | `NumpyArrayHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | +| `ndarray_v1` | the array value hash (§8) | `NumpyArrayHandler.HASH_LABEL`, `hashing/semantic_hashing/builtin_handlers.py` | | `polars_expr_v1` | the Polars expression hash (§1c) | `POLARS_EXPR_HASH_LABEL`, `hashing/semantic_hashing/polars_expr.py` | | `sha256` | the file content hash: SHA-256 of the file's bytes | the file hasher's `algorithm` | @@ -816,63 +816,149 @@ in any data-level preimage; they serve logging, observability, and result inspec --- -## 8. Array Value Digest +## 8. Array Value Hash **Value hash labels.** A type whose values are hashed by a dedicated handler labels its hash -`_v`, for example `ndarray_v1` (and later `tensor_v1`). The label is the -`ContentHash.method`; the type's implementer chooses it. The bytes produced under an existing -label never change. A new definition gets a new label, so a hash always says which definition -produced it. The label is not part of any hash input (see -[The hash label rule](#the-hash-label-rule)). +`_v`, for example `ndarray_v1`. The label is the `ContentHash.method`; the type's +implementer chooses it. The bytes produced under an existing label never change. A new +definition gets a new label, so a hash always says which definition produced it. The label +is not part of any hash input (see [The hash label rule](#the-hash-label-rule)). -A `numpy.ndarray` value is hashed by value (since v0.3, ITL-888). The semantic hasher's -`NumpyArrayHandler` returns `ContentHash(method="ndarray_v1", digest=D)`, where `D` is the -version-1 array digest of the array's stored row. Inside a table hash, an array cell becomes -the token `b"orcapod.ndarray;7b2276657273696f6e223a317d::" + D`: the logical type's identifier (the extension name, `;`, the extension metadata `{"version":1}` in hex), then the digest, without the label. The stored row, the dtype tags and the canonical form are described in -[NumPy Arrays](../concepts/arrays.md). +A `numpy.ndarray` is stored as a `.npy` file and hashed by value (ITL-981). The hash is +defined on the array, never on the stored bytes. The semantic hasher's `NumpyArrayHandler` +returns `ContentHash(method="ndarray_v1", digest=D)`. What is stored and what reads back is +described in [NumPy Arrays](../concepts/arrays.md). -For a stored row with dtype tag `T`, shape `(d0, ..., dn-1)` and data bytes `B`: +`D` is computed in two steps. Notation: `u8` is one byte, `u64le` and `i64le` are 8-byte +little-endian unsigned and signed integers, and `lp(x) = u64le(len(utf8(x))) || utf8(x)`. + +**Step 1, the element digest `E`.** Take the array in canonical form: C order, +little-endian, structured fields packed in declared order. With dtype tag `T` (below), shape +`(d0, ..., dn-1)` and the canonical element bytes `B`: ``` -P = u64le(len(utf8(T))) || utf8(T) - || u64le(n) || i64le(d0) || ... || i64le(dn-1) - || u64le(len(B)) || B -D = SHA-256(P) +E = SHA-256( lp(T) || u64le(n) || i64le(d0) || ... || i64le(dn-1) || u64le(len(B)) || B ) ``` -`u64le` and `i64le` are 8-byte little-endian unsigned and signed integers. Every field is -length-prefixed, so two different rows never share a preimage. `D` depends only on the stored -row, so it can be recomputed from a stored table by any SHA-256 implementation. The label -`ndarray_v1` names this definition; a different definition gets a different label. A -`torch.Tensor` type is to share the digest under its own label. +`E` covers the dtype, the shape and every element bit, and nothing else. It is shared with +other array types (a tensor type computes the same `E`). + +**Step 2, the value hash `D`.** + +``` +D = SHA-256( lp("orcapod.numpy.ndarray") || E || u8(memory_order) + || lp(byte_order) || lp(struct_layout) ) +``` -| Variation | Same `D`? | +| Field | Value | Canonical value (written when the property is ignored) | +|---|---|---| +| leading text | the type's extension name, `orcapod.numpy.ndarray` | | +| `memory_order` | `1` when the array is Fortran-contiguous and not C-contiguous, else `0` | `0` | +| `byte_order` | one character per scalar field of the dtype, depth first in declared field order, a subarray field counting once: `<` little-endian, `>` big-endian, `\|` for a type without byte order (`bool`, 8-bit integers, `S`, `V`). NumPy's native marker `=` is resolved to the machine's order. `float32` gives `<` | the same text with every `>` replaced by `<` | +| `struct_layout` | JSON without whitespace of `L(dtype)`: `null` for an unstructured dtype, otherwise the list `[itemsize, [offset_1, L(field_1)], [offset_2, L(field_2)], ...]` in declared field order; a subarray field uses its base dtype. `float32` gives `null` | `L` of the packed dtype | + +Every field is always present and length-prefixed or of fixed width, so two different +inputs never share a preimage. + +**The ignore list.** The three properties `memory_order`, `byte_order` and `struct_layout` +count by default. A data context can list any of them as ignored for the type. An ignored +property's field is still written, with its canonical value, so arrays that differ only in +ignored properties have equal `D`, and with all three ignored `D` equals the default `D` of +the canonical array. The list itself is not in the preimage. It is in the type's extension +metadata, `{"ignore":[...],"version":1}` (names sorted, keys sorted, no whitespace), and so +in the logical type identifier: two lists are two logical types. A bare array is hashed with +the data context's list; a cell is hashed with the list of its own column. + +**Cells.** Inside a table hash, an array cell becomes the token +`b"orcapod.numpy.ndarray;7b2269676e6f7265223a5b5d2c2276657273696f6e223a317d::" + D`: the +logical type's identifier (the extension name, `;`, the extension metadata +`{"ignore":[],"version":1}` in hex), then the digest, without the label. The cell is decoded +to an array first; the stored `.npy` bytes are never hashed. + +| Variation | Same `D` by default? | |---|---| -| C or Fortran order, strides, views | yes | -| Little- or big-endian byte order | yes | -| Structured offsets, alignment padding | yes | +| Dtype (including width, units, string width), shape, any element bit, structured field names or field order | no | +| C against Fortran memory order | no (`memory_order`) | +| Little- against big-endian byte order | no (`byte_order`) | +| Structured field offsets or item size (aligned against packed) | no (`struct_layout`) | +| Step slices, reversed or broadcast views against their C-contiguous copy | yes | | `np.matrix`, `np.recarray`, `np.memmap` against the plain array | yes | -| Dtype (including width), shape, any element bit, structured field names or field order | no | +| The `writeable` flag, dtype `metadata`, the `align` flag, padding bytes | yes | + +The last three rows are what the `.npy` storage does not keep: a value read back from a +store must hash like the value that was written. -The reference implementation below uses only the standard library, and the vectors are pinned: +### Dtype tags + +The tag names the element type without a byte order. For a structured dtype, the tag is a +JSON list with one `[name, tag]` entry per field, or `[name, tag, [dims]]` for a subarray +field, in declared field order. It is written with no whitespace and with non-ASCII +characters escaped (`json.dumps(..., separators=(",", ":"), ensure_ascii=True)`). + +| NumPy | Tag | Canonical element bytes | +|---|---|---| +| `bool` | `bool` | one byte, 0 or 1 | +| `int8` ... `int64`, `uint8` ... `uint64` | `int8` ... `uint64` | two's complement or unsigned, little-endian | +| `float16`, `float32`, `float64` | same names | IEEE 754, little-endian | +| `complex64`, `complex128` | same names | real part, then imaginary part | +| `datetime64[ns]`, `datetime64[10ms]`, ... | `datetime64[ns]`, `datetime64[10ms]`, ... | `int64`, little-endian (NaT is the minimum) | +| `timedelta64[s]`, ... | `timedelta64[s]`, ... | `int64`, little-endian | +| `U5` | `ucs4[5]` | five UCS-4 code units, each `uint32` little-endian, NUL-padded | +| `S3` | `bytes[3]` | three raw bytes, NUL-padded | +| `V4` | `void[4]` | four raw bytes | +| `[("b", "f8"), ("a", "i4", (2,))]` | `[["b","float64"],["a","int32",[2]]]` | fields packed in tag order | + +### Reference implementation + +The implementation below uses only the standard library, and the vectors are pinned: ```python import hashlib import struct -def array_digest_v1(tag: str, shape: tuple[int, ...], data: bytes) -> str: - t = tag.encode("utf-8") - p = struct.pack(" bytes: + data = text.encode("utf-8") + return struct.pack(" bytes: + p = lp(tag) + struct.pack(" str: + p = lp("orcapod.numpy.ndarray") + element_digest(tag, shape, data) + p += bytes([memory_order]) + lp(byte_order) + lp(struct_layout) return hashlib.sha256(p).hexdigest() -print(array_digest_v1("float32", (2, 3), struct.pack("<6f", 0, 1, 2, 3, 4, 5))) +six = struct.pack("<6f", 0, 1, 2, 3, 4, 5) +print(element_digest("float32", (2, 3), six).hex()) # 318f6e503e8064e89ce6039453fce709f8e01f60c501af2b63bbbe9cf8fbf90e -print(array_digest_v1("float64", (), struct.pack("f4")).to_string()) +# ndarray_v1:1d5b6c2eb0ceaafd3758356256d651c5edf9ca0efa4deecf12c76c4e15bdb637 print(hasher.hash_object(np.array(2.5)).to_string()) -# ndarray_v1:47f182d862f5dd597d88836deddaed609ed94490827302e75e8364e877633cae +# ndarray_v1:642ad052de6aa53ffd059e5cd9a2eeeb5519caa3d3eb12efcf9e0c3fdbeab17b +aligned = np.dtype([("a", "i1"), ("b", " Date: Thu, 12 Nov 2026 13:37:14 -0800 Subject: [PATCH 12/22] docs: changelog fragment, design-issue entry and agent rules for library value types (ITL-981) Co-Authored-By: Claude Fable 5.2 --- .zed/rules | 31 +++++++++ CLAUDE.md | 31 +++++++++ changelog.d/ITL-981.md | 63 +++++++++++++++++++ design_issues/ITL-888-AR1.md | 6 ++ design_issues/ITL-981-LV1.md | 33 ++++++++++ .../specs/2026-11-10-array-format-v0.3.md | 5 +- 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ITL-981.md create mode 100644 design_issues/ITL-981-LV1.md diff --git a/.zed/rules b/.zed/rules index 7e13538e2..0da3f131b 100644 --- a/.zed/rules +++ b/.zed/rules @@ -273,6 +273,37 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension never fall back to the storage type. List an older value in `also_reads` only when the current code decodes that layout. +## Library value types: storage, strict hash, ignore list + +Developer, November 12, 2026 (ITL-981); see `docs/concepts/arrays.md` and +`docs/reference/hashing.md` section 8. + +- Storage and hashing are decided separately. A value of a library type (`numpy.ndarray`, + later tensors and data frames) is stored with the library's own serialization and reads + back as given. Its hash is defined by Orcapod on the content: never hash the stored bytes. +- The hash is strict by default: every property of a value that storage keeps is part of + its identity. A property storage does not restore never counts and has no option name, + because `hash(read(write(v)))` must equal `hash(v)`. +- What the hash may ignore is a closed set of property names per type (`OPTIONS`). The names + a data context ignores are declared once, in `hash_ignore` of + `contexts/logical_types.json`, and referred to as `hash_ignore:` by the + logical type entry and the value handler entry. +- The list is in the extension metadata, `{"ignore":[...],"version":N}` (sorted names, + sorted keys, `ignore` always present), written only by `ignore_list.ignore_metadata`. Two + lists are two logical types. Never put the list into a hash preimage: an ignored property's + field is always written, with its canonical value. +- Build such a type from `logical_types/ignore_list.py` (`IgnoreListLogicalType`, + `IgnoreListHandler`). A bare value is hashed with the handler's configured list; a cell + with the list of its own column (`SemanticHashingVisitor` calls `hash_ignoring`). A data + context reads only columns stored under its own list; anything else raises + `LogicalTypeIdentifierError`. +- Extension names of library value types are lower case `orcapod..` + (`orcapod.numpy.ndarray`). The leading type text of a value hash is the extension name. +- `numpy.ndarray`: one `.npy` file per cell (`numpy.lib.format`, format 3.0, + `allow_pickle=False` on write and read; never raise `max_header_size`). Options: + `memory_order`, `byte_order`, `struct_layout`. Where stored arrays must be ordered, order + them by value hash (`storage_sort_key`), never by stored bytes. + ## User classes: exact and structural A class a user defines is one of two kinds (developer, November 12, 2026, ITL-972; see diff --git a/CLAUDE.md b/CLAUDE.md index 59048c0a1..b6a43e083 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -284,6 +284,37 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension never fall back to the storage type. List an older value in `also_reads` only when the current code decodes that layout. +## Library value types: storage, strict hash, ignore list + +Developer, November 12, 2026 (ITL-981); see `docs/concepts/arrays.md` and +`docs/reference/hashing.md` section 8. + +- Storage and hashing are decided separately. A value of a library type (`numpy.ndarray`, + later tensors and data frames) is stored with the library's own serialization and reads + back as given. Its hash is defined by Orcapod on the content: never hash the stored bytes. +- The hash is strict by default: every property of a value that storage keeps is part of + its identity. A property storage does not restore never counts and has no option name, + because `hash(read(write(v)))` must equal `hash(v)`. +- What the hash may ignore is a closed set of property names per type (`OPTIONS`). The names + a data context ignores are declared once, in `hash_ignore` of + `contexts/logical_types.json`, and referred to as `hash_ignore:` by the + logical type entry and the value handler entry. +- The list is in the extension metadata, `{"ignore":[...],"version":N}` (sorted names, + sorted keys, `ignore` always present), written only by `ignore_list.ignore_metadata`. Two + lists are two logical types. Never put the list into a hash preimage: an ignored property's + field is always written, with its canonical value. +- Build such a type from `logical_types/ignore_list.py` (`IgnoreListLogicalType`, + `IgnoreListHandler`). A bare value is hashed with the handler's configured list; a cell + with the list of its own column (`SemanticHashingVisitor` calls `hash_ignoring`). A data + context reads only columns stored under its own list; anything else raises + `LogicalTypeIdentifierError`. +- Extension names of library value types are lower case `orcapod..` + (`orcapod.numpy.ndarray`). The leading type text of a value hash is the extension name. +- `numpy.ndarray`: one `.npy` file per cell (`numpy.lib.format`, format 3.0, + `allow_pickle=False` on write and read; never raise `max_header_size`). Options: + `memory_order`, `byte_order`, `struct_layout`. Where stored arrays must be ordered, order + them by value hash (`storage_sort_key`), never by stored bytes. + ## User classes: exact and structural A class a user defines is one of two kinds (developer, November 12, 2026, ITL-972; see diff --git a/changelog.d/ITL-981.md b/changelog.d/ITL-981.md new file mode 100644 index 000000000..92f47d10e --- /dev/null +++ b/changelog.d/ITL-981.md @@ -0,0 +1,63 @@ +### Breaking changes and migration + +#### NumPy arrays: stored as `.npy`, hashed strictly, with a per-type ignore list (ITL-981) + +**This replaces the array storage and the array hash described under "NumPy arrays: new +storage format and new hashes" (ITL-888).** That format (the extension type +`orcapod.ndarray` over a `{dtype, shape, data}` struct, with a hash that ignored memory +layout and byte order) was never released; v0.3 ships the one described here. What that +entry says about v0.2 stores still holds: they are not migrated and not read. + +The developer decided on November 12, 2026 that storage and hashing are separate, that the +hash is strict by default, and that what the hash ignores is configured per type. + +- **Storage.** An array column is the Arrow extension type `orcapod.numpy.ndarray` over + `large_binary`. Each cell is one `.npy` file written by NumPy + (`numpy.lib.format.write_array`, format version 3.0, `allow_pickle=False`) and read with + `allow_pickle=False`. An array reads back as it was given: dtype including byte order, + shape, element bits, C or Fortran memory order and structured field offsets and item size + are kept. Not kept: strides other than C or Fortran order (a step slice is stored + C-contiguous), the subclass (read back as plain `np.ndarray`), the `writeable` flag, dtype + `metadata` and the `align` flag of a structured dtype. +- **Hash.** The hash is defined on the value, never on the stored bytes, and keeps the label + `ndarray_v1`. It is `SHA-256(lp("orcapod.numpy.ndarray") || E || memory order || byte + order || structured layout)`, where `E` is the element digest of dtype, shape and element + bits (`docs/reference/hashing.md`, section 8, with a reference implementation and pinned + vectors). By default a C-ordered and a Fortran-ordered array, a little- and a big-endian + array, and a packed and an aligned structured array with equal values now hash + **differently**. What storage does not keep never changes the hash, so a value read back + from a store hashes like the value written. +- **Ignore list.** The three properties `memory_order`, `byte_order` and `struct_layout` + can be ignored. The list of ignored names is set per type in the data context + (`hash_ignore` in `contexts/logical_types.json`; default: empty) and is written into the + extension metadata, `{"ignore":[...],"version":1}`. Two lists are two logical types: + different schema hashes, no shared type in a join, and a data context refuses to read a + column stored under another list (`LogicalTypeIdentifierError`, naming the column and both + lists). A bare value is hashed with the data context's list, a cell with the list of its + own column. v0.3 has no user API to change the list. +- **Newly refused at write** (`UnsupportedValueError`, naming the column and the dtype): a + structured dtype whose fields overlap or are not in offset order (`.npy` has no form for + it; the previous format accepted it), and a dtype whose `.npy` header exceeds 10,000 bytes + (several hundred fields). Masked arrays, object dtypes, `StringDType`, `longdouble`, + unit-less datetimes and field titles are refused as before. +- **`MergeJoin`** orders the arrays it merges by their value hash (it used the stored dtype + tag, shape and bytes). +- **What moves.** Every array value hash, and with the new extension name and metadata the + type, schema and pipeline hashes of anything with an array column and the system tag + column names built from those schema hashes. Nothing released is affected: v0.3 is not + tagged, and v0.2 stores are not read by v0.3. + +### Added + +- **The ignore-list mechanism for library value types** (`orcapod.logical_types.ignore_list`): + `IgnoreListLogicalType` and `IgnoreListHandler`, the extension metadata shape + `{"ignore":[...],"version":N}` (`ignore_metadata`, `parse_ignore_metadata`), and the + `hash_ignore` object of the logical type registration file. `numpy.ndarray` is the first + type built on it. (ITL-981) + +### Changed + +- **`StarfixArrowHasher.hash_schema` hashes a field under the same rule as `hash_table`.** A + logical type with a value handler is hashed as a `large_binary` field carrying the type's + identifier, so its storage type is no longer part of that hash. No library code calls this + method, and no stored or pinned value moves. (ITL-981) diff --git a/design_issues/ITL-888-AR1.md b/design_issues/ITL-888-AR1.md index 3c4afc71a..8b0537595 100644 --- a/design_issues/ITL-888-AR1.md +++ b/design_issues/ITL-888-AR1.md @@ -25,3 +25,9 @@ results. Known compromise: the struct's children are nullable at the Arrow level, because Polars drops nested non-null flags on its way back to Arrow (ITL-842). The logical type enforces non-null instead. Tighten under a new storage version once nested nullability survives Polars. + +**Superseded (ITL-981, November 12, 2026):** the struct storage and the layout-independent hash +described above were replaced before any release by `.npy` storage under +`orcapod.numpy.ndarray` and a strict value hash with a per-type ignore list; see entry LV1 +(`design_issues/ITL-981-LV1.md`). The refusals, the declared-output check and the v0.2 +statements of this entry still hold. diff --git a/design_issues/ITL-981-LV1.md b/design_issues/ITL-981-LV1.md new file mode 100644 index 000000000..6f537df0c --- /dev/null +++ b/design_issues/ITL-981-LV1.md @@ -0,0 +1,33 @@ +## `src/orcapod/logical_types/ignore_list.py`, `src/orcapod/logical_types/numpy_type.py` — library value types + +### LV1 — Library value types: storage and hash decided separately, strict hash, per-type ignore list +**Status:** resolved +**Severity:** high +**Issue:** ITL-981 (supersedes the storage and hash of ITL-888, entry AR1) + +The first v0.3 array format (ITL-888, never released) tied storage to the hash: an array was +stored in Orcapod's own canonical struct (C order, little-endian, packed fields) and hashed +as that row. So an array did not read back as it was given, and the hash silently ignored +memory order and byte order, with no way to say otherwise. The same question was open for +torch tensors, Polars and pandas data frames. + +Developer's decisions (November 12, 2026): storage is the library's own serialization and a +value reads back as given; the hash is defined by Orcapod on the content, never on the stored +bytes; the hash is strict by default (anything about a value that can be changed and that +storage keeps is part of its identity); what the hash ignores is a per-type list of property +names saved in the extension metadata, so two lists are two logical types. A property storage +cannot restore never counts, because a value read back must hash like the value written. + +**Fix:** `logical_types/ignore_list.py` holds the shared mechanism: the metadata shape +`{"ignore":[...],"version":N}`, `IgnoreListLogicalType`, `IgnoreListHandler`, one Arrow +extension class per list. The data context declares each type's list once, in `hash_ignore` +of `contexts/logical_types.json`. The Arrow hasher hashes a cell under the list of its own +column. `numpy.ndarray` is stored as a `.npy` file (format 3.0, no pickle) under +`orcapod.numpy.ndarray` and hashed in two steps: the element digest of dtype, shape and bits, +then the type name with memory order, byte order and structured layout, each ignorable. +Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md`. + +Known limits, by decision: a data context has one list per type and refuses a column stored +under another (the Python schema carries only `np.ndarray`, so a per-column list needs a user +API, which v0.3 does not have); a strided view is stored and hashed as its C-contiguous copy; +unit-less datetimes and field titles stay refused (ITL-911) although `.npy` could store them. diff --git a/superpowers/specs/2026-11-10-array-format-v0.3.md b/superpowers/specs/2026-11-10-array-format-v0.3.md index fb1fffefe..c8d565e26 100644 --- a/superpowers/specs/2026-11-10-array-format-v0.3.md +++ b/superpowers/specs/2026-11-10-array-format-v0.3.md @@ -4,7 +4,10 @@ **Issue:** ITL-888 (fixes ITL-883; refs ITL-874, ITL-871, ITL-465, ITL-882, ITL-884) **Design basis:** ITL-874 review, PR #285 (`superpowers/specs/2026-11-10-array-tensor-storage.md`, candidate 4c, sections C4 and C5; experiments `b0`, `c1`, `c2`, `c3`, `b7`). -**Status:** Implemented in PR #289. +**Status:** Implemented in PR #289. SUPERSEDED on November 12, 2026 by ITL-981 +(`superpowers/specs/2026-11-12-itl-981-library-value-types-design.md`): arrays are stored as +`.npy` under `orcapod.numpy.ndarray` and hashed strictly. Of this spec, the dtype tags and the +digest of section C1 remain, as the element digest of the new hash. **Decisions (developer, November 10 and 11, 2026).** Every design choice in this spec is decided: From ce7a408a1e7aeafd3c1382e10850217053ab79a8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:44:49 -0800 Subject: [PATCH 13/22] fix(hashing): type-clean hash_schema field rule and tests (ITL-981) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/hashing/arrow_hashers.py | 8 ++++---- tests/test_arrays/test_array_value_format.py | 2 +- tests/test_hashing/test_starfix_arrow_hasher.py | 2 ++ tests/test_logical_types/test_ignore_list.py | 8 ++++++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/orcapod/hashing/arrow_hashers.py b/src/orcapod/hashing/arrow_hashers.py index 13a3cabff..382ac8d56 100644 --- a/src/orcapod/hashing/arrow_hashers.py +++ b/src/orcapod/hashing/arrow_hashers.py @@ -165,10 +165,10 @@ def hash_schema(self, schema: "pa.Schema") -> ContentHash: hashed. """ visitor = SemanticHashingVisitor(self._type_converter, self._semantic_hasher) - schema = pa.schema( - [visitor.hashed_field(field) for field in schema], - metadata=schema.metadata, - ) + hashed = pa.schema([visitor.hashed_field(field) for field in schema]) + if schema.metadata is not None: + hashed = hashed.with_metadata(dict(schema.metadata)) + schema = hashed schema = normalize_extension_columns(schema.empty_table()).schema include_meta = has_extension_metadata(schema) or has_extension_type(schema) if include_meta: diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index 3f316bc21..f4e8159b5 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -642,7 +642,7 @@ def test_merge_join_keeps_memory_order_and_byte_order_of_each_side() -> None: right = DictSource( data=[{"k": "a", "x": fortran_big}], tag_columns=["k"], source_id="r" ) - got = next(d.as_dict()["x"] for _, d in MergeJoin()(left, right).iter_data()) + got: Any = next(d.as_dict()["x"] for _, d in MergeJoin()(left, right).iter_data()) assert sorted(x.dtype.str for x in got) == ["f4"] assert sorted(bool(x.flags.f_contiguous) for x in got) == [False, True] diff --git a/tests/test_hashing/test_starfix_arrow_hasher.py b/tests/test_hashing/test_starfix_arrow_hasher.py index 3184554d1..4395753c3 100644 --- a/tests/test_hashing/test_starfix_arrow_hasher.py +++ b/tests/test_hashing/test_starfix_arrow_hasher.py @@ -526,6 +526,7 @@ def test_value_hashed_type_does_not_depend_on_its_storage_type(self): context = get_default_context() hasher = context.arrow_hasher + assert isinstance(hasher, StarfixArrowHasher) for python_type in (op.File, np.ndarray): extension = context.type_converter.python_type_to_arrow_type(python_type) assert isinstance(extension, pa.ExtensionType) @@ -549,6 +550,7 @@ def test_type_without_value_handler_keeps_its_storage_type(self): context = get_default_context() hasher = context.arrow_hasher + assert isinstance(hasher, StarfixArrowHasher) extension = context.type_converter.python_type_to_arrow_type(Path) assert isinstance(extension, pa.ExtensionType) assert hasher.hash_schema( diff --git a/tests/test_logical_types/test_ignore_list.py b/tests/test_logical_types/test_ignore_list.py index f4c80b766..42f9335de 100644 --- a/tests/test_logical_types/test_ignore_list.py +++ b/tests/test_logical_types/test_ignore_list.py @@ -17,6 +17,7 @@ from orcapod.contexts.core import ContextValidationError, DataContext from orcapod.contexts.registry import JSONDataContextRegistry from orcapod.errors import TypeIdentityError +from orcapod.hashing.arrow_hashers import StarfixArrowHasher from orcapod.logical_types.identifier import LogicalTypeIdentifierError from orcapod.logical_types.ignore_list import ( checked_ignore, @@ -171,9 +172,12 @@ def test_two_lists_are_two_types(tmp_path: Path) -> None: [_table(strict, Labelled(1, "a")), _table(lenient, Labelled(1, "a"))] ) # The type hash and the schema hash take the identifier, which holds the list. - assert strict.arrow_hasher.hash_schema( + strict_hasher, lenient_hasher = strict.arrow_hasher, lenient.arrow_hasher + assert isinstance(strict_hasher, StarfixArrowHasher) + assert isinstance(lenient_hasher, StarfixArrowHasher) + assert strict_hasher.hash_schema( pa.schema([("x", strict_type)]) - ) != lenient.arrow_hasher.hash_schema(pa.schema([("x", lenient_type)])) + ) != lenient_hasher.hash_schema(pa.schema([("x", lenient_type)])) assert strict.semantic_hasher.hash_object( Labelled ) != lenient.semantic_hasher.hash_object(Labelled) From 9b0a60b7bec393745b086cc23fa14a6133458b31 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:47:55 -0800 Subject: [PATCH 14/22] docs(plans): record sign-off and progress (ITL-981) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/operators/merge_join.py | 4 +-- .../2026-11-12-itl-981-library-value-types.md | 33 ++++++++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/orcapod/core/operators/merge_join.py b/src/orcapod/core/operators/merge_join.py index 7c5d6b53b..a5fabc65b 100644 --- a/src/orcapod/core/operators/merge_join.py +++ b/src/orcapod/core/operators/merge_join.py @@ -60,8 +60,8 @@ def _storage_sort_key( ) -> Callable[[Any], Any]: """Return the ordering key MergeJoin uses for a column's stored values. - The array logical type defines ``storage_sort_key`` (its stored ``(dtype tag, - shape, data)``). A list of such values, at any depth (an array column after + The array logical type defines ``storage_sort_key`` (the value digest of the + stored array). A list of such values, at any depth (an array column after ``GroupBy`` or ``Batch``), is ordered by the tuple of its elements' keys, with ``None`` first. Every other type is ordered by its stored value as before. General struct ordering is ITL-839. diff --git a/superpowers/plans/2026-11-12-itl-981-library-value-types.md b/superpowers/plans/2026-11-12-itl-981-library-value-types.md index f78983314..19390c82c 100644 --- a/superpowers/plans/2026-11-12-itl-981-library-value-types.md +++ b/superpowers/plans/2026-11-12-itl-981-library-value-types.md @@ -27,8 +27,17 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md dispatches valid lists and raises `LogicalTypeIdentifierError` for anything else, as every built-in type does. The Linear comment on ITL-980 could not be posted from this session (permission denied); the coordinator posts it. - STILL WAITING for the developer's sign-off of the whole design: no library code, no push. -- Nothing is pushed. The design note, this plan and the failing tests are local commits. + (e) DEVELOPER SIGN-OFF received November 12, 2026 (through the coordinator): design approved + as recommended. Label stays `ndarray_v1`; a column under another ignore list is refused on + read; unit-less datetimes and field titles stay refused; metadata keys sorted; the leading + type text of the value hash is the extension name; names are `orcapod..`. + (f) Mechanism pushed at `64c63bfe` for the torch, Polars and pandas workers; the public + names of design note (vi) are stable from that commit. + (g) Implemented and pushed: NumPy `.npy` storage and strict hash, tests, golden (22 values + moved, 8 system-tag entries renamed, 4 added, array entries only), docs, changelog + fragment, design-issue entry, CLAUDE.md and `.zed/rules`. Full suite on 3.12: 10092 passed, + 109 skipped, 23 xfailed; array, hashing, logical type and context tests also pass on 3.11 + and 3.13. Left: independent review findings, PR, CodeRabbit rounds. ## Steps @@ -36,39 +45,39 @@ Design note: `superpowers/specs/2026-11-12-itl-981-library-value-types-design.md 2. [x] Failing tests through the public interface: `tests/test_arrays/test_array_strict_hash.py`. They fail on `main`'s code (the hash ignores memory order and byte order, storage is the struct). -3. [ ] `src/orcapod/logical_types/ignore_list.py`: `ignore_metadata`, +3. [x] `src/orcapod/logical_types/ignore_list.py`: `ignore_metadata`, `parse_ignore_metadata`, `checked_ignore`, `IgnoreListLogicalType`, `IgnoreListHandler`; one Arrow extension class per (name, list), the registered class deserialising any valid list. Polars extension keeps `metadata_str`. -4. [ ] Context loader (`contexts/registry.py`): `hash_ignore` of `logical_types.json` into +4. [x] Context loader (`contexts/registry.py`): `hash_ignore` of `logical_types.json` into the reference table as `hash_ignore:` before anything is built. `logical_types.json`: `"hash_ignore": {"orcapod.numpy.ndarray": []}` and the two `_ref`s. -5. [ ] `numpy_type.py`: storage `large_binary`, `.npy` format 3.0 write +5. [x] `numpy_type.py`: storage `large_binary`, `.npy` format 3.0 write (`numpy.lib.format.write_array`, `allow_pickle=False`), read after checking the magic; refusals (masked, object, StringDType, longdouble, unit-less datetimes, titles, out-of-order or overlapping offsets, header over 10,000 bytes); `storage_sort_key` is the strict `D` of the decoded value. `array_format.py`: drop the storage struct and its metadata constant, keep `array_digest_v1`. -6. [ ] `NumpyArrayHandler` as an `IgnoreListHandler` computing `D`; +6. [x] `NumpyArrayHandler` as an `IgnoreListHandler` computing `D`; `SemanticHashingVisitor.visit_extension` passes the column's list (`handler.hash_ignoring`). -6b. [ ] `StarfixArrowHasher.hash_schema`: apply `SemanticHashingVisitor.hashed_field` to each +6b. [x] `StarfixArrowHasher.hash_schema`: apply `SemanticHashingVisitor.hashed_field` to each field before hashing (same rule as `hash_table`); test that a value-hashed type's schema hash does not depend on its storage type and that no golden value moves. -7. [ ] Rewrite the ITL-888 tests that assert the struct row and layout independence +7. [x] Rewrite the ITL-888 tests that assert the struct row and layout independence (`tests/test_arrays/test_array_value_format.py`, `array_cases.py` (`assert_arrays_identical` requires C order today), `tests/test_logical_types/test_array_format.py`, `test_numpy_type.py`, `tests/test_hashing/test_numpy_handler.py`, `tests/test_hashing/test_hash_label_rule.py` if it names the constant). -8. [ ] Golden: add `value.ndarray.{fortran,big_endian,aligned_struct,strided_view}` to +8. [x] Golden: add `value.ndarray.{fortran,big_endian,aligned_struct,strided_view}` to `tests/test_hashing/golden/cases.py`; regenerate; check the diff moves only the 32 values listed in the design note (iii) and list each by cause in the PR. -9. [ ] Docs: `docs/concepts/arrays.md`, `docs/reference/hashing.md` section 8, +9. [x] Docs: `docs/concepts/arrays.md`, `docs/reference/hashing.md` section 8, `docs/concepts/logical-type-identifiers.md`; "superseded" line in `superpowers/specs/2026-11-10-array-format-v0.3.md`; CLAUDE.md and `.zed/rules` if a rule changes (logical type identifiers: the ignore-list metadata shape). -10. [ ] `changelog.d/ITL-981.md`, `design_issues/ITL-981-LV1.md`. -11. [ ] Checks: `uv run ruff format .`, `uv run ruff check .`, +10. [x] `changelog.d/ITL-981.md`, `design_issues/ITL-981-LV1.md`. +11. [x] Checks: `uv run ruff format .`, `uv run ruff check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/`. 12. [ ] Merge `origin/main` if it moved, push, PR against `main` (`Fixes ITL-981`), `orca linear attach`, `orca worktree set --worktree current --pr `, `worker_done`. From 197b35ab393f7117bda916f2312d4318a0bca193 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:52:18 -0800 Subject: [PATCH 15/22] fix(logical_types): lock the per-list extension type caches; wrap deserialisation errors in the caller (ITL-981) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/logical_types/ignore_list.py | 106 +++++++++++-------- tests/test_logical_types/test_ignore_list.py | 36 +++++++ 2 files changed, 100 insertions(+), 42 deletions(-) diff --git a/src/orcapod/logical_types/ignore_list.py b/src/orcapod/logical_types/ignore_list.py index 55ba99c8b..6ead1d736 100644 --- a/src/orcapod/logical_types/ignore_list.py +++ b/src/orcapod/logical_types/ignore_list.py @@ -31,6 +31,7 @@ import json import re +import threading from collections.abc import Callable, Collection, Iterable from typing import TYPE_CHECKING, Any, ClassVar, cast @@ -136,7 +137,7 @@ def parse_ignore_metadata(metadata: bytes) -> tuple[int, tuple[str, ...]]: """ try: parsed = json.loads(metadata.decode("ascii")) - except (UnicodeDecodeError, ValueError) as exc: + except (UnicodeDecodeError, ValueError, RecursionError) as exc: raise ValueError( f"extension metadata {metadata!r} is not ignore-list metadata: not JSON" ) from exc @@ -189,16 +190,23 @@ def __init__( def arrow_type(self, ignore: tuple[str, ...]) -> "pa.ExtensionType": """Return the Arrow extension type of a checked ignore list.""" - cached = self._arrow_types.get(ignore) - if cached is None: - cached = self._make_arrow_class(ignore)() - self._arrow_types[ignore] = cached - return cached + with _LOCK: + cached = self._arrow_types.get(ignore) + if cached is None: + cached = self._make_arrow_class(ignore)() + self._arrow_types[ignore] = cached + return cached def _deserialize( - self, storage_type: "pa.DataType", serialized: bytes, readable: bytes + self, storage_type: "pa.DataType", serialized: bytes ) -> "pa.ExtensionType": - """Return the type of a stored column, or raise if it cannot be one.""" + """Return the type of a stored column. + + Raises: + TypeIdentityError: If ``storage_type`` is not the family's storage. + ValueError: If ``serialized`` is not the metadata of a valid ignore + list of this family's version. + """ from orcapod.errors import TypeIdentityError from orcapod.logical_types.registry import storage_type_matches @@ -207,16 +215,10 @@ def _deserialize( f"Arrow extension type '{self.extension_name}': expected storage_type " f"{self.storage_type!r} but got {storage_type!r}." ) - try: - version, names = parse_ignore_metadata(serialized) - if version != self.version: - raise ValueError(f"version {version} is not version {self.version}") - ignore = checked_ignore(names, self.options) - except ValueError as exc: - raise LogicalTypeIdentifierError( - LogicalTypeIdentifier(self.extension_name, serialized), (readable,) - ) from exc - return self.arrow_type(ignore) + version, names = parse_ignore_metadata(serialized) + if version != self.version: + raise ValueError(f"version {version} is not version {self.version}") + return self.arrow_type(checked_ignore(names, self.options)) def _make_arrow_class( self, ignore: tuple[str, ...] @@ -236,7 +238,18 @@ def _deserialize( storage_type: "pa.DataType", serialized: bytes, ) -> "pa.ExtensionType": - return family._deserialize(storage_type, serialized, metadata) + from orcapod.errors import TypeIdentityError + + try: + return family._deserialize(storage_type, serialized) + except TypeIdentityError: + raise + except ValueError as exc: + # The class knows what it reads; the family only what is valid. + raise LogicalTypeIdentifierError( + LogicalTypeIdentifier(family.extension_name, serialized), + (metadata,), + ) from exc extension_class = type( f"_ArrowExt_{_sanitize(self.extension_name)}{suffix}", @@ -254,14 +267,15 @@ def _deserialize( def polars_type(self, ignore: tuple[str, ...]) -> "pl.BaseExtension": """Return the Polars extension type of a checked ignore list.""" - cached = self._polars_types.get(ignore) - if cached is None: - if self._polars_class is None: - self._polars_class = self._make_polars_class() - metadata = ignore_metadata(self.version, ignore).decode("ascii") - cached = self._polars_class(metadata) # pyright: ignore[reportCallIssue] # the synthesized class takes the metadata text - self._polars_types[ignore] = cached - return cached + with _LOCK: + cached = self._polars_types.get(ignore) + if cached is None: + if self._polars_class is None: + self._polars_class = self._make_polars_class() + metadata = ignore_metadata(self.version, ignore).decode("ascii") + cached = self._polars_class(metadata) # pyright: ignore[reportCallIssue] # the synthesized class takes the metadata text + self._polars_types[ignore] = cached + return cached def _make_polars_class(self) -> type["pl.BaseExtension"]: name = self.extension_name @@ -291,6 +305,13 @@ def _ext_from_params( _FAMILIES: dict[str, _ExtensionFamily] = {} +_LOCK = threading.RLock() +"""Guards ``_FAMILIES`` and each family's caches. + +Arrow compares extension types by class, so one ignore list must never get two +classes: every cache lookup that may create a class runs under this lock. +""" + def _family( extension_name: str, @@ -304,21 +325,22 @@ def _family( ValueError: If the name was already declared with another storage type, version or set of options. """ - family = _FAMILIES.get(extension_name) - if family is None: - family = _ExtensionFamily(extension_name, storage_type, version, options) - _FAMILIES[extension_name] = family - elif ( - family.storage_type != storage_type - or family.version != version - or family.options != options - ): - raise ValueError( - f"extension name {extension_name!r} is already declared with storage " - f"type {family.storage_type}, version {family.version} and options " - f"{sorted(family.options)}" - ) - return family + with _LOCK: + family = _FAMILIES.get(extension_name) + if family is None: + family = _ExtensionFamily(extension_name, storage_type, version, options) + _FAMILIES[extension_name] = family + elif ( + family.storage_type != storage_type + or family.version != version + or family.options != options + ): + raise ValueError( + f"extension name {extension_name!r} is already declared with storage " + f"type {family.storage_type}, version {family.version} and options " + f"{sorted(family.options)}" + ) + return family class IgnoreListLogicalType(BaseLogicalType): diff --git a/tests/test_logical_types/test_ignore_list.py b/tests/test_logical_types/test_ignore_list.py index 42f9335de..8451d850d 100644 --- a/tests/test_logical_types/test_ignore_list.py +++ b/tests/test_logical_types/test_ignore_list.py @@ -120,6 +120,7 @@ def test_parse_returns_what_was_written() -> None: b'{"ignore":[],"version":"1"}', b'{"ignore":[],"version":0}', b"not json", + b'{"ignore":' + b"[" * 100000 + b"]" * 100000 + b',"version":1}', b"\xff\xfe", b"[]", ], @@ -300,3 +301,38 @@ def test_a_column_stored_under_another_list_is_refused_naming_both( _converter(lenient).apply_logical_types(plain)["x"].type == stored.schema.field("x").type ) + + +def test_concurrent_first_use_gives_one_type_per_list() -> None: + """Threads that first ask for a list at the same time all get the same Arrow type.""" + import sys + import threading + + from orcapod.logical_types.ignore_list import IgnoreListLogicalType + + interval = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + for trial in range(30): + + class Racy(IgnoreListLogicalType): + EXTENSION_NAME = f"orcapod.test.racy{trial}" + STORAGE_TYPE = pa.large_string() + VERSION = 1 + OPTIONS = frozenset({"a"}) + + barrier = threading.Barrier(8) + seen: list[pa.ExtensionType] = [] + + def ask(logical_type: type[IgnoreListLogicalType] = Racy) -> None: + barrier.wait() + seen.append(logical_type(["a"]).get_arrow_extension_type()) + + threads = [threading.Thread(target=ask) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len({id(t) for t in seen}) == 1 + finally: + sys.setswitchinterval(interval) From 598acd9ab538edc1147057943ea0c465fde99e4b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:00:31 -0800 Subject: [PATCH 16/22] docs: a non-default ignore list is not usable end to end in v0.3 (ITL-981) Co-Authored-By: Claude Fable 5.2 --- changelog.d/ITL-981.md | 8 ++++++++ docs/concepts/arrays.md | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/changelog.d/ITL-981.md b/changelog.d/ITL-981.md index 92f47d10e..317d7d897 100644 --- a/changelog.d/ITL-981.md +++ b/changelog.d/ITL-981.md @@ -61,3 +61,11 @@ hash is strict by default, and that what the hash ignores is configured per type logical type with a value handler is hashed as a `large_binary` field carrying the type's identifier, so its storage type is no longer part of that hash. No library code calls this method, and no stored or pinned value moves. (ITL-981) + +### Known limitations + +- **A non-default ignore list is not usable end to end in a pipeline in v0.3.** A data + context built from a custom logical type registration file hashes bare values and Arrow + tables with its list, but streams and datagrams find their data context by key, so their + rows are read by the default context, which refuses a column of another list. (ITL-981, + ITL-965) diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index adf838918..3d1629b87 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -124,6 +124,14 @@ list. The list is configured in the `hash_ignore` object of Orcapod's logical ty registration file (`contexts/logical_types.json`); v0.3 has no other way to change it. A name that is not one of the three above is an error when the data context is built. +!!! note "Known limitation in v0.3" + A list other than the default cannot be used end to end in a pipeline yet. A data + context built from your own registration file hashes bare values and Arrow tables with + its list, but streams and datagrams find their data context by its key, which is the + key of the default context, so their rows are read by the default context, and it + refuses a column of another list (ITL-965). In v0.3 the default list, empty, is the + one that works everywhere. + The same mechanism serves the other library value types, each with its own names. ## What cannot be stored From 2d34857ff9e466627c849a35c17a2033c5d564b2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:01:30 -0800 Subject: [PATCH 17/22] test(arrays): list[np.ndarray] pod outputs on every store and runner; Delta async xfails on ITL-984 (ITL-981) Co-Authored-By: Claude Fable 5.2 --- .../test_arrays/test_array_pipeline_stores.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_arrays/test_array_pipeline_stores.py b/tests/test_arrays/test_array_pipeline_stores.py index 0083c7ef7..c134ab6e2 100644 --- a/tests/test_arrays/test_array_pipeline_stores.py +++ b/tests/test_arrays/test_array_pipeline_stores.py @@ -94,3 +94,46 @@ def test_pipeline_keeps_arrays_exact(tmp_path: Path, store: str, runner: str) -> expected = np.asarray(ARRAYS[name] * 2) assert_arrays_identical(doubled[name], expected) assert totals[name] == pytest.approx(float(np.real(expected).sum())) + + +@function_pod(output_keys="parts") +def split(x: np.ndarray) -> list[np.ndarray]: # pyright: ignore[reportMissingTypeArgument] # the bare np.ndarray annotation is under test + return [x.copy(), np.asarray(x * 2)] + + +@pytest.mark.parametrize( + ("store", "runner"), + [ + ("in_memory", "sync"), + ("in_memory", "async"), + ("delta", "sync"), + pytest.param( + "delta", + "async", + marks=pytest.mark.xfail( + strict=True, + reason=( + "ITL-984: the Delta MERGE cannot coerce large_list " + "against the stored list under the async orchestrator" + ), + ), + ), + ], +) +def test_pod_output_list_of_arrays(tmp_path: Path, store: str, runner: str) -> None: + """A pod output ``list[np.ndarray]`` is stored and read back on every store and runner.""" + source = DictSource( + data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], + tag_columns=["k"], + source_id="numeric", + ) + job = PipelineJob(name="array_lists", store=STORES[store](tmp_path)) + with job: + split.pod(source, label="split") + RUNNERS[runner](job) + + parts = _node(job, "split", "parts") + assert sorted(parts) == sorted(NUMERIC) + for name in NUMERIC: + assert_arrays_identical(parts[name][0], ARRAYS[name]) + assert_arrays_identical(parts[name][1], np.asarray(ARRAYS[name] * 2)) From 960d6fdcea79c1d03854e6e46e5844e0359cb43e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:12:59 -0800 Subject: [PATCH 18/22] test(arrays): compare structured arrays field by field, not their padding bytes (ITL-981) Co-Authored-By: Claude Fable 5.2 --- ...2026-11-12-itl-981-library-value-types-design.md | 3 ++- tests/test_arrays/test_array_strict_hash.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md index a50b9c258..40c3c0f2d 100644 --- a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -44,7 +44,8 @@ Never in the hash (no option): bites: a strided view hashes like its C-contiguous copy. 8. The `align=True` flag of a structured dtype: never / no (measured: `isalignedstruct` reads back `False`; the offsets it produced are kept, property 6). -9. Padding bytes between structured fields: never / yes. They round-trip but NumPy does not +9. Padding bytes between structured fields: never / not reliably (corrected after CI: on + Linux a stored aligned array came back with zeroed padding). NumPy does not define them (`np.empty` leaves garbage, field assignment does not copy them), so they cannot be hashed deterministically. 10. Subclass (`np.matrix`, `np.recarray`, `np.memmap`): never / no (measured: all read back diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py index 228626e7a..6827ac181 100644 --- a/tests/test_arrays/test_array_strict_hash.py +++ b/tests/test_arrays/test_array_strict_hash.py @@ -124,6 +124,17 @@ def _layout(dtype: np.dtype[Any]) -> Any: ] +def _value_bytes(array: npt.NDArray[Any]) -> bytes: + """The bytes of every scalar field in memory order, without padding bytes. + + Padding between structured fields is not compared: NumPy does not define its + content and does not always copy it. + """ + if array.dtype.names is None: + return array.tobytes("A") + return b"".join(_value_bytes(array[name]) for name in array.dtype.names) + + # --- the hash is strict by default ------------------------------------------------ @@ -213,7 +224,7 @@ def test_array_reads_back_exactly(name: str) -> None: assert got.shape == expected.shape assert got.flags.c_contiguous == expected.flags.c_contiguous assert got.flags.f_contiguous == expected.flags.f_contiguous - assert got.tobytes("A") == expected.tobytes("A") + assert _value_bytes(got) == _value_bytes(expected) assert got.flags.writeable From d19fd8bc0854a8333e09e6f0b121bd3a751b3375 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:23:11 -0800 Subject: [PATCH 19/22] fix(arrays): address review: zero struct padding at write, check the declared payload on read, validate hash_ignore (ITL-981) Co-Authored-By: Claude Fable 5.2 --- changelog.d/ITL-981.md | 5 +- docs/concepts/arrays.md | 12 ++- src/orcapod/contexts/registry.py | 9 ++ src/orcapod/logical_types/numpy_type.py | 93 ++++++++++++++++++- ...1-12-itl-981-library-value-types-design.md | 12 ++- tests/test_arrays/test_array_value_format.py | 67 +++++++++++++ tests/test_logical_types/test_ignore_list.py | 12 ++- 7 files changed, 197 insertions(+), 13 deletions(-) diff --git a/changelog.d/ITL-981.md b/changelog.d/ITL-981.md index 317d7d897..6a35ca09c 100644 --- a/changelog.d/ITL-981.md +++ b/changelog.d/ITL-981.md @@ -18,7 +18,10 @@ hash is strict by default, and that what the hash ignores is configured per type shape, element bits, C or Fortran memory order and structured field offsets and item size are kept. Not kept: strides other than C or Fortran order (a step slice is stored C-contiguous), the subclass (read back as plain `np.ndarray`), the `writeable` flag, dtype - `metadata` and the `align` flag of a structured dtype. + `metadata` and the `align` flag of a structured dtype. Padding bytes between structured + fields are not part of the value and are written as zero, so residual process memory + never reaches a store. A cell that does not hold exactly the element bytes its `.npy` + header declares is refused on read. - **Hash.** The hash is defined on the value, never on the stored bytes, and keeps the label `ndarray_v1`. It is `SHA-256(lp("orcapod.numpy.ndarray") || E || memory order || byte order || structured layout)`, where `E` is the element digest of dtype, shape and element diff --git a/docs/concepts/arrays.md b/docs/concepts/arrays.md index 3d1629b87..ae1ab6187 100644 --- a/docs/concepts/arrays.md +++ b/docs/concepts/arrays.md @@ -63,7 +63,7 @@ Part of the hash by default; each has a name and can be put on the type's | Name | What it is | |---|---| | `memory_order` | C or Fortran. An array counts as Fortran when it is Fortran-contiguous and not C-contiguous; every other array counts as C. | -| `byte_order` | Little- or big-endian, for each scalar field. A native array hashes as its actual byte order, so the hash is the same on every machine. | +| `byte_order` | Little- or big-endian, for each scalar field. A native array hashes as the byte order it actually has, never as "native": the same array bytes hash the same on every machine, and an array created natively on a big-endian machine is a big-endian array. | | `struct_layout` | The field offsets and the item size of a structured dtype, at every nesting depth. An aligned (padded) dtype and the packed dtype with the same fields differ. | Never part of the hash, because `.npy` does not keep it. A value read back from a store must @@ -77,7 +77,10 @@ hash like the value that was written, or every cached run would compute again: - **The `writeable` flag.** An array always reads back writeable. - **Dtype `metadata`**, and the `align=True` flag of a structured dtype (the offsets that flag produced are kept and count, as `struct_layout`). -- **Padding bytes between structured fields.** NumPy does not define their content. +- **Padding bytes between structured fields.** NumPy does not define their content (an + `np.empty` array holds whatever the memory held). They are not part of the value: they are + written to the store as zero, so residual memory never reaches a store that may be shared, + and they read back as zero. Every field reads back as written. ```python from orcapod.contexts import get_default_context @@ -177,8 +180,8 @@ Each cell is one `.npy` file, written by NumPy itself (`numpy.lib.format.write_array`, format version 3.0, `allow_pickle=False`) and read with `numpy.lib.format.read_array(..., allow_pickle=False)`. Format 3.0 is the one that stores field names outside Latin-1. Pickle is never written and never read: a cell that holds an -object array, or is not a complete `.npy` file of format 3.0, raises a `ValueError` that -names the column. +object array, is not a complete `.npy` file of format 3.0, or does not hold exactly the +element bytes its header declares, raises a `ValueError` that names the column. The extension metadata is `{"ignore":[],"version":1}`: the ignore list and the version of the type. A null array is a null cell, allowed when the column is annotated @@ -206,6 +209,7 @@ What reads back: | subclass | plain `np.ndarray` | | `writeable` flag | writeable | | dtype `metadata`, the `align` flag | not kept | +| padding bytes between structured fields | zero | ```python big = np.asfortranarray(np.arange(6, dtype=">f4").reshape(2, 3)) diff --git a/src/orcapod/contexts/registry.py b/src/orcapod/contexts/registry.py index db679ec46..ec48bdf9e 100644 --- a/src/orcapod/contexts/registry.py +++ b/src/orcapod/contexts/registry.py @@ -375,6 +375,15 @@ def _create_context_from_spec(self, spec: dict[str, Any]) -> DataContext: # extension name; its logical type and its value handler both take # it as ``{"_ref": "hash_ignore:"}`` (ITL-981). for name, ignore in registration.get("hash_ignore", {}).items(): + # A JSON object or string would iterate as names; only a list + # of names is a list of names. + if not isinstance(ignore, list) or not all( + isinstance(item, str) for item in ignore + ): + raise ValueError( + f"hash_ignore[{name!r}] must be a list of property " + f"names, got {ignore!r}" + ) ref_lut[f"{self._HASH_IGNORE_REF_PREFIX}{name}"] = list(ignore) ref_lut[self._LOGICAL_TYPE_REGISTRY_REF] = parse_objectspec( registration["logical_type_registry"], ref_lut=ref_lut diff --git a/src/orcapod/logical_types/numpy_type.py b/src/orcapod/logical_types/numpy_type.py index f034f39b4..f04690c56 100644 --- a/src/orcapod/logical_types/numpy_type.py +++ b/src/orcapod/logical_types/numpy_type.py @@ -10,7 +10,9 @@ What ``.npy`` does not keep is never part of the hash: strides other than C or Fortran contiguity (a strided view is stored C-contiguous), the subclass, the ``writeable`` flag, dtype metadata, the ``align`` flag of a structured dtype (its -offsets are kept) and padding bytes between structured fields. +offsets are kept) and padding bytes between structured fields (NumPy does not +define their content; they are written as zero, so memory the array happened +to sit on never reaches a store). Value hash ``D`` (label ``ndarray_v1``), in two steps. With ``lp(x) = u64le(len(utf8(x))) || utf8(x)``: @@ -49,9 +51,11 @@ from __future__ import annotations +import ast import hashlib import io import json +import math import re import struct import sys @@ -412,12 +416,34 @@ def _without_metadata(dtype: np.dtype[Any]) -> np.dtype[Any]: return dtype if dtype.metadata is None else np.dtype(dtype.str) +def _has_padding(dtype: np.dtype[Any]) -> bool: + """Return whether a structured dtype has bytes that belong to no field.""" + if dtype.fields is None and dtype.subdtype is None: + return False + return tag_to_dtype(dtype_to_tag(dtype)).itemsize != dtype.itemsize + + +def _with_zeroed_padding(array: npt.NDArray[Any]) -> npt.NDArray[Any]: + """Return a copy of a structured array whose padding bytes are zero. + + Padding between structured fields is not part of the value and NumPy does + not define its content (``np.empty`` leaves whatever the memory held), so it + is never written to a store. The copy has the same dtype, field values and + C or Fortran order; ``array`` is not modified. + """ + fortran = array.flags.f_contiguous and not array.flags.c_contiguous + out = np.zeros(array.shape, dtype=array.dtype, order="F" if fortran else "C") + _copy_by_field_name(out, array) + return out + + def array_to_npy(value: Any) -> bytes: """Return the ``.npy`` file an array is stored as. The array is written as it is: byte order, C or Fortran contiguity and structured offsets are kept. A view that is neither C- nor Fortran-contiguous - is written C-contiguous. ``value`` is not modified. + is written C-contiguous, and padding bytes between structured fields are + written as zero. ``value`` is not modified. Args: value: A ``numpy.ndarray`` (a subclass is stored as a plain array). @@ -434,6 +460,8 @@ def array_to_npy(value: Any) -> bytes: stored_dtype = _without_metadata(plain.dtype) if stored_dtype is not plain.dtype: plain = plain.view(stored_dtype) + if _has_padding(stored_dtype): + plain = _with_zeroed_padding(plain) buffer = io.BytesIO() npy_format.write_array( buffer, plain, version=NPY_FORMAT_VERSION, allow_pickle=False @@ -449,6 +477,63 @@ def array_to_npy(value: Any) -> bytes: return data +def _require_declared_payload(data: bytes) -> None: + """Raise unless the bytes after the header are exactly the declared array. + + ``read_array`` allocates the array its header declares before it reads the + elements, so a small cell that declares a huge shape would ask for that much + memory. The header is checked first: its length, and that dtype item size + times the element count equals the number of bytes that follow it. A header + that declares an object array is left for ``read_array`` to refuse. + + Raises: + ValueError: If the header is too long, incomplete or malformed, or the + payload length differs from what the header declares. + """ + prefix = len(_NPY_MAGIC) + 4 + if len(data) < prefix: + raise ValueError("stored numpy array value is truncated: no .npy header") + (header_bytes,) = struct.unpack_from(" _MAX_NPY_HEADER_BYTES: + raise ValueError( + f"stored numpy array value has a .npy header of {header_bytes} bytes; " + f"at most {_MAX_NPY_HEADER_BYTES} are read" + ) + payload_start = prefix + header_bytes + if len(data) < payload_start: + raise ValueError( + "stored numpy array value is truncated: incomplete .npy header" + ) + try: + header = ast.literal_eval(bytes(data[prefix:payload_start]).decode("utf-8")) + shape = header["shape"] + dtype = npy_format.descr_to_dtype(header["descr"]) + if not isinstance(shape, tuple) or not all( + type(dim) is int and dim >= 0 for dim in shape + ): + raise ValueError(f"shape {shape!r} is not a tuple of non-negative integers") + except ( + ValueError, + TypeError, + KeyError, + SyntaxError, + MemoryError, + RecursionError, + ) as exc: + raise ValueError( + f"stored numpy array value has a malformed .npy header: {exc}" + ) from exc + if dtype.hasobject: + return + declared = dtype.itemsize * math.prod(shape) + stored = len(data) - payload_start + if declared != stored: + raise ValueError( + f"stored numpy array value declares {declared} element bytes (shape " + f"{shape}, dtype {dtype}) but holds {stored}" + ) + + def npy_to_array(data: bytes) -> npt.NDArray[Any]: """Return the array stored in a ``.npy`` file written by ``array_to_npy``. @@ -461,13 +546,15 @@ def npy_to_array(data: bytes) -> npt.NDArray[Any]: Raises: ValueError: If ``data`` is not a ``.npy`` file of format - ``NPY_FORMAT_VERSION``, is truncated, or holds an object array. + ``NPY_FORMAT_VERSION``, is truncated, holds more or fewer element + bytes than its header declares, or holds an object array. """ if bytes(data[: len(_NPY_MAGIC)]) != _NPY_MAGIC: raise ValueError( "stored numpy array value is not a .npy file of format " f"{NPY_FORMAT_VERSION[0]}.{NPY_FORMAT_VERSION[1]}" ) + _require_declared_payload(data) return npy_format.read_array(io.BytesIO(data), allow_pickle=False) diff --git a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md index 40c3c0f2d..2d4508e33 100644 --- a/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md +++ b/superpowers/specs/2026-11-12-itl-981-library-value-types-design.md @@ -1,6 +1,12 @@ -# ITL-981: library value types (design note, awaiting developer sign-off) - -Status: PROPOSAL, November 12, 2026. No library code is changed yet. +# ITL-981: library value types (design note) + +Status: SIGNED OFF by the developer on November 12, 2026 and implemented in PR #328. The +text below is the proposal as it was approved; the four "open decisions" at the end were +decided as proposed (refuse a column of another list; label `ndarray_v1`; keep refusing +unit-less datetimes and field titles; sorted metadata keys), and section (vii) records the +final extension names. Added in review of PR #328: padding bytes between structured fields +are written as zero, and a stored cell must hold exactly the element bytes its header +declares. Measured with NumPy 2.2.6 and pyarrow 23.0.1 on a little-endian machine, writing with `numpy.lib.format.write_array(version=(3, 0), allow_pickle=False)` and reading with `numpy.lib.format.read_array(allow_pickle=False)`. "Measured" below means exactly that. diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index f4e8159b5..a6f42a2fe 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -530,6 +530,73 @@ def test_a_cell_that_is_not_a_stored_array_raises_naming_the_column( data.as_dict() +def _source_of_cell(cell: bytes) -> ArrowTableSource: + converter = get_default_context().type_converter + ext = converter.python_type_to_arrow_type(np.ndarray) + assert isinstance(ext, pa.ExtensionType) + storage = pa.array([cell], type=ext.storage_type) + table = pa.table( + { + "k": pa.array(["r"], pa.large_string()), + "x": pa.ExtensionArray.from_storage(ext, storage), + } + ) + return ArrowTableSource(table, tag_columns=["k"]) + + +def test_a_cell_that_declares_more_than_it_holds_is_refused_before_allocation() -> None: + """A small cell whose header declares a huge array is refused without allocating it.""" + buffer = io.BytesIO() + np.lib.format.write_array_header_2_0( + buffer, {"descr": " None: + """A cell that holds more than the array its header declares is refused.""" + with pytest.raises((ValueError, RuntimeError), match="'x'.*declares"): + for _, data in _source_of_cell(stored(np.arange(4.0)) + b"extra").iter_data(): + data.as_dict() + + +def test_padding_bytes_are_stored_as_zero_and_do_not_change_the_hash() -> None: + """Garbage in structured padding never reaches the store, and is not part of the hash.""" + aligned = np.dtype([("a", "i1"), ("b", " None: """A ``.npy`` file holding a pickled object array is refused on read.""" converter = get_default_context().type_converter diff --git a/tests/test_logical_types/test_ignore_list.py b/tests/test_logical_types/test_ignore_list.py index 8451d850d..8c12b797d 100644 --- a/tests/test_logical_types/test_ignore_list.py +++ b/tests/test_logical_types/test_ignore_list.py @@ -36,7 +36,7 @@ LENIENT = b'{"ignore":["label"],"version":1}' -def _context(tmp_path: Path, ignore: list[str]) -> DataContext: +def _context(tmp_path: Path, ignore: Any) -> DataContext: """Build a data context that registers the toy type with ``ignore``.""" shipped = Path(contexts_package.__file__).parent / "logical_types.json" registration = json.loads(shipped.read_text()) @@ -54,7 +54,8 @@ def _context(tmp_path: Path, ignore: list[str]) -> DataContext: }, ] ) - custom = tmp_path / f"logical_types_{'_'.join(ignore) or 'strict'}.json" + name = "_".join(str(item) for item in ignore) if isinstance(ignore, list) else "" + custom = tmp_path / f"logical_types_{name or 'other'}.json" custom.write_text(json.dumps(registration)) return JSONDataContextRegistry(logical_types_file=custom).get_context() @@ -224,6 +225,13 @@ def test_an_unknown_option_name_fails_the_context_build(tmp_path: Path) -> None: _context(tmp_path, ["colour"]) +@pytest.mark.parametrize("value", [{"label": False}, "label", [1], None]) +def test_hash_ignore_must_be_a_list_of_names(tmp_path: Path, value: Any) -> None: + """Only a JSON list of names is an ignore list; an object or a string is refused.""" + with pytest.raises(ContextValidationError, match="list of property names"): + _context(tmp_path, value) + + # --- Arrow and stored columns ---------------------------------------------------------- From b6de1a4607e07bdb3d007c0ef569a40459d21860 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:40:25 -0800 Subject: [PATCH 20/22] fix(arrays): padding of a structured array read back from a store is zero (ITL-981) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/logical_types/numpy_type.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/orcapod/logical_types/numpy_type.py b/src/orcapod/logical_types/numpy_type.py index f04690c56..9686eebae 100644 --- a/src/orcapod/logical_types/numpy_type.py +++ b/src/orcapod/logical_types/numpy_type.py @@ -542,7 +542,8 @@ def npy_to_array(data: bytes) -> npt.NDArray[Any]: Returns: A new, writeable plain ``np.ndarray`` with the stored dtype (including - byte order and structured offsets), shape and contiguity. + byte order and structured offsets), shape and contiguity; padding + bytes between structured fields are zero. Raises: ValueError: If ``data`` is not a ``.npy`` file of format @@ -555,7 +556,12 @@ def npy_to_array(data: bytes) -> npt.NDArray[Any]: f"{NPY_FORMAT_VERSION[0]}.{NPY_FORMAT_VERSION[1]}" ) _require_declared_payload(data) - return npy_format.read_array(io.BytesIO(data), allow_pickle=False) + array = npy_format.read_array(io.BytesIO(data), allow_pickle=False) + if array.dtype.fields is not None and _has_padding(array.dtype): + # NumPy fills a structured array field by field, so the padding of the + # array it returns is whatever the new memory held, not the stored zeros. + array = _with_zeroed_padding(array) + return array def _byte_order_text(dtype: np.dtype[Any]) -> str: From 21dbc84c08d349595cc745e66ecca6aacd005391 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:28:12 -0800 Subject: [PATCH 21/22] test(arrays): list[np.ndarray] outputs pass on Delta async now that ITL-984 is fixed on main (ITL-981) Co-Authored-By: Claude Fable 5.2 --- tests/test_arrays/test_array_pipeline_stores.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/test_arrays/test_array_pipeline_stores.py b/tests/test_arrays/test_array_pipeline_stores.py index c134ab6e2..e01216d62 100644 --- a/tests/test_arrays/test_array_pipeline_stores.py +++ b/tests/test_arrays/test_array_pipeline_stores.py @@ -107,17 +107,7 @@ def split(x: np.ndarray) -> list[np.ndarray]: # pyright: ignore[reportMissingTy ("in_memory", "sync"), ("in_memory", "async"), ("delta", "sync"), - pytest.param( - "delta", - "async", - marks=pytest.mark.xfail( - strict=True, - reason=( - "ITL-984: the Delta MERGE cannot coerce large_list " - "against the stored list under the async orchestrator" - ), - ), - ), + ("delta", "async"), ], ) def test_pod_output_list_of_arrays(tmp_path: Path, store: str, runner: str) -> None: From fa5cd5653fffd21675de3f8267fafe5e3e291a4b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:55:50 -0800 Subject: [PATCH 22/22] test(arrays): Batch and GroupBy over an array column on every store and runner (ITL-981) Co-Authored-By: Claude Fable 5.2 --- .../test_arrays/test_array_pipeline_stores.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_arrays/test_array_pipeline_stores.py b/tests/test_arrays/test_array_pipeline_stores.py index e01216d62..229870037 100644 --- a/tests/test_arrays/test_array_pipeline_stores.py +++ b/tests/test_arrays/test_array_pipeline_stores.py @@ -11,6 +11,7 @@ from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.operators import Batch, GroupBy from orcapod.pipeline import AsyncPipelineOrchestrator from orcapod.sources import DictSource from tests.test_arrays.array_cases import ARRAYS, NUMERIC, assert_arrays_identical @@ -127,3 +128,37 @@ def test_pod_output_list_of_arrays(tmp_path: Path, store: str, runner: str) -> N for name in NUMERIC: assert_arrays_identical(parts[name][0], ARRAYS[name]) assert_arrays_identical(parts[name][1], np.asarray(ARRAYS[name] * 2)) + + +@function_pod(output_keys="firsts") +def first_elements(x: list[np.ndarray]) -> list[float]: # pyright: ignore[reportMissingTypeArgument] # the bare np.ndarray annotation is under test + assert all(type(a) is np.ndarray and a.shape == (3,) for a in x) + return sorted(float(a[0]) for a in x) + + +@pytest.mark.parametrize("runner", sorted(RUNNERS)) +@pytest.mark.parametrize("store", sorted(STORES)) +@pytest.mark.parametrize("operator", ["batch", "group_by"]) +def test_batch_and_group_by_over_an_array_column( + tmp_path: Path, operator: str, store: str, runner: str +) -> None: + """``Batch`` and ``GroupBy`` over an array column feed a pod on every store and runner.""" + source = DictSource( + data=[ + {"g": "a" if i < 2 else "b", "k": f"r{i}", "x": np.arange(3.0) + i} + for i in range(4) + ], + tag_columns=["g", "k"], + source_id="grouped", + ) + job = PipelineJob(name="array_groups", store=STORES[store](tmp_path)) + with job: + if operator == "batch": + grouped = Batch(batch_size=2)(source) + else: + grouped = GroupBy(["g"])(source) + first_elements.pod(grouped, label="firsts") + RUNNERS[runner](job) + + firsts = sorted(d.as_dict()["firsts"] for _, d in job.nodes["firsts"].iter_data()) + assert firsts == [[0.0, 1.0], [2.0, 3.0]]