From 9ae08c9cfe9e283cac1d8fad52dbfee8a19b3d6e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 19:03:38 -0800 Subject: [PATCH 1/3] docs(specs): review NumPy array storage and compare tensor storage designs (ITL-874) Adds the ITL-874 design document, the experiment scripts that back every claim in it, the plan, and a DESIGN_ISSUES.md entry (AT1). No product code changes. Refs ITL-874 Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 20 + ...2026-11-10-itl-874-array-tensor-storage.md | 34 ++ .../specs/2026-11-10-array-tensor-storage.md | 452 ++++++++++++++++++ .../a1_stored_and_hashed_bytes.py | 243 ++++++++++ .../a1b_numpy_versions.py | 128 +++++ .../a2_dtype_coverage.py | 208 ++++++++ .../a3_cached_rerun.py | 91 ++++ .../a3_cached_rerun_child.py | 129 +++++ .../a3_common.py | 69 +++ .../a3_operators.py | 394 +++++++++++++++ .../a3_pipeline_stores.py | 175 +++++++ .../a4_annotations.py | 136 ++++++ .../a4_foreign_readers.py | 252 ++++++++++ .../b0_common.py | 417 ++++++++++++++++ .../b1_starfix_hashing.py | 271 +++++++++++ .../b2_polars_roundtrip.py | 135 ++++++ .../b3_databases.py | 169 +++++++ .../b4_dtype_strictness.py | 106 ++++ .../b5_size_zero_copy_readers.py | 154 ++++++ .../b6_torch_mapping.py | 207 ++++++++ .../b7_migration.py | 173 +++++++ .../b8_vst_support_and_out_of_table.py | 152 ++++++ .../c1_canonical_digest_prototype.py | 107 +++++ 23 files changed, 4222 insertions(+) create mode 100644 superpowers/plans/2026-11-10-itl-874-array-tensor-storage.md create mode 100644 superpowers/specs/2026-11-10-array-tensor-storage.md create mode 100644 superpowers/specs/array-tensor-storage-experiments/a1_stored_and_hashed_bytes.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a1b_numpy_versions.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a2_dtype_coverage.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a3_cached_rerun.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a3_cached_rerun_child.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a3_common.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a3_operators.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a3_pipeline_stores.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a4_annotations.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/a4_foreign_readers.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b0_common.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b1_starfix_hashing.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b2_polars_roundtrip.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b3_databases.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b4_dtype_strictness.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b5_size_zero_copy_readers.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b6_torch_mapping.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b7_migration.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/b8_vst_support_and_out_of_table.py create mode 100644 superpowers/specs/array-tensor-storage-experiments/c1_canonical_digest_prototype.py diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 012708cab..bf5100d3b 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2109,3 +2109,23 @@ equals the materialized one. Hashes of pipelines that already worked are unchang Regressions: `tests/test_core/operators/test_chained_aggregation.py`. Open: on the async barrier path, per-row source info is still row 0's (ITL-845); that assertion is a strict xfail. + +--- + +## `src/orcapod/logical_types/numpy_type.py` — array and tensor storage + +### AT1 — NumPy array hash commits to the `.npy` file, not the array value; no tensor format yet +**Status:** open +**Severity:** high +**Issue:** ITL-874 + +`LogicalNumpyArray` stores each array as its `np.save` bytes, and `NumpyArrayHandler` hashes +SHA-256 of those bytes. Storage is lossless and stable across processes and NumPy 1.26/2.x. +The hash, however, changes with memory layout (C vs Fortran), byte order and structured +field order, contrary to ITL-465. The column type carries no dtype or shape, so +`NDArray[np.float32]` is rejected and MergeJoin merges float32 with float64 arrays. Masked +arrays silently lose their mask. A pod declared `-> np.ndarray` may return an unchecked NumPy +scalar. Any change of format or hash must ship as an explicit migration to new tables: in +Delta, new-format rows under an unchanged pipeline path hide the old rows. Review, +experiments and options for v0.3 and PyTorch (ITL-871): +`superpowers/specs/2026-11-10-array-tensor-storage.md`. Awaiting the developer's decision. diff --git a/superpowers/plans/2026-11-10-itl-874-array-tensor-storage.md b/superpowers/plans/2026-11-10-itl-874-array-tensor-storage.md new file mode 100644 index 000000000..e7c98b709 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-874-array-tensor-storage.md @@ -0,0 +1,34 @@ +# ITL-874 plan: array and tensor storage review and design + +Issue: ITL-874. Branch: `eywalker/itl-874-design-array-and-tensor-storage-numpy`. +Review and design only. No change under `src/` or `tests/`. + +## Deliverables + +1. `superpowers/specs/2026-11-10-array-tensor-storage.md`: the design document. +2. `superpowers/specs/array-tensor-storage-experiments/*.py`: experiment scripts. Each one runs + with `uv run python