From 7feece980ba47a55993ac01eff0c2ac6823c6e18 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:46:24 -0800 Subject: [PATCH 1/9] docs(plans): ITL-1122 evaluation of large normalization Co-Authored-By: Claude Opus 5.5 --- ...2026-11-14-itl-1122-large-normalization.md | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1122-large-normalization.md diff --git a/superpowers/plans/2026-11-14-itl-1122-large-normalization.md b/superpowers/plans/2026-11-14-itl-1122-large-normalization.md new file mode 100644 index 000000000..bdf2d4ae0 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1122-large-normalization.md @@ -0,0 +1,123 @@ +# ITL-1122: large normalization applied unevenly — evaluation and plan + +Status: EVALUATION DONE, waiting for the coordinator's answers (Q1–Q3) before any code. + +## The rule (developer, November 14, 2026) + +"any arrow table WE create as part of orcapod should always get things normalized to large +variant. However for things where 1) user directly creates arrow table with specific data +types and 2) some source actually make distinction (in recoverable fashion) of large and +non-large variants: for both of these we dont want to normalize the variants between large and +non-large." + +Clauses used below: [O] Orcapod creates the table -> large; [U] the user created the Arrow +table -> kept; [S] the store keeps the distinction recoverably -> kept. + +## Measurement + +Scripts (scratchpad, re-runnable): `measure.py` (source kind x operator x plain / job sync / +job async, populated and zero-row) and `measure2.py` (function pod, DerivedSource, +PollingSource, Spiral fake, store round trips). Base `origin/main` 3a7f3850. + +Columns measured: tag `t: string`, tag `k: int64`, data `s: string`, `b: binary`, +`l: list`, `n: struct, y: string>` (CSV: `t, k, s` only). Source-info +and system-tag columns are large everywhere (Orcapod builds them). + +### Store round trips (write `string` and `large_string`, `binary` and `large_binary`, `list` and `large_list`; read back) + +| Store | Result | Recoverable? | +|---|---|---| +| Delta Lake (deltalake directly) | Delta schema has one `string` / `binary` / `array` type for both; read back both as `string` (or both `large_string` with `as_large_types=True`, which Orcapod uses) | no | +| `DeltaTableDatabase` | all large | no | +| `InMemoryArrowDatabase` | each comes back as written | yes (in memory, not persistent) | +| SQLite | `TEXT` / `BLOB` for both; connector maps to `large_string` / `large_binary` | no | +| PostgreSQL | `text` / `bytea` for both; connector maps to large (by code; no server here) | no | +| SpiralDB (fake models observed ITL-40) | `large_string` written comes back `string`; connector normalizes to large | no | +| CSV file | no type in the file; `read_csv` gives `string` for both | no | + +### Sources (the source's own table = the job's source node, sync and async) + +| Source | Populated | Zero-row | Verdict | +|---|---|---|---| +| `ArrowTableSource` (user's table) | kept (`string`, ...) | kept | CONSISTENT [U] | +| `DictSource` Python rows / Python schema | large | large | CONSISTENT [O] | +| `DictSource(data_schema=pa.Schema)` with `string` | kept | kept | Q2 (CONSISTENT [U] if an explicit Arrow schema counts) | +| `ListSource` | large | n/a | CONSISTENT [O] | +| `CSVSource` (no schema) | `string` | ERROR (header-only CSV: null-typed columns; separate bug) | INCONSISTENT (CSV keeps no distinction) | +| `CSVSource(schema=pa.Schema)` with `string` | kept | kept | Q2 | +| `DataFrameSource` polars / pandas | large | large | CONSISTENT [O] | +| `DataFrameSource(pa.Table)` with `string` | large (through Polars) | large | Q3 | +| `DeltaTableSource` | large | large | CONSISTENT [O] (not recoverable) | +| `SQLiteTableSource` / `PostgreSQLTableSource` / `DBTableSource` | large | SQLite: "Table is empty" by design | CONSISTENT [O] | +| `SpiralDBTableSource` (fake) | large | - | CONSISTENT [O] | +| `CachedSource` over a `string` source, `InMemoryArrowDatabase` | kept `string` | large | INCONSISTENT (zero-row != populated) | +| `CachedSource` over a `string` source, `DeltaTableDatabase` | large | large | INCONSISTENT with the in-memory cache: type depends on the cache store | +| `DerivedSource` (function node, operator node, `tags_as_data`) | large | large | CONSISTENT [O] | +| `PollingSource` (fetch dict or polars) | large | - | CONSISTENT [O]; fetch returning a `pa.Table`: Q3 | +| Function pod output (incl. passed-through tags), plain and job | large | large | CONSISTENT [O] | + +### Operators over a non-large input (`ArrowTableSource`, `DictSource(pa.Schema)`, `CSVSource`, in-memory `CachedSource`) + +Inside a job (sync and async): every operator gives large. CONSISTENT [O]. + +On a plain stream (populated and zero-row identical): + +* Already large: `PolarsFilter` with a predicate, `DropNulls` (any form), `MergeJoin`, `Batch`, + `GroupBy` (member columns), `Index`. +* Pass non-large through: `SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, + `DropDataColumns`, `SelectColumns`, `DropColumns`, `MapTags`, `MapData`, `PolarsFilter()` + (no-op), `Join` (every column), `SemiJoin`, `GroupBy`'s key tag (`GroupBy(by=["t"])`: `t` + stays `string`). +* `Join` of a large input and a `string` input on shared tag `t` picks a side, and which side + depends on the canonical input order (pipeline hash): `DictSource ⋈ ArrowTableSource` gives + `large_string`, `DataFrameSource ⋈ ArrowTableSource` gives `string`. + +Verdict: INCONSISTENT (given Q1 = large): an operator's output is a table Orcapod creates; the +same operator gives different types inside and outside a job, and operators disagree among +themselves on the plain path. + +## Questions to the coordinator + +* Q1 (open): a pass-through operator (select, map, no-op filter, join, semijoin) over a user's + Arrow table: large, or the user's types? Recommendation: large. +* Q2: an explicit Arrow schema (`DictSource(data_schema=pa.Schema)`, `CSVSource(schema=)`, + `DataFrameSource(schema=)`) with `string`: the user chose the Arrow types, so kept [U]? + Recommendation: kept. +* Q3: `DataFrameSource` given a `pa.Table`, `PollingSource` whose fetch returns a `pa.Table`: + both convert through Polars, so large. Recommendation: leave (Polars-frame sources; + `ArrowTableSource` is the source that keeps a user's Arrow types). + +## Proposed fixes (one place each) + +* F1 operators: one helper applied to every operator's sync result in the three base + `static_process` overrides (`UnaryOperator`, `BinaryOperator`, `NonZeroInputOperator`), + beside `without_input_meta`: if any column of the result's table (with source, system-tag + and context columns) differs from `normalize_to_large_types(type, keep_null=True)`, rebuild + the `ArrowTableStream` with those columns re-laid out to the large type + (`arrow_utils.restore_column_types`, as `filters.py` does). Width, nullability, field + metadata and extension types are untouched (`normalize_to_large_types` keeps them). The + async/job path is already large; tests pin it. +* F2 `CSVSource`: normalize the table `read_csv` gives when no `schema=` is passed. +* F3 `CachedSource`: normalize the table it presents in `_build_merged_stream` (read side only; + stored rows unchanged), so in-memory == Delta and zero-row == populated. + +What moves: no hash (large and non-large hash identically; golden suite and a direct test). No +stored schema: operator caches are written only inside jobs (already large); `CachedSource` +writes as before; Delta keeps no distinction. To be confirmed by a test that appends after the +change to a table written before it. + +## Out of scope findings (report in worker_done) + +* A header-only CSV (zero rows) cannot be read: columns are typed `null` and + `infer_schema_nullable` marks them non-null -> `ValueError: A null type field may not be + non-nullable`. + +## Implementation steps (after the answers) + +1. Failing tests: `tests/test_core/operators/test_large_normalization.py` (every operator, + plain stream and job sync/async, populated == zero-row), CSVSource, CachedSource, hash + equality. +2. F1, F2, F3. +3. Golden suite, full suite, stored-schema append check. +4. CLAUDE.md + `.zed/rules` "Arrow types", docs, `changelog.d/ITL-1122.md`, + `design_issues/ITL-1122-LN1.md`. From 1f86f11268f5d39d7650d72023b20b9c6e9c6e62 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:49:27 -0800 Subject: [PATCH 2/9] fix(sources): CSVSource and CachedSource present large Arrow types (ITL-1122) A CSV file and a cache store keep no recoverable large / non-large distinction, so the tables these sources build are Orcapod's and take the large variants. An Arrow schema the user gives CSVSource is kept. normalize_schema_to_large_types and normalize_table_to_large_types take keep_null, as the operators and sources need. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/sources/cached_source.py | 6 + src/orcapod/core/sources/csv_source.py | 7 + src/orcapod/utils/arrow_utils.py | 23 +- .../test_large_normalization_sources.py | 228 ++++++++++++++++++ 4 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 tests/test_core/sources/test_large_normalization_sources.py diff --git a/src/orcapod/core/sources/cached_source.py b/src/orcapod/core/sources/cached_source.py index e7171e04c..4a6596d72 100644 --- a/src/orcapod/core/sources/cached_source.py +++ b/src/orcapod/core/sources/cached_source.py @@ -27,6 +27,7 @@ from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.system_constants import SOURCE_DB_SCHEMA_VERSION, constants from orcapod.types import ColumnConfig, Schema +from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -356,6 +357,11 @@ def _build_merged_stream(self) -> ArrowTableStream: all_records = self._read_cached_records() if all_records is None: all_records = self._empty_table() + # The rows read back are a table Orcapod creates: large variants, whatever + # the cache store keeps and whether it has rows (ITL-1122). + all_records = arrow_utils.normalize_table_to_large_types( + all_records, keep_null=True + ) tag_keys = self._source.keys()[0] # Cached rows are this source's: give them its data context, not the default. diff --git a/src/orcapod/core/sources/csv_source.py b/src/orcapod/core/sources/csv_source.py index ba110effd..48991bec4 100644 --- a/src/orcapod/core/sources/csv_source.py +++ b/src/orcapod/core/sources/csv_source.py @@ -21,6 +21,11 @@ class CSVSource(RootSource): The file is read once at construction time using PyArrow's CSV reader, converted to an Arrow table, and enriched by ``SourceStreamBuilder`` (source-info, schema-hash, system tags). + + A CSV file keeps no distinction between large and non-large Arrow types, so + the table read from it is Orcapod's and takes the large variants (``string`` + is ``large_string``; ITL-1122). An Arrow ``schema`` given by the user is + applied as it is: the user chose those types. """ def __init__( @@ -46,6 +51,8 @@ def __init__( table = table.cast(schema) else: table = table.cast(arrow_utils.infer_schema_nullable(table)) + # The file has no large / non-large distinction: Orcapod's table is large. + table = arrow_utils.normalize_table_to_large_types(table, keep_null=True) builder = SourceStreamBuilder(self.data_context, self.orcapod_config) result = builder.build( diff --git a/src/orcapod/utils/arrow_utils.py b/src/orcapod/utils/arrow_utils.py index 5630eec1a..912a53c71 100644 --- a/src/orcapod/utils/arrow_utils.py +++ b/src/orcapod/utils/arrow_utils.py @@ -194,7 +194,9 @@ def _normalize_field_to_large_types( return field.with_type(normalize_to_large_types(field.type, keep_null=keep_null)) -def normalize_schema_to_large_types(schema: "pa.Schema") -> "pa.Schema": +def normalize_schema_to_large_types( + schema: "pa.Schema", *, keep_null: bool = False +) -> "pa.Schema": """ Convert a schema to use large variants of data types. @@ -203,6 +205,9 @@ def normalize_schema_to_large_types(schema: "pa.Schema") -> "pa.Schema": Args: schema: Arrow schema to normalize + keep_null: Passed to ``normalize_to_large_types``. ``True`` keeps a + ``null`` column ``null``, as operators and sources do; the default + ``False`` maps it to ``large_string``, as ``Data`` does. Returns: New schema with large type variants, or same schema if no changes needed @@ -220,7 +225,7 @@ def normalize_schema_to_large_types(schema: "pa.Schema") -> "pa.Schema": schema_changed = False for field in schema: - normalized_type = normalize_to_large_types(field.type) + normalized_type = normalize_to_large_types(field.type, keep_null=keep_null) # Check if the type actually changed if normalized_type != field.type: @@ -275,16 +280,22 @@ def shared_arrow_type( return None -def normalize_table_to_large_types(table: "pa.Table") -> "pa.Table": +def normalize_table_to_large_types( + table: "pa.Table", *, keep_null: bool = False +) -> "pa.Table": """ Normalize table schema to use large type variants. Uses cast() which should be zero-copy for large variant conversions since they have identical binary representations, but ensures proper - type validation and handles any edge cases safely. + type validation and handles any edge cases safely. Only layouts change: + width, nullability, field metadata and logical (extension) types are kept. Args: table: Arrow table to normalize + keep_null: Passed to ``normalize_to_large_types``. ``True`` keeps a + ``null`` column ``null``, as operators and sources do; the default + ``False`` maps it to ``large_string``, as ``Data`` does. Returns: Table with normalized schema, or same table if no changes needed @@ -296,7 +307,9 @@ def normalize_table_to_large_types(table: "pa.Table") -> "pa.Table": name: large_string age: int64 """ - normalized_schema = normalize_schema_to_large_types(table.schema) + normalized_schema = normalize_schema_to_large_types( + table.schema, keep_null=keep_null + ) # If schema didn't change, return original table if normalized_schema is table.schema: diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py new file mode 100644 index 000000000..353cfbbf8 --- /dev/null +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -0,0 +1,228 @@ +"""Sources present large Arrow types unless the user chose the types (ITL-1122). + +The rule (developer, November 14, 2026): every Arrow table Orcapod creates has its string, +binary and list types in their large variants. Two exceptions keep the types as given: an +Arrow table (or Arrow schema) the user made, and a store that keeps the large / non-large +distinction in a recoverable way. + +* ``CSVSource`` reads a file that has no such distinction, so its table is Orcapod's: large. + An Arrow ``schema=`` given by the user is kept. +* ``CachedSource`` presents the rows it read from its cache store, a table Orcapod creates: + large whatever the store (in memory or Delta) and whether it has rows or not. +""" + +from __future__ import annotations + +from pathlib import Path + +import pyarrow as pa +import pyarrow.csv as pa_csv +import pytest + +from orcapod.core.sources import ArrowTableSource, CachedSource, CSVSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.operators import SelectTagColumns +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import SourceProtocol +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.utils import arrow_utils + +ALL_COLUMNS = {"source": True, "system_tags": True, "context": True} +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +NESTED = pa.struct([pa.field("x", pa.list_(pa.string())), pa.field("y", pa.string())]) +SMALL_SCHEMA = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("s", pa.string(), nullable=False), + pa.field("b", pa.binary(), nullable=False), + pa.field("l", pa.list_(pa.string()), nullable=False), + pa.field("n", NESTED, nullable=False), + ] +) +ROWS = [ + {"t": "a", "s": "x", "b": b"1", "l": ["p"], "n": {"x": ["q"], "y": "r"}}, + {"t": "b", "s": "y", "b": b"2", "l": [], "n": {"x": [], "y": "s"}}, +] + + +def _non_large(table: pa.Table) -> dict[str, pa.DataType]: + """The columns of ``table`` whose type is not its large-normalized form.""" + return { + field.name: field.type + for field in table.schema + if arrow_utils.normalize_to_large_types(field.type, keep_null=True) + != field.type + } + + +def _small_source(n_rows: int, source_id: str = "small") -> ArrowTableSource: + """A user's Arrow table with string, binary, list and nested columns.""" + return ArrowTableSource( + pa.Table.from_pylist(ROWS[:n_rows], schema=SMALL_SCHEMA), + tag_columns=["t"], + source_id=source_id, + ) + + +def _source_node_table(source: SourceProtocol, orchestrator: str) -> pa.Table: + """The table of ``source``'s node in a job run on ``orchestrator``.""" + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + SelectTagColumns(["t"])(source, label="selected") + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + # The job's two nodes: the source's node and ``selected``. + (source_label,) = [label for label in job.nodes if label != "selected"] + return job.nodes[source_label].as_table(columns=ALL_COLUMNS) + + +# --------------------------------------------------------------------------- +# CSVSource +# --------------------------------------------------------------------------- + + +@pytest.fixture +def csv_path(tmp_path: Path) -> Path: + path = tmp_path / "rows.csv" + pa_csv.write_csv( + pa.table({"t": ["a", "b"], "k": [0, 1], "s": ["x", "y"]}), + path, + ) + return path + + +class TestCSVSource: + def test_table_is_large(self, csv_path: Path) -> None: + source = CSVSource(str(csv_path), tag_columns=["t"], source_id="csv") + table = source.as_table(columns=ALL_COLUMNS) + assert _non_large(table) == {} + assert table.schema.field("s").type == pa.large_string() + + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_source_node_in_a_job_is_large( + self, csv_path: Path, orchestrator: str + ) -> None: + source = CSVSource(str(csv_path), tag_columns=["t"], source_id="csv") + assert _non_large(_source_node_table(source, orchestrator)) == {} + + def test_explicit_arrow_schema_is_kept(self, csv_path: Path) -> None: + """The user chose the Arrow types: ``string`` stays ``string``.""" + schema = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("k", pa.int64(), nullable=False), + pa.field("s", pa.string(), nullable=False), + ] + ) + source = CSVSource( + str(csv_path), tag_columns=["t"], source_id="csv", schema=schema + ) + table = source.as_table() + assert table.schema.field("t").type == pa.string() + assert table.schema.field("s").type == pa.string() + + def test_row_hashes_equal_the_explicit_string_schema(self, csv_path: Path) -> None: + """Large and non-large variants hash identically: no row hash moves.""" + schema = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("k", pa.int64(), nullable=False), + pa.field("s", pa.string(), nullable=False), + ] + ) + large = CSVSource(str(csv_path), tag_columns=["t"], source_id="csv") + small = CSVSource( + str(csv_path), tag_columns=["t"], source_id="csv", schema=schema + ) + assert large.content_hash() == small.content_hash() + assert large.pipeline_hash() == small.pipeline_hash() + large_rows = [ + (tag.content_hash(), data.content_hash()) for tag, data in large.iter_data() + ] + small_rows = [ + (tag.content_hash(), data.content_hash()) for tag, data in small.iter_data() + ] + assert large_rows == small_rows + + +# --------------------------------------------------------------------------- +# CachedSource +# --------------------------------------------------------------------------- + + +def _store(kind: str, tmp_path: Path) -> ArrowDatabaseProtocol: + if kind == "in_memory": + return InMemoryArrowDatabase() + return DeltaTableDatabase(base_path=tmp_path / "cache") + + +@pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) +class TestCachedSource: + @pytest.mark.parametrize("n_rows", [2, 0]) + def test_table_is_large(self, store_kind: str, n_rows: int, tmp_path: Path) -> None: + cached = CachedSource( + _small_source(n_rows), cache_database=_store(store_kind, tmp_path) + ) + table = cached.as_table(columns=ALL_COLUMNS) + assert table.num_rows == n_rows + assert _non_large(table) == {} + + def test_zero_row_schema_equals_populated( + self, store_kind: str, tmp_path: Path + ) -> None: + populated = CachedSource( + _small_source(2), cache_database=_store(store_kind, tmp_path / "a") + ).as_table(columns=ALL_COLUMNS) + empty = CachedSource( + _small_source(0), cache_database=_store(store_kind, tmp_path / "b") + ).as_table(columns=ALL_COLUMNS) + user_columns = [field.name for field in SMALL_SCHEMA] + assert ( + empty.select(user_columns).schema == populated.select(user_columns).schema + ) + + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_source_node_in_a_job_is_large( + self, store_kind: str, orchestrator: str, tmp_path: Path + ) -> None: + cached = CachedSource( + _small_source(2), + cache_database=_store(store_kind, tmp_path), + ) + assert _non_large(_source_node_table(cached, orchestrator)) == {} + + def test_row_hashes_equal_the_wrapped_source( + self, store_kind: str, tmp_path: Path + ) -> None: + """Large and non-large variants hash identically: no row hash moves.""" + source = _small_source(2) + cached = CachedSource(source, cache_database=_store(store_kind, tmp_path)) + source_rows = sorted( + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in source.iter_data() + ) + cached_rows = sorted( + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in cached.iter_data() + ) + assert cached_rows == source_rows + + def test_a_cache_written_before_is_read_and_appended( + self, store_kind: str, tmp_path: Path + ) -> None: + """A second cached source on the same store reads the stored rows and adds its own. + + The cache's write path is unchanged: it stores what the wrapped source gives. + """ + store = _store(store_kind, tmp_path) + first = CachedSource(_small_source(1, source_id="small"), cache_database=store) + assert first.as_table().num_rows == 1 + second = CachedSource(_small_source(2, source_id="small"), cache_database=store) + table = second.as_table(columns=ALL_COLUMNS) + assert table.num_rows == 2 + assert _non_large(table) == {} From dd737d0bf003984dd98037d9c2d4b15fbf1e1e68 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:57:19 -0800 Subject: [PATCH 3/9] test(operators): take the string key from a user's Arrow table, not a CSV (ITL-1122) A CSVSource now presents large_string, so the two tests of a string key against a large_string key build the string side from a user's own Arrow table, the source that keeps it. Co-Authored-By: Claude Opus 5.5 --- .../operators/test_merge_join_arrow_types.py | 23 +++++++---- .../operators/test_semijoin_nested.py | 41 +++++++++++++------ 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/tests/test_core/operators/test_merge_join_arrow_types.py b/tests/test_core/operators/test_merge_join_arrow_types.py index 53721091d..4ee5dd3a2 100644 --- a/tests/test_core/operators/test_merge_join_arrow_types.py +++ b/tests/test_core/operators/test_merge_join_arrow_types.py @@ -33,7 +33,7 @@ from orcapod.contexts import get_default_context, get_default_type_converter from orcapod.hashing.arrow_hashers import StarfixArrowHasher from orcapod.core.operators import MergeJoin, PolarsFilter -from orcapod.core.sources import CSVSource, DictSource +from orcapod.core.sources import DictSource from orcapod.core.sources.arrow_table_source import ArrowTableSource from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError @@ -517,18 +517,25 @@ def test_shared_tag_with_other_difference_raises( with pytest.raises(InputValidationError, match=r"'k'"): MergeJoin().static_process(first, second) - def test_csv_and_dict_sources_on_a_string_column(self, tmp_path: Path) -> None: - """A CSV ``string`` column colliding with a dict ``large_string`` one merges.""" - csv_path = tmp_path / "labels.csv" - csv_path.write_text("id,label\n1,a\n2,b\n") - csv_source = CSVSource(str(csv_path), tag_columns=["id"]) + def test_user_string_and_dict_sources_on_a_string_column(self) -> None: + """A user's ``string`` column colliding with a dict ``large_string`` one merges. + + A CSVSource gave ``string`` here until ITL-1122 (a CSV file keeps no large / + non-large distinction, so its table is now large); a user's own Arrow table + is the source that keeps ``string``. + """ + user_source = ArrowTableSource( + _table({"id": (pa.int64(), [1, 2]), "label": (pa.string(), ["a", "b"])}), + tag_columns=["id"], + source_id="user", + ) dict_source = DictSource( [{"id": 1, "label": "x"}, {"id": 2, "label": "y"}], tag_columns=["id"] ) - assert csv_source.as_table().schema.field("label").type == pa.string() + assert user_source.as_table().schema.field("label").type == pa.string() assert dict_source.as_table().schema.field("label").type == pa.large_string() - table = MergeJoin()(csv_source, dict_source).as_table() + table = MergeJoin()(user_source, dict_source).as_table() assert table.schema.field("label").type == pa.large_list(pa.large_string()) labels = table.sort_by("id").column("label").to_pylist() assert labels == [["a", "x"], ["b", "y"]] diff --git a/tests/test_core/operators/test_semijoin_nested.py b/tests/test_core/operators/test_semijoin_nested.py index 7d8d3c6d8..e3e778aed 100644 --- a/tests/test_core/operators/test_semijoin_nested.py +++ b/tests/test_core/operators/test_semijoin_nested.py @@ -5,7 +5,7 @@ field"), so a list, large_list, struct or struct-backed logical column on either input failed. ITL-829 (first case): Acero also rejected a key that is ``string`` on one side and ``large_string`` on the other ("Incompatible data types for -corresponding join field keys"), e.g. a ``CSVSource`` against a ``DictSource``. +corresponding join field keys"), e.g. a user's ``string`` table against a ``DictSource``. The result is the left input's rows that match a right row, in left order, with the left input's Arrow schema exactly (types, top-level and nested nullability, @@ -26,7 +26,7 @@ from orcapod.channels import Channel from orcapod.core.operators import SemiJoin -from orcapod.core.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.core.sources import ArrowTableSource, DictSource from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import StreamProtocol @@ -352,24 +352,41 @@ def test_null_scalar_key_never_matches() -> None: # ── key types ─────────────────────────────────────────────────────────────── -def test_string_vs_large_string_key_csv_vs_dict(tmp_path: Path) -> None: - """ITL-829 (first case): a CSVSource's ``string`` key against a DictSource's - ``large_string`` key, in both directions.""" - csv_path = tmp_path / "a.csv" - _ = csv_path.write_text("id,x\nc,3\na,1\nb,2\n") - csv = CSVSource(str(csv_path), tag_columns=["id"]) +def test_string_vs_large_string_key_user_vs_dict() -> None: + """ITL-829 (first case): a user's ``string`` key against a DictSource's + ``large_string`` key, in both directions. + + A CSVSource gave the ``string`` key until ITL-1122 (a CSV file keeps no large / + non-large distinction, so its table is now large); a user's own Arrow table is + the source that keeps ``string``. + """ + user_source = _source( + { + "id": pa.array(["c", "a", "b"], pa.string()), + "x": pa.array([3, 1, 2], pa.int64()), + }, + tags=("id",), + source_id="user", + ) dict_source = DictSource( data=[{"id": "a", "y": 1}, {"id": "c", "y": 3}], tag_columns=["id"] ) - assert csv.as_table().schema.field("id").type == pa.string() + assert user_source.as_table().schema.field("id").type == pa.string() assert dict_source.as_table().schema.field("id").type == pa.large_string() assert_semijoin_result( - csv, dict_source, expected_rows(csv, lambda row: row["id"] in ("a", "c")) + user_source, + dict_source, + expected_rows(user_source, lambda row: row["id"] in ("a", "c")), + ) + assert_semijoin_result( + dict_source, user_source, expected_rows(dict_source, lambda _: True) ) - assert_semijoin_result(dict_source, csv, expected_rows(dict_source, lambda _: True)) # The stream convenience method takes the same path. - assert csv.semi_join(dict_source).as_table().column("id").to_pylist() == ["c", "a"] + assert user_source.semi_join(dict_source).as_table().column("id").to_pylist() == [ + "c", + "a", + ] def mismatched_key_sources() -> tuple[ArrowTableSource, ArrowTableSource]: From 7a0c9a0e5aae77a0d19a25fe973c528fd512f21b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:06:40 -0800 Subject: [PATCH 4/9] docs(arrow-types): state who normalizes to large, with the developer's two exceptions (ITL-1122) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 +++++++++++++ CLAUDE.md | 13 +++++++++++++ changelog.d/ITL-1122.md | 3 +++ design_issues/ITL-1122-LN1.md | 20 ++++++++++++++++++++ docs/concepts/sources.md | 20 ++++++++++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 changelog.d/ITL-1122.md create mode 100644 design_issues/ITL-1122-LN1.md diff --git a/.zed/rules b/.zed/rules index af3ce036a..fdd98bfc5 100644 --- a/.zed/rules +++ b/.zed/rules @@ -708,6 +708,19 @@ side when its inputs' types disagree; a result's type never depends on data valu order or the path the data took; and inferring a type from Python values is a last resort for when no type is known. +Who normalizes (developer, November 14, 2026, ITL-1122): every Arrow table Orcapod creates has +its types in their large variants. That covers a source's table built from Python values, a data +frame, a file or a store, the rows a CachedSource reads back from its cache, and every operator's +and function pod's output. Two exceptions keep the types as given, with no normalization between +large and non-large: (1) an Arrow table or Arrow schema the user made: ArrowTableSource, and an +Arrow schema= / data_schema= given to CSVSource, DictSource or DataFrameSource, kept at +that source; (2) a store that keeps the large / non-large distinction recoverably (written both +ways, each reads back as written). No shipped source store does (measured in ITL-1122): a CSV +file, Delta Lake, SQLite, PostgreSQL and SpiralDB give one type for both, so their sources are +large. DataFrameSource and PollingSource convert through Polars, which has only large layouts, +so an Arrow table handed to them comes out large; ArrowTableSource is the source that keeps a +user's Arrow types. + Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large for consistency). Infer an Arrow type from Python values only when no Arrow type is known, and diff --git a/CLAUDE.md b/CLAUDE.md index d66e4d9c6..d0c85459b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -750,6 +750,19 @@ side when its inputs' types disagree; a result's type never depends on data valu order or the path the data took; and inferring a type from Python values is a last resort for when no type is known. +Who normalizes (developer, November 14, 2026, ITL-1122): every Arrow table Orcapod creates has +its types in their large variants. That covers a source's table built from Python values, a data +frame, a file or a store, the rows a `CachedSource` reads back from its cache, and every operator's +and function pod's output. Two exceptions keep the types as given, with no normalization between +large and non-large: (1) an Arrow table or Arrow schema the user made: `ArrowTableSource`, and an +Arrow `schema=` / `data_schema=` given to `CSVSource`, `DictSource` or `DataFrameSource`, kept at +that source; (2) a store that keeps the large / non-large distinction recoverably (written both +ways, each reads back as written). No shipped source store does (measured in ITL-1122): a CSV +file, Delta Lake, SQLite, PostgreSQL and SpiralDB give one type for both, so their sources are +large. `DataFrameSource` and `PollingSource` convert through Polars, which has only large layouts, +so an Arrow table handed to them comes out large; `ArrowTableSource` is the source that keeps a +user's Arrow types. + Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large for consistency). Infer an Arrow type from Python values only when no Arrow type is known, and diff --git a/changelog.d/ITL-1122.md b/changelog.d/ITL-1122.md new file mode 100644 index 000000000..f1130272f --- /dev/null +++ b/changelog.d/ITL-1122.md @@ -0,0 +1,3 @@ +### Changed + +- `CSVSource` and `CachedSource` give the large Arrow variants of string, binary and list types (ITL-1122). A CSV file keeps no distinction between `string` and `large_string`, so `CSVSource` gave `string` from a file only because of how PyArrow reads it; it now gives `large_string`, as `DictSource`, `DataFrameSource`, `DeltaTableSource` and the SQL and SpiralDB sources do. An Arrow `schema=` given to `CSVSource` is still applied as given. `CachedSource` gave the types its cache store happened to keep (`string` from an in-memory cache, `large_string` from a Delta cache, and `large_string` with no rows); it now gives large types from every store, with rows or without. Large and non-large variants hash identically: no content hash, pipeline hash, row hash or record id moves, and no stored table changes (a cache keeps what it stored, and a Delta table written with either variant accepts rows of the other). `ArrowTableSource` and an Arrow schema the user gives keep their types. diff --git a/design_issues/ITL-1122-LN1.md b/design_issues/ITL-1122-LN1.md new file mode 100644 index 000000000..f20e93ce9 --- /dev/null +++ b/design_issues/ITL-1122-LN1.md @@ -0,0 +1,20 @@ +## Cross-cutting + +### LN1 — Large normalization applied unevenly across sources and operators +**Status:** in progress +**Severity:** medium +**Issue:** ITL-1122 + +The rule (developer, November 14, 2026): every Arrow table Orcapod creates has its string, binary +and list types in their large variants; an Arrow table or schema the user made, and a store that +keeps the large / non-large distinction recoverably, keep the types as given. Measured on `main` +3a7f3850: `CSVSource` gave `string` (a CSV file keeps no distinction); `CachedSource` gave the +types its cache store kept (`string` from an in-memory cache, `large_string` from Delta and with +no rows); on a plain stream, the operators that pass rows through (select, drop, map, a filter +with nothing to filter, `Join`, `SemiJoin`, `GroupBy`'s key) kept a source's `string` while every +operator inside a job gave `large_string`, and `Join` of a `string` and a `large_string` input on +a shared tag took the type of whichever input came first in canonical order. No shipped source +store (CSV, Delta Lake, SQLite, PostgreSQL, SpiralDB) keeps the distinction. + +**Fix:** `CSVSource` (without `schema=`) and `CachedSource` normalize the table they present +(`arrow_utils.normalize_table_to_large_types(..., keep_null=True)`). diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 83d4a8319..fd32e6c22 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -48,6 +48,26 @@ Data with no rows has no columns to read, so `DictSource(data=[], ...)` raises `ValueError: Table is empty`. To build a source with no rows, pass `data_schema=` so the columns are known. +### Arrow types of a source + +Orcapod gives every Arrow table it creates the large variants of string, binary and list types +(`large_string`, `large_binary`, `large_list`, recursively). Large and non-large variants hash +identically, so this never changes a hash. Two cases keep the types you gave: + +- **An Arrow table or Arrow schema you made.** `ArrowTableSource` keeps the table's types, so a + `string` column stays `string`. An Arrow schema passed as `schema=` to `CSVSource` or + `DataFrameSource`, or as `data_schema=` to `DictSource`, is applied as given. +- **A store that keeps the distinction.** None of the stores the sources read does: a CSV + file, Delta Lake, SQLite, PostgreSQL and SpiralDB each give one type for `string` and + `large_string`. So `CSVSource` (without `schema=`), `DeltaTableSource`, the SQL sources and + `SpiralDBTableSource` give large types. + +Every other source builds its own table and gives large types: `DictSource` and `ListSource` +from Python values, `DataFrameSource` and `PollingSource` through Polars (which has only large +layouts, so an Arrow table handed to them comes out large too), `CachedSource` from the rows it +reads back from its cache, and `DerivedSource` from stored records. To keep your own Arrow +types, use `ArrowTableSource`. + ### SpiralDB tables `SpiralDBTableSource` exposes a table in a [SpiralDB](https://spiraldb.com) project as a From 6a4d6f0059542469ca7dfb21164a3e40f385357b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:34:32 -0800 Subject: [PATCH 5/9] fix(sources): a CachedSource's head before the read shows the large types of the read (ITL-1122) ITL-912 gave the unread head the wrapped source's own zero-row table; the read now presents large types, so the head takes them too, through the one helper both use. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/sources/cached_source.py | 19 ++++++++++++++----- .../test_large_normalization_sources.py | 13 +++++++++++++ .../test_zero_row_non_large_inputs.py | 3 ++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/orcapod/core/sources/cached_source.py b/src/orcapod/core/sources/cached_source.py index 690e10d67..4dfdc45ae 100644 --- a/src/orcapod/core/sources/cached_source.py +++ b/src/orcapod/core/sources/cached_source.py @@ -1,6 +1,7 @@ from __future__ import annotations +import dataclasses import logging from collections.abc import Iterator from typing import TYPE_CHECKING, Any @@ -361,11 +362,7 @@ def _build_merged_stream(self) -> ArrowTableStream: all_records = self._read_cached_records() if all_records is None: all_records = self._empty_table() - # The rows read back are a table Orcapod creates: large variants, whatever - # the cache store keeps and whether it has rows (ITL-1122). - all_records = arrow_utils.normalize_table_to_large_types( - all_records, keep_null=True - ) + all_records = self._presented(all_records) tag_keys = self._source.keys()[0] # Cached rows are this source's: give them its data context, not the default. @@ -373,6 +370,16 @@ def _build_merged_stream(self) -> ArrowTableStream: all_records, tag_columns=tag_keys, data_context=self.data_context ) + @staticmethod + def _presented(table: pa.Table) -> pa.Table: + """The table as this source presents it: in its large variants (ITL-1122). + + The rows read back from the cache are a table Orcapod creates, so they take + the large variants whatever the cache store keeps; the columns ``head()`` + shows before the cache is read take the same types. + """ + return arrow_utils.normalize_table_to_large_types(table, keep_null=True) + def _empty_table(self) -> pa.Table: """Build an empty Arrow table matching the source's output schema. @@ -464,6 +471,8 @@ def head( wrapped = head_of( self._source.head(0, columns=columns, all_info=all_info), self ) + # The wrapped source's columns in the types the read gives them (ITL-1122). + wrapped = dataclasses.replace(wrapped, table=self._presented(wrapped.table)) if wrapped.status is not ViewStatus.ROWS: return wrapped return StreamView.without_rows( diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py index 353cfbbf8..e235faab9 100644 --- a/tests/test_core/sources/test_large_normalization_sources.py +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -226,3 +226,16 @@ def test_a_cache_written_before_is_read_and_appended( table = second.as_table(columns=ALL_COLUMNS) assert table.num_rows == 2 assert _non_large(table) == {} + + @pytest.mark.parametrize("n_rows", [2, 0]) + def test_head_before_the_read_has_the_types_of_the_read( + self, store_kind: str, n_rows: int, tmp_path: Path + ) -> None: + """The columns ``head()`` shows before the cache is read are large too.""" + cached = CachedSource( + _small_source(n_rows), cache_database=_store(store_kind, tmp_path) + ) + unread = cached.head(columns=ALL_COLUMNS).table + assert _non_large(unread) == {} + read = cached.as_table(columns=ALL_COLUMNS) + assert unread.schema.equals(read.schema, check_metadata=False) diff --git a/tests/test_pipeline/test_zero_row_non_large_inputs.py b/tests/test_pipeline/test_zero_row_non_large_inputs.py index b8e5c91b9..aeabbe453 100644 --- a/tests/test_pipeline/test_zero_row_non_large_inputs.py +++ b/tests/test_pipeline/test_zero_row_non_large_inputs.py @@ -163,7 +163,8 @@ def deliver(ctx: object, x: int) -> None: def test_cached_source_over_an_empty_source_equals_its_head() -> None: - """A cache with nothing stored reads with the wrapped source's own Arrow types.""" + """A cache with nothing stored: its head before the read has the types of the read + (the large form of the wrapped source's Arrow types, ITL-1122).""" from orcapod.core.sources import CachedSource empty = ArrowTableSource( From 64b2c874434e79e6a2e1ecf46107042fc1510ce6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:56:00 -0800 Subject: [PATCH 6/9] fix(operators): every operator's output takes the large Arrow types (ITL-1122) An operator's output is a table Orcapod creates, so each column takes the large-normalized form of its input type, also when the operator passes rows through unchanged (select, map, a no-op filter, Join, SemiJoin, GroupBy's key). One helper, core/operators/large_types.with_large_types, is applied beside without_input_meta in UnaryOperator, BinaryOperator and Join, so the plain-stream path gives the types the job path already gave, and Join of a string and a large_string input no longer takes a side. The no-op PolarsFilter's zero-row special case goes: it takes the shared path. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 +- CLAUDE.md | 5 +- changelog.d/ITL-1122.md | 1 + design_issues/ITL-1122-LN1.md | 5 +- docs/concepts/operators.md | 4 + src/orcapod/core/operators/base.py | 17 +- src/orcapod/core/operators/filters.py | 13 -- src/orcapod/core/operators/join.py | 15 +- src/orcapod/core/operators/large_types.py | 41 ++++ src/orcapod/utils/arrow_utils.py | 9 +- .../operators/test_large_normalization.py | 200 ++++++++++++++++++ .../operators/test_semijoin_nested.py | 17 +- 12 files changed, 298 insertions(+), 34 deletions(-) create mode 100644 src/orcapod/core/operators/large_types.py create mode 100644 tests/test_core/operators/test_large_normalization.py diff --git a/.zed/rules b/.zed/rules index 003916c92..21173098d 100644 --- a/.zed/rules +++ b/.zed/rules @@ -769,6 +769,9 @@ data. The rules: operator that round-trips through Polars casts its columns back to the normalized input types (arrow_utils.restore_column_types, or restore_schema_nullability when nullability is restored too). + This holds for every operator, also one that passes its rows through unchanged (select, + map, a no-op filter, Join, SemiJoin), on a plain stream as inside a job (ITL-1122): one + place, core/operators/large_types.with_large_types, applied beside without_input_meta. - List columns built by Batch, GroupBy and MergeJoin are typed by arrow_utils.aggregated_list_type (directly or through build_aggregated_table): large_list, or for a logical T the logical list type from the type converter @@ -864,7 +867,7 @@ data. The rules: nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output is the left - input's schema as it is. The rule is `core/operators/key_nullability.py` + input's schema as it is (its nullability; types in their large variants, ITL-1122). The rule is `core/operators/key_nullability.py` (`reconcile_key_nullability` on tag schemas before they are combined, `key_nullable` for the Arrow field flag, set by re-wrapping arrays, never by a cast). Equal nullability takes the unchanged path; any other type difference, and every data column, behaves as before. diff --git a/CLAUDE.md b/CLAUDE.md index 3918eea1e..1fba7b6c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -811,6 +811,9 @@ data. The rules: operator that round-trips through Polars casts its columns back to the normalized input types (`arrow_utils.restore_column_types`, or `restore_schema_nullability` when nullability is restored too). + This holds for every operator, also one that passes its rows through unchanged (select, + map, a no-op filter, `Join`, `SemiJoin`), on a plain stream as inside a job (ITL-1122): one + place, `core/operators/large_types.with_large_types`, applied beside `without_input_meta`. - List columns built by `Batch`, `GroupBy` and `MergeJoin` are typed by `arrow_utils.aggregated_list_type` (directly or through `build_aggregated_table`): `large_list`, or for a logical `T` the logical list type from the type converter @@ -908,7 +911,7 @@ data. The rules: nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output is the left - input's schema as it is. The rule is `core/operators/key_nullability.py` + input's schema as it is (its nullability; types in their large variants, ITL-1122). The rule is `core/operators/key_nullability.py` (`reconcile_key_nullability` on tag schemas before they are combined, `key_nullable` for the Arrow field flag, set by re-wrapping arrays, never by a cast). Equal nullability takes the unchanged path; any other type difference, and every data column, behaves as before. diff --git a/changelog.d/ITL-1122.md b/changelog.d/ITL-1122.md index f1130272f..802175ef9 100644 --- a/changelog.d/ITL-1122.md +++ b/changelog.d/ITL-1122.md @@ -1,3 +1,4 @@ ### Changed - `CSVSource` and `CachedSource` give the large Arrow variants of string, binary and list types (ITL-1122). A CSV file keeps no distinction between `string` and `large_string`, so `CSVSource` gave `string` from a file only because of how PyArrow reads it; it now gives `large_string`, as `DictSource`, `DataFrameSource`, `DeltaTableSource` and the SQL and SpiralDB sources do. An Arrow `schema=` given to `CSVSource` is still applied as given. `CachedSource` gave the types its cache store happened to keep (`string` from an in-memory cache, `large_string` from a Delta cache, and `large_string` with no rows); it now gives large types from every store, with rows or without. Large and non-large variants hash identically: no content hash, pipeline hash, row hash or record id moves, and no stored table changes (a cache keeps what it stored, and a Delta table written with either variant accepts rows of the other). `ArrowTableSource` and an Arrow schema the user gives keep their types. +- Every operator's output has large Arrow types on a plain stream too, as it already had inside a job (ITL-1122). The operators that pass rows through unchanged (`SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, `DropDataColumns`, `SelectColumns`, `DropColumns`, `MapTags`, `MapData`, `PolarsFilter` with no predicate, `Join`, `SemiJoin`, and `GroupBy`'s key columns) kept a `string`, `binary` or `list` column of a user's Arrow table as it was; they now give `large_string`, `large_binary` and `large_list`, with rows and without. `Join` of an input with a `string` tag and an input with the same tag as `large_string` gave the type of whichever input came first in its canonical order; it now gives `large_string` whatever the order. No hash moves. diff --git a/design_issues/ITL-1122-LN1.md b/design_issues/ITL-1122-LN1.md index f20e93ce9..1a8b2b5f8 100644 --- a/design_issues/ITL-1122-LN1.md +++ b/design_issues/ITL-1122-LN1.md @@ -17,4 +17,7 @@ a shared tag took the type of whichever input came first in canonical order. No store (CSV, Delta Lake, SQLite, PostgreSQL, SpiralDB) keeps the distinction. **Fix:** `CSVSource` (without `schema=`) and `CachedSource` normalize the table they present -(`arrow_utils.normalize_table_to_large_types(..., keep_null=True)`). +(`arrow_utils.normalize_table_to_large_types(..., keep_null=True)`). Every operator's sync result goes through +`core/operators/large_types.with_large_types` (beside `without_input_meta`, in `UnaryOperator`, +`BinaryOperator` and `Join`), so an operator gives large types on a plain stream as inside a +job, and `Join` no longer takes a side. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index c6983495c..89f6606fe 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -634,6 +634,10 @@ silent widening; convert explicitly. - **Types are normalized to their large variants.** `string` becomes `large_string`, `binary` becomes `large_binary` and `list` becomes `large_list`, recursively (also inside structs). The Arrow hasher treats each pair as identical, so this never changes a hash. + Every operator does this, also one that passes its rows through unchanged (a select, a + rename, a filter with nothing to filter, a join), on a plain stream as inside a job: an + operator's output is a table orcapod creates, even over an Arrow table of your own whose + `string` columns `ArrowTableSource` keeps. - **Width, nullability and logical types are kept.** An `int8` column stays `int8`, a `float32` column stays `float32`, and a logical type such as `Path` keeps its logical type. - **List columns built by `MergeJoin`, `Batch` and `GroupBy`** are `large_list`, where `T` diff --git a/src/orcapod/core/operators/base.py b/src/orcapod/core/operators/base.py index f31276679..b9601a51a 100644 --- a/src/orcapod/core/operators/base.py +++ b/src/orcapod/core/operators/base.py @@ -5,6 +5,7 @@ from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.large_types import with_large_types from orcapod.core.operators.meta_columns import ( without_input_meta, without_meta_fields, @@ -140,8 +141,11 @@ def validate_inputs(self, *streams: StreamProtocol) -> None: def static_process(self, *streams: StreamProtocol) -> StreamProtocol: """Forward to ``unary_static_process`` with the single input stream.""" stream = streams[0] - # No operator carries its input's Data meta columns (ITL-1080). - return without_input_meta(self.unary_static_process(stream), streams) + # No operator carries its input's Data meta columns (ITL-1080), and every + # output column takes its large variant (ITL-1122). + return with_large_types( + without_input_meta(self.unary_static_process(stream), streams) + ) def output_schema( self, @@ -240,9 +244,12 @@ def is_commutative(self) -> bool: def static_process(self, *streams: StreamProtocol) -> StreamProtocol: """Forward to ``binary_static_process`` with two input streams.""" left_stream, right_stream = streams - # No operator carries its inputs' Data meta columns (ITL-1080). - return without_input_meta( - self.binary_static_process(left_stream, right_stream), streams + # No operator carries its inputs' Data meta columns (ITL-1080), and every + # output column takes its large variant (ITL-1122). + return with_large_types( + without_input_meta( + self.binary_static_process(left_stream, right_stream), streams + ) ) def output_schema( diff --git a/src/orcapod/core/operators/filters.py b/src/orcapod/core/operators/filters.py index 5234c4019..55678da84 100644 --- a/src/orcapod/core/operators/filters.py +++ b/src/orcapod/core/operators/filters.py @@ -46,19 +46,6 @@ def __init__( self.constraints = constraints if constraints is not None else {} super().__init__(**kwargs) - def zero_row_result(self, *inputs: StreamProtocol) -> StreamProtocol: - """This filter's result over a zero-row input, for a table with no rows (ITL-912). - - A filter with no predicate and no constraint passes its input through - unaltered (``unary_static_process``), so its result is the input with the types - it has. Any other filter gives its input's columns in the large-normalized - form a run gives them, without evaluating the predicate. - """ - if len(self.predicates) == 0 and len(self.constraints) == 0: - (only,) = inputs - return only - return super().zero_row_result(*inputs) - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: if len(self.predicates) == 0 and len(self.constraints) == 0: logger.info( diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index 727c6ca55..1a83cc1b1 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -3,6 +3,7 @@ from collections.abc import Callable, Collection, Mapping, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.large_types import with_large_types from orcapod.core.operators.meta_columns import ( WithoutMetaWriter, without_input_meta, @@ -197,7 +198,8 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: shared tag key has different Arrow types in two inputs. """ if len(streams) == 1: - return without_input_meta(streams[0], streams) + # Every output column takes its large variant (ITL-1122). + return with_large_types(without_input_meta(streams[0], streams)) # Canonically order streams by pipeline_hash for deterministic # system tag column names regardless of input order (Join is commutative) @@ -294,9 +296,14 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: + data_order + [col for col in table.column_names if col not in listed] ) - return ArrowTableStream( - result_table, - tag_columns=tuple(tag_order), + # Every output column takes its large variant (ITL-1122): a column of an + # input with a non-large type, and a shared key whose inputs differ only in + # layout, never keep the type of whichever input came first. + return with_large_types( + ArrowTableStream( + result_table, + tag_columns=tuple(tag_order), + ) ) @staticmethod diff --git a/src/orcapod/core/operators/large_types.py b/src/orcapod/core/operators/large_types.py new file mode 100644 index 000000000..a5563337a --- /dev/null +++ b/src/orcapod/core/operators/large_types.py @@ -0,0 +1,41 @@ +"""Every operator's output takes the large Arrow types (ITL-1122). + +An operator's output is a table Orcapod creates, so each of its columns takes the +large-normalized form of its input type (``arrow_utils.normalize_to_large_types`` with +``keep_null=True``: ``string`` to ``large_string``, ``binary`` to ``large_binary``, +``list`` to ``large_list``, recursively). That holds also when the operator passes its +rows through unchanged (a select, a rename, a filter with nothing to filter, a join), so +an operator gives the same types on a plain stream as inside a job, whichever input a +column came from. Only layouts change: width, nullability, field metadata and logical +(extension) types are kept, and large and non-large variants hash identically. +""" + +from __future__ import annotations + +from orcapod.core.streams.arrow_table_stream import ArrowTableStream +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.utils import arrow_utils + + +def with_large_types(stream: StreamProtocol) -> StreamProtocol: + """Return an operator's sync result with every column in its large variant. + + Args: + stream: What the operator built (or the input it returned unaltered). + + Returns: + ``stream`` itself when every column is already large; otherwise the same rows + (tags, system tags, data, source info, data context) with the non-large + columns re-laid out as their large variants. + """ + table = stream.as_table( + columns={"source": True, "system_tags": True, "context": True} + ) + normalized = arrow_utils.normalize_table_to_large_types(table, keep_null=True) + if normalized is table: + return stream + return ArrowTableStream( + normalized, + tag_columns=stream.keys()[0], + data_context=stream.data_context_key, + ) diff --git a/src/orcapod/utils/arrow_utils.py b/src/orcapod/utils/arrow_utils.py index 912a53c71..67da68714 100644 --- a/src/orcapod/utils/arrow_utils.py +++ b/src/orcapod/utils/arrow_utils.py @@ -307,12 +307,13 @@ def normalize_table_to_large_types( name: large_string age: int64 """ - normalized_schema = normalize_schema_to_large_types( - table.schema, keep_null=keep_null - ) + # ``table.schema`` makes a new Schema object on every access: read it once, + # so the identity check below can find an unchanged schema. + schema = table.schema + normalized_schema = normalize_schema_to_large_types(schema, keep_null=keep_null) # If schema didn't change, return original table - if normalized_schema is table.schema: + if normalized_schema is schema: return table # Use cast() for safety - should be zero-copy for large variant conversions diff --git a/tests/test_core/operators/test_large_normalization.py b/tests/test_core/operators/test_large_normalization.py new file mode 100644 index 000000000..2b7d6c0eb --- /dev/null +++ b/tests/test_core/operators/test_large_normalization.py @@ -0,0 +1,200 @@ +"""Every operator's output takes the large Arrow types, on every path (ITL-1122). + +The rule (developer, November 14, 2026): every Arrow table Orcapod creates has its string, +binary and list types in their large variants; only a user's own Arrow table, and a store +that keeps the distinction recoverably, keep a non-large type. An operator's output is a +table Orcapod creates, also when the operator passes its rows through unchanged (a select, +a rename, a filter with nothing to filter): its columns take the large-normalized form of +their input types, on a plain stream as inside a job, sync and async, with rows and without. +Large and non-large variants hash identically, so no hash moves. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import polars as pl +import pyarrow as pa +import pytest + +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.operators import ( + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + GroupBy, + Index, + Join, + MapData, + MapTags, + MergeJoin, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, +) +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.utils import arrow_utils + +ALL_COLUMNS = {"source": True, "system_tags": True, "context": True} +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +NESTED = pa.struct([pa.field("x", pa.list_(pa.string())), pa.field("y", pa.string())]) +SMALL_SCHEMA = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("k", pa.int64(), nullable=False), + pa.field("s", pa.string(), nullable=False), + pa.field("b", pa.binary(), nullable=False), + pa.field("l", pa.list_(pa.string()), nullable=False), + pa.field("n", NESTED, nullable=False), + ] +) +ROWS = [ + {"t": "a", "k": 0, "s": "x", "b": b"1", "l": ["p"], "n": {"x": ["q"], "y": "r"}}, + {"t": "b", "k": 1, "s": "y", "b": b"2", "l": ["p", "q"], "n": {"x": [], "y": "s"}}, +] + + +def _non_large(table: pa.Table) -> dict[str, pa.DataType]: + """The columns of ``table`` whose type is not its large-normalized form.""" + return { + field.name: field.type + for field in table.schema + if arrow_utils.normalize_to_large_types(field.type, keep_null=True) + != field.type + } + + +def _source(n_rows: int, *, large: bool = False) -> ArrowTableSource: + """A user's Arrow table: non-large string, binary, list and nested columns. + + With ``large``, the same rows with every type in its large variant. + """ + schema = ( + arrow_utils.normalize_schema_to_large_types(SMALL_SCHEMA, keep_null=True) + if large + else SMALL_SCHEMA + ) + return ArrowTableSource( + pa.Table.from_pylist(ROWS[:n_rows], schema=schema), + tag_columns=["t", "k"], + source_id="main", + ) + + +def _other(n_rows: int, data_column: str) -> ArrowTableSource: + """A second user's table: tag ``t`` and one data column, both ``string``.""" + table = pa.table( + { + "t": pa.array(["a", "b"][:n_rows], pa.string()), + data_column: pa.array(["u", "v"][:n_rows], pa.string()), + } + ) + table = table.cast( + pa.schema([pa.field(field.name, field.type, False) for field in table.schema]) + ) + return ArrowTableSource(table, tag_columns=["t"], source_id=f"other_{data_column}") + + +Apply = Callable[[StreamProtocol, int, str | None], StreamProtocol] + +OPERATORS: dict[str, Apply] = { + "SelectTagColumns": lambda s, n, label: SelectTagColumns(["t"])(s, label=label), + "DropTagColumns": lambda s, n, label: DropTagColumns(["k"])(s, label=label), + "SelectDataColumns": lambda s, n, label: SelectDataColumns(["s", "l"])( + s, label=label + ), + "DropDataColumns": lambda s, n, label: DropDataColumns(["b"])(s, label=label), + "SelectColumns": lambda s, n, label: SelectColumns(["t", "k", "s", "n"])( + s, label=label + ), + "DropColumns": lambda s, n, label: DropColumns(["b"])(s, label=label), + "MapTags": lambda s, n, label: MapTags({"t": "t2"})(s, label=label), + "MapData": lambda s, n, label: MapData({"s": "s2"})(s, label=label), + "PolarsFilter(predicate)": lambda s, n, label: PolarsFilter( + predicates=[pl.col("k") >= 0] + )(s, label=label), + "PolarsFilter()": lambda s, n, label: PolarsFilter()(s, label=label), + "DropNulls()": lambda s, n, label: DropNulls()(s, label=label), + "DropNulls([s])": lambda s, n, label: DropNulls(["s"])(s, label=label), + "Join": lambda s, n, label: Join()(s, _other(n, "z"), label=label), + "MergeJoin": lambda s, n, label: MergeJoin()(s, _other(n, "s"), label=label), + "SemiJoin": lambda s, n, label: SemiJoin()(s, _other(n, "z"), label=label), + "Batch": lambda s, n, label: Batch(batch_size=2)(s, label=label), + "GroupBy(t)": lambda s, n, label: GroupBy(by=["t"])(s, label=label), + "GroupBy(k)": lambda s, n, label: GroupBy(by=["k"])(s, label=label), + "Index": lambda s, n, label: Index("l", 0)(s, label=label), +} + + +@pytest.mark.parametrize("name", list(OPERATORS)) +class TestEveryOperator: + @pytest.mark.parametrize("n_rows", [2, 0]) + def test_plain_stream_output_is_large(self, name: str, n_rows: int) -> None: + out = OPERATORS[name](_source(n_rows), n_rows, None) + table = out.as_table(columns=ALL_COLUMNS) + assert _non_large(table) == {} + + def test_zero_row_schema_equals_populated(self, name: str) -> None: + populated = OPERATORS[name](_source(2), 2, None).as_table(columns=ALL_COLUMNS) + empty = OPERATORS[name](_source(0), 0, None).as_table(columns=ALL_COLUMNS) + # The user tag and data columns: which system columns a zero-row result + # carries is ITL-912's subject, not this rule's. + user_columns = [name for name in populated.column_names if name[0] != "_"] + assert ( + empty.select(user_columns).schema == populated.select(user_columns).schema + ) + + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_job_output_is_large_and_equals_plain( + self, name: str, orchestrator: str + ) -> None: + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + OPERATORS[name](_source(2), 2, "op") + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + job_table = job.nodes["op"].as_table() + plain_table = OPERATORS[name](_source(2), 2, None).as_table() + assert _non_large(job_table) == {} + assert job_table.schema == plain_table.schema + + def test_row_hashes_equal_the_large_input(self, name: str) -> None: + """Large and non-large variants hash identically: no row hash moves.""" + small = OPERATORS[name](_source(2), 2, None) + large = OPERATORS[name](_source(2, large=True), 2, None) + assert small.content_hash() == large.content_hash() + assert small.pipeline_hash() == large.pipeline_hash() + + def row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: + return sorted( + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in stream.iter_data() + ) + + assert row_hashes(small) == row_hashes(large) + + +class TestJoinNeverPicksASide: + """A shared tag that is large in one input and ``string`` in the other is large.""" + + @pytest.mark.parametrize("n_rows", [2, 0]) + def test_either_argument_order_gives_large(self, n_rows: int) -> None: + large = _source(n_rows, large=True) + small = _other(n_rows, "z") + forward = Join()(large, small).as_table(columns=ALL_COLUMNS) + backward = Join()(small, large).as_table(columns=ALL_COLUMNS) + assert forward.schema.field("t").type == pa.large_string() + assert backward.schema.field("t").type == pa.large_string() + assert _non_large(forward) == {} + assert _non_large(backward) == {} diff --git a/tests/test_core/operators/test_semijoin_nested.py b/tests/test_core/operators/test_semijoin_nested.py index e3e778aed..e9ca0e152 100644 --- a/tests/test_core/operators/test_semijoin_nested.py +++ b/tests/test_core/operators/test_semijoin_nested.py @@ -8,8 +8,9 @@ corresponding join field keys"), e.g. a user's ``string`` table against a ``DictSource``. The result is the left input's rows that match a right row, in left order, with -the left input's Arrow schema exactly (types, top-level and nested nullability, -logical types). The sync and async paths yield the same rows. +the left input's Arrow schema in its large variants (ITL-1122: ``string`` is +``large_string``, ``list`` is ``large_list``; top-level and nested nullability and +logical types exactly as they were). The sync and async paths yield the same rows. """ from __future__ import annotations @@ -30,6 +31,7 @@ from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.utils import arrow_utils from tests.test_core.operators.test_async_sync_equivalence import ( STRUCT, Column, @@ -146,7 +148,9 @@ def expected_rows( def assert_semijoin_result( left: StreamProtocol, right: StreamProtocol, expected: pa.Table ) -> None: - """The sync result equals ``expected`` exactly, and async yields the same rows.""" + """The sync result equals ``expected`` in its large variants (ITL-1122), and async + yields the same rows.""" + expected = arrow_utils.normalize_table_to_large_types(expected, keep_null=True) result = SemiJoin()(left, right) table = result.as_table(columns=ALL_COLUMNS) assert table.schema == expected.schema @@ -258,8 +262,11 @@ def test_nested_non_nullable_fields_keep_their_flag() -> None: nullable=("s",), ) result = SemiJoin()(left, scalar_right()).as_table(columns=ALL_COLUMNS) - assert result.schema.field("l").type == items - assert result.schema.field("s").type == pa.string() + # The large variants (ITL-1122); the item keeps its non-null flag. + assert result.schema.field("l").type == pa.large_list( + pa.field("item", pa.int8(), nullable=False) + ) + assert result.schema.field("s").type == pa.large_string() assert result.schema.field("st").type == strict_struct assert result.column("k").to_pylist() == [2] From 51835d3ff667870ccf5c5dca20c24e6ca26d52db Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:15:23 -0800 Subject: [PATCH 7/9] fix(streams)!: every table Orcapod presents has large Arrow types, in one place (ITL-1122) Developer, November 14, 2026: every column of every table Orcapod presents or stores is the large variant, with no exception for a user's Arrow table or an explicit Arrow schema. ArrowTableStream.__init__ normalizes the table every source and every operator result becomes a stream from, so the per-source (CSVSource, CachedSource) and per-operator normalizations go. Tests sweep every source kind x operator x plain / job sync / job async x populated / zero-row with a recursive type walk, and pin that string and large_string inputs give equal row, content and pipeline hashes and the same default source id. Tests that built a non-large stream on purpose now compare the layouts with the Arrow hasher on the tables. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 27 +- CLAUDE.md | 27 +- changelog.d/ITL-1122.md | 7 +- design_issues/ITL-1122-LN1.md | 21 +- docs/concepts/operators.md | 6 +- docs/concepts/sources.md | 24 +- src/orcapod/core/operators/base.py | 17 +- src/orcapod/core/operators/join.py | 15 +- src/orcapod/core/operators/large_types.py | 41 -- src/orcapod/core/sources/cached_source.py | 15 - src/orcapod/core/sources/csv_source.py | 7 - .../core/streams/arrow_table_stream.py | 10 + tests/test_core/large_types_cases.py | 281 ++++++++++++ .../operators/test_aggregated_list_types.py | 28 +- .../operators/test_large_normalization.py | 264 ++++++----- .../operators/test_merge_join_arrow_types.py | 39 +- .../operators/test_semijoin_nested.py | 15 +- .../test_large_normalization_sources.py | 431 +++++++++++------- 18 files changed, 821 insertions(+), 454 deletions(-) delete mode 100644 src/orcapod/core/operators/large_types.py create mode 100644 tests/test_core/large_types_cases.py diff --git a/.zed/rules b/.zed/rules index 21173098d..eaa6c71bc 100644 --- a/.zed/rules +++ b/.zed/rules @@ -712,18 +712,17 @@ side when its inputs' types disagree; a result's type never depends on data valu order or the path the data took; and inferring a type from Python values is a last resort for when no type is known. -Who normalizes (developer, November 14, 2026, ITL-1122): every Arrow table Orcapod creates has -its types in their large variants. That covers a source's table built from Python values, a data -frame, a file or a store, the rows a CachedSource reads back from its cache, and every operator's -and function pod's output. Two exceptions keep the types as given, with no normalization between -large and non-large: (1) an Arrow table or Arrow schema the user made: ArrowTableSource, and an -Arrow schema= / data_schema= given to CSVSource, DictSource or DataFrameSource, kept at -that source; (2) a store that keeps the large / non-large distinction recoverably (written both -ways, each reads back as written). No shipped source store does (measured in ITL-1122): a CSV -file, Delta Lake, SQLite, PostgreSQL and SpiralDB give one type for both, so their sources are -large. DataFrameSource and PollingSource convert through Polars, which has only large layouts, -so an Arrow table handed to them comes out large; ArrowTableSource is the source that keeps a -user's Arrow types. +Large variants everywhere (developer, November 14, 2026, ITL-1122): every column of every table +Orcapod presents or stores has its string, binary and list types in their large variants +(large_string, large_binary, large_list, recursively), with no exception: a user's own +Arrow table (ArrowTableSource, a raw ArrowTableStream) and an explicit Arrow schema= / +data_schema= are normalized too. The one place is ArrowTableStream.__init__ +(arrow_utils.normalize_table_to_large_types(table, keep_null=True)), where every source and +every operator result becomes a stream; never add a per-source or per-operator normalization. +Names, nullability (top level and nested), field metadata, widths and logical (extension) types +are untouched, and large and non-large variants hash identically, so no hash or default source +id moves. Not covered: what is inside a library value (numpy, pandas, polars, torch) keeps its +library's own distinction. Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large @@ -770,8 +769,8 @@ data. The rules: to the normalized input types (arrow_utils.restore_column_types, or restore_schema_nullability when nullability is restored too). This holds for every operator, also one that passes its rows through unchanged (select, - map, a no-op filter, Join, SemiJoin), on a plain stream as inside a job (ITL-1122): one - place, core/operators/large_types.with_large_types, applied beside without_input_meta. + map, a no-op filter, Join, SemiJoin), on a plain stream as inside a job: its result + is an ArrowTableStream (ITL-1122). - List columns built by Batch, GroupBy and MergeJoin are typed by arrow_utils.aggregated_list_type (directly or through build_aggregated_table): large_list, or for a logical T the logical list type from the type converter diff --git a/CLAUDE.md b/CLAUDE.md index 1fba7b6c7..bf9ec4e1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -754,18 +754,17 @@ side when its inputs' types disagree; a result's type never depends on data valu order or the path the data took; and inferring a type from Python values is a last resort for when no type is known. -Who normalizes (developer, November 14, 2026, ITL-1122): every Arrow table Orcapod creates has -its types in their large variants. That covers a source's table built from Python values, a data -frame, a file or a store, the rows a `CachedSource` reads back from its cache, and every operator's -and function pod's output. Two exceptions keep the types as given, with no normalization between -large and non-large: (1) an Arrow table or Arrow schema the user made: `ArrowTableSource`, and an -Arrow `schema=` / `data_schema=` given to `CSVSource`, `DictSource` or `DataFrameSource`, kept at -that source; (2) a store that keeps the large / non-large distinction recoverably (written both -ways, each reads back as written). No shipped source store does (measured in ITL-1122): a CSV -file, Delta Lake, SQLite, PostgreSQL and SpiralDB give one type for both, so their sources are -large. `DataFrameSource` and `PollingSource` convert through Polars, which has only large layouts, -so an Arrow table handed to them comes out large; `ArrowTableSource` is the source that keeps a -user's Arrow types. +Large variants everywhere (developer, November 14, 2026, ITL-1122): every column of every table +Orcapod presents or stores has its string, binary and list types in their large variants +(`large_string`, `large_binary`, `large_list`, recursively), with no exception: a user's own +Arrow table (`ArrowTableSource`, a raw `ArrowTableStream`) and an explicit Arrow `schema=` / +`data_schema=` are normalized too. The one place is `ArrowTableStream.__init__` +(`arrow_utils.normalize_table_to_large_types(table, keep_null=True)`), where every source and +every operator result becomes a stream; never add a per-source or per-operator normalization. +Names, nullability (top level and nested), field metadata, widths and logical (extension) types +are untouched, and large and non-large variants hash identically, so no hash or default source +id moves. Not covered: what is inside a library value (numpy, pandas, polars, torch) keeps its +library's own distinction. Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large @@ -812,8 +811,8 @@ data. The rules: to the normalized input types (`arrow_utils.restore_column_types`, or `restore_schema_nullability` when nullability is restored too). This holds for every operator, also one that passes its rows through unchanged (select, - map, a no-op filter, `Join`, `SemiJoin`), on a plain stream as inside a job (ITL-1122): one - place, `core/operators/large_types.with_large_types`, applied beside `without_input_meta`. + map, a no-op filter, `Join`, `SemiJoin`), on a plain stream as inside a job: its result + is an `ArrowTableStream` (ITL-1122). - List columns built by `Batch`, `GroupBy` and `MergeJoin` are typed by `arrow_utils.aggregated_list_type` (directly or through `build_aggregated_table`): `large_list`, or for a logical `T` the logical list type from the type converter diff --git a/changelog.d/ITL-1122.md b/changelog.d/ITL-1122.md index 802175ef9..cf1fcf8ba 100644 --- a/changelog.d/ITL-1122.md +++ b/changelog.d/ITL-1122.md @@ -1,4 +1,7 @@ +### Breaking changes and migration + +- Every table Orcapod presents or stores has the large Arrow variants of string, binary and list types (`large_string`, `large_binary`, `large_list`, recursively), with no exception (ITL-1122). A `string`, `binary` or `list` column of an Arrow table you give `ArrowTableSource` (or a raw `ArrowTableStream`), or of an Arrow schema you pass as `schema=` to `CSVSource` or `DataFrameSource` or as `data_schema=` to `DictSource`, now comes back as `large_string`, `large_binary` or `large_list`. Names, nullability, widths and logical types are kept. Code that compared an Arrow type to `pa.string()` compares it to `pa.large_string()` now. No hash moves: large and non-large variants hash identically, so row hashes, content hashes, pipeline hashes, record ids and default source ids stay as they were, and a Delta table written with either variant accepts rows of the other. + ### Changed -- `CSVSource` and `CachedSource` give the large Arrow variants of string, binary and list types (ITL-1122). A CSV file keeps no distinction between `string` and `large_string`, so `CSVSource` gave `string` from a file only because of how PyArrow reads it; it now gives `large_string`, as `DictSource`, `DataFrameSource`, `DeltaTableSource` and the SQL and SpiralDB sources do. An Arrow `schema=` given to `CSVSource` is still applied as given. `CachedSource` gave the types its cache store happened to keep (`string` from an in-memory cache, `large_string` from a Delta cache, and `large_string` with no rows); it now gives large types from every store, with rows or without. Large and non-large variants hash identically: no content hash, pipeline hash, row hash or record id moves, and no stored table changes (a cache keeps what it stored, and a Delta table written with either variant accepts rows of the other). `ArrowTableSource` and an Arrow schema the user gives keep their types. -- Every operator's output has large Arrow types on a plain stream too, as it already had inside a job (ITL-1122). The operators that pass rows through unchanged (`SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, `DropDataColumns`, `SelectColumns`, `DropColumns`, `MapTags`, `MapData`, `PolarsFilter` with no predicate, `Join`, `SemiJoin`, and `GroupBy`'s key columns) kept a `string`, `binary` or `list` column of a user's Arrow table as it was; they now give `large_string`, `large_binary` and `large_list`, with rows and without. `Join` of an input with a `string` tag and an input with the same tag as `large_string` gave the type of whichever input came first in its canonical order; it now gives `large_string` whatever the order. No hash moves. +- Large types no longer depend on the source kind or the path (ITL-1122). `CSVSource` gave `string` from a file. `CachedSource` gave the types its cache store kept: `string` from an in-memory cache, `large_string` from a Delta cache and with no rows. On a plain stream, the operators that pass rows through unchanged (`SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, `DropDataColumns`, `SelectColumns`, `DropColumns`, `MapTags`, `MapData`, `PolarsFilter` with no predicate, `Join`, `SemiJoin`, and `GroupBy`'s key columns) kept a `string` column, while the same operator inside a job gave `large_string`. `Join` of a `string` and a `large_string` input on one tag gave the type of whichever input came first in its canonical order. All of them now give large types, with rows and without, sync and async. diff --git a/design_issues/ITL-1122-LN1.md b/design_issues/ITL-1122-LN1.md index 1a8b2b5f8..0d75f9339 100644 --- a/design_issues/ITL-1122-LN1.md +++ b/design_issues/ITL-1122-LN1.md @@ -1,14 +1,15 @@ ## Cross-cutting ### LN1 — Large normalization applied unevenly across sources and operators -**Status:** in progress +**Status:** resolved **Severity:** medium **Issue:** ITL-1122 -The rule (developer, November 14, 2026): every Arrow table Orcapod creates has its string, binary -and list types in their large variants; an Arrow table or schema the user made, and a store that -keeps the large / non-large distinction recoverably, keep the types as given. Measured on `main` -3a7f3850: `CSVSource` gave `string` (a CSV file keeps no distinction); `CachedSource` gave the +The rule (developer, November 14, 2026, replacing an earlier form with exceptions for a user's +Arrow table and for a store that keeps the distinction): every column of every table Orcapod +presents or stores has its string, binary and list types in their large variants, with no +exception. Measured on `main` 3a7f3850: `ArrowTableSource` and an explicit Arrow schema kept +`string`; `CSVSource` gave `string` (a CSV file keeps no distinction); `CachedSource` gave the types its cache store kept (`string` from an in-memory cache, `large_string` from Delta and with no rows); on a plain stream, the operators that pass rows through (select, drop, map, a filter with nothing to filter, `Join`, `SemiJoin`, `GroupBy`'s key) kept a source's `string` while every @@ -16,8 +17,8 @@ operator inside a job gave `large_string`, and `Join` of a `string` and a `large a shared tag took the type of whichever input came first in canonical order. No shipped source store (CSV, Delta Lake, SQLite, PostgreSQL, SpiralDB) keeps the distinction. -**Fix:** `CSVSource` (without `schema=`) and `CachedSource` normalize the table they present -(`arrow_utils.normalize_table_to_large_types(..., keep_null=True)`). Every operator's sync result goes through -`core/operators/large_types.with_large_types` (beside `without_input_meta`, in `UnaryOperator`, -`BinaryOperator` and `Join`), so an operator gives large types on a plain stream as inside a -job, and `Join` no longer takes a side. +**Fix:** one place, `ArrowTableStream.__init__`, normalizes the table every source and every +operator result becomes a stream from (`arrow_utils.normalize_table_to_large_types(..., +keep_null=True)`). Swept by `tests/test_core/operators/test_large_normalization.py` and +`tests/test_core/sources/test_large_normalization_sources.py` (every source kind x operator x +plain / job sync / job async x populated / zero-row). diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 89f6606fe..4a16b1ed2 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -635,9 +635,9 @@ silent widening; convert explicitly. `binary` becomes `large_binary` and `list` becomes `large_list`, recursively (also inside structs). The Arrow hasher treats each pair as identical, so this never changes a hash. Every operator does this, also one that passes its rows through unchanged (a select, a - rename, a filter with nothing to filter, a join), on a plain stream as inside a job: an - operator's output is a table orcapod creates, even over an Arrow table of your own whose - `string` columns `ArrowTableSource` keeps. + rename, a filter with nothing to filter, a join), on a plain stream as inside a job. Every + source gives large types too, so an operator's inputs already have them (see + [Arrow types of a source](sources.md#arrow-types-of-a-source)). - **Width, nullability and logical types are kept.** An `int8` column stays `int8`, a `float32` column stays `float32`, and a logical type such as `Path` keeps its logical type. - **List columns built by `MergeJoin`, `Batch` and `GroupBy`** are `large_list`, where `T` diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index fd32e6c22..97776af25 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -50,23 +50,13 @@ columns are known. ### Arrow types of a source -Orcapod gives every Arrow table it creates the large variants of string, binary and list types -(`large_string`, `large_binary`, `large_list`, recursively). Large and non-large variants hash -identically, so this never changes a hash. Two cases keep the types you gave: - -- **An Arrow table or Arrow schema you made.** `ArrowTableSource` keeps the table's types, so a - `string` column stays `string`. An Arrow schema passed as `schema=` to `CSVSource` or - `DataFrameSource`, or as `data_schema=` to `DictSource`, is applied as given. -- **A store that keeps the distinction.** None of the stores the sources read does: a CSV - file, Delta Lake, SQLite, PostgreSQL and SpiralDB each give one type for `string` and - `large_string`. So `CSVSource` (without `schema=`), `DeltaTableSource`, the SQL sources and - `SpiralDBTableSource` give large types. - -Every other source builds its own table and gives large types: `DictSource` and `ListSource` -from Python values, `DataFrameSource` and `PollingSource` through Polars (which has only large -layouts, so an Arrow table handed to them comes out large too), `CachedSource` from the rows it -reads back from its cache, and `DerivedSource` from stored records. To keep your own Arrow -types, use `ArrowTableSource`. +Every source gives the large variants of string, binary and list types (`large_string`, +`large_binary`, `large_list`, recursively), whatever its input: an Arrow table of your own +(`ArrowTableSource`), an Arrow schema you pass (`schema=` or `data_schema=`), Python values, +a data frame, a CSV file or a database table. A `string` column you give comes back as +`large_string`. Only the layout changes: names, nullability, widths (`int8` stays `int8`) and +logical types are kept, and the two variants hash identically, so no hash depends on which +one you gave. ### SpiralDB tables diff --git a/src/orcapod/core/operators/base.py b/src/orcapod/core/operators/base.py index b9601a51a..f31276679 100644 --- a/src/orcapod/core/operators/base.py +++ b/src/orcapod/core/operators/base.py @@ -5,7 +5,6 @@ from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any -from orcapod.core.operators.large_types import with_large_types from orcapod.core.operators.meta_columns import ( without_input_meta, without_meta_fields, @@ -141,11 +140,8 @@ def validate_inputs(self, *streams: StreamProtocol) -> None: def static_process(self, *streams: StreamProtocol) -> StreamProtocol: """Forward to ``unary_static_process`` with the single input stream.""" stream = streams[0] - # No operator carries its input's Data meta columns (ITL-1080), and every - # output column takes its large variant (ITL-1122). - return with_large_types( - without_input_meta(self.unary_static_process(stream), streams) - ) + # No operator carries its input's Data meta columns (ITL-1080). + return without_input_meta(self.unary_static_process(stream), streams) def output_schema( self, @@ -244,12 +240,9 @@ def is_commutative(self) -> bool: def static_process(self, *streams: StreamProtocol) -> StreamProtocol: """Forward to ``binary_static_process`` with two input streams.""" left_stream, right_stream = streams - # No operator carries its inputs' Data meta columns (ITL-1080), and every - # output column takes its large variant (ITL-1122). - return with_large_types( - without_input_meta( - self.binary_static_process(left_stream, right_stream), streams - ) + # No operator carries its inputs' Data meta columns (ITL-1080). + return without_input_meta( + self.binary_static_process(left_stream, right_stream), streams ) def output_schema( diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index 1a83cc1b1..727c6ca55 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -3,7 +3,6 @@ from collections.abc import Callable, Collection, Mapping, Sequence from typing import TYPE_CHECKING, Any -from orcapod.core.operators.large_types import with_large_types from orcapod.core.operators.meta_columns import ( WithoutMetaWriter, without_input_meta, @@ -198,8 +197,7 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: shared tag key has different Arrow types in two inputs. """ if len(streams) == 1: - # Every output column takes its large variant (ITL-1122). - return with_large_types(without_input_meta(streams[0], streams)) + return without_input_meta(streams[0], streams) # Canonically order streams by pipeline_hash for deterministic # system tag column names regardless of input order (Join is commutative) @@ -296,14 +294,9 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: + data_order + [col for col in table.column_names if col not in listed] ) - # Every output column takes its large variant (ITL-1122): a column of an - # input with a non-large type, and a shared key whose inputs differ only in - # layout, never keep the type of whichever input came first. - return with_large_types( - ArrowTableStream( - result_table, - tag_columns=tuple(tag_order), - ) + return ArrowTableStream( + result_table, + tag_columns=tuple(tag_order), ) @staticmethod diff --git a/src/orcapod/core/operators/large_types.py b/src/orcapod/core/operators/large_types.py deleted file mode 100644 index a5563337a..000000000 --- a/src/orcapod/core/operators/large_types.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Every operator's output takes the large Arrow types (ITL-1122). - -An operator's output is a table Orcapod creates, so each of its columns takes the -large-normalized form of its input type (``arrow_utils.normalize_to_large_types`` with -``keep_null=True``: ``string`` to ``large_string``, ``binary`` to ``large_binary``, -``list`` to ``large_list``, recursively). That holds also when the operator passes its -rows through unchanged (a select, a rename, a filter with nothing to filter, a join), so -an operator gives the same types on a plain stream as inside a job, whichever input a -column came from. Only layouts change: width, nullability, field metadata and logical -(extension) types are kept, and large and non-large variants hash identically. -""" - -from __future__ import annotations - -from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.protocols.core_protocols import StreamProtocol -from orcapod.utils import arrow_utils - - -def with_large_types(stream: StreamProtocol) -> StreamProtocol: - """Return an operator's sync result with every column in its large variant. - - Args: - stream: What the operator built (or the input it returned unaltered). - - Returns: - ``stream`` itself when every column is already large; otherwise the same rows - (tags, system tags, data, source info, data context) with the non-large - columns re-laid out as their large variants. - """ - table = stream.as_table( - columns={"source": True, "system_tags": True, "context": True} - ) - normalized = arrow_utils.normalize_table_to_large_types(table, keep_null=True) - if normalized is table: - return stream - return ArrowTableStream( - normalized, - tag_columns=stream.keys()[0], - data_context=stream.data_context_key, - ) diff --git a/src/orcapod/core/sources/cached_source.py b/src/orcapod/core/sources/cached_source.py index 4dfdc45ae..f1453e51e 100644 --- a/src/orcapod/core/sources/cached_source.py +++ b/src/orcapod/core/sources/cached_source.py @@ -1,7 +1,6 @@ from __future__ import annotations -import dataclasses import logging from collections.abc import Iterator from typing import TYPE_CHECKING, Any @@ -32,7 +31,6 @@ from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.system_constants import SOURCE_DB_SCHEMA_VERSION, constants from orcapod.types import ColumnConfig, Schema -from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -362,7 +360,6 @@ def _build_merged_stream(self) -> ArrowTableStream: all_records = self._read_cached_records() if all_records is None: all_records = self._empty_table() - all_records = self._presented(all_records) tag_keys = self._source.keys()[0] # Cached rows are this source's: give them its data context, not the default. @@ -370,16 +367,6 @@ def _build_merged_stream(self) -> ArrowTableStream: all_records, tag_columns=tag_keys, data_context=self.data_context ) - @staticmethod - def _presented(table: pa.Table) -> pa.Table: - """The table as this source presents it: in its large variants (ITL-1122). - - The rows read back from the cache are a table Orcapod creates, so they take - the large variants whatever the cache store keeps; the columns ``head()`` - shows before the cache is read take the same types. - """ - return arrow_utils.normalize_table_to_large_types(table, keep_null=True) - def _empty_table(self) -> pa.Table: """Build an empty Arrow table matching the source's output schema. @@ -471,8 +458,6 @@ def head( wrapped = head_of( self._source.head(0, columns=columns, all_info=all_info), self ) - # The wrapped source's columns in the types the read gives them (ITL-1122). - wrapped = dataclasses.replace(wrapped, table=self._presented(wrapped.table)) if wrapped.status is not ViewStatus.ROWS: return wrapped return StreamView.without_rows( diff --git a/src/orcapod/core/sources/csv_source.py b/src/orcapod/core/sources/csv_source.py index 48991bec4..ba110effd 100644 --- a/src/orcapod/core/sources/csv_source.py +++ b/src/orcapod/core/sources/csv_source.py @@ -21,11 +21,6 @@ class CSVSource(RootSource): The file is read once at construction time using PyArrow's CSV reader, converted to an Arrow table, and enriched by ``SourceStreamBuilder`` (source-info, schema-hash, system tags). - - A CSV file keeps no distinction between large and non-large Arrow types, so - the table read from it is Orcapod's and takes the large variants (``string`` - is ``large_string``; ITL-1122). An Arrow ``schema`` given by the user is - applied as it is: the user chose those types. """ def __init__( @@ -51,8 +46,6 @@ def __init__( table = table.cast(schema) else: table = table.cast(arrow_utils.infer_schema_nullable(table)) - # The file has no large / non-large distinction: Orcapod's table is large. - table = arrow_utils.normalize_table_to_large_types(table, keep_null=True) builder = SourceStreamBuilder(self.data_context, self.orcapod_config) result = builder.build( diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 64b3da113..5084816e0 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -39,6 +39,13 @@ class ArrowTableStream(StreamBase): This is useful for ensuring that the data in the stream remains consistent and unchanging. The types of the tag and data columns are inferred from the PyArrow Table schema. + + Every column takes the large variant of its Arrow type (developer, November 14, + 2026, ITL-1122): ``string`` is ``large_string``, ``binary`` is ``large_binary``, + ``list`` is ``large_list``, recursively. This is the one place a table becomes a + stream, so every source (each builds its stream here, from a user's Arrow table + too) and every operator result presents large types. Width, nullability, field + metadata and logical types are kept; large and non-large variants hash identically. """ # The table's rows are held, not computed: a run() view says "all N rows", as @@ -60,6 +67,9 @@ def __init__( self._producer = producer self._upstreams = upstreams + # Every table Orcapod presents is in its large variants (ITL-1122). + table = arrow_utils.normalize_table_to_large_types(table, keep_null=True) + data_table, data_context_table = arrow_utils.split_by_column_groups( table, [constants.CONTEXT_KEY] ) diff --git a/tests/test_core/large_types_cases.py b/tests/test_core/large_types_cases.py new file mode 100644 index 000000000..d7a67332c --- /dev/null +++ b/tests/test_core/large_types_cases.py @@ -0,0 +1,281 @@ +"""Shared cases for the large-type tests (ITL-1122): every source kind, built from non-large data. + +The rule (developer, November 14, 2026): every column of every table Orcapod presents or +stores has its string, binary and list types in their large variants, recursively, with no +exception for a user's Arrow table or an explicit Arrow schema. These helpers build each +source kind from data that holds ``string``, ``binary``, ``list`` and a nested +``struct, string>`` wherever the kind can carry them, and find any non-large +type in a table by walking its Arrow types. +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import deltalake +import polars as pl +import pyarrow as pa +import pyarrow.csv as pa_csv + +from orcapod.core.sources import ( + ArrowTableSource, + CachedSource, + CSVSource, + DataFrameSource, + DeltaTableSource, + DictSource, + SQLiteTableSource, +) +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.protocols.core_protocols import StreamProtocol + +ALL_COLUMNS = {"source": True, "system_tags": True, "context": True} + +NESTED = pa.struct([pa.field("x", pa.list_(pa.string())), pa.field("y", pa.string())]) +FULL_SCHEMA = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("k", pa.int64(), nullable=False), + pa.field("s", pa.string(), nullable=False), + pa.field("u", pa.string(), nullable=False), + pa.field("b", pa.binary(), nullable=False), + pa.field("l", pa.list_(pa.string()), nullable=False), + pa.field("n", NESTED, nullable=False), + ] +) +ROWS: list[dict[str, Any]] = [ + { + "t": "a", + "k": 0, + "s": "x", + "u": "p", + "b": b"1", + "l": ["p"], + "n": {"x": ["q"], "y": "r"}, + }, + { + "t": "b", + "k": 1, + "s": "y", + "u": "q", + "b": b"2", + "l": ["p", "q"], + "n": {"x": [], "y": "s"}, + }, +] +TAGS = ["t", "k"] + + +def non_large_paths(arrow_type: pa.DataType, path: str) -> list[str]: + """Name every place in ``arrow_type`` that holds a non-large string, binary or list. + + Walks list items, struct children, map keys and items, dictionary values and + extension storage, so a ``string`` nested at any depth is found. Written without + ``normalize_to_large_types``, so it checks that function's work rather than repeat it. + + Args: + arrow_type: The type to walk. + path: The name of the place ``arrow_type`` is at, for the result. + + Returns: + ``": "`` for each non-large place, outermost first. + """ + found: list[str] = [] + if pa.types.is_string(arrow_type) or pa.types.is_binary(arrow_type): + found.append(f"{path}: {arrow_type}") + elif pa.types.is_list(arrow_type): + found.append(f"{path}: {arrow_type}") + found += non_large_paths(arrow_type.value_type, f"{path}[]") + elif pa.types.is_large_list(arrow_type) or pa.types.is_fixed_size_list(arrow_type): + found += non_large_paths(arrow_type.value_type, f"{path}[]") + elif pa.types.is_struct(arrow_type): + for field in arrow_type: + found += non_large_paths(field.type, f"{path}.{field.name}") + elif pa.types.is_map(arrow_type): + found += non_large_paths(arrow_type.key_type, f"{path}{{key}}") + found += non_large_paths(arrow_type.item_type, f"{path}{{item}}") + elif pa.types.is_dictionary(arrow_type): + found += non_large_paths(arrow_type.value_type, f"{path}{{value}}") + elif isinstance(arrow_type, pa.ExtensionType): + found += non_large_paths(arrow_type.storage_type, f"{path}") + return found + + +def non_large_columns(table: pa.Table) -> list[str]: + """Every non-large place in every column of ``table`` (``non_large_paths``).""" + found: list[str] = [] + for field in table.schema: + found += non_large_paths(field.type, field.name) + return found + + +def _rows(n_rows: int, columns: list[str]) -> list[dict[str, Any]]: + return [{name: row[name] for name in columns} for row in ROWS[:n_rows]] + + +def _table(n_rows: int, columns: list[str] | None = None) -> pa.Table: + """The rows as a user's Arrow table in the non-large types of ``FULL_SCHEMA``.""" + names = columns or FULL_SCHEMA.names + schema = pa.schema([FULL_SCHEMA.field(name) for name in names]) + return pa.Table.from_pylist(_rows(n_rows, names), schema=schema) + + +SourceFactory = Callable[[int, Path], StreamProtocol] + + +def _arrow_table_source(n_rows: int, tmp: Path) -> StreamProtocol: + return ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="arrow") + + +def _arrow_table_stream(n_rows: int, tmp: Path) -> StreamProtocol: + return ArrowTableStream(_table(n_rows), tag_columns=TAGS) + + +def _dict_source_python(n_rows: int, tmp: Path) -> StreamProtocol: + columns = ["t", "k", "s", "u", "b", "l"] + if n_rows == 0: + return DictSource( + [], + tag_columns=TAGS, + data_schema={ + "t": str, + "k": int, + "s": str, + "u": str, + "b": bytes, + "l": list[str], + }, + source_id="dict", + ) + return DictSource(_rows(n_rows, columns), tag_columns=TAGS, source_id="dict") + + +def _dict_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: + return DictSource( + _rows(n_rows, FULL_SCHEMA.names), + tag_columns=TAGS, + data_schema=FULL_SCHEMA, + source_id="dict_schema", + ) + + +def _csv_path(n_rows: int, tmp: Path) -> Path: + path = tmp / f"rows{n_rows}.csv" + pa_csv.write_csv(_table(n_rows, ["t", "k", "s", "u"]), path) + return path + + +def _csv_source(n_rows: int, tmp: Path) -> StreamProtocol: + return CSVSource(str(_csv_path(n_rows, tmp)), tag_columns=TAGS, source_id="csv") + + +def _csv_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: + return CSVSource( + str(_csv_path(n_rows, tmp)), + tag_columns=TAGS, + source_id="csv_schema", + schema=pa.schema([FULL_SCHEMA.field(name) for name in ["t", "k", "s", "u"]]), + ) + + +def _data_frame_source_polars(n_rows: int, tmp: Path) -> StreamProtocol: + return DataFrameSource( + pl.from_arrow(_table(n_rows)), tag_columns=TAGS, source_id="polars" + ) + + +def _data_frame_source_pandas(n_rows: int, tmp: Path) -> StreamProtocol: + # No bytes column: an empty pandas object column has no values to tell bytes + # from str, so a zero-row frame could not give the populated frame's type. + return DataFrameSource( + _table(n_rows, ["t", "k", "s", "u"]).to_pandas(), + tag_columns=TAGS, + source_id="pandas", + ) + + +def _data_frame_source_arrow_table(n_rows: int, tmp: Path) -> StreamProtocol: + return DataFrameSource(_table(n_rows), tag_columns=TAGS, source_id="df_arrow") + + +def _data_frame_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: + return DataFrameSource( + pl.from_arrow(_table(n_rows)), + tag_columns=TAGS, + source_id="df_schema", + schema=FULL_SCHEMA, + ) + + +def _delta_table_source(n_rows: int, tmp: Path) -> StreamProtocol: + path = tmp / f"delta{n_rows}" + if not path.exists(): + deltalake.write_deltalake(str(path), _table(n_rows)) + return DeltaTableSource(path, tag_columns=TAGS, source_id="delta") + + +def _sqlite_table_source(n_rows: int, tmp: Path) -> StreamProtocol: + path = tmp / f"rows{n_rows}.sqlite" + if not path.exists(): + connection = sqlite3.connect(path) + _ = connection.execute( + "CREATE TABLE r (t TEXT NOT NULL, k INTEGER NOT NULL, s TEXT NOT NULL, " + "u TEXT NOT NULL, b BLOB NOT NULL, PRIMARY KEY (t, k))" + ) + for row in ROWS[:n_rows]: + _ = connection.execute( + "INSERT INTO r VALUES (?, ?, ?, ?, ?)", + (row["t"], row["k"], row["s"], row["u"], row["b"]), + ) + connection.commit() + connection.close() + return SQLiteTableSource(path, "r", tag_columns=TAGS, source_id="sqlite") + + +def _cached_source_in_memory(n_rows: int, tmp: Path) -> StreamProtocol: + return CachedSource( + ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="cached_mem"), + cache_database=InMemoryArrowDatabase(), + ) + + +def _cached_source_delta(n_rows: int, tmp: Path) -> StreamProtocol: + return CachedSource( + ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="cached_delta"), + cache_database=DeltaTableDatabase(base_path=tmp / f"cache{n_rows}"), + ) + + +# Every source kind that carries tags ``t``, ``k`` and data ``s``, ``u`` (so every +# operator applies), with the row counts it can be built with. A header-only CSV and +# an empty SQLite table cannot be read (separate issues), so those kinds have rows. +SOURCES: dict[str, tuple[SourceFactory, tuple[int, ...]]] = { + "ArrowTableSource": (_arrow_table_source, (2, 0)), + "ArrowTableStream": (_arrow_table_stream, (2, 0)), + "DictSource(python)": (_dict_source_python, (2, 0)), + "DictSource(arrow schema)": (_dict_source_arrow_schema, (2, 0)), + "CSVSource": (_csv_source, (2,)), + "CSVSource(arrow schema)": (_csv_source_arrow_schema, (2, 0)), + "DataFrameSource(polars)": (_data_frame_source_polars, (2, 0)), + "DataFrameSource(pandas)": (_data_frame_source_pandas, (2, 0)), + "DataFrameSource(arrow table)": (_data_frame_source_arrow_table, (2, 0)), + "DataFrameSource(arrow schema)": (_data_frame_source_arrow_schema, (2, 0)), + "DeltaTableSource": (_delta_table_source, (2, 0)), + "SQLiteTableSource": (_sqlite_table_source, (2,)), + "CachedSource(in memory)": (_cached_source_in_memory, (2, 0)), + "CachedSource(delta)": (_cached_source_delta, (2, 0)), +} + +SOURCE_CASES = [ + (name, n_rows) for name, (_, counts) in SOURCES.items() for n_rows in counts +] + + +def build_source(name: str, n_rows: int, tmp: Path) -> StreamProtocol: + """Build the source kind ``name`` with ``n_rows`` rows, its files under ``tmp``.""" + factory, _ = SOURCES[name] + return factory(n_rows, tmp) diff --git a/tests/test_core/operators/test_aggregated_list_types.py b/tests/test_core/operators/test_aggregated_list_types.py index 3b115bca2..00de01aee 100644 --- a/tests/test_core/operators/test_aggregated_list_types.py +++ b/tests/test_core/operators/test_aggregated_list_types.py @@ -171,8 +171,8 @@ def _row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: ] -def _in_list_layout(stream: StreamProtocol) -> ArrowTableStream: - """Rebuild ``stream`` with every top-level ``large_list`` column as ``list``. +def _list_layout_table(stream: StreamProtocol) -> pa.Table: + """Return ``stream``'s table with every top-level ``large_list`` column as ``list``. This is the layout ``Batch`` and ``GroupBy`` emitted before ITL-832 (``pa.list_(input type)``); logical list columns were already extension @@ -187,8 +187,16 @@ def _in_list_layout(stream: StreamProtocol) -> ArrowTableStream: for field in table.schema ] ) + return table.cast(schema) + + +def _in_list_layout(stream: StreamProtocol) -> ArrowTableStream: + """Rebuild ``stream`` from its ``list``-layout table (``_list_layout_table``). + + The stream presents ``large_list`` again (ITL-1122). + """ tag_columns, _ = stream.keys() - return ArrowTableStream(table.cast(schema), tag_columns=tag_columns) + return ArrowTableStream(_list_layout_table(stream), tag_columns=tag_columns) class TestAggregatedContentHashes: @@ -199,9 +207,19 @@ def test_row_hashes_match_the_list_layout( ) -> None: """Every row hashes the same in the ``large_list`` and ``list`` layouts.""" _, _, result = aggregated + # A stream presents large layouts only (ITL-1122): the old layout exists as a + # table, which the Arrow hasher hashes as the new one, row by row. + old_table = _list_layout_table(result) + new_table = result.as_table(columns={"source": True, "system_tags": True}) + assert pa.types.is_list(old_table.schema.field("n").type) + assert pa.types.is_large_list(new_table.schema.field("n").type) + hasher = result.data_context.arrow_hasher + for i in range(new_table.num_rows): + assert hasher.hash_table(old_table.slice(i, 1)) == hasher.hash_table( + new_table.slice(i, 1) + ) before = _in_list_layout(result) - assert pa.types.is_list(before.as_table().schema.field("n").type) - assert pa.types.is_large_list(result.as_table().schema.field("n").type) + assert pa.types.is_large_list(before.as_table().schema.field("n").type) assert _row_hashes(result) == _row_hashes(before) def test_stream_content_hash_matches_the_list_layout( diff --git a/tests/test_core/operators/test_large_normalization.py b/tests/test_core/operators/test_large_normalization.py index 2b7d6c0eb..9428e726c 100644 --- a/tests/test_core/operators/test_large_normalization.py +++ b/tests/test_core/operators/test_large_normalization.py @@ -1,17 +1,17 @@ -"""Every operator's output takes the large Arrow types, on every path (ITL-1122). - -The rule (developer, November 14, 2026): every Arrow table Orcapod creates has its string, -binary and list types in their large variants; only a user's own Arrow table, and a store -that keeps the distinction recoverably, keep a non-large type. An operator's output is a -table Orcapod creates, also when the operator passes its rows through unchanged (a select, -a rename, a filter with nothing to filter): its columns take the large-normalized form of -their input types, on a plain stream as inside a job, sync and async, with rows and without. -Large and non-large variants hash identically, so no hash moves. +"""No operator output has a non-large Arrow type, from any source, on any path (ITL-1122). + +The rule (developer, November 14, 2026): every column of every table Orcapod presents or +stores has its string, binary and list types in their large variants, recursively, with no +exception for a user's Arrow table. So every operator over every source kind gives large +types: on a plain stream and inside a job, sync and async, with rows and without, and a +zero-row result has the schema of a populated one. Large and non-large variants hash +identically, so no hash moves. """ from __future__ import annotations from collections.abc import Callable +from pathlib import Path import polars as pl import pyarrow as pa @@ -41,57 +41,22 @@ from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator from orcapod.protocols.core_protocols import StreamProtocol -from orcapod.utils import arrow_utils +from tests.test_core.large_types_cases import ( + ALL_COLUMNS, + FULL_SCHEMA, + ROWS, + SOURCE_CASES, + SOURCES, + TAGS, + build_source, + non_large_columns, +) -ALL_COLUMNS = {"source": True, "system_tags": True, "context": True} ORCHESTRATORS = { "sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator, } -NESTED = pa.struct([pa.field("x", pa.list_(pa.string())), pa.field("y", pa.string())]) -SMALL_SCHEMA = pa.schema( - [ - pa.field("t", pa.string(), nullable=False), - pa.field("k", pa.int64(), nullable=False), - pa.field("s", pa.string(), nullable=False), - pa.field("b", pa.binary(), nullable=False), - pa.field("l", pa.list_(pa.string()), nullable=False), - pa.field("n", NESTED, nullable=False), - ] -) -ROWS = [ - {"t": "a", "k": 0, "s": "x", "b": b"1", "l": ["p"], "n": {"x": ["q"], "y": "r"}}, - {"t": "b", "k": 1, "s": "y", "b": b"2", "l": ["p", "q"], "n": {"x": [], "y": "s"}}, -] - - -def _non_large(table: pa.Table) -> dict[str, pa.DataType]: - """The columns of ``table`` whose type is not its large-normalized form.""" - return { - field.name: field.type - for field in table.schema - if arrow_utils.normalize_to_large_types(field.type, keep_null=True) - != field.type - } - - -def _source(n_rows: int, *, large: bool = False) -> ArrowTableSource: - """A user's Arrow table: non-large string, binary, list and nested columns. - - With ``large``, the same rows with every type in its large variant. - """ - schema = ( - arrow_utils.normalize_schema_to_large_types(SMALL_SCHEMA, keep_null=True) - if large - else SMALL_SCHEMA - ) - return ArrowTableSource( - pa.Table.from_pylist(ROWS[:n_rows], schema=schema), - tag_columns=["t", "k"], - source_id="main", - ) - def _other(n_rows: int, data_column: str) -> ArrowTableSource: """A second user's table: tag ``t`` and one data column, both ``string``.""" @@ -112,14 +77,10 @@ def _other(n_rows: int, data_column: str) -> ArrowTableSource: OPERATORS: dict[str, Apply] = { "SelectTagColumns": lambda s, n, label: SelectTagColumns(["t"])(s, label=label), "DropTagColumns": lambda s, n, label: DropTagColumns(["k"])(s, label=label), - "SelectDataColumns": lambda s, n, label: SelectDataColumns(["s", "l"])( - s, label=label - ), - "DropDataColumns": lambda s, n, label: DropDataColumns(["b"])(s, label=label), - "SelectColumns": lambda s, n, label: SelectColumns(["t", "k", "s", "n"])( - s, label=label - ), - "DropColumns": lambda s, n, label: DropColumns(["b"])(s, label=label), + "SelectDataColumns": lambda s, n, label: SelectDataColumns(["s"])(s, label=label), + "DropDataColumns": lambda s, n, label: DropDataColumns(["u"])(s, label=label), + "SelectColumns": lambda s, n, label: SelectColumns(["t", "k", "s"])(s, label=label), + "DropColumns": lambda s, n, label: DropColumns(["u"])(s, label=label), "MapTags": lambda s, n, label: MapTags({"t": "t2"})(s, label=label), "MapData": lambda s, n, label: MapData({"s": "s2"})(s, label=label), "PolarsFilter(predicate)": lambda s, n, label: PolarsFilter( @@ -134,67 +95,152 @@ def _other(n_rows: int, data_column: str) -> ArrowTableSource: "Batch": lambda s, n, label: Batch(batch_size=2)(s, label=label), "GroupBy(t)": lambda s, n, label: GroupBy(by=["t"])(s, label=label), "GroupBy(k)": lambda s, n, label: GroupBy(by=["k"])(s, label=label), - "Index": lambda s, n, label: Index("l", 0)(s, label=label), } -@pytest.mark.parametrize("name", list(OPERATORS)) -class TestEveryOperator: - @pytest.mark.parametrize("n_rows", [2, 0]) - def test_plain_stream_output_is_large(self, name: str, n_rows: int) -> None: - out = OPERATORS[name](_source(n_rows), n_rows, None) - table = out.as_table(columns=ALL_COLUMNS) - assert _non_large(table) == {} - - def test_zero_row_schema_equals_populated(self, name: str) -> None: - populated = OPERATORS[name](_source(2), 2, None).as_table(columns=ALL_COLUMNS) - empty = OPERATORS[name](_source(0), 0, None).as_table(columns=ALL_COLUMNS) - # The user tag and data columns: which system columns a zero-row result - # carries is ITL-912's subject, not this rule's. - user_columns = [name for name in populated.column_names if name[0] != "_"] - assert ( - empty.select(user_columns).schema == populated.select(user_columns).schema - ) +def _cases(row_counts: tuple[int, ...]) -> list[tuple[str, str, int]]: + """Every (source kind, operator, rows) with rows among ``row_counts``.""" + return [ + (source, operator, n_rows) + for source, n_rows in SOURCE_CASES + if n_rows in row_counts + for operator in OPERATORS + ] + + +def _job_tables( + source: str, operator: str, orchestrator: str, tmp_path: Path +) -> tuple[pa.Table, list[pa.Table]]: + """The operator's node table, and every other node's, after a job run.""" + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + OPERATORS[operator](build_source(source, 2, tmp_path), 2, "op") + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + tables = { + label: node.as_table(columns=ALL_COLUMNS) for label, node in job.nodes.items() + } + return tables.pop("op"), list(tables.values()) - @pytest.mark.parametrize("orchestrator", ["sync", "async"]) - def test_job_output_is_large_and_equals_plain( - self, name: str, orchestrator: str - ) -> None: - job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) - with job: - OPERATORS[name](_source(2), 2, "op") - job.run(orchestrator=ORCHESTRATORS[orchestrator]()) - job_table = job.nodes["op"].as_table() - plain_table = OPERATORS[name](_source(2), 2, None).as_table() - assert _non_large(job_table) == {} - assert job_table.schema == plain_table.schema - - def test_row_hashes_equal_the_large_input(self, name: str) -> None: - """Large and non-large variants hash identically: no row hash moves.""" - small = OPERATORS[name](_source(2), 2, None) - large = OPERATORS[name](_source(2, large=True), 2, None) - assert small.content_hash() == large.content_hash() - assert small.pipeline_hash() == large.pipeline_hash() - - def row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: - return sorted( - (tag.content_hash().to_hex(), data.content_hash().to_hex()) - for tag, data in stream.iter_data() + +def _user_schema(table: pa.Table) -> pa.Schema: + """The schema of ``table``'s user tag and data columns.""" + return table.select([name for name in table.column_names if name[0] != "_"]).schema + + +@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2, 0))) +def test_plain_stream_has_no_non_large_type( + source: str, operator: str, n_rows: int, tmp_path: Path +) -> None: + out = OPERATORS[operator](build_source(source, n_rows, tmp_path), n_rows, None) + assert non_large_columns(out.as_table(columns=ALL_COLUMNS)) == [] + assert non_large_columns(out.head(columns=ALL_COLUMNS).table) == [] + + +@pytest.mark.parametrize( + ("source", "operator"), + [(source, operator) for source, operator, _ in _cases((0,))], +) +def test_zero_row_schema_equals_populated( + source: str, operator: str, tmp_path: Path +) -> None: + populated = OPERATORS[operator](build_source(source, 2, tmp_path), 2, None) + empty = OPERATORS[operator](build_source(source, 0, tmp_path), 0, None) + populated_table = populated.as_table(columns=ALL_COLUMNS) + empty_table = empty.as_table(columns=ALL_COLUMNS) + # The user tag and data columns: which system columns a zero-row result + # carries is ITL-912's subject, not this rule's. + assert _user_schema(empty_table) == _user_schema(populated_table) + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2,))) +def test_job_has_no_non_large_type_and_equals_plain( + source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path +) -> None: + op_table, other_tables = _job_tables(source, operator, orchestrator, tmp_path) + assert non_large_columns(op_table) == [] + for table in other_tables: + assert non_large_columns(table) == [] + plain = OPERATORS[operator](build_source(source, 2, tmp_path), 2, None) + assert _user_schema(op_table) == _user_schema(plain.as_table(columns=ALL_COLUMNS)) + + +@pytest.mark.parametrize("operator", list(OPERATORS)) +def test_row_hashes_equal_for_large_and_non_large_input( + operator: str, tmp_path: Path +) -> None: + """A ``string`` source and a ``large_string`` source of the same values give equal + row, content and pipeline hashes after every operator.""" + small = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=FULL_SCHEMA), tag_columns=TAGS + ) + large_schema = pa.schema( + [ + field.with_type( + pa.large_string() + if field.type == pa.string() + else pa.large_binary() + if field.type == pa.binary() + else pa.large_list(pa.large_string()) + if field.type == pa.list_(pa.string()) + else pa.struct( + [ + pa.field("x", pa.large_list(pa.large_string())), + pa.field("y", pa.large_string()), + ] + ) + if field.name == "n" + else field.type ) + for field in FULL_SCHEMA + ] + ) + large = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=large_schema), tag_columns=TAGS + ) + small_out = OPERATORS[operator](small, 2, None) + large_out = OPERATORS[operator](large, 2, None) + assert small_out.content_hash() == large_out.content_hash() + assert small_out.pipeline_hash() == large_out.pipeline_hash() + + def row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: + return sorted( + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in stream.iter_data() + ) + + assert row_hashes(small_out) == row_hashes(large_out) + - assert row_hashes(small) == row_hashes(large) +def test_index_over_a_list_column_is_large(tmp_path: Path) -> None: + """``Index`` reads a list column (the kinds above that carry one: ArrowTableSource).""" + for n_rows in (2, 0): + out = Index("l", 0)(build_source("ArrowTableSource", n_rows, tmp_path)) + assert non_large_columns(out.as_table(columns=ALL_COLUMNS)) == [] class TestJoinNeverPicksASide: - """A shared tag that is large in one input and ``string`` in the other is large.""" + """A shared tag that is ``string`` in one user table and ``large_string`` in the + other is ``large_string``, whichever comes first.""" @pytest.mark.parametrize("n_rows", [2, 0]) def test_either_argument_order_gives_large(self, n_rows: int) -> None: - large = _source(n_rows, large=True) + large = ArrowTableSource( + pa.table( + {"t": pa.array(["a", "b"][:n_rows], pa.large_string())} + ).append_column("w", pa.array([1, 2][:n_rows], pa.int64())), + tag_columns=["t"], + source_id="large", + ) small = _other(n_rows, "z") forward = Join()(large, small).as_table(columns=ALL_COLUMNS) backward = Join()(small, large).as_table(columns=ALL_COLUMNS) assert forward.schema.field("t").type == pa.large_string() assert backward.schema.field("t").type == pa.large_string() - assert _non_large(forward) == {} - assert _non_large(backward) == {} + assert non_large_columns(forward) == [] + assert non_large_columns(backward) == [] + + +def test_every_source_kind_is_swept() -> None: + """The sweep covers every source kind of ``large_types_cases``.""" + assert {source for source, _ in SOURCE_CASES} == set(SOURCES) diff --git a/tests/test_core/operators/test_merge_join_arrow_types.py b/tests/test_core/operators/test_merge_join_arrow_types.py index 4ee5dd3a2..5c7a83a41 100644 --- a/tests/test_core/operators/test_merge_join_arrow_types.py +++ b/tests/test_core/operators/test_merge_join_arrow_types.py @@ -518,11 +518,11 @@ def test_shared_tag_with_other_difference_raises( MergeJoin().static_process(first, second) def test_user_string_and_dict_sources_on_a_string_column(self) -> None: - """A user's ``string`` column colliding with a dict ``large_string`` one merges. + """A user's ``string`` column colliding with a dict one merges. - A CSVSource gave ``string`` here until ITL-1122 (a CSV file keeps no large / - non-large distinction, so its table is now large); a user's own Arrow table - is the source that keeps ``string``. + The ``string`` / ``large_string`` meeting of ITL-829 no longer reaches an + operator: every stream presents the large variant (ITL-1122), so the user's + ``string`` column is ``large_string`` at its source already. """ user_source = ArrowTableSource( _table({"id": (pa.int64(), [1, 2]), "label": (pa.string(), ["a", "b"])}), @@ -532,7 +532,7 @@ def test_user_string_and_dict_sources_on_a_string_column(self) -> None: dict_source = DictSource( [{"id": 1, "label": "x"}, {"id": 2, "label": "y"}], tag_columns=["id"] ) - assert user_source.as_table().schema.field("label").type == pa.string() + assert user_source.as_table().schema.field("label").type == pa.large_string() assert dict_source.as_table().schema.field("label").type == pa.large_string() table = MergeJoin()(user_source, dict_source).as_table() @@ -745,10 +745,10 @@ def _small_layout(arrow_type: pa.DataType) -> pa.DataType: return arrow_type -def _relayout( +def _relayout_table( stream: StreamProtocol, overrides: dict[str, pa.DataType] | None = None -) -> ArrowTableStream: - """Rebuild ``stream`` in the small layouts, with optional per-column types. +) -> pa.Table: + """Return ``stream``'s table in the small layouts, with optional per-column types. The small layouts are what MergeJoin emitted for a merged column of ``int64`` or ``string`` values before this change (``list<...>``). @@ -761,8 +761,19 @@ def _relayout( for field in table.schema ] ) + return table.cast(schema) + + +def _relayout( + stream: StreamProtocol, overrides: dict[str, pa.DataType] | None = None +) -> ArrowTableStream: + """Rebuild ``stream`` from its table in the small layouts (``_relayout_table``). + + The stream presents the large layouts again (ITL-1122); what stays is any width + an override changes. + """ tag_columns, _ = stream.keys() - return ArrowTableStream(table.cast(schema), tag_columns=tag_columns) + return ArrowTableStream(_relayout_table(stream, overrides), tag_columns=tag_columns) def _stream_hash(stream: StreamProtocol) -> str: @@ -855,8 +866,16 @@ def test_layout_change_keeps_merge_join_hashes( """ values = [1, 2] if arrow_type == pa.int64() else ["a", "b"] result = MergeJoin().static_process(*_pair(arrow_type, arrow_type, values, 2)) + # A stream presents large layouts only (ITL-1122): the old layout exists as a + # table, which the Arrow hasher hashes as the new one. + old_table = _relayout_table(result) + assert pa.types.is_list(old_table.schema.field("x").type) + hasher = _starfix_hasher() + assert hasher.hash_table(old_table) == hasher.hash_table( + result.as_table(columns={"source": True, "system_tags": True}) + ) old_layout = _relayout(result) - assert pa.types.is_list(old_layout.as_table().schema.field("x").type) + assert pa.types.is_large_list(old_layout.as_table().schema.field("x").type) assert _row_hashes(result) == _row_hashes(old_layout) assert _stream_hash(result) == _stream_hash(old_layout) diff --git a/tests/test_core/operators/test_semijoin_nested.py b/tests/test_core/operators/test_semijoin_nested.py index e9ca0e152..a2fdbff14 100644 --- a/tests/test_core/operators/test_semijoin_nested.py +++ b/tests/test_core/operators/test_semijoin_nested.py @@ -5,7 +5,8 @@ field"), so a list, large_list, struct or struct-backed logical column on either input failed. ITL-829 (first case): Acero also rejected a key that is ``string`` on one side and ``large_string`` on the other ("Incompatible data types for -corresponding join field keys"), e.g. a user's ``string`` table against a ``DictSource``. +corresponding join field keys"), e.g. a ``CSVSource`` against a ``DictSource``; since +ITL-1122 every stream presents the large variant, so that meeting no longer occurs. The result is the left input's rows that match a right row, in left order, with the left input's Arrow schema in its large variants (ITL-1122: ``string`` is @@ -360,12 +361,12 @@ def test_null_scalar_key_never_matches() -> None: def test_string_vs_large_string_key_user_vs_dict() -> None: - """ITL-829 (first case): a user's ``string`` key against a DictSource's - ``large_string`` key, in both directions. + """ITL-829 (first case): a user's ``string`` key against a DictSource's key, in + both directions. - A CSVSource gave the ``string`` key until ITL-1122 (a CSV file keeps no large / - non-large distinction, so its table is now large); a user's own Arrow table is - the source that keeps ``string``. + The ``string`` / ``large_string`` meeting no longer reaches an operator: every + stream presents the large variant (ITL-1122), so the user's ``string`` key is + ``large_string`` at its source already. """ user_source = _source( { @@ -378,7 +379,7 @@ def test_string_vs_large_string_key_user_vs_dict() -> None: dict_source = DictSource( data=[{"id": "a", "y": 1}, {"id": "c", "y": 3}], tag_columns=["id"] ) - assert user_source.as_table().schema.field("id").type == pa.string() + assert user_source.as_table().schema.field("id").type == pa.large_string() assert dict_source.as_table().schema.field("id").type == pa.large_string() assert_semijoin_result( diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py index e235faab9..835e99051 100644 --- a/tests/test_core/sources/test_large_normalization_sources.py +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -1,241 +1,318 @@ -"""Sources present large Arrow types unless the user chose the types (ITL-1122). - -The rule (developer, November 14, 2026): every Arrow table Orcapod creates has its string, -binary and list types in their large variants. Two exceptions keep the types as given: an -Arrow table (or Arrow schema) the user made, and a store that keeps the large / non-large -distinction in a recoverable way. - -* ``CSVSource`` reads a file that has no such distinction, so its table is Orcapod's: large. - An Arrow ``schema=`` given by the user is kept. -* ``CachedSource`` presents the rows it read from its cache store, a table Orcapod creates: - large whatever the store (in memory or Delta) and whether it has rows or not. +"""Every source presents large Arrow types, with no exception (ITL-1122). + +The rule (developer, November 14, 2026): every column of every table Orcapod presents or +stores has its string, binary and list types in their large variants, recursively. A +user's own Arrow table and an explicit Arrow schema are normalized too. The one place is +where a table becomes a stream (``ArrowTableStream``), so a source, its job node, its +head and a cache over it agree. Large and non-large variants hash identically, so no +row hash, content hash, pipeline hash or default source id moves. """ from __future__ import annotations from pathlib import Path +from typing import Any import pyarrow as pa -import pyarrow.csv as pa_csv import pytest -from orcapod.core.sources import ArrowTableSource, CachedSource, CSVSource +from orcapod import function_pod +from orcapod.core.sources import ( + ArrowTableSource, + CachedSource, + DictSource, + ListSource, + PollingSource, +) from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.operators import SelectTagColumns from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator -from orcapod.protocols.core_protocols import SourceProtocol +from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.types import Cursor, PollingConfig from orcapod.utils import arrow_utils +from tests.test_core.large_types_cases import ( + ALL_COLUMNS, + FULL_SCHEMA, + ROWS, + SOURCE_CASES, + TAGS, + build_source, + non_large_columns, +) -ALL_COLUMNS = {"source": True, "system_tags": True, "context": True} ORCHESTRATORS = { "sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator, } +LARGE_SCHEMA = arrow_utils.normalize_schema_to_large_types(FULL_SCHEMA, keep_null=True) -NESTED = pa.struct([pa.field("x", pa.list_(pa.string())), pa.field("y", pa.string())]) -SMALL_SCHEMA = pa.schema( - [ - pa.field("t", pa.string(), nullable=False), - pa.field("s", pa.string(), nullable=False), - pa.field("b", pa.binary(), nullable=False), - pa.field("l", pa.list_(pa.string()), nullable=False), - pa.field("n", NESTED, nullable=False), - ] -) -ROWS = [ - {"t": "a", "s": "x", "b": b"1", "l": ["p"], "n": {"x": ["q"], "y": "r"}}, - {"t": "b", "s": "y", "b": b"2", "l": [], "n": {"x": [], "y": "s"}}, -] +def _job_node_tables(source: StreamProtocol, orchestrator: str) -> list[pa.Table]: + """Every node's table after a job over ``source`` runs on ``orchestrator``.""" + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + _ = SelectTagColumns(["t"])(source, label="selected") + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + return [node.as_table(columns=ALL_COLUMNS) for node in job.nodes.values()] -def _non_large(table: pa.Table) -> dict[str, pa.DataType]: - """The columns of ``table`` whose type is not its large-normalized form.""" - return { - field.name: field.type - for field in table.schema - if arrow_utils.normalize_to_large_types(field.type, keep_null=True) - != field.type - } +@pytest.mark.parametrize(("name", "n_rows"), SOURCE_CASES) +class TestEverySourceKind: + def test_table_and_head_have_no_non_large_type( + self, name: str, n_rows: int, tmp_path: Path + ) -> None: + source = build_source(name, n_rows, tmp_path) + table = source.as_table(columns=ALL_COLUMNS) + assert table.num_rows == n_rows + assert non_large_columns(table) == [] + assert non_large_columns(source.head(columns=ALL_COLUMNS).table) == [] -def _small_source(n_rows: int, source_id: str = "small") -> ArrowTableSource: - """A user's Arrow table with string, binary, list and nested columns.""" - return ArrowTableSource( - pa.Table.from_pylist(ROWS[:n_rows], schema=SMALL_SCHEMA), - tag_columns=["t"], - source_id=source_id, + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_job_nodes_have_no_non_large_type( + self, name: str, n_rows: int, orchestrator: str, tmp_path: Path + ) -> None: + tables = _job_node_tables(build_source(name, n_rows, tmp_path), orchestrator) + for table in tables: + assert non_large_columns(table) == [] + + +def test_explicit_arrow_schema_is_normalized(tmp_path: Path) -> None: + """An Arrow schema the user gives is taken in its large variants.""" + table = build_source("CSVSource(arrow schema)", 2, tmp_path).as_table() + assert table.schema.field("t").type == pa.large_string() + assert table.schema.field("s").type == pa.large_string() + dict_table = build_source("DictSource(arrow schema)", 2, tmp_path).as_table() + arrow_table = build_source("ArrowTableSource", 2, tmp_path).as_table() + assert dict_table.schema == arrow_table.schema + + +def test_user_arrow_table_keeps_width_and_nullability() -> None: + """Only layouts change: widths, nullability (top level and nested), field names.""" + items = pa.list_(pa.field("item", pa.int8(), nullable=False)) + schema = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("l", items, nullable=False), + pa.field("s", pa.string(), nullable=True), + ] + ) + table = pa.Table.from_pylist([{"t": "a", "l": [1], "s": None}], schema=schema) + out = ArrowTableSource(table, tag_columns=["t"], source_id="u").as_table() + assert out.schema.field("t") == pa.field("t", pa.large_string(), nullable=False) + assert out.schema.field("l") == pa.field( + "l", pa.large_list(pa.field("item", pa.int8(), nullable=False)), nullable=False + ) + assert out.schema.field("s") == pa.field("s", pa.large_string(), nullable=True) + + +def test_list_source_is_large() -> None: + source = ListSource("v", ["a", "b"]) + assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == [] + + +class _Impl: + """A polling implementation that fetches one batch, as a dict or an Arrow table.""" + + def __init__(self, as_arrow: bool) -> None: + self.as_arrow = as_arrow + + def identity(self) -> str: + return f"impl-{self.as_arrow}" + + def to_config(self) -> None: + return None + + @classmethod + def from_config(cls, config: Any) -> _Impl: + return cls(False) + + def schema(self) -> None: + return None + + async def poll(self, cursor: Cursor[int] | None = None) -> bool: + return cursor is None + + async def fetch(self, cursor: Cursor[int] | None = None) -> tuple[Cursor[int], Any]: + if cursor is not None: + return Cursor(value=1), {} + if self.as_arrow: + return Cursor(value=1), pa.table( + {"t": pa.array(["a"], pa.string()), "s": pa.array(["x"], pa.string())} + ) + return Cursor(value=1), {"t": ["a"], "s": ["x"]} + + async def close(self) -> None: + return None + + +@pytest.mark.parametrize("as_arrow", [False, True]) +def test_polling_source_is_large(as_arrow: bool) -> None: + source = PollingSource( + _Impl(as_arrow), tag_columns="t", polling_config=PollingConfig(duration=0.01) ) + assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == [] + + +@function_pod(output_keys="v") +def _upper(s: str) -> str: + return s.upper() -def _source_node_table(source: SourceProtocol, orchestrator: str) -> pa.Table: - """The table of ``source``'s node in a job run on ``orchestrator``.""" +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize("n_rows", [2, 0]) +def test_function_nodes_and_derived_sources_are_large( + orchestrator: str, n_rows: int, tmp_path: Path +) -> None: + """A function node passing a ``string`` tag through, and derived sources.""" job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) with job: - SelectTagColumns(["t"])(source, label="selected") - job.run(orchestrator=ORCHESTRATORS[orchestrator]()) - # The job's two nodes: the source's node and ``selected``. - (source_label,) = [label for label in job.nodes if label != "selected"] - return job.nodes[source_label].as_table(columns=ALL_COLUMNS) + source = build_source("ArrowTableSource", n_rows, tmp_path) + _ = _upper.pod(source, label="upper") + _ = SelectTagColumns(["t"])(source, label="selected") + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + for label in ("upper", "selected"): + node = job.nodes[label] + assert non_large_columns(node.as_table(columns=ALL_COLUMNS)) == [] + derived = node.as_source() + assert non_large_columns(derived.as_table(columns=ALL_COLUMNS)) == [] + with_tags = node.as_source(tags_as_data={"t": "t_data"}) + assert non_large_columns(with_tags.as_table(columns=ALL_COLUMNS)) == [] + + +@pytest.mark.parametrize("n_rows", [2, 0]) +def test_function_pod_stream_is_large(n_rows: int, tmp_path: Path) -> None: + stream = _upper.pod(build_source("ArrowTableSource", n_rows, tmp_path)) + assert non_large_columns(stream.as_table(columns=ALL_COLUMNS)) == [] + + +def test_spiraldb_table_source_is_large(tmp_path: Path) -> None: + from orcapod.sources import SpiralDBTableSource + from tests.test_databases import fake_pyspiral + + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project("p") + table = project.create_table( + "default.r", key_schema=pa.schema([pa.field("t", pa.string())]) + ) + table.write( + pa.table( + { + "t": pa.array(["a", "b"], pa.string()), + "s": pa.array(["x", "y"], pa.string()), + "b": pa.array([b"1", b"2"], pa.binary()), + "l": pa.array([["x"], []], pa.list_(pa.string())), + } + ) + ) + source = SpiralDBTableSource("p", "r", tag_columns=["t"]) + assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == [] # --------------------------------------------------------------------------- -# CSVSource +# What does not move # --------------------------------------------------------------------------- -@pytest.fixture -def csv_path(tmp_path: Path) -> Path: - path = tmp_path / "rows.csv" - pa_csv.write_csv( - pa.table({"t": ["a", "b"], "k": [0, 1], "s": ["x", "y"]}), - path, - ) - return path +def _row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: + return [ + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in stream.iter_data() + ] -class TestCSVSource: - def test_table_is_large(self, csv_path: Path) -> None: - source = CSVSource(str(csv_path), tag_columns=["t"], source_id="csv") - table = source.as_table(columns=ALL_COLUMNS) - assert _non_large(table) == {} - assert table.schema.field("s").type == pa.large_string() +@pytest.mark.parametrize("source_id", [None, "named"]) +def test_string_and_large_string_sources_hash_alike(source_id: str | None) -> None: + """The same values as ``string`` and as ``large_string`` give equal row, content and + pipeline hashes and the same (default) source id.""" + small = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=FULL_SCHEMA), + tag_columns=TAGS, + source_id=source_id, + ) + large = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=LARGE_SCHEMA), + tag_columns=TAGS, + source_id=source_id, + ) + assert small.source_id == large.source_id + assert small.content_hash() == large.content_hash() + assert small.pipeline_hash() == large.pipeline_hash() + assert _row_hashes(small) == _row_hashes(large) + assert small.as_table(columns=ALL_COLUMNS).equals( + large.as_table(columns=ALL_COLUMNS) + ) - @pytest.mark.parametrize("orchestrator", ["sync", "async"]) - def test_source_node_in_a_job_is_large( - self, csv_path: Path, orchestrator: str - ) -> None: - source = CSVSource(str(csv_path), tag_columns=["t"], source_id="csv") - assert _non_large(_source_node_table(source, orchestrator)) == {} - - def test_explicit_arrow_schema_is_kept(self, csv_path: Path) -> None: - """The user chose the Arrow types: ``string`` stays ``string``.""" - schema = pa.schema( - [ - pa.field("t", pa.string(), nullable=False), - pa.field("k", pa.int64(), nullable=False), - pa.field("s", pa.string(), nullable=False), - ] - ) - source = CSVSource( - str(csv_path), tag_columns=["t"], source_id="csv", schema=schema - ) - table = source.as_table() - assert table.schema.field("t").type == pa.string() - assert table.schema.field("s").type == pa.string() - - def test_row_hashes_equal_the_explicit_string_schema(self, csv_path: Path) -> None: - """Large and non-large variants hash identically: no row hash moves.""" - schema = pa.schema( - [ - pa.field("t", pa.string(), nullable=False), - pa.field("k", pa.int64(), nullable=False), - pa.field("s", pa.string(), nullable=False), - ] - ) - large = CSVSource(str(csv_path), tag_columns=["t"], source_id="csv") - small = CSVSource( - str(csv_path), tag_columns=["t"], source_id="csv", schema=schema - ) - assert large.content_hash() == small.content_hash() - assert large.pipeline_hash() == small.pipeline_hash() - large_rows = [ - (tag.content_hash(), data.content_hash()) for tag, data in large.iter_data() - ] - small_rows = [ - (tag.content_hash(), data.content_hash()) for tag, data in small.iter_data() - ] - assert large_rows == small_rows + +def test_dict_source_arrow_schema_hashes_alike() -> None: + """A ``DictSource`` given ``string`` or ``large_string`` in its Arrow schema.""" + small = DictSource(ROWS, tag_columns=TAGS, data_schema=FULL_SCHEMA) + large = DictSource(ROWS, tag_columns=TAGS, data_schema=LARGE_SCHEMA) + assert small.source_id == large.source_id + assert small.content_hash() == large.content_hash() + assert small.pipeline_hash() == large.pipeline_hash() + assert _row_hashes(small) == _row_hashes(large) # --------------------------------------------------------------------------- -# CachedSource +# Stored tables # --------------------------------------------------------------------------- +def _user_source(n_rows: int) -> ArrowTableSource: + """A user's Arrow table in the non-large types, as a source.""" + return ArrowTableSource( + pa.Table.from_pylist(ROWS[:n_rows], schema=FULL_SCHEMA), + tag_columns=TAGS, + source_id="user", + ) + + def _store(kind: str, tmp_path: Path) -> ArrowDatabaseProtocol: if kind == "in_memory": return InMemoryArrowDatabase() - return DeltaTableDatabase(base_path=tmp_path / "cache") + return DeltaTableDatabase(base_path=tmp_path / "store") @pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) -class TestCachedSource: - @pytest.mark.parametrize("n_rows", [2, 0]) - def test_table_is_large(self, store_kind: str, n_rows: int, tmp_path: Path) -> None: - cached = CachedSource( - _small_source(n_rows), cache_database=_store(store_kind, tmp_path) - ) - table = cached.as_table(columns=ALL_COLUMNS) - assert table.num_rows == n_rows - assert _non_large(table) == {} - - def test_zero_row_schema_equals_populated( - self, store_kind: str, tmp_path: Path - ) -> None: - populated = CachedSource( - _small_source(2), cache_database=_store(store_kind, tmp_path / "a") - ).as_table(columns=ALL_COLUMNS) - empty = CachedSource( - _small_source(0), cache_database=_store(store_kind, tmp_path / "b") - ).as_table(columns=ALL_COLUMNS) - user_columns = [field.name for field in SMALL_SCHEMA] - assert ( - empty.select(user_columns).schema == populated.select(user_columns).schema - ) - - @pytest.mark.parametrize("orchestrator", ["sync", "async"]) - def test_source_node_in_a_job_is_large( - self, store_kind: str, orchestrator: str, tmp_path: Path - ) -> None: +class TestStoredTables: + def test_cached_records_are_large(self, store_kind: str, tmp_path: Path) -> None: cached = CachedSource( - _small_source(2), + _user_source(2), cache_database=_store(store_kind, tmp_path), ) - assert _non_large(_source_node_table(cached, orchestrator)) == {} - - def test_row_hashes_equal_the_wrapped_source( - self, store_kind: str, tmp_path: Path - ) -> None: - """Large and non-large variants hash identically: no row hash moves.""" - source = _small_source(2) - cached = CachedSource(source, cache_database=_store(store_kind, tmp_path)) - source_rows = sorted( - (tag.content_hash().to_hex(), data.content_hash().to_hex()) - for tag, data in source.iter_data() - ) - cached_rows = sorted( - (tag.content_hash().to_hex(), data.content_hash().to_hex()) - for tag, data in cached.iter_data() - ) - assert cached_rows == source_rows - - def test_a_cache_written_before_is_read_and_appended( - self, store_kind: str, tmp_path: Path - ) -> None: - """A second cached source on the same store reads the stored rows and adds its own. - - The cache's write path is unchanged: it stores what the wrapped source gives. - """ - store = _store(store_kind, tmp_path) - first = CachedSource(_small_source(1, source_id="small"), cache_database=store) - assert first.as_table().num_rows == 1 - second = CachedSource(_small_source(2, source_id="small"), cache_database=store) - table = second.as_table(columns=ALL_COLUMNS) - assert table.num_rows == 2 - assert _non_large(table) == {} + _ = cached.as_table() + stored = cached.get_all_records() + assert stored is not None + assert non_large_columns(stored) == [] @pytest.mark.parametrize("n_rows", [2, 0]) - def test_head_before_the_read_has_the_types_of_the_read( + def test_cache_head_before_the_read_has_the_types_of_the_read( self, store_kind: str, n_rows: int, tmp_path: Path ) -> None: - """The columns ``head()`` shows before the cache is read are large too.""" cached = CachedSource( - _small_source(n_rows), cache_database=_store(store_kind, tmp_path) + _user_source(n_rows), + cache_database=_store(store_kind, tmp_path), ) unread = cached.head(columns=ALL_COLUMNS).table - assert _non_large(unread) == {} read = cached.as_table(columns=ALL_COLUMNS) + assert non_large_columns(unread) == [] assert unread.schema.equals(read.schema, check_metadata=False) + + +def test_a_delta_table_written_with_string_accepts_large_rows(tmp_path: Path) -> None: + """A Delta table a build before this change wrote with ``string`` columns takes the + ``large_string`` rows written after it (Delta keeps one string type). An in-memory + store keeps no data across processes, so no such table of it can exist.""" + store = _store("delta", tmp_path) + before = pa.Table.from_pylist(ROWS[:1], schema=FULL_SCHEMA) + after = pa.Table.from_pylist(ROWS[1:], schema=LARGE_SCHEMA) + _ = store.add_records(("t",), before, record_id_column="b") + store.flush() + _ = store.add_records(("t",), after, record_id_column="b") + store.flush() + stored = store.get_all_records(("t",), record_id_column="b") + assert stored is not None + assert stored.num_rows == 2 From 9296644e2dd123c678824c9188ea56aba6c69185 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:26:00 -0800 Subject: [PATCH 8/9] docs(plans): ITL-1122 plan records the final rule Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-14-itl-1122-large-normalization.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-14-itl-1122-large-normalization.md b/superpowers/plans/2026-11-14-itl-1122-large-normalization.md index bdf2d4ae0..c0579e0df 100644 --- a/superpowers/plans/2026-11-14-itl-1122-large-normalization.md +++ b/superpowers/plans/2026-11-14-itl-1122-large-normalization.md @@ -1,6 +1,10 @@ # ITL-1122: large normalization applied unevenly — evaluation and plan -Status: EVALUATION DONE, waiting for the coordinator's answers (Q1–Q3) before any code. +Status: DONE (PR #389). The rule below was REPLACED the same day by the developer: "let's go +ahead and always convert all non-large variants into large variants everywhere" (no exception for +a user's Arrow table or explicit schema). Implemented in one place, `ArrowTableStream.__init__`; +the per-source / per-operator fixes F1-F3 planned below were folded into it and removed. The +measurements below remain the record of what was uneven on `main` 3a7f3850. ## The rule (developer, November 14, 2026) From 1e27e20ad2948617011374df31061f34c0a0560c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:24:01 -0800 Subject: [PATCH 9/9] test(large-types): zero-row inputs in the job sweep; pin list_view as unsupported (ITL-1122) CodeRabbit round 1. The sync and async job sweep now covers zero-row inputs and compares each with the zero-row plain result. list_view and large_list_view stay unsupported (ITL-1129): a test pins today's behaviour and the docs say every supported Arrow type. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 20 +++++++++-------- CLAUDE.md | 20 +++++++++-------- docs/concepts/sources.md | 3 ++- .../operators/test_large_normalization.py | 17 +++++++++----- .../test_large_normalization_sources.py | 22 +++++++++++++++++++ 5 files changed, 57 insertions(+), 25 deletions(-) diff --git a/.zed/rules b/.zed/rules index eaa6c71bc..a2e1375b5 100644 --- a/.zed/rules +++ b/.zed/rules @@ -714,15 +714,17 @@ when no type is known. Large variants everywhere (developer, November 14, 2026, ITL-1122): every column of every table Orcapod presents or stores has its string, binary and list types in their large variants -(large_string, large_binary, large_list, recursively), with no exception: a user's own -Arrow table (ArrowTableSource, a raw ArrowTableStream) and an explicit Arrow schema= / -data_schema= are normalized too. The one place is ArrowTableStream.__init__ -(arrow_utils.normalize_table_to_large_types(table, keep_null=True)), where every source and -every operator result becomes a stream; never add a per-source or per-operator normalization. -Names, nullability (top level and nested), field metadata, widths and logical (extension) types -are untouched, and large and non-large variants hash identically, so no hash or default source -id moves. Not covered: what is inside a library value (numpy, pandas, polars, torch) keeps its -library's own distinction. +(large_string, large_binary, large_list, recursively), for every supported Arrow type, with +no exception: a user's own Arrow table (ArrowTableSource, a raw ArrowTableStream) and an +explicit Arrow schema= / data_schema= are normalized too. The one place is +ArrowTableStream.__init__ (arrow_utils.normalize_table_to_large_types(table, keep_null=True)), +where every source and every operator result becomes a stream; never add a per-source or +per-operator normalization. Names, nullability (top level and nested), field metadata, widths and +logical (extension) types are untouched, and large and non-large variants hash identically, so no +hash or default source id moves. Not covered: what is inside a library value (numpy, pandas, +polars, torch) keeps its library's own distinction. list_view and large_list_view are not +supported: a source refuses them (the Arrow hasher cannot hash them) and a raw ArrowTableStream +holds them as given (ITL-1129). Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large diff --git a/CLAUDE.md b/CLAUDE.md index bf9ec4e1d..9eef8c1dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -756,15 +756,17 @@ when no type is known. Large variants everywhere (developer, November 14, 2026, ITL-1122): every column of every table Orcapod presents or stores has its string, binary and list types in their large variants -(`large_string`, `large_binary`, `large_list`, recursively), with no exception: a user's own -Arrow table (`ArrowTableSource`, a raw `ArrowTableStream`) and an explicit Arrow `schema=` / -`data_schema=` are normalized too. The one place is `ArrowTableStream.__init__` -(`arrow_utils.normalize_table_to_large_types(table, keep_null=True)`), where every source and -every operator result becomes a stream; never add a per-source or per-operator normalization. -Names, nullability (top level and nested), field metadata, widths and logical (extension) types -are untouched, and large and non-large variants hash identically, so no hash or default source -id moves. Not covered: what is inside a library value (numpy, pandas, polars, torch) keeps its -library's own distinction. +(`large_string`, `large_binary`, `large_list`, recursively), for every supported Arrow type, with +no exception: a user's own Arrow table (`ArrowTableSource`, a raw `ArrowTableStream`) and an +explicit Arrow `schema=` / `data_schema=` are normalized too. The one place is +`ArrowTableStream.__init__` (`arrow_utils.normalize_table_to_large_types(table, keep_null=True)`), +where every source and every operator result becomes a stream; never add a per-source or +per-operator normalization. Names, nullability (top level and nested), field metadata, widths and +logical (extension) types are untouched, and large and non-large variants hash identically, so no +hash or default source id moves. Not covered: what is inside a library value (numpy, pandas, +polars, torch) keeps its library's own distinction. `list_view` and `large_list_view` are not +supported: a source refuses them (the Arrow hasher cannot hash them) and a raw `ArrowTableStream` +holds them as given (ITL-1129). Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 97776af25..f3dba4b6a 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -56,7 +56,8 @@ Every source gives the large variants of string, binary and list types (`large_s a data frame, a CSV file or a database table. A `string` column you give comes back as `large_string`. Only the layout changes: names, nullability, widths (`int8` stays `int8`) and logical types are kept, and the two variants hash identically, so no hash depends on which -one you gave. +one you gave. The Arrow `list_view` and `large_list_view` types are not supported: a source +refuses a column of either type. ### SpiralDB tables diff --git a/tests/test_core/operators/test_large_normalization.py b/tests/test_core/operators/test_large_normalization.py index 9428e726c..5abc10c86 100644 --- a/tests/test_core/operators/test_large_normalization.py +++ b/tests/test_core/operators/test_large_normalization.py @@ -109,12 +109,13 @@ def _cases(row_counts: tuple[int, ...]) -> list[tuple[str, str, int]]: def _job_tables( - source: str, operator: str, orchestrator: str, tmp_path: Path + source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path ) -> tuple[pa.Table, list[pa.Table]]: - """The operator's node table, and every other node's, after a job run.""" + """The operator's node table, and every other node's, after a job run over + inputs of ``n_rows`` rows.""" job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) with job: - OPERATORS[operator](build_source(source, 2, tmp_path), 2, "op") + OPERATORS[operator](build_source(source, n_rows, tmp_path), n_rows, "op") _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) tables = { label: node.as_table(columns=ALL_COLUMNS) for label, node in job.nodes.items() @@ -153,15 +154,19 @@ def test_zero_row_schema_equals_populated( @pytest.mark.parametrize("orchestrator", ["sync", "async"]) -@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2,))) +@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2, 0))) def test_job_has_no_non_large_type_and_equals_plain( source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path ) -> None: - op_table, other_tables = _job_tables(source, operator, orchestrator, tmp_path) + """With rows and without, a job node equals the same operator on a plain stream.""" + op_table, other_tables = _job_tables( + source, operator, n_rows, orchestrator, tmp_path + ) + assert (op_table.num_rows > 0) == (n_rows > 0) assert non_large_columns(op_table) == [] for table in other_tables: assert non_large_columns(table) == [] - plain = OPERATORS[operator](build_source(source, 2, tmp_path), 2, None) + plain = OPERATORS[operator](build_source(source, n_rows, tmp_path), n_rows, None) assert _user_schema(op_table) == _user_schema(plain.as_table(columns=ALL_COLUMNS)) diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py index 835e99051..a878c41ce 100644 --- a/tests/test_core/sources/test_large_normalization_sources.py +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -24,6 +24,7 @@ ListSource, PollingSource, ) +from orcapod.core.streams import ArrowTableStream from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.operators import SelectTagColumns from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator @@ -108,6 +109,27 @@ def test_user_arrow_table_keeps_width_and_nullability() -> None: assert out.schema.field("s") == pa.field("s", pa.large_string(), nullable=True) +def test_list_view_is_not_supported_today() -> None: + """Pins today's behaviour: ``list_view`` is not a supported type (ITL-1129). + + A source refuses a ``list_view`` column (the Arrow hasher cannot hash it); a raw + ``ArrowTableStream`` holds it as given, untyped and unhashable. Supporting it + (normalized to ``large_list``) is a deliberate later change. + """ + table = pa.table( + { + "t": pa.array(["a"], pa.large_string()), + "lv": pa.array([["x"]], pa.list_view(pa.string())), + } + ) + with pytest.raises(ValueError, match="list_view"): + _ = ArrowTableSource(table, tag_columns=["t"], source_id="lv") + stream = ArrowTableStream(table, tag_columns=["t"]) + assert stream.as_table().schema.field("lv").type == pa.list_view(pa.string()) + with pytest.raises(ValueError, match="list_view"): + _ = stream.content_hash() + + def test_list_source_is_large() -> None: source = ListSource("v", ["a", "b"]) assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == []