diff --git a/.zed/rules b/.zed/rules index b801c0552..a2e1375b5 100644 --- a/.zed/rules +++ b/.zed/rules @@ -712,6 +712,20 @@ side when its inputs' types disagree; a result's type never depends on data valu order or the path the data took; and inferring a type from Python values is a last resort for when no type is known. +Large variants everywhere (developer, November 14, 2026, ITL-1122): every column of every table +Orcapod presents or stores has its string, binary and list types in their large variants +(large_string, large_binary, large_list, recursively), for every supported Arrow type, with +no exception: a user's own Arrow table (ArrowTableSource, a raw ArrowTableStream) and an +explicit Arrow schema= / data_schema= are normalized too. The one place is +ArrowTableStream.__init__ (arrow_utils.normalize_table_to_large_types(table, keep_null=True)), +where every source and every operator result becomes a stream; never add a per-source or +per-operator normalization. Names, nullability (top level and nested), field metadata, widths and +logical (extension) types are untouched, and large and non-large variants hash identically, so no +hash or default source id moves. Not covered: what is inside a library value (numpy, pandas, +polars, torch) keeps its library's own distinction. list_view and large_list_view are not +supported: a source refuses them (the Arrow hasher cannot hash them) and a raw ArrowTableStream +holds them as given (ITL-1129). + Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large for consistency). Infer an Arrow type from Python values only when no Arrow type is known, and @@ -756,6 +770,9 @@ data. The rules: operator that round-trips through Polars casts its columns back to the normalized input types (arrow_utils.restore_column_types, or restore_schema_nullability when nullability is restored too). + This holds for every operator, also one that passes its rows through unchanged (select, + map, a no-op filter, Join, SemiJoin), on a plain stream as inside a job: its result + is an ArrowTableStream (ITL-1122). - List columns built by Batch, GroupBy and MergeJoin are typed by arrow_utils.aggregated_list_type (directly or through build_aggregated_table): large_list, or for a logical T the logical list type from the type converter @@ -851,7 +868,7 @@ data. The rules: nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output is the left - input's schema as it is. The rule is `core/operators/key_nullability.py` + input's schema as it is (its nullability; types in their large variants, ITL-1122). The rule is `core/operators/key_nullability.py` (`reconcile_key_nullability` on tag schemas before they are combined, `key_nullable` for the Arrow field flag, set by re-wrapping arrays, never by a cast). Equal nullability takes the unchanged path; any other type difference, and every data column, behaves as before. diff --git a/CLAUDE.md b/CLAUDE.md index c2c43bbf3..9eef8c1dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -754,6 +754,20 @@ side when its inputs' types disagree; a result's type never depends on data valu order or the path the data took; and inferring a type from Python values is a last resort for when no type is known. +Large variants everywhere (developer, November 14, 2026, ITL-1122): every column of every table +Orcapod presents or stores has its string, binary and list types in their large variants +(`large_string`, `large_binary`, `large_list`, recursively), for every supported Arrow type, with +no exception: a user's own Arrow table (`ArrowTableSource`, a raw `ArrowTableStream`) and an +explicit Arrow `schema=` / `data_schema=` are normalized too. The one place is +`ArrowTableStream.__init__` (`arrow_utils.normalize_table_to_large_types(table, keep_null=True)`), +where every source and every operator result becomes a stream; never add a per-source or +per-operator normalization. Names, nullability (top level and nested), field metadata, widths and +logical (extension) types are untouched, and large and non-large variants hash identically, so no +hash or default source id moves. Not covered: what is inside a library value (numpy, pandas, +polars, torch) keeps its library's own distinction. `list_view` and `large_list_view` are not +supported: a source refuses them (the Arrow hasher cannot hash them) and a raw `ArrowTableStream` +holds them as given (ITL-1129). + Respect the Arrow types the inputs already have, normalized to their large variants (developer, November 10, 2026: large and non-large variants hash identically, so always normalize to large for consistency). Infer an Arrow type from Python values only when no Arrow type is known, and @@ -798,6 +812,9 @@ data. The rules: operator that round-trips through Polars casts its columns back to the normalized input types (`arrow_utils.restore_column_types`, or `restore_schema_nullability` when nullability is restored too). + This holds for every operator, also one that passes its rows through unchanged (select, + map, a no-op filter, `Join`, `SemiJoin`), on a plain stream as inside a job: its result + is an `ArrowTableStream` (ITL-1122). - List columns built by `Batch`, `GroupBy` and `MergeJoin` are typed by `arrow_utils.aggregated_list_type` (directly or through `build_aggregated_table`): `large_list`, or for a logical `T` the logical list type from the type converter @@ -895,7 +912,7 @@ data. The rules: nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output is the left - input's schema as it is. The rule is `core/operators/key_nullability.py` + input's schema as it is (its nullability; types in their large variants, ITL-1122). The rule is `core/operators/key_nullability.py` (`reconcile_key_nullability` on tag schemas before they are combined, `key_nullable` for the Arrow field flag, set by re-wrapping arrays, never by a cast). Equal nullability takes the unchanged path; any other type difference, and every data column, behaves as before. diff --git a/changelog.d/ITL-1122.md b/changelog.d/ITL-1122.md new file mode 100644 index 000000000..cf1fcf8ba --- /dev/null +++ b/changelog.d/ITL-1122.md @@ -0,0 +1,7 @@ +### Breaking changes and migration + +- Every table Orcapod presents or stores has the large Arrow variants of string, binary and list types (`large_string`, `large_binary`, `large_list`, recursively), with no exception (ITL-1122). A `string`, `binary` or `list` column of an Arrow table you give `ArrowTableSource` (or a raw `ArrowTableStream`), or of an Arrow schema you pass as `schema=` to `CSVSource` or `DataFrameSource` or as `data_schema=` to `DictSource`, now comes back as `large_string`, `large_binary` or `large_list`. Names, nullability, widths and logical types are kept. Code that compared an Arrow type to `pa.string()` compares it to `pa.large_string()` now. No hash moves: large and non-large variants hash identically, so row hashes, content hashes, pipeline hashes, record ids and default source ids stay as they were, and a Delta table written with either variant accepts rows of the other. + +### Changed + +- Large types no longer depend on the source kind or the path (ITL-1122). `CSVSource` gave `string` from a file. `CachedSource` gave the types its cache store kept: `string` from an in-memory cache, `large_string` from a Delta cache and with no rows. On a plain stream, the operators that pass rows through unchanged (`SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, `DropDataColumns`, `SelectColumns`, `DropColumns`, `MapTags`, `MapData`, `PolarsFilter` with no predicate, `Join`, `SemiJoin`, and `GroupBy`'s key columns) kept a `string` column, while the same operator inside a job gave `large_string`. `Join` of a `string` and a `large_string` input on one tag gave the type of whichever input came first in its canonical order. All of them now give large types, with rows and without, sync and async. diff --git a/design_issues/ITL-1122-LN1.md b/design_issues/ITL-1122-LN1.md new file mode 100644 index 000000000..0d75f9339 --- /dev/null +++ b/design_issues/ITL-1122-LN1.md @@ -0,0 +1,24 @@ +## Cross-cutting + +### LN1 — Large normalization applied unevenly across sources and operators +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1122 + +The rule (developer, November 14, 2026, replacing an earlier form with exceptions for a user's +Arrow table and for a store that keeps the distinction): every column of every table Orcapod +presents or stores has its string, binary and list types in their large variants, with no +exception. Measured on `main` 3a7f3850: `ArrowTableSource` and an explicit Arrow schema kept +`string`; `CSVSource` gave `string` (a CSV file keeps no distinction); `CachedSource` gave the +types its cache store kept (`string` from an in-memory cache, `large_string` from Delta and with +no rows); on a plain stream, the operators that pass rows through (select, drop, map, a filter +with nothing to filter, `Join`, `SemiJoin`, `GroupBy`'s key) kept a source's `string` while every +operator inside a job gave `large_string`, and `Join` of a `string` and a `large_string` input on +a shared tag took the type of whichever input came first in canonical order. No shipped source +store (CSV, Delta Lake, SQLite, PostgreSQL, SpiralDB) keeps the distinction. + +**Fix:** one place, `ArrowTableStream.__init__`, normalizes the table every source and every +operator result becomes a stream from (`arrow_utils.normalize_table_to_large_types(..., +keep_null=True)`). Swept by `tests/test_core/operators/test_large_normalization.py` and +`tests/test_core/sources/test_large_normalization_sources.py` (every source kind x operator x +plain / job sync / job async x populated / zero-row). diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index c6983495c..4a16b1ed2 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -634,6 +634,10 @@ silent widening; convert explicitly. - **Types are normalized to their large variants.** `string` becomes `large_string`, `binary` becomes `large_binary` and `list` becomes `large_list`, recursively (also inside structs). The Arrow hasher treats each pair as identical, so this never changes a hash. + Every operator does this, also one that passes its rows through unchanged (a select, a + rename, a filter with nothing to filter, a join), on a plain stream as inside a job. Every + source gives large types too, so an operator's inputs already have them (see + [Arrow types of a source](sources.md#arrow-types-of-a-source)). - **Width, nullability and logical types are kept.** An `int8` column stays `int8`, a `float32` column stays `float32`, and a logical type such as `Path` keeps its logical type. - **List columns built by `MergeJoin`, `Batch` and `GroupBy`** are `large_list`, where `T` diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 83d4a8319..f3dba4b6a 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -48,6 +48,17 @@ Data with no rows has no columns to read, so `DictSource(data=[], ...)` raises `ValueError: Table is empty`. To build a source with no rows, pass `data_schema=` so the columns are known. +### Arrow types of a source + +Every source gives the large variants of string, binary and list types (`large_string`, +`large_binary`, `large_list`, recursively), whatever its input: an Arrow table of your own +(`ArrowTableSource`), an Arrow schema you pass (`schema=` or `data_schema=`), Python values, +a data frame, a CSV file or a database table. A `string` column you give comes back as +`large_string`. Only the layout changes: names, nullability, widths (`int8` stays `int8`) and +logical types are kept, and the two variants hash identically, so no hash depends on which +one you gave. The Arrow `list_view` and `large_list_view` types are not supported: a source +refuses a column of either type. + ### SpiralDB tables `SpiralDBTableSource` exposes a table in a [SpiralDB](https://spiraldb.com) project as a diff --git a/src/orcapod/core/operators/filters.py b/src/orcapod/core/operators/filters.py index 5234c4019..55678da84 100644 --- a/src/orcapod/core/operators/filters.py +++ b/src/orcapod/core/operators/filters.py @@ -46,19 +46,6 @@ def __init__( self.constraints = constraints if constraints is not None else {} super().__init__(**kwargs) - def zero_row_result(self, *inputs: StreamProtocol) -> StreamProtocol: - """This filter's result over a zero-row input, for a table with no rows (ITL-912). - - A filter with no predicate and no constraint passes its input through - unaltered (``unary_static_process``), so its result is the input with the types - it has. Any other filter gives its input's columns in the large-normalized - form a run gives them, without evaluating the predicate. - """ - if len(self.predicates) == 0 and len(self.constraints) == 0: - (only,) = inputs - return only - return super().zero_row_result(*inputs) - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: if len(self.predicates) == 0 and len(self.constraints) == 0: logger.info( diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 64b3da113..5084816e0 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -39,6 +39,13 @@ class ArrowTableStream(StreamBase): This is useful for ensuring that the data in the stream remains consistent and unchanging. The types of the tag and data columns are inferred from the PyArrow Table schema. + + Every column takes the large variant of its Arrow type (developer, November 14, + 2026, ITL-1122): ``string`` is ``large_string``, ``binary`` is ``large_binary``, + ``list`` is ``large_list``, recursively. This is the one place a table becomes a + stream, so every source (each builds its stream here, from a user's Arrow table + too) and every operator result presents large types. Width, nullability, field + metadata and logical types are kept; large and non-large variants hash identically. """ # The table's rows are held, not computed: a run() view says "all N rows", as @@ -60,6 +67,9 @@ def __init__( self._producer = producer self._upstreams = upstreams + # Every table Orcapod presents is in its large variants (ITL-1122). + table = arrow_utils.normalize_table_to_large_types(table, keep_null=True) + data_table, data_context_table = arrow_utils.split_by_column_groups( table, [constants.CONTEXT_KEY] ) diff --git a/src/orcapod/utils/arrow_utils.py b/src/orcapod/utils/arrow_utils.py index 5630eec1a..67da68714 100644 --- a/src/orcapod/utils/arrow_utils.py +++ b/src/orcapod/utils/arrow_utils.py @@ -194,7 +194,9 @@ def _normalize_field_to_large_types( return field.with_type(normalize_to_large_types(field.type, keep_null=keep_null)) -def normalize_schema_to_large_types(schema: "pa.Schema") -> "pa.Schema": +def normalize_schema_to_large_types( + schema: "pa.Schema", *, keep_null: bool = False +) -> "pa.Schema": """ Convert a schema to use large variants of data types. @@ -203,6 +205,9 @@ def normalize_schema_to_large_types(schema: "pa.Schema") -> "pa.Schema": Args: schema: Arrow schema to normalize + keep_null: Passed to ``normalize_to_large_types``. ``True`` keeps a + ``null`` column ``null``, as operators and sources do; the default + ``False`` maps it to ``large_string``, as ``Data`` does. Returns: New schema with large type variants, or same schema if no changes needed @@ -220,7 +225,7 @@ def normalize_schema_to_large_types(schema: "pa.Schema") -> "pa.Schema": schema_changed = False for field in schema: - normalized_type = normalize_to_large_types(field.type) + normalized_type = normalize_to_large_types(field.type, keep_null=keep_null) # Check if the type actually changed if normalized_type != field.type: @@ -275,16 +280,22 @@ def shared_arrow_type( return None -def normalize_table_to_large_types(table: "pa.Table") -> "pa.Table": +def normalize_table_to_large_types( + table: "pa.Table", *, keep_null: bool = False +) -> "pa.Table": """ Normalize table schema to use large type variants. Uses cast() which should be zero-copy for large variant conversions since they have identical binary representations, but ensures proper - type validation and handles any edge cases safely. + type validation and handles any edge cases safely. Only layouts change: + width, nullability, field metadata and logical (extension) types are kept. Args: table: Arrow table to normalize + keep_null: Passed to ``normalize_to_large_types``. ``True`` keeps a + ``null`` column ``null``, as operators and sources do; the default + ``False`` maps it to ``large_string``, as ``Data`` does. Returns: Table with normalized schema, or same table if no changes needed @@ -296,10 +307,13 @@ def normalize_table_to_large_types(table: "pa.Table") -> "pa.Table": name: large_string age: int64 """ - normalized_schema = normalize_schema_to_large_types(table.schema) + # ``table.schema`` makes a new Schema object on every access: read it once, + # so the identity check below can find an unchanged schema. + schema = table.schema + normalized_schema = normalize_schema_to_large_types(schema, keep_null=keep_null) # If schema didn't change, return original table - if normalized_schema is table.schema: + if normalized_schema is schema: return table # Use cast() for safety - should be zero-copy for large variant conversions diff --git a/superpowers/plans/2026-11-14-itl-1122-large-normalization.md b/superpowers/plans/2026-11-14-itl-1122-large-normalization.md new file mode 100644 index 000000000..c0579e0df --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1122-large-normalization.md @@ -0,0 +1,127 @@ +# ITL-1122: large normalization applied unevenly — evaluation and plan + +Status: DONE (PR #389). The rule below was REPLACED the same day by the developer: "let's go +ahead and always convert all non-large variants into large variants everywhere" (no exception for +a user's Arrow table or explicit schema). Implemented in one place, `ArrowTableStream.__init__`; +the per-source / per-operator fixes F1-F3 planned below were folded into it and removed. The +measurements below remain the record of what was uneven on `main` 3a7f3850. + +## The rule (developer, November 14, 2026) + +"any arrow table WE create as part of orcapod should always get things normalized to large +variant. However for things where 1) user directly creates arrow table with specific data +types and 2) some source actually make distinction (in recoverable fashion) of large and +non-large variants: for both of these we dont want to normalize the variants between large and +non-large." + +Clauses used below: [O] Orcapod creates the table -> large; [U] the user created the Arrow +table -> kept; [S] the store keeps the distinction recoverably -> kept. + +## Measurement + +Scripts (scratchpad, re-runnable): `measure.py` (source kind x operator x plain / job sync / +job async, populated and zero-row) and `measure2.py` (function pod, DerivedSource, +PollingSource, Spiral fake, store round trips). Base `origin/main` 3a7f3850. + +Columns measured: tag `t: string`, tag `k: int64`, data `s: string`, `b: binary`, +`l: list`, `n: struct, y: string>` (CSV: `t, k, s` only). Source-info +and system-tag columns are large everywhere (Orcapod builds them). + +### Store round trips (write `string` and `large_string`, `binary` and `large_binary`, `list` and `large_list`; read back) + +| Store | Result | Recoverable? | +|---|---|---| +| Delta Lake (deltalake directly) | Delta schema has one `string` / `binary` / `array` type for both; read back both as `string` (or both `large_string` with `as_large_types=True`, which Orcapod uses) | no | +| `DeltaTableDatabase` | all large | no | +| `InMemoryArrowDatabase` | each comes back as written | yes (in memory, not persistent) | +| SQLite | `TEXT` / `BLOB` for both; connector maps to `large_string` / `large_binary` | no | +| PostgreSQL | `text` / `bytea` for both; connector maps to large (by code; no server here) | no | +| SpiralDB (fake models observed ITL-40) | `large_string` written comes back `string`; connector normalizes to large | no | +| CSV file | no type in the file; `read_csv` gives `string` for both | no | + +### Sources (the source's own table = the job's source node, sync and async) + +| Source | Populated | Zero-row | Verdict | +|---|---|---|---| +| `ArrowTableSource` (user's table) | kept (`string`, ...) | kept | CONSISTENT [U] | +| `DictSource` Python rows / Python schema | large | large | CONSISTENT [O] | +| `DictSource(data_schema=pa.Schema)` with `string` | kept | kept | Q2 (CONSISTENT [U] if an explicit Arrow schema counts) | +| `ListSource` | large | n/a | CONSISTENT [O] | +| `CSVSource` (no schema) | `string` | ERROR (header-only CSV: null-typed columns; separate bug) | INCONSISTENT (CSV keeps no distinction) | +| `CSVSource(schema=pa.Schema)` with `string` | kept | kept | Q2 | +| `DataFrameSource` polars / pandas | large | large | CONSISTENT [O] | +| `DataFrameSource(pa.Table)` with `string` | large (through Polars) | large | Q3 | +| `DeltaTableSource` | large | large | CONSISTENT [O] (not recoverable) | +| `SQLiteTableSource` / `PostgreSQLTableSource` / `DBTableSource` | large | SQLite: "Table is empty" by design | CONSISTENT [O] | +| `SpiralDBTableSource` (fake) | large | - | CONSISTENT [O] | +| `CachedSource` over a `string` source, `InMemoryArrowDatabase` | kept `string` | large | INCONSISTENT (zero-row != populated) | +| `CachedSource` over a `string` source, `DeltaTableDatabase` | large | large | INCONSISTENT with the in-memory cache: type depends on the cache store | +| `DerivedSource` (function node, operator node, `tags_as_data`) | large | large | CONSISTENT [O] | +| `PollingSource` (fetch dict or polars) | large | - | CONSISTENT [O]; fetch returning a `pa.Table`: Q3 | +| Function pod output (incl. passed-through tags), plain and job | large | large | CONSISTENT [O] | + +### Operators over a non-large input (`ArrowTableSource`, `DictSource(pa.Schema)`, `CSVSource`, in-memory `CachedSource`) + +Inside a job (sync and async): every operator gives large. CONSISTENT [O]. + +On a plain stream (populated and zero-row identical): + +* Already large: `PolarsFilter` with a predicate, `DropNulls` (any form), `MergeJoin`, `Batch`, + `GroupBy` (member columns), `Index`. +* Pass non-large through: `SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, + `DropDataColumns`, `SelectColumns`, `DropColumns`, `MapTags`, `MapData`, `PolarsFilter()` + (no-op), `Join` (every column), `SemiJoin`, `GroupBy`'s key tag (`GroupBy(by=["t"])`: `t` + stays `string`). +* `Join` of a large input and a `string` input on shared tag `t` picks a side, and which side + depends on the canonical input order (pipeline hash): `DictSource ⋈ ArrowTableSource` gives + `large_string`, `DataFrameSource ⋈ ArrowTableSource` gives `string`. + +Verdict: INCONSISTENT (given Q1 = large): an operator's output is a table Orcapod creates; the +same operator gives different types inside and outside a job, and operators disagree among +themselves on the plain path. + +## Questions to the coordinator + +* Q1 (open): a pass-through operator (select, map, no-op filter, join, semijoin) over a user's + Arrow table: large, or the user's types? Recommendation: large. +* Q2: an explicit Arrow schema (`DictSource(data_schema=pa.Schema)`, `CSVSource(schema=)`, + `DataFrameSource(schema=)`) with `string`: the user chose the Arrow types, so kept [U]? + Recommendation: kept. +* Q3: `DataFrameSource` given a `pa.Table`, `PollingSource` whose fetch returns a `pa.Table`: + both convert through Polars, so large. Recommendation: leave (Polars-frame sources; + `ArrowTableSource` is the source that keeps a user's Arrow types). + +## Proposed fixes (one place each) + +* F1 operators: one helper applied to every operator's sync result in the three base + `static_process` overrides (`UnaryOperator`, `BinaryOperator`, `NonZeroInputOperator`), + beside `without_input_meta`: if any column of the result's table (with source, system-tag + and context columns) differs from `normalize_to_large_types(type, keep_null=True)`, rebuild + the `ArrowTableStream` with those columns re-laid out to the large type + (`arrow_utils.restore_column_types`, as `filters.py` does). Width, nullability, field + metadata and extension types are untouched (`normalize_to_large_types` keeps them). The + async/job path is already large; tests pin it. +* F2 `CSVSource`: normalize the table `read_csv` gives when no `schema=` is passed. +* F3 `CachedSource`: normalize the table it presents in `_build_merged_stream` (read side only; + stored rows unchanged), so in-memory == Delta and zero-row == populated. + +What moves: no hash (large and non-large hash identically; golden suite and a direct test). No +stored schema: operator caches are written only inside jobs (already large); `CachedSource` +writes as before; Delta keeps no distinction. To be confirmed by a test that appends after the +change to a table written before it. + +## Out of scope findings (report in worker_done) + +* A header-only CSV (zero rows) cannot be read: columns are typed `null` and + `infer_schema_nullable` marks them non-null -> `ValueError: A null type field may not be + non-nullable`. + +## Implementation steps (after the answers) + +1. Failing tests: `tests/test_core/operators/test_large_normalization.py` (every operator, + plain stream and job sync/async, populated == zero-row), CSVSource, CachedSource, hash + equality. +2. F1, F2, F3. +3. Golden suite, full suite, stored-schema append check. +4. CLAUDE.md + `.zed/rules` "Arrow types", docs, `changelog.d/ITL-1122.md`, + `design_issues/ITL-1122-LN1.md`. diff --git a/tests/test_core/large_types_cases.py b/tests/test_core/large_types_cases.py new file mode 100644 index 000000000..d7a67332c --- /dev/null +++ b/tests/test_core/large_types_cases.py @@ -0,0 +1,281 @@ +"""Shared cases for the large-type tests (ITL-1122): every source kind, built from non-large data. + +The rule (developer, November 14, 2026): every column of every table Orcapod presents or +stores has its string, binary and list types in their large variants, recursively, with no +exception for a user's Arrow table or an explicit Arrow schema. These helpers build each +source kind from data that holds ``string``, ``binary``, ``list`` and a nested +``struct, string>`` wherever the kind can carry them, and find any non-large +type in a table by walking its Arrow types. +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import deltalake +import polars as pl +import pyarrow as pa +import pyarrow.csv as pa_csv + +from orcapod.core.sources import ( + ArrowTableSource, + CachedSource, + CSVSource, + DataFrameSource, + DeltaTableSource, + DictSource, + SQLiteTableSource, +) +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.protocols.core_protocols import StreamProtocol + +ALL_COLUMNS = {"source": True, "system_tags": True, "context": True} + +NESTED = pa.struct([pa.field("x", pa.list_(pa.string())), pa.field("y", pa.string())]) +FULL_SCHEMA = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("k", pa.int64(), nullable=False), + pa.field("s", pa.string(), nullable=False), + pa.field("u", pa.string(), nullable=False), + pa.field("b", pa.binary(), nullable=False), + pa.field("l", pa.list_(pa.string()), nullable=False), + pa.field("n", NESTED, nullable=False), + ] +) +ROWS: list[dict[str, Any]] = [ + { + "t": "a", + "k": 0, + "s": "x", + "u": "p", + "b": b"1", + "l": ["p"], + "n": {"x": ["q"], "y": "r"}, + }, + { + "t": "b", + "k": 1, + "s": "y", + "u": "q", + "b": b"2", + "l": ["p", "q"], + "n": {"x": [], "y": "s"}, + }, +] +TAGS = ["t", "k"] + + +def non_large_paths(arrow_type: pa.DataType, path: str) -> list[str]: + """Name every place in ``arrow_type`` that holds a non-large string, binary or list. + + Walks list items, struct children, map keys and items, dictionary values and + extension storage, so a ``string`` nested at any depth is found. Written without + ``normalize_to_large_types``, so it checks that function's work rather than repeat it. + + Args: + arrow_type: The type to walk. + path: The name of the place ``arrow_type`` is at, for the result. + + Returns: + ``": "`` for each non-large place, outermost first. + """ + found: list[str] = [] + if pa.types.is_string(arrow_type) or pa.types.is_binary(arrow_type): + found.append(f"{path}: {arrow_type}") + elif pa.types.is_list(arrow_type): + found.append(f"{path}: {arrow_type}") + found += non_large_paths(arrow_type.value_type, f"{path}[]") + elif pa.types.is_large_list(arrow_type) or pa.types.is_fixed_size_list(arrow_type): + found += non_large_paths(arrow_type.value_type, f"{path}[]") + elif pa.types.is_struct(arrow_type): + for field in arrow_type: + found += non_large_paths(field.type, f"{path}.{field.name}") + elif pa.types.is_map(arrow_type): + found += non_large_paths(arrow_type.key_type, f"{path}{{key}}") + found += non_large_paths(arrow_type.item_type, f"{path}{{item}}") + elif pa.types.is_dictionary(arrow_type): + found += non_large_paths(arrow_type.value_type, f"{path}{{value}}") + elif isinstance(arrow_type, pa.ExtensionType): + found += non_large_paths(arrow_type.storage_type, f"{path}") + return found + + +def non_large_columns(table: pa.Table) -> list[str]: + """Every non-large place in every column of ``table`` (``non_large_paths``).""" + found: list[str] = [] + for field in table.schema: + found += non_large_paths(field.type, field.name) + return found + + +def _rows(n_rows: int, columns: list[str]) -> list[dict[str, Any]]: + return [{name: row[name] for name in columns} for row in ROWS[:n_rows]] + + +def _table(n_rows: int, columns: list[str] | None = None) -> pa.Table: + """The rows as a user's Arrow table in the non-large types of ``FULL_SCHEMA``.""" + names = columns or FULL_SCHEMA.names + schema = pa.schema([FULL_SCHEMA.field(name) for name in names]) + return pa.Table.from_pylist(_rows(n_rows, names), schema=schema) + + +SourceFactory = Callable[[int, Path], StreamProtocol] + + +def _arrow_table_source(n_rows: int, tmp: Path) -> StreamProtocol: + return ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="arrow") + + +def _arrow_table_stream(n_rows: int, tmp: Path) -> StreamProtocol: + return ArrowTableStream(_table(n_rows), tag_columns=TAGS) + + +def _dict_source_python(n_rows: int, tmp: Path) -> StreamProtocol: + columns = ["t", "k", "s", "u", "b", "l"] + if n_rows == 0: + return DictSource( + [], + tag_columns=TAGS, + data_schema={ + "t": str, + "k": int, + "s": str, + "u": str, + "b": bytes, + "l": list[str], + }, + source_id="dict", + ) + return DictSource(_rows(n_rows, columns), tag_columns=TAGS, source_id="dict") + + +def _dict_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: + return DictSource( + _rows(n_rows, FULL_SCHEMA.names), + tag_columns=TAGS, + data_schema=FULL_SCHEMA, + source_id="dict_schema", + ) + + +def _csv_path(n_rows: int, tmp: Path) -> Path: + path = tmp / f"rows{n_rows}.csv" + pa_csv.write_csv(_table(n_rows, ["t", "k", "s", "u"]), path) + return path + + +def _csv_source(n_rows: int, tmp: Path) -> StreamProtocol: + return CSVSource(str(_csv_path(n_rows, tmp)), tag_columns=TAGS, source_id="csv") + + +def _csv_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: + return CSVSource( + str(_csv_path(n_rows, tmp)), + tag_columns=TAGS, + source_id="csv_schema", + schema=pa.schema([FULL_SCHEMA.field(name) for name in ["t", "k", "s", "u"]]), + ) + + +def _data_frame_source_polars(n_rows: int, tmp: Path) -> StreamProtocol: + return DataFrameSource( + pl.from_arrow(_table(n_rows)), tag_columns=TAGS, source_id="polars" + ) + + +def _data_frame_source_pandas(n_rows: int, tmp: Path) -> StreamProtocol: + # No bytes column: an empty pandas object column has no values to tell bytes + # from str, so a zero-row frame could not give the populated frame's type. + return DataFrameSource( + _table(n_rows, ["t", "k", "s", "u"]).to_pandas(), + tag_columns=TAGS, + source_id="pandas", + ) + + +def _data_frame_source_arrow_table(n_rows: int, tmp: Path) -> StreamProtocol: + return DataFrameSource(_table(n_rows), tag_columns=TAGS, source_id="df_arrow") + + +def _data_frame_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: + return DataFrameSource( + pl.from_arrow(_table(n_rows)), + tag_columns=TAGS, + source_id="df_schema", + schema=FULL_SCHEMA, + ) + + +def _delta_table_source(n_rows: int, tmp: Path) -> StreamProtocol: + path = tmp / f"delta{n_rows}" + if not path.exists(): + deltalake.write_deltalake(str(path), _table(n_rows)) + return DeltaTableSource(path, tag_columns=TAGS, source_id="delta") + + +def _sqlite_table_source(n_rows: int, tmp: Path) -> StreamProtocol: + path = tmp / f"rows{n_rows}.sqlite" + if not path.exists(): + connection = sqlite3.connect(path) + _ = connection.execute( + "CREATE TABLE r (t TEXT NOT NULL, k INTEGER NOT NULL, s TEXT NOT NULL, " + "u TEXT NOT NULL, b BLOB NOT NULL, PRIMARY KEY (t, k))" + ) + for row in ROWS[:n_rows]: + _ = connection.execute( + "INSERT INTO r VALUES (?, ?, ?, ?, ?)", + (row["t"], row["k"], row["s"], row["u"], row["b"]), + ) + connection.commit() + connection.close() + return SQLiteTableSource(path, "r", tag_columns=TAGS, source_id="sqlite") + + +def _cached_source_in_memory(n_rows: int, tmp: Path) -> StreamProtocol: + return CachedSource( + ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="cached_mem"), + cache_database=InMemoryArrowDatabase(), + ) + + +def _cached_source_delta(n_rows: int, tmp: Path) -> StreamProtocol: + return CachedSource( + ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="cached_delta"), + cache_database=DeltaTableDatabase(base_path=tmp / f"cache{n_rows}"), + ) + + +# Every source kind that carries tags ``t``, ``k`` and data ``s``, ``u`` (so every +# operator applies), with the row counts it can be built with. A header-only CSV and +# an empty SQLite table cannot be read (separate issues), so those kinds have rows. +SOURCES: dict[str, tuple[SourceFactory, tuple[int, ...]]] = { + "ArrowTableSource": (_arrow_table_source, (2, 0)), + "ArrowTableStream": (_arrow_table_stream, (2, 0)), + "DictSource(python)": (_dict_source_python, (2, 0)), + "DictSource(arrow schema)": (_dict_source_arrow_schema, (2, 0)), + "CSVSource": (_csv_source, (2,)), + "CSVSource(arrow schema)": (_csv_source_arrow_schema, (2, 0)), + "DataFrameSource(polars)": (_data_frame_source_polars, (2, 0)), + "DataFrameSource(pandas)": (_data_frame_source_pandas, (2, 0)), + "DataFrameSource(arrow table)": (_data_frame_source_arrow_table, (2, 0)), + "DataFrameSource(arrow schema)": (_data_frame_source_arrow_schema, (2, 0)), + "DeltaTableSource": (_delta_table_source, (2, 0)), + "SQLiteTableSource": (_sqlite_table_source, (2,)), + "CachedSource(in memory)": (_cached_source_in_memory, (2, 0)), + "CachedSource(delta)": (_cached_source_delta, (2, 0)), +} + +SOURCE_CASES = [ + (name, n_rows) for name, (_, counts) in SOURCES.items() for n_rows in counts +] + + +def build_source(name: str, n_rows: int, tmp: Path) -> StreamProtocol: + """Build the source kind ``name`` with ``n_rows`` rows, its files under ``tmp``.""" + factory, _ = SOURCES[name] + return factory(n_rows, tmp) diff --git a/tests/test_core/operators/test_aggregated_list_types.py b/tests/test_core/operators/test_aggregated_list_types.py index 3b115bca2..00de01aee 100644 --- a/tests/test_core/operators/test_aggregated_list_types.py +++ b/tests/test_core/operators/test_aggregated_list_types.py @@ -171,8 +171,8 @@ def _row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: ] -def _in_list_layout(stream: StreamProtocol) -> ArrowTableStream: - """Rebuild ``stream`` with every top-level ``large_list`` column as ``list``. +def _list_layout_table(stream: StreamProtocol) -> pa.Table: + """Return ``stream``'s table with every top-level ``large_list`` column as ``list``. This is the layout ``Batch`` and ``GroupBy`` emitted before ITL-832 (``pa.list_(input type)``); logical list columns were already extension @@ -187,8 +187,16 @@ def _in_list_layout(stream: StreamProtocol) -> ArrowTableStream: for field in table.schema ] ) + return table.cast(schema) + + +def _in_list_layout(stream: StreamProtocol) -> ArrowTableStream: + """Rebuild ``stream`` from its ``list``-layout table (``_list_layout_table``). + + The stream presents ``large_list`` again (ITL-1122). + """ tag_columns, _ = stream.keys() - return ArrowTableStream(table.cast(schema), tag_columns=tag_columns) + return ArrowTableStream(_list_layout_table(stream), tag_columns=tag_columns) class TestAggregatedContentHashes: @@ -199,9 +207,19 @@ def test_row_hashes_match_the_list_layout( ) -> None: """Every row hashes the same in the ``large_list`` and ``list`` layouts.""" _, _, result = aggregated + # A stream presents large layouts only (ITL-1122): the old layout exists as a + # table, which the Arrow hasher hashes as the new one, row by row. + old_table = _list_layout_table(result) + new_table = result.as_table(columns={"source": True, "system_tags": True}) + assert pa.types.is_list(old_table.schema.field("n").type) + assert pa.types.is_large_list(new_table.schema.field("n").type) + hasher = result.data_context.arrow_hasher + for i in range(new_table.num_rows): + assert hasher.hash_table(old_table.slice(i, 1)) == hasher.hash_table( + new_table.slice(i, 1) + ) before = _in_list_layout(result) - assert pa.types.is_list(before.as_table().schema.field("n").type) - assert pa.types.is_large_list(result.as_table().schema.field("n").type) + assert pa.types.is_large_list(before.as_table().schema.field("n").type) assert _row_hashes(result) == _row_hashes(before) def test_stream_content_hash_matches_the_list_layout( diff --git a/tests/test_core/operators/test_large_normalization.py b/tests/test_core/operators/test_large_normalization.py new file mode 100644 index 000000000..5abc10c86 --- /dev/null +++ b/tests/test_core/operators/test_large_normalization.py @@ -0,0 +1,251 @@ +"""No operator output has a non-large Arrow type, from any source, on any path (ITL-1122). + +The rule (developer, November 14, 2026): every column of every table Orcapod presents or +stores has its string, binary and list types in their large variants, recursively, with no +exception for a user's Arrow table. So every operator over every source kind gives large +types: on a plain stream and inside a job, sync and async, with rows and without, and a +zero-row result has the schema of a populated one. Large and non-large variants hash +identically, so no hash moves. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import polars as pl +import pyarrow as pa +import pytest + +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.operators import ( + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + GroupBy, + Index, + Join, + MapData, + MapTags, + MergeJoin, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, +) +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol +from tests.test_core.large_types_cases import ( + ALL_COLUMNS, + FULL_SCHEMA, + ROWS, + SOURCE_CASES, + SOURCES, + TAGS, + build_source, + non_large_columns, +) + +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + + +def _other(n_rows: int, data_column: str) -> ArrowTableSource: + """A second user's table: tag ``t`` and one data column, both ``string``.""" + table = pa.table( + { + "t": pa.array(["a", "b"][:n_rows], pa.string()), + data_column: pa.array(["u", "v"][:n_rows], pa.string()), + } + ) + table = table.cast( + pa.schema([pa.field(field.name, field.type, False) for field in table.schema]) + ) + return ArrowTableSource(table, tag_columns=["t"], source_id=f"other_{data_column}") + + +Apply = Callable[[StreamProtocol, int, str | None], StreamProtocol] + +OPERATORS: dict[str, Apply] = { + "SelectTagColumns": lambda s, n, label: SelectTagColumns(["t"])(s, label=label), + "DropTagColumns": lambda s, n, label: DropTagColumns(["k"])(s, label=label), + "SelectDataColumns": lambda s, n, label: SelectDataColumns(["s"])(s, label=label), + "DropDataColumns": lambda s, n, label: DropDataColumns(["u"])(s, label=label), + "SelectColumns": lambda s, n, label: SelectColumns(["t", "k", "s"])(s, label=label), + "DropColumns": lambda s, n, label: DropColumns(["u"])(s, label=label), + "MapTags": lambda s, n, label: MapTags({"t": "t2"})(s, label=label), + "MapData": lambda s, n, label: MapData({"s": "s2"})(s, label=label), + "PolarsFilter(predicate)": lambda s, n, label: PolarsFilter( + predicates=[pl.col("k") >= 0] + )(s, label=label), + "PolarsFilter()": lambda s, n, label: PolarsFilter()(s, label=label), + "DropNulls()": lambda s, n, label: DropNulls()(s, label=label), + "DropNulls([s])": lambda s, n, label: DropNulls(["s"])(s, label=label), + "Join": lambda s, n, label: Join()(s, _other(n, "z"), label=label), + "MergeJoin": lambda s, n, label: MergeJoin()(s, _other(n, "s"), label=label), + "SemiJoin": lambda s, n, label: SemiJoin()(s, _other(n, "z"), label=label), + "Batch": lambda s, n, label: Batch(batch_size=2)(s, label=label), + "GroupBy(t)": lambda s, n, label: GroupBy(by=["t"])(s, label=label), + "GroupBy(k)": lambda s, n, label: GroupBy(by=["k"])(s, label=label), +} + + +def _cases(row_counts: tuple[int, ...]) -> list[tuple[str, str, int]]: + """Every (source kind, operator, rows) with rows among ``row_counts``.""" + return [ + (source, operator, n_rows) + for source, n_rows in SOURCE_CASES + if n_rows in row_counts + for operator in OPERATORS + ] + + +def _job_tables( + source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path +) -> tuple[pa.Table, list[pa.Table]]: + """The operator's node table, and every other node's, after a job run over + inputs of ``n_rows`` rows.""" + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + OPERATORS[operator](build_source(source, n_rows, tmp_path), n_rows, "op") + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + tables = { + label: node.as_table(columns=ALL_COLUMNS) for label, node in job.nodes.items() + } + return tables.pop("op"), list(tables.values()) + + +def _user_schema(table: pa.Table) -> pa.Schema: + """The schema of ``table``'s user tag and data columns.""" + return table.select([name for name in table.column_names if name[0] != "_"]).schema + + +@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2, 0))) +def test_plain_stream_has_no_non_large_type( + source: str, operator: str, n_rows: int, tmp_path: Path +) -> None: + out = OPERATORS[operator](build_source(source, n_rows, tmp_path), n_rows, None) + assert non_large_columns(out.as_table(columns=ALL_COLUMNS)) == [] + assert non_large_columns(out.head(columns=ALL_COLUMNS).table) == [] + + +@pytest.mark.parametrize( + ("source", "operator"), + [(source, operator) for source, operator, _ in _cases((0,))], +) +def test_zero_row_schema_equals_populated( + source: str, operator: str, tmp_path: Path +) -> None: + populated = OPERATORS[operator](build_source(source, 2, tmp_path), 2, None) + empty = OPERATORS[operator](build_source(source, 0, tmp_path), 0, None) + populated_table = populated.as_table(columns=ALL_COLUMNS) + empty_table = empty.as_table(columns=ALL_COLUMNS) + # The user tag and data columns: which system columns a zero-row result + # carries is ITL-912's subject, not this rule's. + assert _user_schema(empty_table) == _user_schema(populated_table) + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2, 0))) +def test_job_has_no_non_large_type_and_equals_plain( + source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path +) -> None: + """With rows and without, a job node equals the same operator on a plain stream.""" + op_table, other_tables = _job_tables( + source, operator, n_rows, orchestrator, tmp_path + ) + assert (op_table.num_rows > 0) == (n_rows > 0) + assert non_large_columns(op_table) == [] + for table in other_tables: + assert non_large_columns(table) == [] + plain = OPERATORS[operator](build_source(source, n_rows, tmp_path), n_rows, None) + assert _user_schema(op_table) == _user_schema(plain.as_table(columns=ALL_COLUMNS)) + + +@pytest.mark.parametrize("operator", list(OPERATORS)) +def test_row_hashes_equal_for_large_and_non_large_input( + operator: str, tmp_path: Path +) -> None: + """A ``string`` source and a ``large_string`` source of the same values give equal + row, content and pipeline hashes after every operator.""" + small = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=FULL_SCHEMA), tag_columns=TAGS + ) + large_schema = pa.schema( + [ + field.with_type( + pa.large_string() + if field.type == pa.string() + else pa.large_binary() + if field.type == pa.binary() + else pa.large_list(pa.large_string()) + if field.type == pa.list_(pa.string()) + else pa.struct( + [ + pa.field("x", pa.large_list(pa.large_string())), + pa.field("y", pa.large_string()), + ] + ) + if field.name == "n" + else field.type + ) + for field in FULL_SCHEMA + ] + ) + large = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=large_schema), tag_columns=TAGS + ) + small_out = OPERATORS[operator](small, 2, None) + large_out = OPERATORS[operator](large, 2, None) + assert small_out.content_hash() == large_out.content_hash() + assert small_out.pipeline_hash() == large_out.pipeline_hash() + + def row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: + return sorted( + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in stream.iter_data() + ) + + assert row_hashes(small_out) == row_hashes(large_out) + + +def test_index_over_a_list_column_is_large(tmp_path: Path) -> None: + """``Index`` reads a list column (the kinds above that carry one: ArrowTableSource).""" + for n_rows in (2, 0): + out = Index("l", 0)(build_source("ArrowTableSource", n_rows, tmp_path)) + assert non_large_columns(out.as_table(columns=ALL_COLUMNS)) == [] + + +class TestJoinNeverPicksASide: + """A shared tag that is ``string`` in one user table and ``large_string`` in the + other is ``large_string``, whichever comes first.""" + + @pytest.mark.parametrize("n_rows", [2, 0]) + def test_either_argument_order_gives_large(self, n_rows: int) -> None: + large = ArrowTableSource( + pa.table( + {"t": pa.array(["a", "b"][:n_rows], pa.large_string())} + ).append_column("w", pa.array([1, 2][:n_rows], pa.int64())), + tag_columns=["t"], + source_id="large", + ) + small = _other(n_rows, "z") + forward = Join()(large, small).as_table(columns=ALL_COLUMNS) + backward = Join()(small, large).as_table(columns=ALL_COLUMNS) + assert forward.schema.field("t").type == pa.large_string() + assert backward.schema.field("t").type == pa.large_string() + assert non_large_columns(forward) == [] + assert non_large_columns(backward) == [] + + +def test_every_source_kind_is_swept() -> None: + """The sweep covers every source kind of ``large_types_cases``.""" + assert {source for source, _ in SOURCE_CASES} == set(SOURCES) diff --git a/tests/test_core/operators/test_merge_join_arrow_types.py b/tests/test_core/operators/test_merge_join_arrow_types.py index 53721091d..5c7a83a41 100644 --- a/tests/test_core/operators/test_merge_join_arrow_types.py +++ b/tests/test_core/operators/test_merge_join_arrow_types.py @@ -33,7 +33,7 @@ from orcapod.contexts import get_default_context, get_default_type_converter from orcapod.hashing.arrow_hashers import StarfixArrowHasher from orcapod.core.operators import MergeJoin, PolarsFilter -from orcapod.core.sources import CSVSource, DictSource +from orcapod.core.sources import DictSource from orcapod.core.sources.arrow_table_source import ArrowTableSource from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError @@ -517,18 +517,25 @@ def test_shared_tag_with_other_difference_raises( with pytest.raises(InputValidationError, match=r"'k'"): MergeJoin().static_process(first, second) - def test_csv_and_dict_sources_on_a_string_column(self, tmp_path: Path) -> None: - """A CSV ``string`` column colliding with a dict ``large_string`` one merges.""" - csv_path = tmp_path / "labels.csv" - csv_path.write_text("id,label\n1,a\n2,b\n") - csv_source = CSVSource(str(csv_path), tag_columns=["id"]) + def test_user_string_and_dict_sources_on_a_string_column(self) -> None: + """A user's ``string`` column colliding with a dict one merges. + + The ``string`` / ``large_string`` meeting of ITL-829 no longer reaches an + operator: every stream presents the large variant (ITL-1122), so the user's + ``string`` column is ``large_string`` at its source already. + """ + user_source = ArrowTableSource( + _table({"id": (pa.int64(), [1, 2]), "label": (pa.string(), ["a", "b"])}), + tag_columns=["id"], + source_id="user", + ) dict_source = DictSource( [{"id": 1, "label": "x"}, {"id": 2, "label": "y"}], tag_columns=["id"] ) - assert csv_source.as_table().schema.field("label").type == pa.string() + assert user_source.as_table().schema.field("label").type == pa.large_string() assert dict_source.as_table().schema.field("label").type == pa.large_string() - table = MergeJoin()(csv_source, dict_source).as_table() + table = MergeJoin()(user_source, dict_source).as_table() assert table.schema.field("label").type == pa.large_list(pa.large_string()) labels = table.sort_by("id").column("label").to_pylist() assert labels == [["a", "x"], ["b", "y"]] @@ -738,10 +745,10 @@ def _small_layout(arrow_type: pa.DataType) -> pa.DataType: return arrow_type -def _relayout( +def _relayout_table( stream: StreamProtocol, overrides: dict[str, pa.DataType] | None = None -) -> ArrowTableStream: - """Rebuild ``stream`` in the small layouts, with optional per-column types. +) -> pa.Table: + """Return ``stream``'s table in the small layouts, with optional per-column types. The small layouts are what MergeJoin emitted for a merged column of ``int64`` or ``string`` values before this change (``list<...>``). @@ -754,8 +761,19 @@ def _relayout( for field in table.schema ] ) + return table.cast(schema) + + +def _relayout( + stream: StreamProtocol, overrides: dict[str, pa.DataType] | None = None +) -> ArrowTableStream: + """Rebuild ``stream`` from its table in the small layouts (``_relayout_table``). + + The stream presents the large layouts again (ITL-1122); what stays is any width + an override changes. + """ tag_columns, _ = stream.keys() - return ArrowTableStream(table.cast(schema), tag_columns=tag_columns) + return ArrowTableStream(_relayout_table(stream, overrides), tag_columns=tag_columns) def _stream_hash(stream: StreamProtocol) -> str: @@ -848,8 +866,16 @@ def test_layout_change_keeps_merge_join_hashes( """ values = [1, 2] if arrow_type == pa.int64() else ["a", "b"] result = MergeJoin().static_process(*_pair(arrow_type, arrow_type, values, 2)) + # A stream presents large layouts only (ITL-1122): the old layout exists as a + # table, which the Arrow hasher hashes as the new one. + old_table = _relayout_table(result) + assert pa.types.is_list(old_table.schema.field("x").type) + hasher = _starfix_hasher() + assert hasher.hash_table(old_table) == hasher.hash_table( + result.as_table(columns={"source": True, "system_tags": True}) + ) old_layout = _relayout(result) - assert pa.types.is_list(old_layout.as_table().schema.field("x").type) + assert pa.types.is_large_list(old_layout.as_table().schema.field("x").type) assert _row_hashes(result) == _row_hashes(old_layout) assert _stream_hash(result) == _stream_hash(old_layout) diff --git a/tests/test_core/operators/test_semijoin_nested.py b/tests/test_core/operators/test_semijoin_nested.py index 7d8d3c6d8..a2fdbff14 100644 --- a/tests/test_core/operators/test_semijoin_nested.py +++ b/tests/test_core/operators/test_semijoin_nested.py @@ -5,11 +5,13 @@ field"), so a list, large_list, struct or struct-backed logical column on either input failed. ITL-829 (first case): Acero also rejected a key that is ``string`` on one side and ``large_string`` on the other ("Incompatible data types for -corresponding join field keys"), e.g. a ``CSVSource`` against a ``DictSource``. +corresponding join field keys"), e.g. a ``CSVSource`` against a ``DictSource``; since +ITL-1122 every stream presents the large variant, so that meeting no longer occurs. The result is the left input's rows that match a right row, in left order, with -the left input's Arrow schema exactly (types, top-level and nested nullability, -logical types). The sync and async paths yield the same rows. +the left input's Arrow schema in its large variants (ITL-1122: ``string`` is +``large_string``, ``list`` is ``large_list``; top-level and nested nullability and +logical types exactly as they were). The sync and async paths yield the same rows. """ from __future__ import annotations @@ -26,10 +28,11 @@ from orcapod.channels import Channel from orcapod.core.operators import SemiJoin -from orcapod.core.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.core.sources import ArrowTableSource, DictSource from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.utils import arrow_utils from tests.test_core.operators.test_async_sync_equivalence import ( STRUCT, Column, @@ -146,7 +149,9 @@ def expected_rows( def assert_semijoin_result( left: StreamProtocol, right: StreamProtocol, expected: pa.Table ) -> None: - """The sync result equals ``expected`` exactly, and async yields the same rows.""" + """The sync result equals ``expected`` in its large variants (ITL-1122), and async + yields the same rows.""" + expected = arrow_utils.normalize_table_to_large_types(expected, keep_null=True) result = SemiJoin()(left, right) table = result.as_table(columns=ALL_COLUMNS) assert table.schema == expected.schema @@ -258,8 +263,11 @@ def test_nested_non_nullable_fields_keep_their_flag() -> None: nullable=("s",), ) result = SemiJoin()(left, scalar_right()).as_table(columns=ALL_COLUMNS) - assert result.schema.field("l").type == items - assert result.schema.field("s").type == pa.string() + # The large variants (ITL-1122); the item keeps its non-null flag. + assert result.schema.field("l").type == pa.large_list( + pa.field("item", pa.int8(), nullable=False) + ) + assert result.schema.field("s").type == pa.large_string() assert result.schema.field("st").type == strict_struct assert result.column("k").to_pylist() == [2] @@ -352,24 +360,41 @@ def test_null_scalar_key_never_matches() -> None: # ── key types ─────────────────────────────────────────────────────────────── -def test_string_vs_large_string_key_csv_vs_dict(tmp_path: Path) -> None: - """ITL-829 (first case): a CSVSource's ``string`` key against a DictSource's - ``large_string`` key, in both directions.""" - csv_path = tmp_path / "a.csv" - _ = csv_path.write_text("id,x\nc,3\na,1\nb,2\n") - csv = CSVSource(str(csv_path), tag_columns=["id"]) +def test_string_vs_large_string_key_user_vs_dict() -> None: + """ITL-829 (first case): a user's ``string`` key against a DictSource's key, in + both directions. + + The ``string`` / ``large_string`` meeting no longer reaches an operator: every + stream presents the large variant (ITL-1122), so the user's ``string`` key is + ``large_string`` at its source already. + """ + user_source = _source( + { + "id": pa.array(["c", "a", "b"], pa.string()), + "x": pa.array([3, 1, 2], pa.int64()), + }, + tags=("id",), + source_id="user", + ) dict_source = DictSource( data=[{"id": "a", "y": 1}, {"id": "c", "y": 3}], tag_columns=["id"] ) - assert csv.as_table().schema.field("id").type == pa.string() + assert user_source.as_table().schema.field("id").type == pa.large_string() assert dict_source.as_table().schema.field("id").type == pa.large_string() assert_semijoin_result( - csv, dict_source, expected_rows(csv, lambda row: row["id"] in ("a", "c")) + user_source, + dict_source, + expected_rows(user_source, lambda row: row["id"] in ("a", "c")), + ) + assert_semijoin_result( + dict_source, user_source, expected_rows(dict_source, lambda _: True) ) - assert_semijoin_result(dict_source, csv, expected_rows(dict_source, lambda _: True)) # The stream convenience method takes the same path. - assert csv.semi_join(dict_source).as_table().column("id").to_pylist() == ["c", "a"] + assert user_source.semi_join(dict_source).as_table().column("id").to_pylist() == [ + "c", + "a", + ] def mismatched_key_sources() -> tuple[ArrowTableSource, ArrowTableSource]: diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py new file mode 100644 index 000000000..a878c41ce --- /dev/null +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -0,0 +1,340 @@ +"""Every source presents large Arrow types, with no exception (ITL-1122). + +The rule (developer, November 14, 2026): every column of every table Orcapod presents or +stores has its string, binary and list types in their large variants, recursively. A +user's own Arrow table and an explicit Arrow schema are normalized too. The one place is +where a table becomes a stream (``ArrowTableStream``), so a source, its job node, its +head and a cache over it agree. Large and non-large variants hash identically, so no +row hash, content hash, pipeline hash or default source id moves. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.sources import ( + ArrowTableSource, + CachedSource, + DictSource, + ListSource, + PollingSource, +) +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.operators import SelectTagColumns +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.types import Cursor, PollingConfig +from orcapod.utils import arrow_utils +from tests.test_core.large_types_cases import ( + ALL_COLUMNS, + FULL_SCHEMA, + ROWS, + SOURCE_CASES, + TAGS, + build_source, + non_large_columns, +) + +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} +LARGE_SCHEMA = arrow_utils.normalize_schema_to_large_types(FULL_SCHEMA, keep_null=True) + + +def _job_node_tables(source: StreamProtocol, orchestrator: str) -> list[pa.Table]: + """Every node's table after a job over ``source`` runs on ``orchestrator``.""" + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + _ = SelectTagColumns(["t"])(source, label="selected") + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + return [node.as_table(columns=ALL_COLUMNS) for node in job.nodes.values()] + + +@pytest.mark.parametrize(("name", "n_rows"), SOURCE_CASES) +class TestEverySourceKind: + def test_table_and_head_have_no_non_large_type( + self, name: str, n_rows: int, tmp_path: Path + ) -> None: + source = build_source(name, n_rows, tmp_path) + table = source.as_table(columns=ALL_COLUMNS) + assert table.num_rows == n_rows + assert non_large_columns(table) == [] + assert non_large_columns(source.head(columns=ALL_COLUMNS).table) == [] + + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_job_nodes_have_no_non_large_type( + self, name: str, n_rows: int, orchestrator: str, tmp_path: Path + ) -> None: + tables = _job_node_tables(build_source(name, n_rows, tmp_path), orchestrator) + for table in tables: + assert non_large_columns(table) == [] + + +def test_explicit_arrow_schema_is_normalized(tmp_path: Path) -> None: + """An Arrow schema the user gives is taken in its large variants.""" + table = build_source("CSVSource(arrow schema)", 2, tmp_path).as_table() + assert table.schema.field("t").type == pa.large_string() + assert table.schema.field("s").type == pa.large_string() + dict_table = build_source("DictSource(arrow schema)", 2, tmp_path).as_table() + arrow_table = build_source("ArrowTableSource", 2, tmp_path).as_table() + assert dict_table.schema == arrow_table.schema + + +def test_user_arrow_table_keeps_width_and_nullability() -> None: + """Only layouts change: widths, nullability (top level and nested), field names.""" + items = pa.list_(pa.field("item", pa.int8(), nullable=False)) + schema = pa.schema( + [ + pa.field("t", pa.string(), nullable=False), + pa.field("l", items, nullable=False), + pa.field("s", pa.string(), nullable=True), + ] + ) + table = pa.Table.from_pylist([{"t": "a", "l": [1], "s": None}], schema=schema) + out = ArrowTableSource(table, tag_columns=["t"], source_id="u").as_table() + assert out.schema.field("t") == pa.field("t", pa.large_string(), nullable=False) + assert out.schema.field("l") == pa.field( + "l", pa.large_list(pa.field("item", pa.int8(), nullable=False)), nullable=False + ) + assert out.schema.field("s") == pa.field("s", pa.large_string(), nullable=True) + + +def test_list_view_is_not_supported_today() -> None: + """Pins today's behaviour: ``list_view`` is not a supported type (ITL-1129). + + A source refuses a ``list_view`` column (the Arrow hasher cannot hash it); a raw + ``ArrowTableStream`` holds it as given, untyped and unhashable. Supporting it + (normalized to ``large_list``) is a deliberate later change. + """ + table = pa.table( + { + "t": pa.array(["a"], pa.large_string()), + "lv": pa.array([["x"]], pa.list_view(pa.string())), + } + ) + with pytest.raises(ValueError, match="list_view"): + _ = ArrowTableSource(table, tag_columns=["t"], source_id="lv") + stream = ArrowTableStream(table, tag_columns=["t"]) + assert stream.as_table().schema.field("lv").type == pa.list_view(pa.string()) + with pytest.raises(ValueError, match="list_view"): + _ = stream.content_hash() + + +def test_list_source_is_large() -> None: + source = ListSource("v", ["a", "b"]) + assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == [] + + +class _Impl: + """A polling implementation that fetches one batch, as a dict or an Arrow table.""" + + def __init__(self, as_arrow: bool) -> None: + self.as_arrow = as_arrow + + def identity(self) -> str: + return f"impl-{self.as_arrow}" + + def to_config(self) -> None: + return None + + @classmethod + def from_config(cls, config: Any) -> _Impl: + return cls(False) + + def schema(self) -> None: + return None + + async def poll(self, cursor: Cursor[int] | None = None) -> bool: + return cursor is None + + async def fetch(self, cursor: Cursor[int] | None = None) -> tuple[Cursor[int], Any]: + if cursor is not None: + return Cursor(value=1), {} + if self.as_arrow: + return Cursor(value=1), pa.table( + {"t": pa.array(["a"], pa.string()), "s": pa.array(["x"], pa.string())} + ) + return Cursor(value=1), {"t": ["a"], "s": ["x"]} + + async def close(self) -> None: + return None + + +@pytest.mark.parametrize("as_arrow", [False, True]) +def test_polling_source_is_large(as_arrow: bool) -> None: + source = PollingSource( + _Impl(as_arrow), tag_columns="t", polling_config=PollingConfig(duration=0.01) + ) + assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == [] + + +@function_pod(output_keys="v") +def _upper(s: str) -> str: + return s.upper() + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize("n_rows", [2, 0]) +def test_function_nodes_and_derived_sources_are_large( + orchestrator: str, n_rows: int, tmp_path: Path +) -> None: + """A function node passing a ``string`` tag through, and derived sources.""" + job = PipelineJob(name="itl1122", database=InMemoryArrowDatabase()) + with job: + source = build_source("ArrowTableSource", n_rows, tmp_path) + _ = _upper.pod(source, label="upper") + _ = SelectTagColumns(["t"])(source, label="selected") + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + for label in ("upper", "selected"): + node = job.nodes[label] + assert non_large_columns(node.as_table(columns=ALL_COLUMNS)) == [] + derived = node.as_source() + assert non_large_columns(derived.as_table(columns=ALL_COLUMNS)) == [] + with_tags = node.as_source(tags_as_data={"t": "t_data"}) + assert non_large_columns(with_tags.as_table(columns=ALL_COLUMNS)) == [] + + +@pytest.mark.parametrize("n_rows", [2, 0]) +def test_function_pod_stream_is_large(n_rows: int, tmp_path: Path) -> None: + stream = _upper.pod(build_source("ArrowTableSource", n_rows, tmp_path)) + assert non_large_columns(stream.as_table(columns=ALL_COLUMNS)) == [] + + +def test_spiraldb_table_source_is_large(tmp_path: Path) -> None: + from orcapod.sources import SpiralDBTableSource + from tests.test_databases import fake_pyspiral + + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project("p") + table = project.create_table( + "default.r", key_schema=pa.schema([pa.field("t", pa.string())]) + ) + table.write( + pa.table( + { + "t": pa.array(["a", "b"], pa.string()), + "s": pa.array(["x", "y"], pa.string()), + "b": pa.array([b"1", b"2"], pa.binary()), + "l": pa.array([["x"], []], pa.list_(pa.string())), + } + ) + ) + source = SpiralDBTableSource("p", "r", tag_columns=["t"]) + assert non_large_columns(source.as_table(columns=ALL_COLUMNS)) == [] + + +# --------------------------------------------------------------------------- +# What does not move +# --------------------------------------------------------------------------- + + +def _row_hashes(stream: StreamProtocol) -> list[tuple[str, str]]: + return [ + (tag.content_hash().to_hex(), data.content_hash().to_hex()) + for tag, data in stream.iter_data() + ] + + +@pytest.mark.parametrize("source_id", [None, "named"]) +def test_string_and_large_string_sources_hash_alike(source_id: str | None) -> None: + """The same values as ``string`` and as ``large_string`` give equal row, content and + pipeline hashes and the same (default) source id.""" + small = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=FULL_SCHEMA), + tag_columns=TAGS, + source_id=source_id, + ) + large = ArrowTableSource( + pa.Table.from_pylist(ROWS, schema=LARGE_SCHEMA), + tag_columns=TAGS, + source_id=source_id, + ) + assert small.source_id == large.source_id + assert small.content_hash() == large.content_hash() + assert small.pipeline_hash() == large.pipeline_hash() + assert _row_hashes(small) == _row_hashes(large) + assert small.as_table(columns=ALL_COLUMNS).equals( + large.as_table(columns=ALL_COLUMNS) + ) + + +def test_dict_source_arrow_schema_hashes_alike() -> None: + """A ``DictSource`` given ``string`` or ``large_string`` in its Arrow schema.""" + small = DictSource(ROWS, tag_columns=TAGS, data_schema=FULL_SCHEMA) + large = DictSource(ROWS, tag_columns=TAGS, data_schema=LARGE_SCHEMA) + assert small.source_id == large.source_id + assert small.content_hash() == large.content_hash() + assert small.pipeline_hash() == large.pipeline_hash() + assert _row_hashes(small) == _row_hashes(large) + + +# --------------------------------------------------------------------------- +# Stored tables +# --------------------------------------------------------------------------- + + +def _user_source(n_rows: int) -> ArrowTableSource: + """A user's Arrow table in the non-large types, as a source.""" + return ArrowTableSource( + pa.Table.from_pylist(ROWS[:n_rows], schema=FULL_SCHEMA), + tag_columns=TAGS, + source_id="user", + ) + + +def _store(kind: str, tmp_path: Path) -> ArrowDatabaseProtocol: + if kind == "in_memory": + return InMemoryArrowDatabase() + return DeltaTableDatabase(base_path=tmp_path / "store") + + +@pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) +class TestStoredTables: + def test_cached_records_are_large(self, store_kind: str, tmp_path: Path) -> None: + cached = CachedSource( + _user_source(2), + cache_database=_store(store_kind, tmp_path), + ) + _ = cached.as_table() + stored = cached.get_all_records() + assert stored is not None + assert non_large_columns(stored) == [] + + @pytest.mark.parametrize("n_rows", [2, 0]) + def test_cache_head_before_the_read_has_the_types_of_the_read( + self, store_kind: str, n_rows: int, tmp_path: Path + ) -> None: + cached = CachedSource( + _user_source(n_rows), + cache_database=_store(store_kind, tmp_path), + ) + unread = cached.head(columns=ALL_COLUMNS).table + read = cached.as_table(columns=ALL_COLUMNS) + assert non_large_columns(unread) == [] + assert unread.schema.equals(read.schema, check_metadata=False) + + +def test_a_delta_table_written_with_string_accepts_large_rows(tmp_path: Path) -> None: + """A Delta table a build before this change wrote with ``string`` columns takes the + ``large_string`` rows written after it (Delta keeps one string type). An in-memory + store keeps no data across processes, so no such table of it can exist.""" + store = _store("delta", tmp_path) + before = pa.Table.from_pylist(ROWS[:1], schema=FULL_SCHEMA) + after = pa.Table.from_pylist(ROWS[1:], schema=LARGE_SCHEMA) + _ = store.add_records(("t",), before, record_id_column="b") + store.flush() + _ = store.add_records(("t",), after, record_id_column="b") + store.flush() + stored = store.get_all_records(("t",), record_id_column="b") + assert stored is not None + assert stored.num_rows == 2 diff --git a/tests/test_pipeline/test_zero_row_non_large_inputs.py b/tests/test_pipeline/test_zero_row_non_large_inputs.py index b8e5c91b9..aeabbe453 100644 --- a/tests/test_pipeline/test_zero_row_non_large_inputs.py +++ b/tests/test_pipeline/test_zero_row_non_large_inputs.py @@ -163,7 +163,8 @@ def deliver(ctx: object, x: int) -> None: def test_cached_source_over_an_empty_source_equals_its_head() -> None: - """A cache with nothing stored reads with the wrapped source's own Arrow types.""" + """A cache with nothing stored: its head before the read has the types of the read + (the large form of the wrapped source's Arrow types, ITL-1122).""" from orcapod.core.sources import CachedSource empty = ArrowTableSource(