diff --git a/.zed/rules b/.zed/rules index af3ce036a..b801c0552 100644 --- a/.zed/rules +++ b/.zed/rules @@ -501,7 +501,11 @@ Key methods: output_schema(), keys(), iter_data(), as_table(). Every stream, source and node has run() with no arguments (developer, November 13, 2026, ITL-1081): it does the work as_table() does (a job node runs as in a job) and returns a StreamView of every row; the view's head/tail/sort_by_tags choose what is shown. -head() and the notebook preview never compute. Never give run() arguments. +head() and the notebook preview never compute (ITL-991, ITL-912): they never compute, poll or deliver a row (they read only what is held or stored, or a bounded table read), never call user code (a function, a predicate, a side effect) and never +write. With no rows to show they learn the column types by applying operators to zero-row +tables, which processes no data; an operator whose output schema is its input's +(OUTPUT_SCHEMA_IS_INPUT_SCHEMA, PolarsFilter) is not applied at all. A user-defined operator's static_process is applied to zero-row tables and must not have effects. Never give run() +arguments. Source (core/sources/) — produces a stream from external data. ArrowTableSource is the core implementation; CSV/Delta/DataFrame/Dict/List sources all delegate to it internally. Each @@ -716,6 +720,21 @@ pa.Table.from_pylist(rows) without a type for a column whose input type is known inference widens int8 to int64, float32 to double, and takes decimal precision from the data. The rules: +- The Arrow schema is never computed (developer, November 13, 2026, ITL-912): "Generally + Orcapod should never have to compute anything to get the arrow schema, all function pods and + operator pods can deterministically derive the output stream schema from the input stream + schemas, and ultimately no source can be created without having schema specified somehow." + A table with no rows (before a run, after clear_cache(), a zero-row result or input) has + exactly the types of the rows: get it from streams/base.typed_zero_row_table, never from + the Python schema (except when a stream cannot type itself without rows: an unbound input or a + PollingSource not yet polled, ITL-1089, which fall back to the declared schema), and never by + computing, reading or polling a row or calling a function. + An operator's is its result over its inputs' zero-row tables; a function pod's is the + input's tags plus the declared outputs (function_zero_row_table); a source's is the table + or schema it holds. A new stream kind implements _typed_zero_row_table. A head with no + rows is built in one place, StreamBase._head_without_rows, from the stream's + _zero_row_table(config), which equals as_table(columns=config) with no rows; a kind + whose as_table shapes its table its own way overrides _zero_row_table with that shaping. - Type strictness (developer, November 10, 2026): Orcapod is type strict. It never expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on diff --git a/CLAUDE.md b/CLAUDE.md index d66e4d9c6..c2c43bbf3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -516,7 +516,11 @@ Key methods: `output_schema()`, `keys()`, `iter_data()`, `as_table()`. Every stream, source and node has `run()` with no arguments (developer, November 13, 2026, ITL-1081): it does the work `as_table()` does (a job node runs as in a job) and returns a `StreamView` of every row; the view's `head`/`tail`/`sort_by_tags` choose what is shown. -`head()` and the notebook preview never compute. Never give `run()` arguments. +`head()` and the notebook preview never compute (ITL-991, ITL-912): they never compute, poll or deliver a row (they read only what is held or stored, or a bounded table read), never call user code (a function, a predicate, a side effect) and never +write. With no rows to show they learn the column types by applying operators to zero-row +tables, which processes no data; an operator whose output schema is its input's +(`OUTPUT_SCHEMA_IS_INPUT_SCHEMA`, `PolarsFilter`) is not applied at all. A user-defined operator's `static_process` is applied to zero-row tables and must not have effects. Never give `run()` +arguments. **Source** (`core/sources/`) — produces a stream from external data. `ArrowTableSource` is the core implementation; CSV/Delta/DataFrame/Dict/List sources all delegate to it internally. Each @@ -758,6 +762,21 @@ then through the data context's type converter — never call `pa.array(values)` inference widens `int8` to `int64`, `float32` to `double`, and takes decimal precision from the data. The rules: +- **The Arrow schema is never computed** (developer, November 13, 2026, ITL-912): "Generally + Orcapod should never have to compute anything to get the arrow schema, all function pods and + operator pods can deterministically derive the output stream schema from the input stream + schemas, and ultimately no source can be created without having schema specified somehow." + A table with no rows (before a run, after `clear_cache()`, a zero-row result or input) has + exactly the types of the rows: get it from `streams/base.typed_zero_row_table`, never from + the Python schema (except when a stream cannot type itself without rows: an unbound input or a + `PollingSource` not yet polled, ITL-1089, which fall back to the declared schema), and never by + computing, reading or polling a row or calling a function. + An operator's is its result over its inputs' zero-row tables; a function pod's is the + input's tags plus the declared outputs (`function_zero_row_table`); a source's is the table + or schema it holds. A new stream kind implements `_typed_zero_row_table`. A head with no + rows is built in one place, `StreamBase._head_without_rows`, from the stream's + `_zero_row_table(config)`, which equals `as_table(columns=config)` with no rows; a kind + whose `as_table` shapes its table its own way overrides `_zero_row_table` with that shaping. - **Type strictness** (developer, November 10, 2026): Orcapod is type strict. It never expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index 92b1d7e93..6d650d3b1 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -22,10 +22,8 @@ - An operator node whose async run produces no row keeps the sync run's columns and Arrow types (`int8` stays `int8`; system-tag and source-info columns are present), instead of a table built from the Python output schema (ITL-1080). When an input itself delivered no row, that - input is rebuilt from its upstream's Python output schema on both paths alike (ITL-912), so a - `Join` of such an input with a non-empty one whose key is, say, `int8` now raises - `InputValidationError` on an async run as it already did on a sync run, instead of finishing - with a mistyped empty result. + input is rebuilt the same way on both paths alike, from its upstream's zero-row table + (ITL-912). - `DropDataColumns` and `SelectDataColumns` raise `InputValidationError` when their result would have no data column (every data column dropped, or `strict=False` matching none), when the operator is applied on the sync path and on the first row on the async path, with one message diff --git a/changelog.d/ITL-912.md b/changelog.d/ITL-912.md new file mode 100644 index 000000000..c5a7011b3 --- /dev/null +++ b/changelog.d/ITL-912.md @@ -0,0 +1,62 @@ +### Changed + +- `head()` contract (ITL-912): `head()` and the notebook preview never compute, poll or deliver a row (they read only what is held or stored, or make a bounded table read), never call user code (a function, a predicate, a side effect) and never write. + With no rows to show, they now give the columns and Arrow types the rows would have: they + apply each operator to empty (zero-row) tables, which processes no data (a chain of N + operators applies up to N operators to empty tables, about 60 ms for 20), and a filter is not + applied at all, since its columns are its input's. A user-defined operator's `static_process` is applied to zero-row tables and must not have effects. So a pipeline whose inputs disagree on a + column's Arrow type (`int8` against `int64`) now raises from `head()` the + `InputValidationError` its run raises; the notebook preview shows it as "could not read". A + filter whose predicate does not fit its columns shows a table in `head()` and raises when it + runs. + +### Fixed + +#### A table with no rows has the types of the rows (ITL-912) + +An operator node read before its run, after `clear_cache()`, or after a run that produced no +row, returned a zero-row table built from its Python output schema: `int8` became `int64`, +a non-nullable list item became nullable, source-info columns changed nullability and the +system-tag columns were missing. A function pod's stream and a function node did the same for +their tag columns, and left out the `_context_key` and source-info columns; a side-effect +pod's stream or node that delivered no row did the same, and so did `head()` and the notebook +preview of a stream or node with no rows to show. Both orchestrators built an +operator node's input the same way when an upstream node delivered no row, so a `Join` of +such an input with a non-empty one whose key is, say, `int8` raised `InputValidationError`. + +Each of these tables now has exactly the columns, Arrow types and nullability of the rows, +on both orchestrators, and that `Join` completes with typed zero rows on both paths. Nothing +is computed, read or called to get them: an operator applies +itself to its inputs' zero-row tables, a function pod combines its input's tags with its +declared output types, a side-effect pod gives its input's, and a source gives the table or +schema it already holds. Nothing is +stored for an empty table, so no hash, stored table or path changes. + +### Known limitations + +- A zero-row table is still built from the Python output schema (`int8` reads as `int64`, + system-tag columns are absent) when an upstream cannot say its Arrow types without rows: a + `PollingSource` before its first poll and an unbound input slot or `SourceProxy` (both are + declared with a Python schema only), a `DerivedSource`, and a loaded job node with no live + upstream and no stored table. `Index` and `Pick` over zero rows still lack the system-tag + columns (ITL-985). These cases are tracked in ITL-1089. (ITL-912) +- A source-info column (`_source_*`) of an operator node's rows is nullable only when some row's + provenance token is `None`, a rule that depends on the rows. A zero-row table has no token, + so it declares the column non-nullable, as rows with a known token do; when the rows have an + unknown (`None`) token, the field's nullability differs between the zero-row table and the + rows. A row-independent rule would change the operator node's stored record hashes. Tracked in + ITL-1089. (ITL-912) + +#### Zero-row tables over non-large inputs (ITL-912) + +A source holding `string`, `binary` or `list` columns gave a function node's or operator +node's table with no rows the non-large types, while the table with rows has the large ones +(`large_string`, ...). The zero-row table now takes its input's types in the large-normalized +form, as the populated table does, and an operator node's zero-row table lists its columns in +the order its rows do (a join's system tags before its data). A source's own table is not +changed. + +A side-effect pod's stream takes its input's zero-row types in the same large form; a +`PolarsFilter()` with no predicate shows its input's types, as it passes the input through; and +a `CachedSource` over a source with no rows, with nothing stored, reads with the source's own +Arrow types and system-tag columns, as its `head()` shows them. diff --git a/changelog.d/ITL-991.md b/changelog.d/ITL-991.md index 21830317a..9279098ae 100644 --- a/changelog.d/ITL-991.md +++ b/changelog.d/ITL-991.md @@ -18,8 +18,8 @@ the operator, displaying a side-effect stream delivered every side effect, and d - `head(n=10, columns=..., all_info=...)` on every stream, source and node returns a `StreamHead` (`orcapod.streams`): at most `n` rows as a PyArrow table with their exact types, the row count when it is known without computing, and a `status` (`HeadStatus`) that - tells an empty result from one that is not computed. It never calls a function, runs an - operator, delivers a side effect, polls a source or writes. The notebook preview is + tells an empty result from one that is not computed. It never computes, polls or delivers a row (it reads only what is held or stored, or makes a bounded table read), never calls user code and never writes (see ITL-912 for how a head with no + rows learns its column types). The notebook preview is `head()` rendered, with large cells (arrays, bytes, long text, long lists) summarised. A job node with nothing in memory reads its whole stored slice to show its first rows. (ITL-991) diff --git a/design_issues/ITL-912-ZR1.md b/design_issues/ITL-912-ZR1.md new file mode 100644 index 000000000..496945ab1 --- /dev/null +++ b/design_issues/ITL-912-ZR1.md @@ -0,0 +1,45 @@ +## Cross-cutting + +### ZR1 — Zero-row tables were typed from the Python schema +**Status:** resolved +**Severity:** high +**Issue:** ITL-912 + +The table a stream or node gives when it has no row was built from its Python output schema +(`arrow_utils.make_empty_table`): an operator node before its run, after `clear_cache()`, with +no stored table, or after a zero-row async run (`OperatorJobNode._make_empty_table`); the +sync orchestrator's input for an upstream that produced no row +(`SyncPipelineOrchestrator._materialize_as_stream`); and a function pod's stream or node with +no row (`_table_from_pairs([])`); and a side-effect pod's stream or node that delivered no row +(`side_effects._table_of_rows`); and the table of a `head()` with no rows to show +(`StreamBase._head_without_rows`). So `int8` read as `int64`, list items and source-info +columns changed nullability, system-tag columns went missing (operator nodes) and a function +pod lost its input's tag types and its context and source-info columns. Zero-row and +non-empty results must have the same types, and the developer's rule (November 13, 2026) is +that an Arrow schema is never computed: every pod derives it from its inputs' schemas and every +source has one. + +**Fix:** one function, `streams/base.typed_zero_row_table(stream)`, asks a stream for +`_typed_zero_row_table()`: zero rows, user tags, system tags, data and source info, with the +types its rows have, or `None` when it cannot know them without rows. An `ArrowTableStream` +cuts its table; a source gives the table or schema it holds (`_stream`; cached sources +delegate to the wrapped source, a polling source to what it polled, a proxy to its delegate); +an operator node or `DynamicPodStream` applies its operator (`static_process`) to its inputs' +zero-row tables; a function pod's stream or node joins its input's tag columns to the zero-row +form of its output `Data` (`function_zero_row_table`, `Data.function_output_zero_row_table`, +built from the same converter call and field builders a computed row uses); a side-effect +pod's stream or node gives its input's (it passes rows through), with the context column. Inputs a job +builds with no rows use `StaticOutputOperatorPod._zero_row_input` (row-built column order, +`_rows_to_table`'s source-info nullability rule), through `_materialize_upstream` on both +orchestrators. A head with no rows is built in one place, `StreamBase._head_without_rows`, +from the stream's `_zero_row_table(config)`: `as_table(columns=config)` with no rows. Nothing is stored, so no hash, stored schema or +path changes; nothing is computed, read, polled or called. Regressions: +`tests/test_pipeline/test_operator_node_empty_table_types.py` and the un-marked xfails in +`tests/test_pipeline/test_operator_node_async_readback.py`. + +Still built from the Python schema (no Arrow schema exists without rows): a `PollingSource` +before its first poll and an unbound input slot or `SourceProxy` (declared with a Python +`Schema` only; reported to the developer as a design gap), a `DerivedSource` (its +`_empty_origin_table` is left as it is), and a loaded node with no live upstream and no stored +table. `Index` and `Pick` over zero rows lack system-tag columns (ITL-985). These remaining +cases are tracked in ITL-1089. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 868cad97f..c6983495c 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -653,8 +653,8 @@ silent widening; convert explicitly. the outputs of other operators and function pods. An input that delivers no rows gives no rows on both paths. One limit remains: an async channel carries rows, not a schema, so where the sync path returns an empty stream with typed columns the async path emits - nothing, and an operator node that received no rows reads back columns typed from its - Python output schema (ITL-858, ITL-912). + nothing. An operator node that received no rows still reads back the sync path's columns + and Arrow types: its operator applied to its inputs' zero-row tables (ITL-858, ITL-912). ```python import pyarrow as pa diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 311431c80..e6ff8356a 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -106,9 +106,19 @@ In a notebook, a stream, source or job node shown at the end of a cell renders a table: its first rows in tag order, with a line saying what they are. That preview is `head()` asked for `sort_by_tags` and rendered, and `head(n, columns={"sort_by_tags": True})` gives the same rows to work with. Neither ever computes anything: they show -only what is readily available, and never call a function, run an operator, deliver a side -effect, poll a source or write to a database. `run()` computes (see "Computing a stream -outside a job" below). +only what is readily available, never compute, poll or deliver a row (they read only what is held or stored, or make a bounded table read), never call your +code (a function, a predicate, a side effect) and never write to a database. `run()` computes +(see "Computing a stream outside a job" below). + +With no rows to show, a head still has the columns and Arrow types the rows would have. To +learn them it applies each operator to empty (zero-row) tables, which processes no data: a +chain of N operators applies up to N operators to empty tables (about 60 ms for 20). A filter +is not applied at all, since its columns are its input's. A user-defined operator's +`static_process` is applied to zero-row tables and must not have effects. Two consequences: a pipeline whose +inputs disagree on a column's Arrow type (say `int8` against `int64`) raises from `head()` +the same `InputValidationError` its run raises (the notebook preview shows it as "could not +read"), and a filter whose predicate does not fit its columns shows a table in `head()` and +raises when it runs. ```python from orcapod import function_pod diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index e1000b33b..2fcce1538 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -9,10 +9,14 @@ from typing import TYPE_CHECKING, Any from orcapod.core.datagrams import Datagram -from orcapod.core.function_pod import FunctionPodStream, WrappedFunctionPod +from orcapod.core.function_pod import ( + FunctionPodStream, + WrappedFunctionPod, + function_zero_row_table, +) from orcapod.core.streams.view import StreamView from orcapod.system_constants import constants -from orcapod.types import ColumnConfig +from orcapod.types import ColumnConfig, Schema from orcapod.contexts.stored import StaleTables from orcapod.core.result_cache import ResultCache from orcapod.hooks import InvocationStatus @@ -458,6 +462,15 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: for tag, data in super().iter_data(): yield tag, _without_stored_meta(data) + def _zero_row_output_table(self) -> pa.Table | None: + """The table of this stream's rows with no row, ``__computed`` included.""" + return function_zero_row_table( + self._input_stream, + self.output_schema()[1], + self.data_context, + Schema({CachedFunctionPod.RESULT_COMPUTED_FLAG: bool}), + ) + def _computed_prefix( self, limit: int, contiguous: bool = True ) -> tuple[list[tuple[TagProtocol, DataProtocol]], int]: diff --git a/src/orcapod/core/datagrams/datagram.py b/src/orcapod/core/datagrams/datagram.py index bcc3ee9fe..623d73803 100644 --- a/src/orcapod/core/datagrams/datagram.py +++ b/src/orcapod/core/datagrams/datagram.py @@ -46,6 +46,11 @@ logger = logging.getLogger(__name__) +def context_key_field() -> pa.Field[Any]: + """The field of a datagram's data context column (``_context_key``).""" + return pa.field(constants.CONTEXT_KEY, pa.large_string(), nullable=False) + + class Datagram(ContentIdentifiableBase): """ Immutable datagram backed by either an Arrow table or a Python dict. @@ -297,9 +302,7 @@ def _ensure_context_table(self) -> pa.Table: Ensure context table is materialized and then returned (relevant for Arrow representation) """ if self._context_table is None: - schema = pa.schema( - [pa.field(constants.CONTEXT_KEY, pa.large_string(), nullable=False)] - ) + schema = pa.schema([context_key_field()]) self._context_table = pa.Table.from_pylist( [{constants.CONTEXT_KEY: self._data_context.context_key}], schema=schema, diff --git a/src/orcapod/core/datagrams/tag_data.py b/src/orcapod/core/datagrams/tag_data.py index 42f8cab0f..5b9f1ccd2 100644 --- a/src/orcapod/core/datagrams/tag_data.py +++ b/src/orcapod/core/datagrams/tag_data.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Self from orcapod import contexts -from orcapod.core.datagrams.datagram import Datagram +from orcapod.core.datagrams.datagram import Datagram, context_key_field from orcapod.semantic_types import infer_python_schema_from_pylist_data from orcapod.system_constants import constants from orcapod.types import ( @@ -50,6 +50,13 @@ # --------------------------------------------------------------------------- +def _source_info_field(name: str, token: SourceInfoValue) -> pa.Field[Any]: + """The field of source-info column ``name`` holding ``token``, as a row's table has it.""" + import pyarrow as _pa + + return _pa.field(name, arrow_utils.source_info_arrow_type(token)) + + class Tag(Datagram): """ Datagram with system-tags support. @@ -379,10 +386,7 @@ def _ensure_source_info_table(self) -> "pa.Table": f"{constants.SOURCE_PREFIX}{k}": v for k, v in current.items() } schema = _pa.schema( - [ - _pa.field(k, arrow_utils.source_info_arrow_type(v)) - for k, v in prefixed.items() - ] + [_source_info_field(k, v) for k, v in prefixed.items()] ) self._source_info_table = _pa.Table.from_pylist( [prefixed], schema=schema @@ -391,6 +395,55 @@ def _ensure_source_info_table(self) -> "pa.Table": self._source_info_table = _pa.table({}) return self._source_info_table + @staticmethod + def function_output_zero_row_table( + python_schema: Schema, + data_context: contexts.DataContext, + meta_python_schema: Schema | None = None, + ) -> pa.Table: + """The table a function's output rows give, with no row. + + The zero-row form of ``Data.as_table(columns={"context": True, "source": + True})`` for the ``Data`` a data function returns: built with + ``python_schema=`` its declared output schema and a string source-info + token for every column. Each part comes from the code that types a + computed row: the data fields from the converter's + ``python_schema_to_arrow_schema`` (``Datagram._ensure_arrow_schema``), the + context field from ``context_key_field``, the meta fields from the same + conversion of their Python types (``Datagram._ensure_meta_table``) and the + source-info fields from ``_source_info_field`` with a string token + (ITL-912). The columns are in the order ``Data.arrow_schema`` gives them. + + Args: + python_schema: The function's declared output schema. + data_context: The data context the rows are typed in. + meta_python_schema: The meta columns every output row carries (a + cached pod's ``__computed`` flag), if any. + + Returns: + A zero-row table: data columns, ``_context_key``, meta columns, + source-info columns. + """ + import pyarrow as _pa + + fields = list( + data_context.type_converter.python_schema_to_arrow_schema(python_schema) + ) + fields.append(context_key_field()) + if meta_python_schema: + fields.extend( + data_context.type_converter.python_schema_to_arrow_schema( + meta_python_schema + ) + ) + # A data function's source-info token is always a string (see + # ``PythonDataFunction``): the type is the one a string token gives. + fields.extend( + _source_info_field(f"{constants.SOURCE_PREFIX}{name}", "") + for name in python_schema + ) + return _pa.schema(fields).empty_table() + # ------------------------------------------------------------------ # Source-info API # ------------------------------------------------------------------ diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index a2bc9e6c7..874f80ab9 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -15,9 +15,11 @@ from orcapod.core.base import TraceableBase from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, StreamBase, check_head_size, schema_requires_io, + typed_zero_row_input, ) from orcapod.core.streams.view import ( SCHEMA_NEEDS_POLL_NOTE, @@ -966,6 +968,56 @@ def from_config( ) +def function_zero_row_table( + input_stream: StreamProtocol, + output_schema: Schema, + data_context: contexts.DataContext, + meta_python_schema: Schema | None = None, +) -> pa.Table | None: + """The table a function pod's output rows give over ``input_stream``, with no row. + + A function pod passes its input's tags through untouched, so the user tags and + system tags are the input's zero-row table's (``typed_zero_row_table``). The data + columns are those of the ``Data`` the data function returns, with no row + (``Data.function_output_zero_row_table``): the declared output types, the data + context column and the source-info columns. Nothing is computed and the function + is not called (ITL-912). This is the zero-row form of the table the function pod's + stream and node build from computed rows. + + Args: + input_stream: The function pod's input. + output_schema: The function's declared output schema. + data_context: The data context the output rows are typed in. + meta_python_schema: The meta columns every output row carries, if any. + + Returns: + User tags, system tags, data, ``_context_key``, meta and source-info columns, or + ``None`` when the input cannot say its Arrow types without rows. + """ + from orcapod.core.datagrams.tag_data import Data + + input_table = typed_zero_row_input(input_stream) + if input_table is None: + return None + system_tags = [ + name + for name in input_table.column_names + if name.startswith(constants.SYSTEM_TAG_PREFIX) + ] + tags = input_table.select([*input_stream.keys()[0], *system_tags]) + return arrow_utils.hstack_tables( + tags, + Data.function_output_zero_row_table( + output_schema, data_context, meta_python_schema + ), + ) + + +def _no_content_hashes() -> pa.Array[Any]: + """The content hashes of no row.""" + return pa.array([], type=pa.large_string()) + + class FunctionPodStream(StreamBase): """Recomputable stream wrapping a data function.""" @@ -1326,15 +1378,47 @@ async def _gather() -> list[ if data is not None: yield tag, data + def _zero_row_output_table(self) -> pa.Table | None: + """This stream's rows' table with no row; see ``function_zero_row_table``. + + A cached data function's rows carry the result cache's ``__computed`` meta + column, so its zero-row table does too. + """ + meta = None + if isinstance(self._function_pod.data_function, CachedDataFunction): + from orcapod.core.result_cache import ResultCache + + meta = Schema({ResultCache.RESULT_COMPUTED_FLAG: bool}) + return function_zero_row_table( + self._input_stream, self.output_schema()[1], self.data_context, meta + ) + + def _typed_zero_row_table(self) -> pa.Table | None: + """This stream's zero-row table, without calling the function (ITL-912).""" + return self._zero_row_table(ZERO_ROW_INPUT_COLUMNS) + + def _zero_row_table(self, config: ColumnConfig) -> pa.Table | None: + """``as_table(columns=config)`` with no rows, shaped as ``as_table`` shapes it.""" + table = self._zero_row_output_table() + if table is None: + return None + return self._shape_table(table, config, _no_content_hashes) + def _table_from_pairs( self, pairs: list[tuple[TagProtocol, DataProtocol]] ) -> pa.Table: """Return the output rows ``pairs`` as one table with every column group. - With no rows the table has the declared columns (tags, system tags and data), - typed from the declared schema, as the rest of a head without rows is. + With no rows the table is ``function_zero_row_table``: the input's tags and + system tags and the declared data, context and source-info columns, with the + types computed rows have (ITL-912). Only when the input cannot say its Arrow + types without rows is it built from the declared Python schema. """ if not pairs: + typed = self._zero_row_output_table() + if typed is not None: + return typed + # The input cannot say its Arrow types without rows: the declared schema. tag_schema, data_schema = self.output_schema(all_info=True) return arrow_utils.make_empty_table( {**tag_schema, **data_schema}, self.data_context.type_converter diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index ab84ed88c..5357b1c2c 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -31,6 +31,7 @@ from orcapod.core.cached_function_pod import CachedFunctionPod from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, StreamBase, check_head_size, ) @@ -594,6 +595,27 @@ def all_content_hashes() -> "pa.Array[Any]": self._cached_output_table, column_config, all_content_hashes ) + def _zero_row_output_table(self) -> "pa.Table | None": + """This node's rows' table with no row; see ``function_zero_row_table``.""" + from orcapod.core.function_pod import function_zero_row_table + + return function_zero_row_table( + self._input_stream, self.output_schema()[1], self.data_context + ) + + def _typed_zero_row_table(self) -> "pa.Table | None": + """This node's zero-row table, without calling the function (ITL-912).""" + return self._zero_row_table(ZERO_ROW_INPUT_COLUMNS) + + def _zero_row_table(self, config: ColumnConfig) -> "pa.Table | None": + """``as_table(columns=config)`` with no rows, shaped as ``as_table`` shapes it.""" + table = self._zero_row_output_table() + if table is None: + return None + return self._shape_table( + table, config, lambda: pa.array([], type=pa.large_string()) + ) + def _table_from_pairs( self, pairs: "list[tuple[TagProtocol, DataProtocol]]" ) -> "pa.Table": @@ -603,8 +625,11 @@ def _table_from_pairs( row has: a tag the type of the input's tag, never a type inferred from Python values (ITL-928). A row's meta columns (a computed row's ``__computed`` flag) are its own bookkeeping, not part of the node's - output, as for the pod's stream outside a job. With no rows the table has - the declared columns, typed from the declared output schema. + output, as for the pod's stream outside a job. With no rows the table is + ``function_zero_row_table``: the input's tags and system tags and the + declared data, context and source-info columns, typed as computed rows are + (ITL-912); only when the input cannot say its Arrow types without rows is + it built from the declared Python schema. Raises: InputValidationError: If two rows give one column different Arrow @@ -612,9 +637,11 @@ def _table_from_pairs( """ if not pairs: # No data was produced (node has not been run yet or produced zero - # outputs). Build a zero-row table whose schema is derived from the - # pod's declared output so that callers always receive a well-formed - # schema regardless of whether the node has actually executed. + # outputs): the table computed rows would have, with no row (ITL-912). + typed = self._zero_row_output_table() + if typed is not None: + return typed + # The input cannot say its Arrow types without rows: the declared schema. tag_python_schema, data_python_schema = self.output_schema(all_info=True) return pa.Table.from_pylist( [], diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 2a167e48e..2b98053e5 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -28,6 +28,7 @@ from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, StreamBase, check_head_size, ) @@ -1226,16 +1227,62 @@ def _read_current_context_records(self) -> "pa.Table | None": ) return records.drop([constants.DATA_CONTEXT_COL]) + def _typed_zero_row_table(self) -> "pa.Table | None": + """This node's zero-row table: its operator applied to zero-row inputs. + + The sync path's rule (ITL-912): each input is the upstream's zero-row table + (``StaticOutputOperatorPod._zero_row_input``), and the operator's result over + those is this node's table with no rows, with the Arrow types, nullability, + system-tag and source-info columns its rows have. No row is computed or read, + no function is called and nothing is stored, and a result the node holds is + not consulted: the types follow from the inputs' types alone. + + Returns: + The zero-row table with system tags and source info, or ``None`` when the + node has no operator or an input cannot say its Arrow types without rows + (see ``typed_zero_row_table``). + """ + if self._operator is None: + return None + inputs: list[StreamProtocol] = [] + for upstream in self._input_streams: + zero_rows = StaticOutputOperatorPod._zero_row_input(upstream) + if zero_rows is None: + return None + inputs.append(zero_rows) + # zero_row_result calls static_process, not process: an invocation recorded by + # a tracker would add a step to whatever pipeline is being built while this + # table is read. + operator = cast(StaticOutputOperatorPod, self._operator) + result = operator.zero_row_result(*inputs) + table = result.as_table(columns=ZERO_ROW_INPUT_COLUMNS) + # The column order of the node's rows (``_rows_to_table``): user tags, system + # tags, data, source info; an operator's own table may order them otherwise + # (a join lists its data before its system tags). + tag_keys, data_keys = result.keys() + names = table.column_names + system_tags = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] + source_info = [n for n in names if n.startswith(constants.SOURCE_PREFIX)] + data = [n for n in data_keys if n in names] + return table.select([*tag_keys, *system_tags, *data, *source_info]) + def _make_empty_table(self) -> "pa.Table": - """Build a zero-row PyArrow table matching this node's full output schema. + """Build the zero-row table this node reads as when it holds no rows. - Uses ``arrow_utils.make_empty_table`` so field nullability is preserved: - plain types produce ``nullable=False`` fields and ``T | None`` types - produce ``nullable=True`` fields. + The node's operator applied to its inputs' zero-row tables + (``_typed_zero_row_table``), so the columns, Arrow types and nullability are + those of the node's rows. When an input cannot say its Arrow types without + rows (a known limitation, see ``as_table``), the table is built from the + node's Python output schema with ``arrow_utils.make_empty_table``: plain + types give ``nullable=False`` fields and ``T | None`` types + ``nullable=True`` fields. Requires ``self._operator is not None`` (pre-existing limitation shared with ``_replay_from_cache``). """ + typed = self._typed_zero_row_table() + if typed is not None: + return typed tag_schema, data_schema = self.output_schema() return arrow_utils.make_empty_table( {**tag_schema, **data_schema}, @@ -1531,6 +1578,16 @@ def as_table( ``CacheMode.REPLAY`` with a database returns the rows stored there; in any other mode it returns a zero-row table with the node's columns. + That zero-row table has the columns, Arrow types and nullability of the + node's rows, system tags and source info included: it is the operator's + result over its inputs' zero-row tables, got without computing or reading + a row (ITL-912). Known limitation: when an input cannot say its Arrow + types without rows (a ``PollingSource`` before its first poll, an unbound + input slot or ``SourceProxy``, a ``DerivedSource``, or a loaded node with + no live upstream), the table is built from the node's Python output + schema, as before: an ``int8`` column reads as ``int64`` and the + system-tag columns are absent (ITL-1089, ITL-985). + Loaded nodes raise when no readable stored cache is available. """ if self._cached_output_stream is not None: @@ -1560,7 +1617,7 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the first rows the node holds or has stored; never runs the operator. + """Return the first rows the node holds or has stored; never runs it over rows. After a run (or ``execute``) the rows are the node's output in memory. Before that, a node in ``CacheMode.REPLAY`` shows the first of its stored rows, read @@ -1841,8 +1898,8 @@ async def forward() -> None: # inputs rebuilt with no rows, exactly as the sync orchestrator # rebuilds them, so the node reads as computed and empty with the # sync path's columns and Arrow types (ITL-1080). An input that - # delivered no row is rebuilt from its Python output schema, as on - # the sync path. Nothing is stored for it. + # delivered no row is its upstream's zero-row table, as on the sync + # path (ITL-912). Nothing is stored for it. empty_result = operator.process( *( StaticOutputOperatorPod._materialize_upstream( diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8d7cf8923..fdee2631d 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -16,7 +16,7 @@ from orcapod import contexts from orcapod.config import OrcapodConfig from orcapod.core.base import TraceableBase -from orcapod.core.streams.base import schema_requires_io +from orcapod.core.streams.base import schema_requires_io, typed_zero_row_table from orcapod.core.streams.view import ( ViewStatus, StreamView, @@ -634,6 +634,13 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: ) return self._bound_source.iter_data() + def _typed_zero_row_table(self) -> "pa.Table | None": + """The bound source's zero-row table; ``None`` when unbound or unknown. + + See ``typed_zero_row_table``. + """ + return typed_zero_row_table(self._bound_source) + def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll: as for the bound source.""" return self._bound_source is not None and schema_requires_io( diff --git a/src/orcapod/core/operators/filters.py b/src/orcapod/core/operators/filters.py index 51f96afa0..5234c4019 100644 --- a/src/orcapod/core/operators/filters.py +++ b/src/orcapod/core/operators/filters.py @@ -29,6 +29,9 @@ class PolarsFilter(UnaryOperator): Operator that applies Polars filtering to a stream """ + # A filter chooses rows and never changes a column (ITL-912). + OUTPUT_SCHEMA_IS_INPUT_SCHEMA = True + def __init__( self, predicates: Collection[ @@ -43,6 +46,19 @@ def __init__( self.constraints = constraints if constraints is not None else {} super().__init__(**kwargs) + def zero_row_result(self, *inputs: StreamProtocol) -> StreamProtocol: + """This filter's result over a zero-row input, for a table with no rows (ITL-912). + + A filter with no predicate and no constraint passes its input through + unaltered (``unary_static_process``), so its result is the input with the types + it has. Any other filter gives its input's columns in the large-normalized + form a run gives them, without evaluating the predicate. + """ + if len(self.predicates) == 0 and len(self.constraints) == 0: + (only,) = inputs + return only + return super().zero_row_result(*inputs) + def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: if len(self.predicates) == 0 and len(self.constraints) == 0: logger.info( diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 2f75582b1..467ad80c6 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -5,7 +5,7 @@ from abc import abstractmethod from collections.abc import Collection, Iterator, Sequence from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from orcapod.channels import ReadableChannel, WritableChannel from orcapod import contexts @@ -15,8 +15,11 @@ from orcapod.core.function_pod import FunctionPodStream from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, StreamBase, check_head_size, + typed_zero_row_input, + typed_zero_row_table, ) from orcapod.core.streams.view import ( STALE_NOTE, @@ -54,6 +57,14 @@ class StaticOutputOperatorPod(TraceableBase): tracked by the tracker manager. """ + OUTPUT_SCHEMA_IS_INPUT_SCHEMA: ClassVar[bool] = False + """Whether the output has exactly its single input's columns and Arrow types. + + Declared by an operator that only chooses which rows exist (``PolarsFilter``). Its + zero-row table is then its input's (``zero_row_result``), got without evaluating a + predicate, so no user code runs to learn the column types (ITL-912). + """ + def __init__( self, tracker_manager: TrackerManagerProtocol | None = None, **kwargs ) -> None: @@ -228,7 +239,6 @@ def _rows_to_table( Raises: ValueError: If ``rows`` is empty. """ - from orcapod.system_constants import constants from orcapod.utils import arrow_utils if not rows: @@ -242,17 +252,112 @@ def _rows_to_table( combined_tags = arrow_utils.concat_tables_by_name(tag_tables) combined_data = arrow_utils.concat_tables_by_name(data_tables) - data_schema = combined_data.schema - for i, field in enumerate(data_schema): - if field.name.startswith(constants.SOURCE_PREFIX): - nullable = combined_data.column(i).null_count > 0 - data_schema = data_schema.set(i, field.with_nullable(nullable)) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized - # Same arrays, new field flags: no cast, so no value or type can change. - combined_data = pa.Table.from_arrays(combined_data.columns, schema=data_schema) + combined_data = StaticOutputOperatorPod._flag_source_info_nullability( + combined_data + ) full_table = arrow_utils.hstack_tables(combined_tags, combined_data) return full_table, tuple(rows[0][0].keys()) + @staticmethod + def _flag_source_info_nullability(table: pa.Table) -> pa.Table: + """Declare each ``_source_*`` field nullable only if some row's token is ``None``. + + The rule ``_rows_to_table`` applies to the rows it concatenates, and that a + zero-row input (``_zero_row_input``) applies to no row: there, no token is + ``None``, so every source-info field is non-nullable. + + Args: + table: A table whose ``_source_*`` columns hold source-info tokens. + + Returns: + The same arrays under the new field flags. + """ + from orcapod.system_constants import constants + + schema = table.schema + for i, field in enumerate(schema): + if field.name.startswith(constants.SOURCE_PREFIX): + nullable = table.column(i).null_count > 0 + schema = schema.set(i, field.with_nullable(nullable)) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + # Same arrays, new field flags: no cast, so no value or type can change. + return pa.Table.from_arrays(table.columns, schema=schema) + + @staticmethod + def _zero_row_input(upstream: StreamProtocol) -> StreamProtocol | None: + """Return ``upstream`` as an operator input with no rows, typed as its rows are. + + The zero-row form of what ``_materialize_upstream`` builds from an upstream's + rows: the upstream's zero-row table (``typed_zero_row_table``: user tags, + system tags, data and source info, with the Arrow types its rows have) in the + column order and with the source-info flags ``_rows_to_table`` gives, as the + upstream's stream (``_as_upstream_stream``), so the stream's identity, + pipeline hash and system-tag column names are the upstream's (ITL-912). + + Args: + upstream: The node or stream an operator reads. + + Returns: + The zero-row stream, or ``None`` when ``upstream`` cannot say its Arrow + types without rows; the caller then builds its input from the Python + output schema. + """ + from orcapod.system_constants import constants + + table = typed_zero_row_input(upstream) + if table is None: + return None + # The column order of a row-built table (``_rows_to_table``): user tags, + # system tags, data, source info. + tag_keys, data_keys = upstream.keys() + names = table.column_names + system_tags = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] + source_info = [n for n in names if n.startswith(constants.SOURCE_PREFIX)] + # A raw stream lists its meta columns as data; the zero-row table has none. + data = [n for n in data_keys if not n.startswith(constants.META_PREFIX)] + table = table.select([*tag_keys, *system_tags, *data, *source_info]) + return StaticOutputOperatorPod._as_upstream_stream( + StaticOutputOperatorPod._flag_source_info_nullability(table), + tag_keys, + upstream, + ) + + def zero_row_result(self, *inputs: StreamProtocol) -> StreamProtocol: + """This operator's result over zero-row inputs, for its zero-row table (ITL-912). + + ``static_process`` over the inputs, or, for an operator whose output schema is + its input's (``OUTPUT_SCHEMA_IS_INPUT_SCHEMA``), the input's columns in the + large-normalized form a run gives them, so no predicate or other user code is + evaluated. A run never comes here: it calls + ``static_process`` (or ``process``) whether its inputs have rows or not. + + Args: + inputs: The inputs as zero-row streams. + + Returns: + A stream with no rows and the columns and Arrow types of the result. + """ + if self.OUTPUT_SCHEMA_IS_INPUT_SCHEMA: + from orcapod.utils import arrow_utils + + (only,) = inputs + # The input's columns, each in the large-normalized form a run gives it. + schema = only.as_table(columns=ZERO_ROW_INPUT_COLUMNS).schema + normalized = pa.schema( + [ + field.with_type( + arrow_utils.normalize_to_large_types(field.type, keep_null=True) + ) + for field in schema + ] + ) + return ArrowTableStream( + normalized.empty_table(), + tag_columns=only.keys()[0], + data_context=only.data_context_key, + ) + return self.static_process(*inputs) + @staticmethod def _materialize_to_stream( rows: Sequence[tuple[TagProtocol, DataProtocol]], @@ -297,9 +402,11 @@ def _materialize_upstream( system-tag column names come out as they would on the upstream itself. With rows, the table is ``_rows_to_table(rows)``: every row keeps its - own system tags and source info. Without rows, the table is built from - the upstream's Python output schema (with system tags and source info), - which cannot recover Arrow types the rows would have had (ITL-912). + own system tags and source info. Without rows, it is the upstream's + zero-row table (``_zero_row_input``), with the Arrow types its rows have + (ITL-912). Only an upstream that cannot say its Arrow types without rows + (see ``typed_zero_row_table``) is built from its Python output schema + (with system tags and source info), a known limitation (ITL-1089). Args: rows: The rows the upstream delivered; may be empty. @@ -309,29 +416,58 @@ def _materialize_upstream( Returns: A stream over the rows (none when ``zero_rows``), in order. """ - from orcapod.core.base import DataContextMixin - from orcapod.system_constants import constants from orcapod.utils import arrow_utils - # The rebuilt stream is in the upstream's data context, as its rows are. - context = ( - upstream.data_context - if isinstance(upstream, DataContextMixin) - else contexts.resolve_context(upstream.data_context_key) - ) if rows: table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) if zero_rows: table = table.slice(0, 0) - else: - # Build an empty table preserving declared field nullability. - tag_schema, data_schema = upstream.output_schema( - columns={"system_tags": True, "source": True} + return StaticOutputOperatorPod._as_upstream_stream( + table, tag_keys, upstream ) - table = arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, context.type_converter - ) - tag_keys = upstream.keys()[0] + typed = StaticOutputOperatorPod._zero_row_input(upstream) + if typed is not None: + return typed + # Build an empty table preserving declared field nullability. + tag_schema, data_schema = upstream.output_schema( + columns={"system_tags": True, "source": True} + ) + table = arrow_utils.make_empty_table( + {**tag_schema, **data_schema}, + StaticOutputOperatorPod._upstream_context(upstream).type_converter, + ) + return StaticOutputOperatorPod._as_upstream_stream( + table, upstream.keys()[0], upstream + ) + + @staticmethod + def _upstream_context(upstream: StreamProtocol) -> DataContext: + """The data context of ``upstream``, which its rebuilt stream is in.""" + from orcapod.core.base import DataContextMixin + + if isinstance(upstream, DataContextMixin): + return upstream.data_context + return contexts.resolve_context(upstream.data_context_key) + + @staticmethod + def _as_upstream_stream( + table: pa.Table, tag_keys: Sequence[str], upstream: StreamProtocol + ) -> StreamProtocol: + """Wrap a table of ``upstream``'s rows (or none) as ``upstream``'s stream. + + Args: + table: User tags, system tags, data and source info, without the + context column. + tag_keys: The user tag columns. + upstream: The node or stream the table stands for. + + Returns: + A stream with ``upstream``'s producer, upstreams and data context. + """ + from orcapod.system_constants import constants + + # The rebuilt stream is in the upstream's data context, as its rows are. + context = StaticOutputOperatorPod._upstream_context(upstream) # The context column names the upstream's context on every row; without # it, the stream would fill in the default context's key. table = table.append_column( @@ -515,6 +651,32 @@ def clear_cache(self) -> None: self._kept_states = None self._reset_generation += 1 + def _typed_zero_row_table(self) -> pa.Table | None: + """The pod's result over its upstreams' zero-row tables (ITL-912). + + The rule an operator node follows (``OperatorJobNode._typed_zero_row_table``), + with each upstream given as the stream it is, as ``static_process`` reads + it: no row is computed or read, and nothing this stream holds is consulted. + """ + inputs: list[StreamProtocol] = [] + for upstream in self._upstreams: + # The upstream itself with no row: the pod reads the upstream stream, + # not rows rebuilt from it, so its columns stay in their own order. + table = typed_zero_row_table(upstream) + if table is None: + return None + inputs.append( + ArrowTableStream( + table, + tag_columns=upstream.keys()[0], + producer=upstream.producer, + upstreams=upstream.upstreams, + data_context=StaticOutputOperatorPod._upstream_context(upstream), + ) + ) + pod = cast(StaticOutputOperatorPod, self._pod) + return pod.zero_row_result(*inputs).as_table(columns=ZERO_ROW_INPUT_COLUMNS) + def keys( self, *, @@ -637,7 +799,7 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the first rows of the operator's kept result; never runs the operator. + """Return the first rows of the operator's kept result; never runs it over rows. The head of an operator is not the operator over the heads of its inputs (a group over ten rows has the wrong members), so before the result is computed, diff --git a/src/orcapod/core/sources/base.py b/src/orcapod/core/sources/base.py index f594bac80..4dd5f1c89 100644 --- a/src/orcapod/core/sources/base.py +++ b/src/orcapod/core/sources/base.py @@ -4,7 +4,7 @@ from orcapod import contexts from orcapod.config import OrcapodConfig -from orcapod.core.streams.base import StreamBase +from orcapod.core.streams.base import StreamBase, typed_zero_row_table from orcapod.core.streams.view import StreamView from orcapod.protocols.core_protocols import StreamProtocol from orcapod.types import ColumnConfig, Schema @@ -171,6 +171,17 @@ def as_table( # A source's rows are its own, not computed: a run() view says "all N rows". _RUN_ORIGIN: str = "held" + def _typed_zero_row_table(self) -> "pa.Table | None": + """The source's zero-row table, from the stream it holds; see ``typed_zero_row_table``. + + A source that reads its table when it is built (Arrow table, dict, list, data + frame, CSV, Delta, SQLite, PostgreSQL) holds it in ``_stream``, and a SpiralDB + source holds a zero-row stream with the table's schema: either is cut to no + rows, so no row is read. A source with no such stream (polling, cached, + derived, proxy) reads or polls its rows later and answers ``None``. + """ + return typed_zero_row_table(getattr(self, "_stream", None)) + def _display_name(self) -> str: """The name a preview header shows: the label when one was given, else the source id (a source id is the source's name), e.g. a table's name. The label diff --git a/src/orcapod/core/sources/cached_source.py b/src/orcapod/core/sources/cached_source.py index e7171e04c..f1453e51e 100644 --- a/src/orcapod/core/sources/cached_source.py +++ b/src/orcapod/core/sources/cached_source.py @@ -9,7 +9,11 @@ from orcapod.config import OrcapodConfig from orcapod.core.sources.base import RootSource from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.core.streams.base import check_head_size, schema_requires_io +from orcapod.core.streams.base import ( + check_head_size, + schema_requires_io, + typed_zero_row_table, +) from orcapod.core.streams.view import ( ViewStatus, StreamView, @@ -364,7 +368,15 @@ def _build_merged_stream(self) -> ArrowTableStream: ) def _empty_table(self) -> pa.Table: - """Build an empty Arrow table matching the source's output schema.""" + """Build an empty Arrow table matching the source's output schema. + + The wrapped source's zero-row table when it gives one (its Arrow types and + system-tag columns, as ``head()`` shows them; ITL-912), otherwise one built + from its Python output schema. + """ + typed = self._typed_zero_row_table() + if typed is not None: + return typed tag_schema, data_schema = self._source.output_schema() merged = dict(tag_schema) merged.update(data_schema) @@ -412,6 +424,13 @@ def as_table( assert self._cached_stream is not None return self._cached_stream.as_table(columns=columns, all_info=all_info) + def _typed_zero_row_table(self) -> "pa.Table | None": + """The wrapped source's zero-row table: the cache stores that source's rows. + + See ``typed_zero_row_table``; neither the cache nor the source is read. + """ + return typed_zero_row_table(self._source) + def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll: as for the wrapped source.""" return schema_requires_io(self._source, config) diff --git a/src/orcapod/core/sources/polling_source.py b/src/orcapod/core/sources/polling_source.py index 466b1d85c..8935554df 100644 --- a/src/orcapod/core/sources/polling_source.py +++ b/src/orcapod/core/sources/polling_source.py @@ -26,6 +26,7 @@ ) from orcapod.core.streams.base import ( check_head_size, + typed_zero_row_table, ) from orcapod.errors import ( CursorInvalidatedError, @@ -443,6 +444,29 @@ def keys( self._sync_poll_and_commit() return self._get_combined_stream().keys(columns=columns, all_info=all_info) + def _typed_zero_row_table(self) -> pa.Table | None: + """The zero-row table of the rows polled so far; ``None`` before a poll. + + See ``typed_zero_row_table``. Never polls. Before the first poll the source + knows only the Python schema ``impl.schema()`` declares, which cannot say an + Arrow type such as ``int8`` (a known limitation, ITL-912). + """ + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + batches = list(self._batches) # snapshot, as in ``_get_combined_stream`` + if not batches: + return None + # Each batch with no rows, combined as the batches themselves are (the same + # schema check and promotion), so the cost does not grow with the rows polled. + result: ArrowTableStream | None = None + for batch in batches: + table = typed_zero_row_table(batch) + if table is None: + return None + zero_rows = ArrowTableStream(table=table, tag_columns=self._tag_columns) + result = zero_rows if result is None else self._combine(result, zero_rows) + return typed_zero_row_table(result) + def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll. diff --git a/src/orcapod/core/sources/source_proxy.py b/src/orcapod/core/sources/source_proxy.py index 713f6131c..75a6a84ff 100644 --- a/src/orcapod/core/sources/source_proxy.py +++ b/src/orcapod/core/sources/source_proxy.py @@ -21,6 +21,7 @@ from orcapod.types import ColumnConfig from orcapod.core.streams.base import ( check_head_size, + typed_zero_row_table, ) from orcapod.protocols.core_protocols import SourceProtocol from orcapod.types import ContentHash, Schema @@ -199,6 +200,14 @@ def _require_delegate(self) -> SourceProtocol: ) return self._delegate + def _typed_zero_row_table(self) -> pa.Table | None: + """The bound delegate's zero-row table; ``None`` while unbound. + + See ``typed_zero_row_table``. An unbound proxy knows only its stored Python + schema (a known limitation, ITL-912). + """ + return typed_zero_row_table(self._delegate) + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: return self._require_delegate().iter_data() diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 2966327d4..64b3da113 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -8,6 +8,7 @@ from orcapod import contexts from orcapod.core.datagrams import Data, Tag from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, StreamBase, check_head_size, ) @@ -279,6 +280,10 @@ def head( total_rows=self._table.num_rows, ) + def _typed_zero_row_table(self) -> "pa.Table": + """This stream's table cut to no rows: its own Arrow types, no row assembled.""" + return self._assemble_table(ZERO_ROW_INPUT_COLUMNS, limit=0) + def _assemble_table( self, columns_config: ColumnConfig, limit: int | None ) -> "pa.Table": diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 12782ba7a..9882ac6e2 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -3,8 +3,9 @@ import logging from abc import abstractmethod from collections.abc import AsyncIterator, Collection, Iterator, Mapping +from contextvars import ContextVar from datetime import datetime -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from orcapod.core.base import TraceableBase from orcapod.core.streams.view import ( @@ -54,6 +55,109 @@ def schema_requires_io(stream: object, config: ColumnConfig) -> bool: return bool(check(config)) if callable(check) else False +ZERO_ROW_INPUT_COLUMNS = ColumnConfig(system_tags=True, source=True) +"""The column groups of a zero-row table: what a node's rows carry downstream.""" + + +_ZERO_ROW_MEMO: ContextVar[dict[int, tuple[object, pa.Table | None]] | None] = ( + ContextVar("_ZERO_ROW_MEMO", default=None) +) +"""The zero-row tables derived during one outermost ``typed_zero_row_table`` call.""" + + +def typed_zero_row_table(stream: object) -> pa.Table | None: + """Return ``stream``'s table with no rows and the Arrow types its rows would have. + + The table has the user tags, system tags, data and source-info columns + (``ZERO_ROW_INPUT_COLUMNS``; never a meta column) in the order the stream's own table has them, with + the Arrow types, nullability and logical types the stream's rows have (ITL-912). Getting it never computes, reads or polls a row: + an in-memory table is sliced, a source answers from the table or schema it already + holds, and an operator node applies its operator to its inputs' zero-row tables, as + the sync path applies it to zero-row inputs. + + A stream that does not say (``_typed_zero_row_table``) cannot know its Arrow types + without rows. The caller then builds the table from the Python output schema. + + Within one outermost call each stream is derived once, so an upstream that several + inputs share is not derived again per path; the memo is discarded when that call + returns and is never shared between threads or tasks. + + Args: + stream: The stream, node or source whose zero-row table is wanted. + + Returns: + The zero-row table, or ``None`` when ``stream`` cannot type it without rows: + a function pod's stream or node (ITL-985), an unbound input slot, a source + that reads, polls or derives its rows later, a pod stream not yet computed. + """ + memo = _ZERO_ROW_MEMO.get() + if memo is None: + # The outermost call: a memo for this derivation only, so an upstream that + # several inputs share (a diamond) is derived once. ``reset`` discards it, and + # a ContextVar keeps it apart from other threads and tasks. + token = _ZERO_ROW_MEMO.set({}) + try: + return typed_zero_row_table(stream) + finally: + _ZERO_ROW_MEMO.reset(token) + known = memo.get(id(stream)) + if known is not None and known[0] is stream: + return known[1] + table = _derive_zero_row_table(stream) + # The stream is held with its table, so its id is not reused during the call. + memo[id(stream)] = (stream, table) + return table + + +def typed_zero_row_input(stream: object) -> pa.Table | None: + """Return ``stream``'s zero-row table as a pod reads it: in the large-normalized form. + + What a function pod or an operator takes from an input's zero-row table + (``typed_zero_row_table``): the input's types in the form its rows have downstream + (``arrow_utils.normalize_to_large_types`` with ``keep_null=True``: ``string`` to + ``large_string``, ``binary`` to ``large_binary``, ``list`` to ``large_list``, + recursively), as the populated path normalizes them (ITL-912). Only layouts change: + width, nullability, field names and logical types are kept, and no value is cast + (there is none). A source's own table is not changed: ``typed_zero_row_table`` still + gives it as it holds it. + + Args: + stream: The input whose zero-row table a pod reads. + + Returns: + The normalized zero-row table, or ``None`` when ``stream`` cannot type it + without rows. + """ + table = typed_zero_row_table(stream) + if table is None: + return None + return table.cast( + pa.schema( + [ + field.with_type( + arrow_utils.normalize_to_large_types(field.type, keep_null=True) + ) + for field in table.schema + ] + ).with_metadata(table.schema.metadata or {}) + ) + + +def _derive_zero_row_table(stream: object) -> pa.Table | None: + """Ask ``stream`` for its zero-row table and drop the meta columns; no memo.""" + method = getattr(stream, "_typed_zero_row_table", None) + table = cast("pa.Table | None", method()) if callable(method) else None + if table is None: + return None + # A row passed downstream carries no meta column (``_rows_to_table`` reads a row's + # data with source info only), so neither does its zero-row table: a cached + # function pod's ``__computed`` stays in that stream's own table. + from orcapod.core.operators.meta_columns import without_meta_columns + + tag_columns = cast(StreamProtocol, stream).keys()[0] + return without_meta_columns(table, tag_columns) + + def check_head_size(n: int) -> None: """Check the row count given to ``head``. @@ -594,7 +698,8 @@ def run(self) -> StreamView: """Compute the whole stream and return a ``StreamView`` of all its rows. The one way to compute any stream outside a job (developer, November 13, 2026, - ITL-1081): ``head()`` and the notebook preview never compute. ``run()`` does + ITL-1081): ``head()`` and the notebook preview never compute a row (see + ``head``). ``run()`` does exactly the work ``as_table()`` does, through the same code path and cache, so a stream already computed is not computed again, and one whose inputs changed is computed again. The view holds every row with the default column configuration; @@ -634,8 +739,18 @@ def head( The rows are ones the stream already holds, has cached from an earlier computation, has stored, or can read with a bounded read of its table. ``head`` - never calls a function, runs an operator, delivers a side effect, polls a source - or writes anything. ``head`` takes the column configuration of ``as_table``, + never computes, polls or delivers a row (it reads only what is held or stored, + or makes a bounded table read), never calls user code (a + function, a predicate, a side effect) and never writes. With no rows to show, it + learns the column types without data (ITL-912): each operator of the stream is + applied to zero-row inputs, which processes no data, so a chain of ``N`` + operators applies up to ``N`` operators to empty tables (about 60 ms for 20); an + operator whose output schema is its input's (``OUTPUT_SCHEMA_IS_INPUT_SCHEMA``, + such as ``PolarsFilter``) is not applied at all. A user-defined operator's + ``static_process`` is applied to zero-row tables and must not have effects. So a pipeline whose inputs + disagree on a column's Arrow type raises from ``head`` the + ``InputValidationError`` its run raises, and a filter with an ill-typed + predicate shows its input's columns: the error appears at the run. ``head`` takes the column configuration of ``as_table``, sorting included, and applies it to the rows the stream has, then takes the first ``n``. The default configuration does not sort, as ``as_table()`` does not, so the rows are the first ``n`` that ``as_table()`` would return. With @@ -678,10 +793,15 @@ def _head_without_rows( status: ViewStatus, note: str | None = None, ) -> StreamView: - """Return a head with no rows: the stream's columns from its declared schema. + """Return a head with no rows: the stream's columns, typed as its rows are. - When reading the schema would poll a source (``schema_requires_io``), the head - has no columns either, and the note says so. + The one place a head's table of no rows is built (ITL-912): the stream's + ``_zero_row_table(config)``, which has the columns and Arrow types + ``as_table(columns=config)`` gives the stream with no rows; getting it + computes, reads, polls and delivers nothing. Only a stream that cannot say its + Arrow types without rows (see ``typed_zero_row_table``; ITL-1089) gets a table + built from its declared Python schema. When reading the schema would poll a source + (``schema_requires_io``), the head has no columns either, and the note says so. """ if schema_requires_io(self, config): return StreamView.without_rows( @@ -692,9 +812,11 @@ def _head_without_rows( note=f"{note or default_note(status)}; {SCHEMA_NEEDS_POLL_NOTE}", ) tag_schema, data_schema = self.output_schema(columns=config) - table = arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, self.data_context.type_converter - ) + table = self._zero_row_table(config) + if table is None: + table = arrow_utils.make_empty_table( + {**tag_schema, **data_schema}, self.data_context.type_converter + ) return StreamView.without_rows( self, status, table, tag_columns=tuple(tag_schema.keys()), note=note ) @@ -712,6 +834,34 @@ def _schema_requires_io(self, config: ColumnConfig) -> bool: """ return any(schema_requires_io(upstream, config) for upstream in self.upstreams) + def _zero_row_table(self, config: ColumnConfig) -> pa.Table | None: + """This stream's table with no rows and the column groups of ``config``. + + What ``as_table(columns=config)`` gives this stream when it has no rows: its + zero-row table (``typed_zero_row_table``) shaped as an ``ArrowTableStream`` + shapes its table. A kind whose ``as_table`` shapes its table another way + overrides this with that shaping. Never computes, reads, polls or delivers. + + Returns: + The table, or ``None`` when the stream cannot type it without rows. + """ + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + typed = typed_zero_row_table(self) + if typed is None: + return None + return ArrowTableStream( + typed, tag_columns=self.keys()[0], data_context=self.data_context_key + ).as_table(columns=config) + + def _typed_zero_row_table(self) -> pa.Table | None: + """This stream's zero-row table; ``None``: unknown without rows. + + See ``typed_zero_row_table``. A stream whose Arrow types are known only from + computed rows (a pod's stream) keeps this default. + """ + return None + def _repr_html_(self) -> str: """Render ``head()`` in tag order for a notebook; never computes.""" return preview_html(self) diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index d19412899..9450f50eb 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -2,8 +2,10 @@ ``head(n)`` on a stream, source or node returns a ``StreamView`` of at most ``n`` rows the object already holds, has cached from an earlier computation, has stored, or can read from -its table with a bounded read. It never computes: no user function is called, no operator -is run, no side effect is delivered, no source is polled and nothing is written. The +its table with a bounded read. It never computes: no row is computed, read, polled or +delivered, no user code (a function, a predicate, a side effect) is called and nothing is +written. With no rows to show, the column types come from applying operators to zero-row +tables, which processes no data (ITL-912; see ``StreamBase.head``). The notebook preview of a stream is its ``head()`` rendered by ``StreamView._repr_html_``. ``run()`` on a stream computes it (the work ``as_table()`` does) and returns a diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index 1b1eeb1e4..b1967f56a 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -217,7 +217,7 @@ def run(self) -> "StreamViewProtocol": Does the work ``as_table()`` does, through the same path and cache; a job node runs as it does in a job. Every stream has it, with no argument: it is the one way to compute a stream outside a job (ITL-1081). ``head()`` and the notebook - preview never compute. + preview never compute a row. Returns: StreamView: Every computed row, with the default column configuration. @@ -234,8 +234,13 @@ def head( """Return at most ``n`` readily available rows, without computing. The rows are ones the stream already holds, has cached from an earlier - computation, has stored, or can read with a bounded read. It never calls a - function, runs an operator, delivers a side effect, polls or writes. + computation, has stored, or can read with a bounded read. It never computes, + polls or delivers a row (it reads only what is held or stored, or makes a bounded + read), never calls user code (a function, a predicate, + a side effect) and never writes. With no rows to show, it may apply operators + to zero-row tables to learn the column types, which processes no data (ITL-912). A + user-defined operator's ``static_process`` is applied to zero-row tables and must + not have effects. Returns: StreamView: The rows, and a status that tells an empty result from one diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index fb9ad3d2e..38526a201 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -23,6 +23,7 @@ StreamBase, check_head_size, schema_requires_io, + typed_zero_row_input, ) from orcapod.core.streams.view import ( SIDE_EFFECTS_NOTE, @@ -49,6 +50,7 @@ from orcapod.system_constants import ( OLD_TRACKING_DB_SCHEMA_VERSIONS, TRACKING_DB_SCHEMA_VERSION, + constants, ) from orcapod.types import ColumnConfig from orcapod.utils import arrow_utils @@ -219,16 +221,39 @@ def as_table( self.iter_data(), column_config, tag_keys=lambda: self.keys()[0], + zero_rows=self._typed_zero_row_table, declared_schema=lambda: self.output_schema(all_info=True), type_converter=self._pod.data_context.type_converter, ) + def _zero_row_table(self, config: ColumnConfig) -> pa.Table | None: + """``as_table(columns=config)`` with no rows; nothing is delivered (ITL-912).""" + typed = self._typed_zero_row_table() + if typed is None: + return None + return _table_of_rows( + iter(()), + config, + tag_keys=lambda: self.keys()[0], + zero_rows=lambda: typed, + declared_schema=lambda: self.output_schema(all_info=True), + type_converter=self._pod.data_context.type_converter, + ) + + def _typed_zero_row_table(self) -> pa.Table | None: + """The input's zero-row table: the pod passes its input's rows through. + + See ``typed_zero_row_table``; the pod's function is not called (ITL-912). + """ + return typed_zero_row_input(self._input_stream) + def _table_of_rows( rows: Iterator[tuple[TagProtocol, DataProtocol]], column_config: ColumnConfig, *, tag_keys: Callable[[], tuple[str, ...]], + zero_rows: Callable[[], pa.Table | None], declared_schema: Callable[[], tuple[Schema, Schema]], type_converter: Any, ) -> pa.Table: @@ -245,8 +270,12 @@ def _table_of_rows( column_config: The column groups to keep, and whether to sort by tags. tag_keys: Returns the names of the regular tag columns. Called after the rows were read: asking a source for its keys can poll it. + zero_rows: Returns the rows' table with no row (``typed_zero_row_table``: user + tags, system tags, data and source info, typed as the rows are), or ``None`` + when it is unknown without rows. Called only for the table of no rows. declared_schema: Returns the declared (tag, data) schemas with every group. Called - only for the table of no rows: asking a source for its schema can poll it. + only for the table of no rows that ``zero_rows`` cannot type: asking a source + for its schema can poll it. type_converter: The data context's type converter, to type the table of no rows. """ tag_tables: list[pa.Table] = [] @@ -266,12 +295,41 @@ def _table_of_rows( ) ) hashes.append(data.content_hash().to_string()) + typed = None if tag_tables else zero_rows() if tag_tables: full = arrow_utils.hstack_tables( pa.concat_tables(tag_tables), pa.concat_tables(data_tables) ) + elif typed is not None: + # No rows: the rows' columns and types (ITL-912), in the order a row's tag and + # data tables give them: tags, system tags, data, context, source info. + from orcapod.core.datagrams.datagram import context_key_field + + names = typed.column_names + source_info = [n for n in names if n.startswith(constants.SOURCE_PREFIX)] + system_tags = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] + parts = [typed.select([*tag_keys(), *system_tags])] + parts.append( + typed.select( + [ + n + for n in names + if n not in parts[0].column_names and n not in source_info + ] + ) + ) + if column_config.context: + parts.append(pa.schema([context_key_field()]).empty_table()) + # A row's data table declares every source-info field nullable + # (``Data.as_table``), and the rows' table is those tables concatenated. + source_table = typed.select(source_info) + nullable = pa.schema([f.with_nullable(True) for f in source_table.schema]) + # Same arrays, new field flags: no cast. + parts.append(pa.Table.from_arrays(source_table.columns, schema=nullable)) + full = arrow_utils.hstack_tables(*parts) else: - # No rows: the declared columns, with the declared field nullability. + # No rows and no Arrow types without them: the declared columns, with the + # declared field nullability. tag_schema, data_schema = declared_schema() full = arrow_utils.make_empty_table( {**tag_schema, **data_schema}, type_converter @@ -896,10 +954,32 @@ def as_table( self.iter_data(), column_config, tag_keys=lambda: self.keys()[0], + zero_rows=self._typed_zero_row_table, declared_schema=lambda: self.output_schema(all_info=True), type_converter=self._pod.data_context.type_converter, ) + def _zero_row_table(self, config: ColumnConfig) -> pa.Table | None: + """``as_table(columns=config)`` with no rows; nothing is delivered (ITL-912).""" + typed = self._typed_zero_row_table() + if typed is None: + return None + return _table_of_rows( + iter(()), + config, + tag_keys=lambda: self.keys()[0], + zero_rows=lambda: typed, + declared_schema=lambda: self.output_schema(all_info=True), + type_converter=self._pod.data_context.type_converter, + ) + + def _typed_zero_row_table(self) -> pa.Table | None: + """The input's zero-row table: the pod passes its input's rows through. + + See ``typed_zero_row_table``; the pod's function is not called (ITL-912). + """ + return typed_zero_row_input(self._input_stream) + @property def node_uri(self) -> tuple[str, ...]: """Canonical URI tuple identifying this side-effect node. diff --git a/superpowers/plans/2026-11-13-itl-912-typed-zero-row-tables.md b/superpowers/plans/2026-11-13-itl-912-typed-zero-row-tables.md new file mode 100644 index 000000000..a1549ad12 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-912-typed-zero-row-tables.md @@ -0,0 +1,129 @@ +# ITL-912: typed zero-row tables for operator nodes — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** An operator node's zero-row table (before a run, after `clear_cache()`, with no stored +table, after a zero-row run, and as an empty input on the sync orchestrator) has exactly the Arrow +types, nullability and system-tag columns the node's rows would have. + +**Architecture:** (Coordinator-approved design (i), November 13, 2026.) One rule, the sync rule: a +node's zero-row table is its operator's result over zero-row inputs. One function, +`typed_zero_row_table(stream) -> pa.Table | None` in `core/streams/base.py` (next to +`schema_requires_io`, same getattr convention), asks a stream for `_typed_zero_row_table()`: +zero rows, system tags and source info, the Arrow types rows would have, or `None` when that cannot +be known without computing or reading rows. No state is stored on the node. Every empty path calls +it and keeps today's Python-schema table only when it returns `None`. + +**Tech Stack:** PyArrow, pytest, uv. + +## Who answers what (`_typed_zero_row_table`) + +| Stream | Answer | Reads rows? | +|---|---|---| +| `StreamBase` (default: function pod streams/nodes, `DynamicPodStream`, ...) | `None` | no | +| `ArrowTableStream` | `_assemble_table(ColumnConfig(system_tags=True, source=True), limit=0)` | no (slice of the in-memory table) | +| `RootSource` with an `ArrowTableStream` `_stream` (ArrowTable, Dict, List, DataFrame, CSV, Delta, SQLite, Postgres: table read at construction; SpiralDB: zero-row schema template) | `_stream`'s answer | no | +| `RootSource` without `_stream` (Polling, Cached, Derived, SourceProxy) | `None` | no | +| `SourceJobNode` | bound source's answer, `None` when unbound | no | +| `OperatorJobNode` | held result's table sliced to 0 rows; else `operator.process()` read with system tags + source; `None` if any input answers `None` or no live operator | no input row; operator runs on zero rows | + +## Callers + +1. `OperatorJobNode._make_empty_table` (as_table with nothing held, `_load_cached_stream_from_db` + and `_replay_from_cache` with no stored table, async zero-row run on `main`). +2. `SyncPipelineOrchestrator._materialize_as_stream`, empty buffer. +3. After PR #369 merges: `StaticOutputOperatorPod._materialize_upstream`, empty-rows branch + (#369 moves (2) there). Coordinator tells me when #369 merges. + +## Fallback (unchanged table, documented as known limitation, ITL-985) + +Function node upstream not run / zero rows; unbound input slot; polling/cached/derived source; +loaded node without stored table or live upstreams. + +## Tasks + +### Task 1: failing tests + +**Files:** `tests/test_pipeline/test_operator_node_empty_table_types.py` (new), +`tests/test_pipeline/test_operator_node_async_readback.py` (remove xfail marks). + +- [ ] Matrix test: source with tags `id int8 NN`, `grp large_string NN`; data `x int8 NN`, `p Path`, + `l large_list NN`. Pipelines: filter over source, select after a filter, + batch after a filter. States: before run, after clear_cache, zero-row run (same pipeline, source + with no matching row). Orchestrators sync/async. Assert `as_table(all_info=True).schema` equals + the run-with-rows schema. +- [ ] No-computation proof: source subclass whose `iter_data`/`as_table` raise; operator node over + it reads as typed zero rows before a run. Function pod upstream counts calls: zero calls, and the + node keeps today's table (fallback) — assert the fallback table equals the Python-schema table. +- [ ] Remove xfail from `test_clear_cache_keeps_operator_node_arrow_types` and + `test_operator_node_with_no_rows_keeps_columns_and_types_after_async_run` (the async zero-row + case is fixed on main too, through rule (1)). +- [ ] Run, confirm they fail. Commit (WIP, tests only). + +### Task 2: implement + +**Files:** `src/orcapod/core/streams/base.py`, `src/orcapod/core/streams/arrow_table_stream.py`, +`src/orcapod/core/sources/base.py`, `src/orcapod/core/nodes/source_node.py`, +`src/orcapod/core/nodes/operator_node.py`, `src/orcapod/pipeline/sync_orchestrator.py`. + +```python +# core/streams/base.py +def typed_zero_row_table(stream: object) -> "pa.Table | None": + method = getattr(stream, "_typed_zero_row_table", None) + return method() if callable(method) else None + +class StreamBase: + def _typed_zero_row_table(self) -> "pa.Table | None": + return None +``` + +```python +# OperatorJobNode +def _typed_zero_row_table(self): + if self._cached_output_stream is not None: + return self._cached_output_stream.as_table(columns=_INPUT_COLUMNS).slice(0, 0) + if self._operator is None: + return None + inputs = [] + for upstream in self._input_streams: + table = typed_zero_row_table(upstream) + if table is None: + return None + inputs.append(ArrowTableStream(table, tag_columns=upstream.keys()[0], + producer=upstream.producer, upstreams=upstream.upstreams)) + return self._operator.process(*inputs).as_table(columns=_INPUT_COLUMNS) + +def _make_empty_table(self): + typed = self._typed_zero_row_table() + if typed is not None: + return typed + ... today's code ... +``` + +- [ ] Implement; run the new tests and the readback tests. +- [ ] Commit. + +### Task 3: docs, changelog, design issue, verification + +- [ ] `as_table` docstring of `OperatorJobNode`: the zero-row table rule and the fallback cases. +- [ ] `changelog.d/ITL-912.md`: `### Fixed` + `### Known limitations` (ITL-985). +- [ ] `design_issues/ITL-912-.md` (resolved, with Fix note); `--check`. +- [ ] Targeted tests (pipeline, operators, nodes, golden) in one pytest process on 3.12; ruff; + basedpyright exit code. +- [ ] Push, PR (`Fixes ITL-912`, #369 merges first), link, CodeRabbit loop. + +## Notes for a successor + +- #369 (ITL-1080) touches the same block of `OperatorJobNode.async_execute` and + `_materialize_as_stream`; on merge, keep #369's taps and route its `_materialize_upstream` + empty-rows branch through `typed_zero_row_table` before the Python-schema fallback. + +## Follow-up (coordinator probe, head 45698b4c): non-large inputs + +Finding: a source holding `string` tags/data gives `string` in a function or operator node's +zero-row table, `large_string` once rows exist. Fix: a consumer of an input's zero-row table +(`function_zero_row_table`, `_zero_row_input`, the pod stream's `_typed_zero_row_table`) +normalizes it with `arrow_utils.normalize_to_large_types(keep_null=True)` per field, as the +populated path does (a type-only change; no cast of values). A source's own table is not +touched. Steps: failing test (tests/test_core/streams/test_zero_row_large_types.py) -> fix -> +targeted tests -> one commit/push -> source report (main vs branch). diff --git a/tests/test_channels/test_polling_source.py b/tests/test_channels/test_polling_source.py index 7d95719a9..eccc69523 100644 --- a/tests/test_channels/test_polling_source.py +++ b/tests/test_channels/test_polling_source.py @@ -426,6 +426,32 @@ def test_sync_snapshot_fetch_called_once_on_repeated_access(self): assert len(fake.fetch_cursors) == 1 assert len(fake.poll_cursors) == 1 + def test_zero_row_table_is_the_polled_batches_and_never_polls(self): + """``typed_zero_row_table``: ``None`` before a poll; then the polled table's. + + Non-public on purpose: the test is about the helper's contract (ITL-912). + """ + from orcapod.core.streams.base import typed_zero_row_table + + fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) + src = PollingSource( + fake, tag_columns="id", polling_config=PollingConfig(interval=1.0) + ) + assert typed_zero_row_table(src) is None + assert fake.fetch_cursors == [] and fake.poll_cursors == [] + + list(src.iter_data()) + list(src.iter_data()) # two batches polled and combined + expected = src.as_table(columns={"system_tags": True, "source": True}) + polls = (len(fake.poll_cursors), len(fake.fetch_cursors)) + + table = typed_zero_row_table(src) + + assert table is not None + assert table.num_rows == 0 + assert table.schema == expected.schema + assert (len(fake.poll_cursors), len(fake.fetch_cursors)) == polls + def test_sync_cache_miss_refetches_and_combines(self): """Second iter_data() re-fetches when poll returns True.""" fake = FakeDynamicSource(batches=[_batch(1, 10), _batch(2, 20)]) diff --git a/tests/test_core/streams/test_head_and_preview.py b/tests/test_core/streams/test_head_and_preview.py index e9239cecf..3cd815c22 100644 --- a/tests/test_core/streams/test_head_and_preview.py +++ b/tests/test_core/streams/test_head_and_preview.py @@ -260,6 +260,18 @@ def _grouped_source() -> ArrowTableSource: return ArrowTableSource(table, tag_columns=["id", "grp"], source_id="grouped") +def _calls_over_rows(spy: mock.MagicMock) -> int: + """How many calls of the spied ``static_process`` were given rows. + + A call over inputs with no rows gives only the operator's zero-row table, which a + head with no rows shows without computing a row (ITL-912). + """ + return sum( + any(stream.as_table().num_rows for stream in call.args[1:]) + for call in spy.call_args_list + ) + + OPERATOR_CASES = { "join": (Join, lambda: Join()(_source(), _other_source())), "group_by": (GroupBy, lambda: GroupBy(by=["grp"])(_grouped_source())), @@ -282,13 +294,13 @@ def test_operator_stream_is_not_run_by_head_or_preview(case: str) -> None: ) as spy: head = stream.head() _ = stream._repr_html_() - assert spy.call_count == 0 + assert _calls_over_rows(spy) == 0 assert head.status is ViewStatus.NOT_COMPUTED assert len(head) == 0 table = stream.as_table() - assert spy.call_count == 1 + assert _calls_over_rows(spy) == 1 after = stream.head(2) - assert spy.call_count == 1 + assert _calls_over_rows(spy) == 1 assert after.status is ViewStatus.ROWS assert after.table.equals(table.slice(0, 2)) diff --git a/tests/test_core/streams/test_run.py b/tests/test_core/streams/test_run.py index 5f3796352..4b6c18b80 100644 --- a/tests/test_core/streams/test_run.py +++ b/tests/test_core/streams/test_run.py @@ -362,6 +362,18 @@ def test_cached_function_pod_stream_computes_each_row_once() -> None: assert len(calls) == ROWS +def _calls_over_rows(spy: mock.MagicMock) -> int: + """How many calls of the spied ``static_process`` were given rows. + + A call over inputs with no rows gives only the operator's zero-row table, which a + head with no rows shows without computing a row (ITL-912). + """ + return sum( + any(stream.as_table().num_rows for stream in call.args[1:]) + for call in spy.call_args_list + ) + + def test_operator_stream_run_recomputes_after_its_upstream_changed() -> None: """``run()`` on an operator stream recomputes a stale result, as ``as_table()`` does.""" calls: list[int] = [] @@ -374,13 +386,13 @@ def test_operator_stream_run_recomputes_after_its_upstream_changed() -> None: MapData, "static_process", autospec=True, side_effect=original ) as spy: first = out.run() - assert spy.call_count == 1 + assert _calls_over_rows(spy) == 1 _ = out.run() - assert spy.call_count == 1 + assert _calls_over_rows(spy) == 1 fn_stream.clear_cache() # the upstream is modified after the result was kept assert out.head().status is ViewStatus.NOT_COMPUTED second = out.run() - assert spy.call_count == 2 + assert _calls_over_rows(spy) == 2 assert first.as_table().equals(second.as_table()) assert len(calls) == 2 * ROWS diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index f8a08d6a0..a362b6920 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -314,13 +314,26 @@ def _live_replaying(store: ArrowDatabaseProtocol, job: PipelineJob) -> Any: return node -def _assert_reads_as_live_with_no_rows(loaded_node: Any, live_node: Any) -> None: - """The loaded node reads what a running node reads from the same store: no rows.""" +def _assert_reads_as_live_with_no_rows( + loaded_node: Any, live_node: Any, *, same_types: bool = True +) -> None: + """The loaded node reads what a running node reads from the same store: no rows. + + ``same_types=False`` where the loaded node's input slots are unbound: a running + node types its zero-row table from its sources (ITL-912), while the loaded node has + only their Python schema, so only the user columns are compared (ITL-1089). + """ + user_columns = [*live_node.keys()[0], *live_node.keys()[1]] for config in ({}, {"all_info": True}): table = loaded_node.as_table(**config) live = live_node.as_table(**config) assert table.num_rows == 0 - assert table.schema.equals(live.schema), (table.schema, live.schema) + if same_types: + assert table.schema.equals(live.schema), (table.schema, live.schema) + else: + assert [n for n in table.column_names if n in user_columns] == [ + n for n in live.column_names if n in user_columns + ] assert list(loaded_node.iter_data()) == [] head = loaded_node.head() assert head.status is ViewStatus.NOTHING_STORED @@ -378,7 +391,30 @@ def test_an_absent_cache_table_gives_an_empty_result( node = _load_replaying(_save(job, tmp_path), empty_store).nodes["joined"] # The job file records the run's 2 rows: an advisory warning, not an error. with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): - _assert_reads_as_live_with_no_rows(node, _live_replaying(empty_store, job)) + _assert_reads_as_live_with_no_rows( + node, _live_replaying(empty_store, job), same_types=False + ) + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "ITL-1089: a loaded node over an unbound input slot has only a Python schema, " + "so its zero-row table cannot have the types a running node's has (ITL-912)" + ), +) +def test_an_absent_cache_table_gives_the_running_nodes_types( + tmp_path: pathlib.Path, +) -> None: + """The loaded node's zero-row table should have the running node's exact schema.""" + job = _run(InMemoryArrowDatabase(), "non_null") + empty_store = InMemoryArrowDatabase() + node = _load_replaying(_save(job, tmp_path), empty_store).nodes["joined"] + live = _live_replaying(empty_store, job) + with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): + table = node.as_table(all_info=True) + assert table.schema.equals(live.as_table(all_info=True).schema) def _run_empty_join(store: ArrowDatabaseProtocol) -> PipelineJob: @@ -422,7 +458,11 @@ def test_a_loaded_run_with_an_empty_result_reads_as_empty( job = _run_empty_join(store) assert job.nodes["joined"].as_table().num_rows == 0 node = _load_replaying(_save(job, tmp_path), store).nodes["joined"] - _assert_reads_as_live_with_no_rows(node, _live_replaying(store, job)) + # With other runs' rows the table exists and both read the stored types; without, + # the loaded node's unbound slots leave it the Python schema (ITL-1089). + _assert_reads_as_live_with_no_rows( + node, _live_replaying(store, job), same_types=other_runs_stored + ) def test_an_empty_nullable_list_means_every_column_is_non_null( diff --git a/tests/test_pipeline/test_node_head.py b/tests/test_pipeline/test_node_head.py index 7cbea6a69..a12320d24 100644 --- a/tests/test_pipeline/test_node_head.py +++ b/tests/test_pipeline/test_node_head.py @@ -23,6 +23,7 @@ from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.streams import ViewStatus from orcapod.types import CacheMode, Schema @@ -81,7 +82,18 @@ def _job(store: ArrowDatabaseProtocol) -> PipelineJob: def test_nodes_before_a_run_show_no_rows_and_compute_nothing() -> None: """Before a run, nodes show no rows and run neither the function nor the operator.""" job = _job(InMemoryArrowDatabase()) - # The operator must not run: either entry point raises if it is called. + original = SelectDataColumns.static_process + + def static_process_over_no_rows( + operator: SelectDataColumns, *streams: StreamProtocol + ) -> StreamProtocol: + # Over inputs with no rows it gives only the zero-row table a head shows + # (ITL-912); over rows it would compute them. + if any(stream.as_table().num_rows for stream in streams): + raise AssertionError("static_process ran over rows") + return original(operator, *streams) + + # The operator must not run over rows: either entry point raises if it does. with ( mock.patch.object( SelectDataColumns, "process", side_effect=AssertionError("process ran") @@ -89,7 +101,8 @@ def test_nodes_before_a_run_show_no_rows_and_compute_nothing() -> None: mock.patch.object( SelectDataColumns, "static_process", - side_effect=AssertionError("static_process ran"), + autospec=True, + side_effect=static_process_over_no_rows, ), ): fn_head = job.nodes["triple"].head() diff --git a/tests/test_pipeline/test_operator_node_async_readback.py b/tests/test_pipeline/test_operator_node_async_readback.py index 3dee317cc..c2db09a9a 100644 --- a/tests/test_pipeline/test_operator_node_async_readback.py +++ b/tests/test_pipeline/test_operator_node_async_readback.py @@ -39,7 +39,6 @@ from orcapod.core.nodes import OperatorJobNode from orcapod.core.sources import ArrowTableSource from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase -from orcapod.errors import InputValidationError from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator @@ -431,8 +430,8 @@ def _no_match(stream: StreamProtocol) -> StreamProtocol: Case( "batch_all_after_no_rows", lambda a, b, c: Batch()(_no_match(a), label="op"), 0 ), - # Both inputs empty: a zero-row operator node reads back types built from Python - # types (ITL-912), so one empty and one non-empty input disagree on ``id`` on sync. + # Both inputs empty: each is its upstream's zero-row table, typed as its rows are + # (ITL-912). Case( "merge_join_after_no_rows", lambda a, b, c: MergeJoin()(_no_match(a), _no_match(c), label="op"), @@ -533,8 +532,9 @@ def test_operator_node_with_no_rows_reads_back_empty_after_async_run( 0, ), # An input that delivers no row at all: both paths build that input from the - # upstream's Python output schema (ITL-912), so they agree with each other. - *AFTER_NO_ROWS_CASES[:4], + # upstream's zero-row table (ITL-912), so they agree with each other. The last + # case is a function node, not an operator node. + *AFTER_NO_ROWS_CASES[:-1], ] @@ -629,13 +629,6 @@ def test_clear_cache_returns_operator_node_to_not_yet_run(tmp_path: Path) -> Non assert async_cleared.schema == sync_cleared.schema -@pytest.mark.xfail( - strict=True, - reason=( - "ITL-912: the zero-row table built from the Python output schema widens " - "int8 to int64 and changes nullability" - ), -) @pytest.mark.parametrize("orchestrator", ["sync", "async"]) def test_clear_cache_keeps_operator_node_arrow_types( orchestrator: str, tmp_path: Path @@ -698,23 +691,26 @@ def test_operator_node_reads_back_only_the_latest_run( @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) -def test_join_with_an_input_that_delivered_no_row_fails_alike_on_both_paths( +def test_join_with_an_input_that_delivered_no_row_completes_alike_on_both_paths( orchestrator: str, tmp_path: Path ) -> None: - """An input with no row is rebuilt from its Python output schema on both paths. + """An input with no row is its upstream's zero-row table on both paths. - So an ``int8`` key comes back ``int64`` (ITL-912) and ``Join`` with a non-empty - ``int8`` input refuses the disagreement, on the async path as on the sync path - (ITL-1080); before, the async node finished with a mistyped zero-row result. - When ITL-912 keeps the Arrow types of an empty input, both paths complete. + So an ``int8`` key stays ``int8`` (ITL-912), and ``Join`` with a non-empty ``int8`` + input completes with no row, on the async path as on the sync path (ITL-1080). """ case = Case( "join_after_no_rows", lambda a, b, c: Join()(_no_match(a), b, label="op"), 0, ) - with pytest.raises(InputValidationError, match="'id'"): - _ = _run_job(case, orchestrator, "in_memory", tmp_path) + job = _run_job(case, orchestrator, "in_memory", tmp_path) + node: object = job.nodes["op"] + assert isinstance(node, StreamProtocol) + assert list(node.iter_data()) == [] + table = node.as_table(all_info=True) + assert table.num_rows == 0 + assert table.schema.field("id").type == pa.int8() def test_rebuilt_upstream_keeps_the_upstream_data_context() -> None: diff --git a/tests/test_pipeline/test_operator_node_empty_table_types.py b/tests/test_pipeline/test_operator_node_empty_table_types.py new file mode 100644 index 000000000..2608f8651 --- /dev/null +++ b/tests/test_pipeline/test_operator_node_empty_table_types.py @@ -0,0 +1,767 @@ +"""An operator node's zero-row table has the Arrow types its rows have (ITL-912). + +Before a run, after ``clear_cache()`` and after a run that produced no row, an operator node +reads as a zero-row table. That table must have exactly the columns, Arrow types and +nullability the node's table has after a run with rows: an ``int8`` stays ``int8``, a +non-nullable column stays non-nullable, a list keeps its item type and the system-tag +columns are present. The node gets those types without computing or reading a row: it +applies its operator to zero-row inputs, as the sync path does. +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from typing import Any +from unittest import mock +from dataclasses import dataclass +from pathlib import Path + +import polars as pl +import pyarrow as pa +import pytest + +from orcapod import contexts, function_pod +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.nodes import OperatorJobNode +from orcapod.core.operators import ( + Batch, + Join, + PolarsFilter, + SelectDataColumns, + SemiJoin, +) +from orcapod.core.sources import ArrowTableSource, DeltaTableSource +from orcapod.core.sources.cached_source import CachedSource +from orcapod.core.streams import ArrowTableStream +from orcapod.core.streams.base import typed_zero_row_table +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.side_effects import SideEffectPod +from orcapod.types import ColumnConfig + +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +WITH_ROWS = ("g1", "g1", "g2") +NO_MATCH = ("g2", "g2", "g3") + + +def _path_column() -> pa.ChunkedArray[Any]: + """Three ``Path`` values in the logical type the data context stores them with.""" + converter = contexts.resolve_context(None).type_converter + table = converter.python_dicts_to_arrow_table( + [{"p": Path("/a")}, {"p": Path("/b")}, {"p": Path("/c")}], + python_schema={"p": Path}, + ) + return table.column("p") + + +def _table(groups: tuple[str, ...]) -> pa.Table: + """Non-nullable ``int8`` tag and data, a logical ``Path`` column and a list column.""" + path = _path_column() + schema = pa.schema( + [ + pa.field("id", pa.int8(), nullable=False), + pa.field("grp", pa.large_string(), nullable=False), + pa.field("x", pa.int8(), nullable=False), + pa.field("p", path.type, nullable=False), + pa.field( + "l", + pa.large_list(pa.field("item", pa.int8(), nullable=False)), + nullable=False, + ), + ] + ) + return pa.table( + [ + pa.array([1, 2, 3], pa.int8()), + pa.array(list(groups), pa.large_string()), + pa.array([10, 20, 30], pa.int8()), + path.combine_chunks(), + pa.array([[1], [2, 3], []], schema.field("l").type), + ], + schema=schema, + ) + + +def _source(groups: tuple[str, ...] = WITH_ROWS) -> ArrowTableSource: + """The source every pipeline here reads; same schema whatever ``groups`` is.""" + return ArrowTableSource(_table(groups), tag_columns=["id", "grp"], source_id="a") + + +def _g1(stream: StreamProtocol) -> StreamProtocol: + """``stream`` behind a filter keeping group ``g1``.""" + return PolarsFilter(constraints={"grp": "g1"})(stream) + + +PIPELINES: dict[str, Callable[[StreamProtocol], object]] = { + "filter": lambda a: PolarsFilter(constraints={"grp": "g1"})(a, label="op"), + "select_after_filter": lambda a: SelectDataColumns(["x", "p", "l"])( + _g1(a), label="op" + ), + "batch_after_filter": lambda a: Batch(batch_size=2)(_g1(a), label="op"), +} + + +def _job(pipeline: str, groups: tuple[str, ...] = WITH_ROWS) -> PipelineJob: + """A job holding ``pipeline`` over a source with the given groups; not run.""" + job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with job: + _ = PIPELINES[pipeline](_source(groups)) + return job + + +def _op(job: PipelineJob) -> OperatorJobNode: + """The job's operator node labelled ``op``.""" + node = job.nodes["op"] + assert isinstance(node, OperatorJobNode) + return node + + +def _schema_after_run(pipeline: str, orchestrator: str) -> pa.Schema: + """The node's full schema after a run with rows: the reference.""" + job = _job(pipeline) + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + table = _op(job).as_table(all_info=True) + assert table.num_rows > 0 + return table.schema + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +@pytest.mark.parametrize("pipeline", list(PIPELINES)) +def test_not_yet_run_table_has_the_run_types(pipeline: str, orchestrator: str) -> None: + """Before a run, the node's zero-row table has the types of its rows.""" + expected = _schema_after_run(pipeline, orchestrator) + + table = _op(_job(pipeline)).as_table(all_info=True) + + assert table.num_rows == 0 + assert table.schema == expected + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +@pytest.mark.parametrize("pipeline", list(PIPELINES)) +def test_cleared_table_has_the_run_types(pipeline: str, orchestrator: str) -> None: + """After ``clear_cache()``, the node's zero-row table has the run's types.""" + job = _job(pipeline) + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + node = _op(job) + expected = node.as_table(all_info=True).schema + + node.clear_cache() + table = node.as_table(all_info=True) + + assert table.num_rows == 0 + assert table.schema == expected + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +@pytest.mark.parametrize("pipeline", list(PIPELINES)) +def test_zero_row_run_table_has_the_run_types(pipeline: str, orchestrator: str) -> None: + """A run that produces no row reads with the types of a run with rows. + + ``select_after_filter`` and ``batch_after_filter`` take their input from a node that + produced no row, so they check the empty input each orchestrator builds as well. + """ + expected = _schema_after_run(pipeline, orchestrator) + job = _job(pipeline, NO_MATCH) + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + + table = _op(job).as_table(all_info=True) + + assert table.num_rows == 0 + assert table.schema == expected + + +class _RowsMustNotBeRead(ArrowTableSource): + """A source whose rows must not be read: reading them raises.""" + + def iter_data(self): + raise AssertionError("the source's rows were read") + + def as_table( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> pa.Table: + raise AssertionError("the source's rows were read") + + +def test_not_yet_run_table_reads_no_source_row() -> None: + """The node types its zero-row table from the source's schema, not its rows.""" + expected = _schema_after_run("select_after_filter", "sync") + job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with job: + _ = PIPELINES["select_after_filter"]( + _RowsMustNotBeRead( + _table(WITH_ROWS), tag_columns=["id", "grp"], source_id="a" + ) + ) + + table = _op(job).as_table(all_info=True) + + assert table.num_rows == 0 + assert table.schema == expected + + +@dataclass(frozen=True) +class Point: + """A user class a function returns.""" + + a: int + b: str + + +CALLS: list[int] = [] + + +@function_pod(output_keys=("n", "maybe", "where", "items", "point")) +def describe_x(x: int) -> tuple[int, int | None, Path, list[int], Point]: + """Outputs of each kind: plain, optional, logical, list and user class.""" + CALLS.append(x) + return x, None, Path(f"/{x}"), [x, x], Point(x, str(x)) + + +def _function_stream(groups: tuple[str, ...]) -> StreamProtocol: + """``describe_x`` over the ``g1`` rows of a source with the given groups.""" + return describe_x.pod(_g1(_source(groups))) + + +def _cached_function_stream(groups: tuple[str, ...]) -> StreamProtocol: + """``describe_x`` through a ``CachedFunctionPod`` over the ``g1`` rows.""" + # The decorated function's pod, as the hashing tests pass it (typed loosely). + inner: Any = describe_x.pod + pod = CachedFunctionPod(inner, result_database=InMemoryArrowDatabase()) + return pod(_g1(_source(groups))) + + +@pytest.mark.parametrize( + "make", [_function_stream, _cached_function_stream], ids=["plain", "cached"] +) +def test_function_pod_stream_zero_rows_has_the_computed_types( + make: Callable[[tuple[str, ...]], StreamProtocol], +) -> None: + """A function pod over zero rows gives the table of its computed rows, no row.""" + expected = make(WITH_ROWS).as_table(all_info=True) + assert expected.num_rows > 0 + CALLS.clear() + + table = make(NO_MATCH).as_table(all_info=True) + + assert CALLS == [] + assert table.num_rows == 0 + assert table.schema == expected.schema + + +def _function_job(groups: tuple[str, ...] = WITH_ROWS) -> PipelineJob: + """A job: ``describe_x`` over the ``g1`` rows, then an operator labelled ``op``.""" + job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with job: + _ = SelectDataColumns(["n", "where", "items"])( + describe_x.pod(_g1(_source(groups)), label="fn"), label="op" + ) + return job + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_function_node_zero_row_tables_have_the_computed_types( + orchestrator: str, +) -> None: + """A function node, and an operator after it, type their zero-row tables. + + Before a run and after a run with no row, both tables equal the tables of a run + with rows, and the function is never called for them. + """ + ran = _function_job() + _ = ran.run(orchestrator=ORCHESTRATORS[orchestrator]()) + expected_fn = ran.nodes["fn"].as_table(all_info=True).schema + expected_op = _op(ran).as_table(all_info=True).schema + CALLS.clear() + + not_run = _function_job() + assert not_run.nodes["fn"].as_table(all_info=True).schema == expected_fn + assert _op(not_run).as_table(all_info=True).schema == expected_op + assert CALLS == [] + + no_rows = _function_job(NO_MATCH) + _ = no_rows.run(orchestrator=ORCHESTRATORS[orchestrator]()) + fn_table = no_rows.nodes["fn"].as_table(all_info=True) + op_table = _op(no_rows).as_table(all_info=True) + assert fn_table.num_rows == op_table.num_rows == 0 + assert fn_table.schema == expected_fn + assert op_table.schema == expected_op + assert CALLS == [] + + +def _int8_table() -> pa.Table: + """A small table of non-nullable ``int8`` tag and data columns.""" + return pa.table( + [pa.array([1, 2], pa.int8()), pa.array([10, 20], pa.int8())], + schema=pa.schema( + [ + pa.field("id", pa.int8(), nullable=False), + pa.field("x", pa.int8(), nullable=False), + ] + ), + ) + + +def _arrow_table_source(tmp_path: Path) -> ArrowTableSource: + """An in-memory source; ``tmp_path`` is unused (every maker takes it).""" + del tmp_path + return ArrowTableSource(_int8_table(), tag_columns=["id"], source_id="kinds") + + +def _cached_source(tmp_path: Path) -> StreamProtocol: + """A cached source over an in-memory source; its cache is never read here.""" + del tmp_path + return CachedSource( + ArrowTableSource(_int8_table(), tag_columns=["id"], source_id="kinds"), + cache_database=InMemoryArrowDatabase(), + ) + + +def _delta_source(tmp_path: Path) -> StreamProtocol: + """A Delta table source; its table is read when it is built.""" + from deltalake import write_deltalake + + write_deltalake(tmp_path / "delta", _int8_table(), mode="overwrite") + return DeltaTableSource(tmp_path / "delta", tag_columns=["id"], source_id="kinds") + + +@pytest.mark.parametrize( + "make_source", + [_arrow_table_source, _cached_source, _delta_source], + ids=["arrow_table", "cached", "delta"], +) +def test_operator_node_over_each_source_kind_types_its_zero_row_table( + make_source: Callable[[Path], StreamProtocol], tmp_path: Path +) -> None: + """Before a run, the node's zero-row table has the types of the run's rows.""" + + def job() -> PipelineJob: + """A job selecting ``x`` from a fresh source of this kind.""" + new = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with new: + _ = SelectDataColumns(["x"])(make_source(tmp_path), label="op") + return new + + table = _op(job()).as_table(all_info=True) + ran = job() + _ = ran.run(orchestrator=SyncPipelineOrchestrator()) + expected = _op(ran).as_table(all_info=True) + + assert expected.num_rows == 2 + assert table.num_rows == 0 + assert table.schema == expected.schema + + +def test_a_filter_after_a_cached_function_pod_has_no_meta_column_with_zero_rows() -> ( + None +): + """A zero-row input carries no meta column, as a row passed downstream does not. + + A cached function pod's own table has ``__computed``; an operator after it never + shows it, with rows or without. + """ + + def job(groups: tuple[str, ...]) -> PipelineJob: + """A filter keeping every column, after a cached function pod.""" + new = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with new: + _ = PolarsFilter(constraints={"grp": "g1"})( + _cached_function_stream(groups), label="op" + ) + return new + + ran = job(WITH_ROWS) + _ = ran.run(orchestrator=SyncPipelineOrchestrator()) + in_job = _op(ran).as_table(all_info=True) + not_run = _op(job(WITH_ROWS)).as_table(all_info=True) + + def plain(groups: tuple[str, ...]) -> pa.Table: + """The same filter over the cached function pod's stream, outside a job.""" + stream = PolarsFilter(constraints={"grp": "g1"})( + _cached_function_stream(groups) + ) + return stream.as_table(all_info=True) + + plain_rows = plain(WITH_ROWS) + plain_zero_rows = plain(NO_MATCH) + + assert in_job.num_rows > 0 and plain_rows.num_rows > 0 + assert not_run.num_rows == plain_zero_rows.num_rows == 0 + assert not_run.schema == in_job.schema + assert plain_zero_rows.schema == plain_rows.schema + assert not any(n.startswith("__") for n in not_run.column_names) + + +def test_a_zero_row_table_never_carries_a_meta_column() -> None: + """The zero-row table of a cached function pod's stream has no ``__computed``. + + Non-public on purpose: the test is about ``typed_zero_row_table``'s contract (user + tags, system tags, data, source info; no meta), which no public read shows today. + """ + stream = _cached_function_stream(NO_MATCH) + assert "__computed" in stream.as_table(all_info=True).column_names + + table = typed_zero_row_table(stream) + + assert table is not None + assert not [name for name in table.column_names if name.startswith("__")] + + +def _side_effect_job() -> tuple[PipelineJob, list[int]]: + """A job with a side-effect node ``delivered`` over ``x`` and ``p``; not run.""" + sent: list[int] = [] + + def deliver(ctx: Any, x: int, p: Path) -> None: + sent.append(x) + + job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with job: + selected = SelectDataColumns(["x", "p"])(_g1(_source())) + _ = SideEffectPod(deliver)(selected, label="delivered") + return job, sent + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_side_effect_node_not_run_reads_as_zero_rows_typed_as_its_rows( + orchestrator: str, +) -> None: + """A side-effect job node with no rows from a run has its input's zero-row table. + + It holds the rows of its last run (ITL-1091); without them it reads as zero rows + with the types its rows have (ITL-912), and nothing is delivered to get them. + """ + ran, _ = _side_effect_job() + _ = ran.run(orchestrator=ORCHESTRATORS[orchestrator]()) + after_run = ran.nodes["delivered"].as_table(all_info=True) + assert after_run.num_rows > 0 + + job, sent = _side_effect_job() + not_run = job.nodes["delivered"].as_table(all_info=True) + + assert not_run.num_rows == 0 + assert not_run.schema == after_run.schema + assert sent == [] + + +@function_pod(output_keys=("y", "q")) +def must_not_run(x: int, p: Path) -> tuple[int, Path]: + """A function that must not be called: calling it raises.""" + raise AssertionError("the function was called") + + +def must_not_deliver(ctx: Any, x: int, p: Path) -> None: + """A side effect that must not be delivered: delivering it raises.""" + raise AssertionError("the side effect was delivered") + + +def _every_kind_job(source: ArrowTableSource) -> PipelineJob: + """A job not run: source ``a``, operator ``op``, function ``fn``, side effect ``se``.""" + job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with job: + selected = SelectDataColumns(["x", "p"])(_g1(source), label="op") + _ = must_not_run.pod(selected, label="fn") + _ = SideEffectPod(must_not_deliver)(selected, label="se") + return job + + +HEAD_CONFIGS: dict[str, dict[str, Any] | None] = { + "default": {}, + "system_tags": {"system_tags": True}, + "source": {"source": True}, + "context": {"context": True}, + "meta": {"meta": True}, + "content_hash": {"content_hash": True}, + "sort_by_tags": {"sort_by_tags": True}, + "all_info": None, +} + + +@pytest.mark.parametrize("config", list(HEAD_CONFIGS)) +@pytest.mark.parametrize("node", ["op", "fn", "se"]) +def test_head_of_a_node_not_run_has_the_types_of_its_table( + node: str, config: str +) -> None: + """A head with no rows has the columns and Arrow types ``as_table`` gives. + + Under every column configuration, for an operator, function and side-effect job + node (a source node shows the source's rows): neither read calls the function or + delivers the side effect. + """ + columns = HEAD_CONFIGS[config] + options: dict[str, Any] = ( + {"all_info": True} if columns is None else {"columns": columns} + ) + stream = _every_kind_job(_source(NO_MATCH)).nodes[node] + + head = stream.head(**options).table + table = stream.as_table(**options) + + assert head.num_rows == 0 + assert head.schema == table.schema + + +@pytest.mark.parametrize("node", ["op", "fn", "se"]) +def test_head_of_a_node_not_run_reads_no_source_row(node: str) -> None: + """A head with no rows takes its types from the source's schema, not its rows.""" + source = _RowsMustNotBeRead( + _table(WITH_ROWS), tag_columns=["id", "grp"], source_id="a" + ) + + head = _every_kind_job(source).nodes[node].head(all_info=True).table + + assert head.num_rows == 0 + assert head.schema.field("id").type == pa.int8() + assert any(name.startswith("_tag_") for name in head.column_names) + + +UDF_CALLS: list[int] = [] + + +def _record_and_keep(batch: pl.Series) -> pl.Series: + """A user predicate: records the batch length and keeps every row.""" + UDF_CALLS.append(len(batch)) + return pl.Series([True] * len(batch)) + + +def _udf_filter(groups: tuple[str, ...]) -> StreamProtocol: + """A standalone filter whose predicate is user code (a job refuses to hash it).""" + return PolarsFilter([pl.col("x").map_batches(_record_and_keep, pl.Boolean)])( + _g1(_source(groups)) + ) + + +def test_head_of_a_filter_never_calls_its_predicate() -> None: + """A filter's zero-row table is its input's: the predicate is not evaluated.""" + UDF_CALLS.clear() + + head = _udf_filter(WITH_ROWS).head(all_info=True).table + + assert UDF_CALLS == [] + assert head.num_rows == 0 + assert head.schema == _g1(_source(WITH_ROWS)).as_table(all_info=True).schema + + +def test_a_zero_row_run_of_a_filter_still_evaluates_its_predicate() -> None: + """A run with no input row evaluates the predicate once, over an empty batch.""" + UDF_CALLS.clear() + + table = _udf_filter(NO_MATCH).as_table(all_info=True) + + assert UDF_CALLS == [0] + assert table.num_rows == 0 + + +def test_an_ill_typed_predicate_raises_at_run_not_at_head() -> None: + """``head()`` shows the input's columns; the run, even with no row, raises.""" + stream = PolarsFilter([pl.col("grp") > 1])(_g1(_source(NO_MATCH))) + + head = stream.head() + + assert head.table.num_rows == 0 + with pytest.raises(pl.exceptions.PolarsError): + _ = stream.as_table() + + +def test_head_of_a_join_whose_keys_disagree_raises_as_its_run_does() -> None: + """Inputs that disagree on a key's Arrow type: ``head`` raises the run's error. + + The notebook preview (``_repr_html_``, the notebook's display hook) never raises: it + shows the error. + """ + wide = pa.table( + { + "id": pa.array([1, 2, 3], pa.int64()), + "grp": pa.array(list(WITH_ROWS), pa.large_string()), + "y": pa.array([4, 5, 6], pa.int8()), + } + ) + joined = Join()( + _source(), ArrowTableSource(wide, tag_columns=["id", "grp"], source_id="b") + ) + + with pytest.raises(InputValidationError, match="'id'") as at_head: + _ = joined.head() + with pytest.raises(InputValidationError) as at_run: + _ = joined.run() + + assert str(at_head.value) == str(at_run.value) + assert "could not read" in joined._repr_html_() + + +@function_pod(output_keys="y", result_database=InMemoryArrowDatabase()) +def cached_double(x: int) -> int: + """A function pod with a result cache: its rows carry ``__computed``.""" + return 2 * x + + +def test_a_cached_data_function_has_its_meta_column_with_zero_rows() -> None: + """With and without rows, a cached data function's table has the same columns.""" + with_rows = cached_double.pod(_g1(_source(WITH_ROWS))).as_table(all_info=True) + no_rows = cached_double.pod(_g1(_source(NO_MATCH))).as_table(all_info=True) + + assert with_rows.num_rows > 0 + assert no_rows.num_rows == 0 + assert no_rows.schema == with_rows.schema + + +def _string_stream(groups: tuple[str, ...]) -> StreamProtocol: + """A raw stream whose ``grp`` tag is a ``string`` (not ``large_string``) column.""" + return ArrowTableStream( + pa.table( + { + "id": pa.array([1, 2, 3], pa.int8()), + "grp": pa.array(list(groups), pa.string()), + "x": pa.array([10, 20, 30], pa.int8()), + } + ), + tag_columns=["id", "grp"], + ) + + +def test_head_of_a_filter_has_the_large_types_its_run_gives() -> None: + """A filter's run normalizes ``string`` to ``large_string``; its head does too.""" + keep_g1 = PolarsFilter(constraints={"grp": "g1"}) + + head = keep_g1(_string_stream(WITH_ROWS)).head(all_info=True).table + ran = keep_g1(_string_stream(WITH_ROWS)).as_table(all_info=True) + + assert head.schema.field("grp").type == pa.large_string() + assert head.schema == ran.schema + + +def test_an_input_with_a_meta_data_column_gives_a_zero_row_table() -> None: + """A raw stream lists a ``__`` column as data; an operator node over it still + reads as zero rows before a run, without that column.""" + raw = ArrowTableStream( + pa.table( + { + "id": pa.array([1, 2], pa.int8()), + "x": pa.array([1, 2], pa.int8()), + "__debug": pa.array(["a", "b"], pa.large_string()), + } + ), + tag_columns=["id"], + ) + job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) + with job: + _ = SelectDataColumns(["x"])(raw, label="op") + node = _op(job) + + head = node.head(all_info=True).table + table = node.as_table(all_info=True) + + assert head.num_rows == table.num_rows == 0 + assert head.schema == table.schema + assert table.schema.field("x").type == pa.int8() + assert not any("__debug" in name for name in table.column_names) + + +# Both branches share the data columns, which SemiJoin ignores: not under test here. +@pytest.mark.filterwarnings("ignore::orcapod.errors.SemiJoinIgnoredColumnWarning") +def test_a_shared_upstream_is_derived_once_per_zero_row_table() -> None: + """Ten stacked diamonds derive the source's zero-row table once, not 2**10 times.""" + stream: StreamProtocol = _source() + for _ in range(10): + stream = SemiJoin()( + PolarsFilter(constraints={"grp": "g1"})(stream), + PolarsFilter(constraints={"grp": "g2"})(stream), + ) + original = ArrowTableSource._typed_zero_row_table + # A spy on the non-public hook: the test is about how often it is asked. + with mock.patch.object( + ArrowTableSource, "_typed_zero_row_table", autospec=True, side_effect=original + ) as spy: + head = stream.head(all_info=True).table + assert spy.call_count == 1 + _ = stream.head(all_info=True) + # A new outermost call derives again: nothing is kept between calls. + assert spy.call_count == 2 + assert head.schema.field("id").type == pa.int8() + + +class _PerThreadTable(ArrowTableSource): + """A source whose zero-row table names the thread that asked for it.""" + + def _typed_zero_row_table(self) -> pa.Table: + """A zero-row table with an ``id`` column and a column named after the thread. + + Returns: + The table, which tells which thread's derivation made it. + """ + name = threading.current_thread().name + return pa.table({"id": pa.array([], pa.int8()), name: pa.array([], pa.int8())}) + + +class _AsksTwice(ArrowTableSource): + """Asks for ``inner``'s zero-row table, waits for the other thread, asks again.""" + + def __init__(self, inner: StreamProtocol, gate: threading.Barrier, **kwargs: Any): + """Hold the stream to ask and the barrier the two threads meet at. + + Args: + inner: The stream whose zero-row table each thread asks for twice. + gate: Synchronizes the two threads: each waits there between its two + asks, so one thread's first ask happens before the other's second. + **kwargs: As for ``ArrowTableSource``. + """ + super().__init__(**kwargs) + self.inner = inner + self.gate = gate + # The column names each thread got from its two asks, by thread name. + self.seen: dict[str, list[list[str]]] = {} + + def _typed_zero_row_table(self) -> pa.Table | None: + """Ask ``inner`` twice around the barrier and record what came back. + + Returns: + ``None`` on purpose: the answers are recorded in ``seen``. + """ + name = threading.current_thread().name + first = typed_zero_row_table(self.inner) + self.gate.wait() + second = typed_zero_row_table(self.inner) + assert first is not None and second is not None + self.seen[name] = [first.column_names, second.column_names] + return None + + +def test_two_concurrent_zero_row_derivations_never_share_a_memo() -> None: + """Each thread's outermost call has its own memo: no table crosses threads.""" + gate = threading.Barrier(2, timeout=10) + inner = _PerThreadTable(_int8_table(), tag_columns=["id"], source_id="inner") + outer = _AsksTwice( + inner, gate, table=_int8_table(), tag_columns=["id"], source_id="outer" + ) + errors: list[BaseException] = [] + + def derive() -> None: + try: + _ = typed_zero_row_table(outer) + except BaseException as error: # noqa: BLE001 - reported by the test below + errors.append(error) + gate.abort() # the other thread stops waiting at once + + threads = [threading.Thread(target=derive, name=name) for name in ("t1", "t2")] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + assert not any(thread.is_alive() for thread in threads) + assert errors == [] + assert outer.seen == { + "t1": [["id", "t1"], ["id", "t1"]], + "t2": [["id", "t2"], ["id", "t2"]], + } diff --git a/tests/test_pipeline/test_zero_row_non_large_inputs.py b/tests/test_pipeline/test_zero_row_non_large_inputs.py new file mode 100644 index 000000000..b8e5c91b9 --- /dev/null +++ b/tests/test_pipeline/test_zero_row_non_large_inputs.py @@ -0,0 +1,177 @@ +"""A zero-row table has the full Arrow schema of the populated one, for non-large inputs (ITL-912). + +A source may hold ``string``, ``binary`` and ``list`` columns; every downstream table has +their large forms. The table with no rows must have them too: names, types, nullable +flags and order, with every column group. +""" + +from __future__ import annotations + +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.operators import ( + Batch, + GroupBy, + Join, + MapTags, + PolarsFilter, + SelectDataColumns, +) +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline.job import PipelineJob + +NESTED = pa.struct([pa.field("a", pa.string()), pa.field("l", pa.list_(pa.string()))]) + + +@function_pod(output_keys="y") +def half(x: int) -> float: + """Half of ``x``.""" + return x / 2 + + +def _source(source_id: str = "s", data_prefix: str = "") -> ArrowTableSource: + """Non-large tag and data columns, a non-null narrow int tag.""" + schema = pa.schema( + [ + pa.field("k", pa.int8(), nullable=False), + pa.field("g", pa.string()), + pa.field("tb", pa.binary()), + pa.field(data_prefix + "x", pa.int32(), nullable=False), + pa.field(data_prefix + "s", pa.string()), + pa.field(data_prefix + "b", pa.binary()), + pa.field(data_prefix + "l", pa.list_(pa.string())), + pa.field(data_prefix + "st", NESTED), + ] + ) + rows = { + "k": [1, 2, 3], + "g": ["a", "a", "b"], + "tb": [b"p", b"p", b"q"], + data_prefix + "x": [1, 2, 3], + data_prefix + "s": ["u", "v", "w"], + data_prefix + "b": [b"1", b"2", b"3"], + data_prefix + "l": [["a"], ["b", "c"], []], + data_prefix + "st": [{"a": "z", "l": ["q"]}] * 3, + } + return ArrowTableSource( + pa.Table.from_pydict(rows, schema=schema), + tag_columns=["k", "g", "tb"], + source_id=source_id, + ) + + +def _build(kind: str, job: PipelineJob) -> None: + """Add the node ``n`` of ``kind`` to the open job.""" + src = _source() + if kind == "function": + half.pod(SelectDataColumns(["x"])(src), label="n") + elif kind == "group_by": + GroupBy(["g"])(src, label="n") + elif kind == "batch": + Batch(batch_size=2)(src, label="n") + elif kind == "map": + MapTags({"g": "g2"})(src, label="n") + elif kind == "filter": + PolarsFilter(constraints={"x": 2})(src, label="n") + elif kind == "join": + Join()(src, _source("t", "o_"), label="n") + elif kind == "group_by_function": + grouped = GroupBy(["g"])(src) + SelectDataColumns(["s"])(grouped, label="n") + else: + raise AssertionError(kind) + + +KINDS = ["function", "group_by", "batch", "map", "filter", "join", "group_by_function"] + + +def _schemas(kind: str) -> tuple[pa.Schema, pa.Schema, pa.Schema]: + """(before run, after clear/zero-row, populated) schemas of node ``n``, all info.""" + job = PipelineJob(name="z", database=InMemoryArrowDatabase()) + with job: + _build(kind, job) + node = job.nodes["n"] + before = node.head(all_info=True).table.schema + assert node.head(all_info=True).table.num_rows == 0 + job.run() + populated = node.as_table(all_info=True).schema + assert node.as_table(all_info=True).num_rows > 0 + node.clear_cache() + after = node.head(all_info=True).table.schema + return before, after, populated + + +@pytest.mark.parametrize("kind", KINDS) +def test_zero_row_schema_equals_populated(kind: str) -> None: + """Not run, and after clear_cache: the full schema equals the populated one.""" + before, after, populated = _schemas(kind) + assert before.equals(populated, check_metadata=False), (before, populated) + assert after.equals(populated, check_metadata=False), (after, populated) + + +@pytest.mark.parametrize("kind", KINDS) +def test_plain_stream_zero_row_head_equals_populated(kind: str) -> None: + """A stream outside a job: head() of a fresh stream matches its populated table.""" + src = _source() + stream = { + "function": lambda: half.pod(SelectDataColumns(["x"])(src)), + "group_by": lambda: GroupBy(["g"])(src), + "batch": lambda: Batch(batch_size=2)(src), + "map": lambda: MapTags({"g": "g2"})(src), + "filter": lambda: PolarsFilter(constraints={"x": 2})(src), + "join": lambda: Join()(src, _source("t", "o_")), + "group_by_function": lambda: SelectDataColumns(["s"])(GroupBy(["g"])(src)), + }[kind]() + zero = stream.head(all_info=True).table.schema + populated = stream.as_table(all_info=True).schema + assert zero.equals(populated, check_metadata=False), (zero, populated) + + +def test_zero_row_run_equals_populated() -> None: + """A run whose filter keeps nothing has the populated run's schema.""" + job = PipelineJob(name="z", database=InMemoryArrowDatabase()) + with job: + PolarsFilter(constraints={"x": -1})(_source(), label="n") + job.run() + empty = job.nodes["n"].as_table(all_info=True).schema + _, _, populated = _schemas("filter") + assert empty.equals(populated, check_metadata=False), (empty, populated) + + +def test_no_op_filter_zero_row_equals_populated() -> None: + """A filter with no predicate passes its input through: head matches the table.""" + stream = PolarsFilter()(_source()) + zero = stream.head(all_info=True).table.schema + populated = stream.as_table(all_info=True).schema + assert zero.equals(populated, check_metadata=False), (zero, populated) + + +def test_side_effect_stream_zero_row_equals_populated() -> None: + """A side-effect pod's stream over non-large inputs: head matches its table.""" + from orcapod import SideEffectPod + + def deliver(ctx: object, x: int) -> None: + return None + + stream = SideEffectPod(deliver)(SelectDataColumns(["x"])(_source())) + zero = stream.head(all_info=True).table.schema + populated = stream.as_table(all_info=True).schema + assert zero.equals(populated, check_metadata=False), (zero, populated) + + +def test_cached_source_over_an_empty_source_equals_its_head() -> None: + """A cache with nothing stored reads with the wrapped source's own Arrow types.""" + from orcapod.core.sources import CachedSource + + empty = ArrowTableSource( + _source().as_table().slice(0, 0), + tag_columns=["k", "g", "tb"], + source_id="empty", + ) + cached = CachedSource(empty, cache_database=InMemoryArrowDatabase()) + zero = cached.head(all_info=True).table.schema + populated = cached.as_table(all_info=True).schema + assert zero.equals(populated, check_metadata=False), (zero, populated)