diff --git a/.zed/rules b/.zed/rules index d9dbc0fe2..423e26fcb 100644 --- a/.zed/rules +++ b/.zed/rules @@ -301,6 +301,7 @@ src/orcapod/ column_selection.py — Select/Drop Tag/Data columns mappers.py — MapTags, MapData (rename columns) filters.py — PolarsFilter + drop_nulls.py — DropNulls (drop rows with a null in named or all columns, T | None → T) hashing/ semantic_hashing/ — BaseSemanticHasher, type handlers semantic_types/ — Type conversion (Python ↔ Arrow) @@ -367,7 +368,8 @@ FunctionNode. ### Strict operator / function pod boundary -Operators: inspect tags (never data content), can rename columns, cannot synthesize values. +Operators: inspect tags (never data content, except filters; see below), can rename columns, +cannot synthesize values. Function Pods: inspect data content (never tags), synthesize new values, cached by content. No data element comes from a tag value (developer, November 11, 2026, ITL-955). An operator @@ -380,6 +382,11 @@ GroupBy is the only operator that reduces row count many→one: it collapses N r tag tuple into one row with list-valued members. It synthesizes no new data values and keys only on tags, never on data. Its non-key tags stay tags, list-valued, as Batch keeps them. +Filters are the operators that read data columns (ITL-943): PolarsFilter evaluates its +predicate on values, and DropNulls reads only whether a value is present (its validity), +never the value. A filter decides which rows exist and never synthesizes a value; no other +operator reads data content. + Operator or derived source? (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, the step is not an operator: it is a derived source. So a tag's value is offered as data only @@ -532,6 +539,18 @@ data. The rules: (circular — it delegates back to the pod) and NOT against as_table() (legitimately differs: output_schema ignores columns.source, so _source_*, _content_hash and _context_key appear in the table but never in the schema). +- DropNulls keeps the rows where every named column (tag or data) is non-null and clears + the nullable flag of each named field, so T | None becomes T. Only top-level validity is + read (not list items, struct children or NaN). Its identity is the sorted tuple of unique + names. Both paths apply one rule, DropNulls._narrow_table: an Arrow validity mask and + the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast + other than large normalization. A named column that is already non-nullable is a no-op. +- DropNulls() with no argument (columns=None) means every user tag and data column of + the input (developer, November 12, 2026). The names are resolved from each input + (DropNulls._target_columns: stream.keys(), or the row's tag and data keys on the + async path) and never stored, so its identity is the fixed ("DropNulls", None), + different from any list of names. System-tag, source-info, context and meta columns are + never included. An explicitly empty collection is not "no argument": it raises ValueError. - GroupBy requires every column in `by` to be a scalar tag column; raises InputValidationError for unknown, data, or list-valued columns (list-valued usually means the stream came from Batch — group before batching, not after). Members sort by non-key tag values with system diff --git a/CLAUDE.md b/CLAUDE.md index 97820f8a4..f2f775901 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -312,7 +312,8 @@ src/orcapod/ │ ├── group_by.py # GroupBy (many→one reduction keyed on tag values) │ ├── column_selection.py # Select/Drop Tag/Data columns │ ├── mappers.py # MapTags, MapData (rename columns) -│ └── filters.py # PolarsFilter +│ ├── filters.py # PolarsFilter +│ └── drop_nulls.py # DropNulls (drop rows with a null in named or all columns, T | None → T) ├── hashing/ │ └── semantic_hashing/ # BaseSemanticHasher, type handlers ├── semantic_types/ # Type conversion (Python ↔ Arrow) @@ -384,7 +385,7 @@ FunctionNode. | | Operator | Function Pod | |---|---|---| -| Inspects data content | Never | Yes | +| Inspects data content | Never, except filters (see below) | Yes | | Inspects / uses tags | Yes | No | | Can rename columns | Yes | No | | Synthesizes new values | No | Yes | @@ -401,6 +402,11 @@ tag tuple into one row with list-valued members. It still synthesizes no new dat every emitted element came from an input row — and it keys only on tags, never on data. Its non-key tags stay tags, list-valued, as `Batch` keeps them. +Filters are the operators that read data columns (ITL-943): `PolarsFilter` evaluates its +predicate on values, and `DropNulls` reads only whether a value is present (its validity), +never the value. A filter decides which rows exist and never synthesizes a value; no other +operator reads data content. + **Operator or derived source?** (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, the step is not an operator: it is a derived source. So a tag's value is offered as data only @@ -576,6 +582,18 @@ data. The rules: `as_table()` (which legitimately differs: `ArrowTableStream.output_schema` ignores `columns.source`, so `_source_*`, `_content_hash`, and `_context_key` appear in the table but never in the schema). +- `DropNulls` keeps the rows where every named column (tag or data) is non-null and clears + the nullable flag of each named field, so `T | None` becomes `T`. Only top-level validity is + read (not list items, struct children or NaN). Its identity is the sorted tuple of unique + names. Both paths apply one rule, `DropNulls._narrow_table`: an Arrow validity mask and + the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast + other than large normalization. A named column that is already non-nullable is a no-op. +- `DropNulls()` with no argument (`columns=None`) means every user tag and data column of + the input (developer, November 12, 2026). The names are resolved from each input + (`DropNulls._target_columns`: `stream.keys()`, or the row's tag and data keys on the + async path) and never stored, so its identity is the fixed `("DropNulls", None)`, + different from any list of names. System-tag, source-info, context and meta columns are + never included. An explicitly empty collection is not "no argument": it raises `ValueError`. - `GroupBy` requires every column in `by` to be a scalar tag column; it raises `InputValidationError` for unknown, data, or list-valued columns (the last usually means the stream came from `Batch` — group before batching, not after). Members are sorted by non-key diff --git a/changelog.d/ITL-943.md b/changelog.d/ITL-943.md new file mode 100644 index 000000000..705db8913 --- /dev/null +++ b/changelog.d/ITL-943.md @@ -0,0 +1,21 @@ +### Added + +- **`DropNulls` operator and `stream.drop_nulls(columns)`** — keeps the rows where + every named column is non-null and narrows each named column from `T | None` to `T`, in + `output_schema()` and in the Arrow table, so a pod whose parameter is a plain `T` can + consume a column that is nullable by specification. The names may be tag columns, data + columns or both; a name that is not a column of the stream raises `InputValidationError`, + and a column that is already non-nullable is left as it is. With no argument + (`DropNulls()`, `stream.drop_nulls()`) the columns are every tag and data column of the + input stream: a row with a null in any of them is dropped and all of them become + non-nullable. System-tag, source-info, context and meta columns are never included. On a + source whose columns are all nullable (SpiralDB stores every column nullable) this form can + drop most rows; naming the columns is the precise form. An empty collection is not "no + argument" and raises `ValueError`. Only the top-level value is + tested (a null list or struct; not list items, struct fields or NaN). Other columns, system + tags, source info and row order are unchanged. The operator's identity is the set of names; + the no-argument form has one fixed identity, different from that of any list of names. + A surviving row's data hash (or tag hash, for a tag column) changes when the named column + was nullable, because nullability is part of the row's Arrow schema; it becomes the hash + the row has in a stream that declared the column non-nullable. No existing hash changes. + (ITL-943) diff --git a/design_issues/ITL-943-DN1.md b/design_issues/ITL-943-DN1.md new file mode 100644 index 000000000..c5c1f03ac --- /dev/null +++ b/design_issues/ITL-943-DN1.md @@ -0,0 +1,35 @@ +## `src/orcapod/core/operators/drop_nulls.py` — null rows and nullable columns (ITL-943) + +### DN1 — No way to turn a `T | None` column into `T` +**Status:** resolved +**Severity:** medium +**Issue:** ITL-943 + +Many value columns are nullable by specification (EDI, Spiral tables), and a pod written as +`def f(x: int)` is refused on an `int | None` column. `PolarsFilter` could drop the null rows, +but the column stayed `T | None`, so the pod was still refused; a nullable key column likewise +could not be joined with a non-nullable one. + +**Fix:** `DropNulls(columns)` and `stream.drop_nulls(columns)` keep the rows where every +named column (tag or data) is non-null and clear the nullable flag of each named field. +Decisions (coordinator, November 11, 2026): one operator for tag and data columns; only +top-level validity is read; a missing column is `InputValidationError`; an already non-nullable +column is a no-op; identity is the sorted tuple of unique names; sync and async paths share one +table rule (`_narrow_table`), pure Arrow. A surviving row's `Data` (or `Tag`) content hash +changes when a named column was nullable, to the hash of the same row declared non-nullable; +no existing golden value moved. + +No-argument form (developer, November 12, 2026: "plain no argument = drop all is fair and as +applied to both tags and data columns"): `DropNulls()` / `stream.drop_nulls()` tests and +narrows every user tag and data column of the input. The names are resolved from each input +(`stream.keys()`; per row on the async path) and never stored, so the identity is the fixed +`("DropNulls", None)` and one operator object works on any stream. `DropNulls()` and +`DropNulls([every column])` give the same rows and schema but are different identities. +System-tag, source-info, context and meta columns are never included (meta columns are +filtered by prefix, because a stream built straight from a table lists them among its data +keys, ITL-846). An explicitly empty collection still raises `ValueError`: an empty list is a +different intent from no argument, and a computed list that came out empty must not drop rows. + +Open: the class and method names (`DropNulls` / `drop_nulls` versus `DropNulls` / +`drop_nulls`) await the developer's decision. Possible follow-up: a variant that raises on a +null instead of dropping the row. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 170cefc88..945ba9ad5 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -13,7 +13,7 @@ This separation is a core Orcapod design principle: | | Operator | Function Pod | |---|---|---| -| Inspects data content | Never | Yes | +| Inspects data content | Never, except filters (see below) | Yes | | Inspects / uses tags | Yes | No | | Can rename columns | Yes | No | | Synthesizes new values | No | Yes | @@ -32,6 +32,10 @@ One rule holds for every operator: **no data element comes from a tag value**. A use tags to join, filter, group or order rows, but it never turns a tag column into a data column, so a function pod never receives tag content. +The filters are the operators that read data columns: `PolarsFilter` evaluates its predicate +on values, and `DropNulls` reads only whether a value is present (its validity), never the +value. Both decide which rows exist; neither synthesizes a value. + ## Operator categories ### `UnaryOperator` -- single input @@ -370,6 +374,145 @@ except ValueError as exc: # Cannot hash the Polars expression 'col("age").python_udf()' for an Orcapod identity ``` +### DropNulls + +`DropNulls(columns)` keeps the rows where every named column is non-null, and narrows +each named column from `T | None` to `T`. `DropNulls()`, with no names, does this for every +tag and data column (see [Every column](#every-column) below). Use it when a column is nullable by specification +but a function needs a value: a pod whose parameter is a plain `int` refuses an `int | None` +column, and `PolarsFilter` can drop the null rows but leaves the type as it was. + +```python +import pyarrow as pa +from orcapod import function_pod +from orcapod.operators import DropNulls +from orcapod.sources import ArrowTableSource + +recordings = ArrowTableSource( + pa.table( + { + "session": pa.array(["s1", "s2", "s3"], pa.large_string()), + "start_ms": pa.array([1000, None, 3000], pa.int64()), + }, + schema=pa.schema( + [ + pa.field("session", pa.large_string(), nullable=False), + pa.field("start_ms", pa.int64(), nullable=True), + ] + ), + ), + tag_columns=["session"], +) +print(recordings.output_schema()[1]) +# Schema({'start_ms': int | None}) + + +@function_pod(output_keys="start_s") +def to_seconds(start_ms: int) -> float: + return start_ms / 1000 + + +try: + to_seconds.pod(recordings) +except ValueError as exc: + print(str(exc).rsplit(": ", 1)[1]) +# column 'start_ms' has type int | None, the function expects int + +present = DropNulls("start_ms")(recordings) +print(present.output_schema()[1]) +# Schema({'start_ms': int}) +print(to_seconds.pod(present).as_table().to_pylist()) +# [{'session': 's1', 'start_s': 1.0}, {'session': 's3', 'start_s': 3.0}] +``` + +The rules: + +- **Columns.** Pass one name or a collection of names; a row survives only if every named + column is non-null. The names may be tag columns, data columns, or both. A name that is + not a tag or data column of the stream raises `InputValidationError`. With no argument + the columns are every tag and data column of the input. An empty collection is not "no + argument": it raises `ValueError`, so a list of names that came out empty never drops + rows by surprise. +- **What counts as null.** Only the top-level value: a null list or a null struct is null. An + empty list, a list that holds nulls, a struct whose fields are null and a floating-point + NaN are not. Nested fields keep their nullability. +- **Already non-nullable columns** are accepted and left as they are, so applying the + operator twice is the same as applying it once. +- **Everything else is unchanged**: other columns' types and nullability, logical types + (a `Path | None` column becomes `Path`), system tags, source info and row order. A result + with no rows has the same narrowed schema. +- **Identity** is the set of names: `DropNulls("a")`, `DropNulls(["a"])` and + `DropNulls({"a"})` are one operator, and the order of names does not matter. + `DropNulls()` has one fixed identity of its own, whatever stream it is applied to. +- **Row hashes.** Nullability is part of a row's Arrow schema, so a surviving row's data + hash changes when a named data column was nullable (its tag hash, when a named tag column + was). The new hash is the one the row has in a stream that declared the column + non-nullable from the start. + +Tag columns work the same way. A stream whose key column is `str | None` cannot be joined +with one whose key is `str` (`Join` never reconciles types), so narrow the key first. The +stream method is `drop_nulls`: + +```python +by_method = recordings.drop_nulls(["session", "start_ms"]) +by_operator = DropNulls(["start_ms", "session"])(recordings) +print(by_method.pipeline_hash() == by_operator.pipeline_hash()) +# True +``` + +#### Every column + +`DropNulls()` and `stream.drop_nulls()`, with no names, drop every row that has a null in +**any** tag or data column, and make every tag and data column non-nullable. System-tag, +source-info, context and meta columns are never tested or changed. + +```python +visits = ArrowTableSource( + pa.table( + { + "session": pa.array(["s1", "s2", "s3", "s4"], pa.large_string()), + "start_ms": pa.array([1000, None, 3000, 4000], pa.int64()), + "notes": pa.array([None, "noisy", None, "ok"], pa.large_string()), + } + ), + tag_columns=["session"], +) +print(visits.output_schema()) +# (Schema({'session': str | None}), Schema({'start_ms': int | None, 'notes': str | None})) + +complete = visits.drop_nulls() +print(complete.output_schema()) +# (Schema({'session': str}), Schema({'start_ms': int, 'notes': str})) +print(complete.as_table().to_pylist()) +# [{'session': 's4', 'start_ms': 4000, 'notes': 'ok'}] + +timed = visits.drop_nulls("start_ms") +print(timed.as_table().to_pylist()) +# [{'session': 's1', 'start_ms': 1000, 'notes': None}, {'session': 's3', 'start_ms': 3000, 'notes': None}, {'session': 's4', 'start_ms': 4000, 'notes': 'ok'}] +``` + +!!! warning "The no-argument form can drop most of a table" + + A null in any tag or data column drops the row, including a column that is empty for a + good reason. Above, `notes` is empty for most sessions, so three of the four rows are + gone although only one of them lacks `start_ms`. Sources whose columns are all nullable + are common: SpiralDB stores every column as nullable. For such a source, name the columns + a downstream step needs (`drop_nulls("start_ms")`); that is the precise form. Use the + no-argument form only when every column must be present. + +The column names are read from each input stream when the operator is applied, so one +`DropNulls()` object works on any stream. On the same stream it gives the rows and schema +of `DropNulls([every tag and data column])`, but the two are different operators: their +identities, and so the pipeline and content hashes of their output streams, differ. + +```python +by_name = DropNulls(["session", "start_ms", "notes"])(visits) +print(complete.as_table() == by_name.as_table()) +# True +print(complete.pipeline_hash() == by_name.pipeline_hash()) +# False +``` + ### Pick and Index `Pick(column, key)` extracts one entry from a struct- or dict-typed data column, and @@ -464,8 +607,8 @@ operators or function pods. Every stream also exposes the operators as methods, which reads well when chaining: `join`, `semi_join`, `batch`, `group_by`, `select_tag_columns`, `select_data_columns`, -`drop_tag_columns`, `drop_data_columns`, `map_tags`, `map_data`, `polars_filter`, `pick` and -`index`. +`drop_tag_columns`, `drop_data_columns`, `map_tags`, `map_data`, `polars_filter`, +`drop_nulls`, `pick` and `index`. ```python import polars as pl diff --git a/src/orcapod/core/operators/__init__.py b/src/orcapod/core/operators/__init__.py index b8e846d5f..c80f6b21f 100644 --- a/src/orcapod/core/operators/__init__.py +++ b/src/orcapod/core/operators/__init__.py @@ -5,6 +5,7 @@ SelectDataColumns, SelectTagColumns, ) +from .drop_nulls import DropNulls from .filters import PolarsFilter from .group_by import GroupBy from .index import Index @@ -27,6 +28,7 @@ "DropTagColumns", "DropDataColumns", "PolarsFilter", + "DropNulls", "Pick", "Index", ] diff --git a/src/orcapod/core/operators/drop_nulls.py b/src/orcapod/core/operators/drop_nulls.py new file mode 100644 index 000000000..b938653f1 --- /dev/null +++ b/src/orcapod/core/operators/drop_nulls.py @@ -0,0 +1,354 @@ +"""DropNulls operator: drop rows with a null in the chosen columns, narrow ``T | None`` to ``T``.""" + +from __future__ import annotations + +import types +import typing +from collections.abc import Collection, Sequence +from typing import TYPE_CHECKING, Any + +from orcapod.channels import ReadableChannel, WritableChannel +from orcapod.core.datagrams import Data, Tag +from orcapod.core.operators.base import UnaryOperator +from orcapod.core.streams.arrow_table_stream import ArrowTableStream +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol +from orcapod.system_constants import constants +from orcapod.types import ColumnConfig, DataType, Schema +from orcapod.utils import arrow_utils +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import pyarrow as pa + import pyarrow.compute as pc +else: + pa = LazyModule("pyarrow") + pc = LazyModule("pyarrow.compute") + + +def _without_none(python_type: DataType) -> DataType: + """Return ``python_type`` with ``None`` removed from a union. + + ``int | None`` gives ``int`` and ``int | str | None`` gives ``int | str``. + A type that does not admit ``None`` is returned unchanged. + + Args: + python_type: A column's Python type. + + Returns: + The type without its ``None`` member. + """ + if typing.get_origin(python_type) not in (typing.Union, types.UnionType): + return python_type + members = tuple(m for m in typing.get_args(python_type) if m is not type(None)) + if len(members) == len(typing.get_args(python_type)) or not members: + return python_type + return typing.Union[members] # noqa: UP007 # built from a runtime tuple of members + + +class DropNulls(UnaryOperator): + """Keep the rows where every named column is non-null; narrow those columns to ``T``. + + A row survives only if all the named columns hold a value. Afterwards each + named column is non-nullable: ``T`` instead of ``T | None`` in + ``output_schema()``, and a non-nullable field in the Arrow table. A pod + whose parameter is a plain ``T`` can then consume the column. + + Without names (``DropNulls()``), the columns are every tag column and every + data column of the input stream: a row with a null in any of them is + dropped, and all of them are non-nullable afterwards. The names are taken + from each input stream when the operator is applied and are never stored, + so one ``DropNulls()`` works on any stream. System-tag, source-info, + context and meta columns are never among them. On a stream whose columns + are all nullable and legitimately empty in places, this form can drop most + rows; naming the columns is the precise form. + + The names may be tag columns, data columns, or both. Only the validity of + the top-level value is read, never the value itself: a null list or a null + struct is null, while an empty list, a list holding nulls, a struct whose + fields are null and a floating-point NaN are not. Nested fields keep their + nullability. + + A named column that is already non-nullable is accepted and left as it is, + so applying the operator twice equals applying it once. + + Everything else passes through unchanged: the other columns' types and + nullability, system tags (names and values), source info and row order. + Because nullability is part of a row's Arrow schema, a surviving row's + ``Data`` content hash changes when a named data column was nullable, and + its ``Tag`` content hash changes when a named tag column was nullable. The + new hash is the one the row has in a stream that declared the column + non-nullable from the start. + + Args: + columns: One column name, a collection of names, or ``None`` (the + default) for every tag and data column of the input stream. + Duplicates and order are ignored: the operator's identity is the + set of names, or one fixed value when no names are given. An empty + collection is not "no argument" and is refused: a list of names + that came out empty is never read as "every column". + + Raises: + ValueError: If ``columns`` is an empty collection. + """ + + def __init__( + self, columns: str | Collection[str] | None = None, **kwargs: Any + ) -> None: + self.columns: tuple[str, ...] | None + if columns is None: + # Every tag and data column of whichever stream the operator is + # applied to; resolved per input, never stored. + self.columns = None + else: + names = (columns,) if isinstance(columns, str) else tuple(columns) + if not names: + raise ValueError( + "DropNulls needs at least one column name; got an empty" + + " collection. Pass no argument to mean every tag and data" + + " column." + ) + # A sorted tuple of unique names, so a list, a tuple and a set of the + # same names (in any order) are one identity. + self.columns = tuple(sorted(set(names))) + super().__init__(**kwargs) + + def identity_structure(self) -> Any: + """Return the operator's identity: its class name and the set of column names. + + Returns: + A ``(class_name, columns)`` tuple, where ``columns`` is the sorted + tuple of unique names, or ``None`` for the no-argument form (every + tag and data column), whatever stream it is applied to. + """ + return (self.__class__.__name__, self.columns) + + def to_config(self) -> dict[str, Any]: + """Serialize this operator to a config dict. + + Returns: + A dict with ``class_name``, ``module_path``, and ``config`` keys, + where ``config`` contains ``columns``: the list of names, or + ``None`` for the no-argument form. + """ + config = super().to_config() + config["config"] = { + "columns": None if self.columns is None else list(self.columns) + } + return config + + def _target_columns( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> tuple[str, ...]: + """Return the columns to test and narrow for an input with these columns. + + Args: + tag_columns: The input's tag column names. + data_columns: The input's data column names. + + Returns: + The operator's names, or every given tag and data column when the + operator was built without names. Meta columns are left out: a + stream built straight from a table lists them among its data + columns (ITL-846), and they are not user columns. + """ + if self.columns is not None: + return self.columns + return tuple( + name + for name in (*tag_columns, *data_columns) + if not name.startswith(constants.META_PREFIX) + ) + + # ------------------------------------------------------------------ + # Validation and schema prediction + # ------------------------------------------------------------------ + + def validate_unary_input(self, stream: StreamProtocol) -> None: + """Check that every named column is a tag or data column of ``stream``. + + Args: + stream: The upstream stream. + + Raises: + InputValidationError: If a named column is not a tag or data column. + """ + if self.columns is None: + return + tag_columns, data_columns = stream.keys() + missing = [ + name + for name in self.columns + if name not in tag_columns and name not in data_columns + ] + if missing: + raise InputValidationError( + f"DropNulls: columns {missing} are not tag or data columns of the" + + f" input stream. Tag columns: {list(tag_columns)}; data columns:" + + f" {list(data_columns)}." + ) + + @staticmethod + def _narrow_schema(schema: Schema, names: Collection[str]) -> Schema: + """Return ``schema`` with ``None`` removed from the type of each column in ``names``.""" + return Schema( + { + name: _without_none(python_type) if name in names else python_type + for name, python_type in schema.items() + }, + optional_fields=schema.optional_fields, + ) + + def unary_output_schema( + self, + stream: StreamProtocol, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """Return the input schemas with the named columns narrowed to ``T``. + + Without names, every tag and data column of ``stream`` is narrowed; + system-tag columns included by ``columns`` are not. + + Args: + stream: The upstream stream. + columns: Column inclusion config passed through to ``output_schema``. + all_info: Whether to include all info columns. + + Returns: + A ``(tag_schema, data_schema)`` tuple. + """ + tag_schema, data_schema = stream.output_schema( + columns=columns, all_info=all_info + ) + names = self._target_columns(*stream.keys()) + return ( + self._narrow_schema(tag_schema, names), + self._narrow_schema(data_schema, names), + ) + + # ------------------------------------------------------------------ + # Execution + # ------------------------------------------------------------------ + + @staticmethod + def _narrow_fields(table: pa.Table, names: Collection[str]) -> pa.Table: + """Flag the columns in ``names`` that are present in ``table`` non-nullable. + + Every column takes the large-normalized form of its input type; no + other type, flag or value changes. The named columns must hold no null. + + Args: + table: A table whose named columns hold no null. + names: The columns to flag. + + Returns: + The same rows with the named fields non-nullable. + """ + table = arrow_utils.restore_column_types( + table, + { + field.name: arrow_utils.normalize_to_large_types( + field.type, keep_null=True + ) + for field in table.schema + }, + ) + schema = table.schema + for name in names: + position = schema.get_field_index(name) + if position >= 0: + schema = schema.set( # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + position, schema.field(position).with_nullable(False) + ) + # Same arrays under new field flags: no cast, so no value or type can change. + return pa.Table.from_arrays(table.columns, schema=schema) + + @staticmethod + def _narrow_table(table: pa.Table, names: Collection[str]) -> pa.Table: + """Drop the rows with a null in a named column and flag those columns non-nullable. + + Only the columns in ``names`` that are present in ``table`` are + considered, so the same rule serves a whole stream table and a single + row's tag or data table. + + Args: + table: A table holding some or all of the named columns. + names: The columns to test and narrow. + + Returns: + The surviving rows, in order, with the named fields non-nullable. + """ + mask = None + for name in names: + if name in table.column_names: + valid = pc.is_valid(table.column(name)) + mask = valid if mask is None else pc.and_(mask, valid) + if mask is not None: + table = table.filter(mask) + return DropNulls._narrow_fields(table, names) + + def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: + """Return the stream's rows whose named columns are all non-null. + + Args: + stream: The upstream stream. + + Returns: + A stream over the surviving rows, in input order, with the named + columns non-nullable. A result with no rows has the same schema. + """ + table = stream.as_table( + columns={"source": True, "system_tags": True, "sort_by_tags": False} + ) + tag_columns, data_columns = stream.keys() + return ArrowTableStream( + self._narrow_table(table, self._target_columns(tag_columns, data_columns)), + tag_columns=tag_columns, + data_context=stream.data_context, # pyright: ignore[reportAttributeAccessIssue] # StreamProtocol omits data_context, which every concrete stream provides (ITL-844) + ) + + async def async_execute( + self, + inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + **kwargs: Any, + ) -> None: + """Streaming: test and narrow each row as it arrives, without materializing. + + Each row's tag and data Arrow tables go through the rule of the sync + path (``_narrow_table``), and a surviving row is rebuilt from those + tables with its own data context. Without names, the columns are the + row's own tag and data keys. + + Args: + inputs: Single-element sequence of readable channels. + output: Channel to send surviving rows to. + **kwargs: Ignored; present for protocol compliance. + """ + try: + async for tag, data in inputs[0]: + names = self._target_columns(tag.keys(), data.keys()) + tag_table = self._narrow_table( + tag.as_table(columns={"system_tags": True}), names + ) + data_table = self._narrow_table( + data.as_table(columns={"source": True, "meta": True}), names + ) + # A table with no columns (a stream without tags) has no rows + # to lose, so only a table that held a named column can drop the row. + if (tag_table.num_columns and tag_table.num_rows == 0) or ( + data_table.num_columns and data_table.num_rows == 0 + ): + continue + new_tag = ( + Tag(tag_table, data_context=tag.data_context_key) + if tag_table.num_columns + else tag + ) + await output.send( + (new_tag, Data(data_table, data_context=data.data_context_key)) + ) + finally: + await output.close() diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 37cc2347a..e6d05cebf 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -187,6 +187,32 @@ def polars_filter( self, label=label ) + def drop_nulls( + self, + columns: str | Collection[str] | None = None, + label: str | None = None, + ) -> StreamBase: + """Keep the rows where every named column is non-null; narrow those columns to ``T``. + + Each named column, tag or data, is non-nullable afterwards (``T`` + instead of ``T | None``), so a pod whose parameter is a plain ``T`` + can consume it. Without names, every tag and data column of this + stream is tested and narrowed: a row with a null in any of them is + dropped. See ``orcapod.core.operators.DropNulls``. + + Args: + columns: One column name, a collection of names, or ``None`` (the + default) for every tag and data column. An empty collection + raises ``ValueError``. + label: Optional node label for the pipeline graph. + + Returns: + A stream over the surviving rows, in input order. + """ + from orcapod.core.operators import DropNulls + + return DropNulls(columns)(self, label=label) + def select_tag_columns( self, tag_columns: str | Collection[str], diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 56b2e46ff..a9f392d1f 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -168,6 +168,7 @@ def _build_operator_registry() -> dict[str, type]: from orcapod.core.operators import ( Batch, DropDataColumns, + DropNulls, DropTagColumns, GroupBy, Join, @@ -193,6 +194,7 @@ def _build_operator_registry() -> dict[str, type]: "MapTags": MapTags, "MapData": MapData, "PolarsFilter": PolarsFilter, + "DropNulls": DropNulls, } diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index b5000e7ef..894e437a3 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -285,6 +285,18 @@ def polars_filter( **constraints: Any, ) -> "StreamProtocol": ... + def drop_nulls( + self, + columns: str | Collection[str] | None = None, + label: str | None = None, + ) -> "StreamProtocol": + """ + Keep the rows where every named column (tag or data) is non-null, and + narrow each named column from ``T | None`` to ``T``. Without names, + every tag and data column is tested and narrowed. + """ + ... + def select_tag_columns( self, tag_columns: str | Collection[str], diff --git a/superpowers/plans/2026-11-11-itl-943-drop-nulls.md b/superpowers/plans/2026-11-11-itl-943-drop-nulls.md new file mode 100644 index 000000000..b19df3578 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-943-drop-nulls.md @@ -0,0 +1,171 @@ +# DropNulls Operator Implementation Plan (ITL-943) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A unary operator `DropNulls(columns)` and `stream.drop_nulls(columns)` that keeps +only rows where every named column is non-null and narrows each named column from `T | None` +to `T` (schema and Arrow field). + +**Architecture:** One operator class in `src/orcapod/core/operators/drop_nulls.py`. Pure +Arrow (validity mask, `Table.filter`, re-wrap the same arrays under a schema whose named +fields are non-nullable); no Polars. A streaming `async_execute` applies the same +table-level narrowing to each datagram's Arrow table. + +**Tech Stack:** pyarrow, pytest, `uv run`. + +--- + +## Design (approved by the coordinator, November 11, 2026) + +Coordinator's answer to the STEP 1 `ask`: (b)-(j) approved as proposed. (a) The NAME was the +developer's call; the developer chose **`DropNulls` / `drop_nulls`** on November 11, 2026, and +the operator (first built under a longer working name) was renamed everywhere, with no alias. + +- **Argument:** `columns: str | Collection[str]`; a bare `str` is one name. Empty collection is + `ValueError` at construction. Stored as `self.columns = tuple(sorted(set(names)))`. +- **Row rule:** a row survives only if every named column is non-null. +- **Missing column:** `InputValidationError` at call time (`validate_unary_input`), naming the + missing names. No `strict` flag. System-tag, source-info, meta and context columns cannot be + named (reported as missing). +- **Already non-nullable column:** accepted, no-op for that column (idempotent; postcondition + semantics; a pipeline's validity must not depend on declared upstream nullability). +- **Tag vs data:** one operator; names may be tag or data columns, mixed. +- **Nested:** only top-level validity is tested (null list / null struct; not an empty list, + list items, struct children, or NaN). Only the top-level field becomes non-nullable. +- **Identity:** `identity_structure() == ("DropNulls", self.columns)`; same for content and + pipeline hash. `"x"`, `["x"]`, `("x",)`, `{"x"}`, `["x", "x"]` are one identity; order is + irrelevant. +- **Hash effects (measured):** nullability is in the Arrow schema hash. A surviving row's + `Data` content hash changes iff a named data column was nullable; its `Tag` content hash + changes iff a named tag column was nullable. The new hash equals the hash the same row has in + a stream that declared the column non-nullable from the start. System-tag names and values + and source-info tokens are unchanged. No existing golden value moves. +- **System tags:** name-preserving. Source-info columns untouched. Row order preserved. +- **Schema prediction:** upstream schema with `None` removed from each named column's type. + Verified against `unary_static_process(stream).output_schema(...)`. Zero-row results keep the + narrowed types. +- **Async:** own streaming `async_execute`; nullness read from each datagram's Arrow table, + surviving rows rebuilt from that table. Added to `test_async_sync_equivalence.py`. +- **Arrow:** every column large-normalized (`keep_null=True`), nothing else changed. +- **Save/load:** `to_config()["config"] == {"columns": [...]}`; registered in + `pipeline/serialization.py`. +- **Boundary note:** the operator reads validity only, never values. Add one line to the + operator table in `CLAUDE.md` and `.zed/rules`; put the wording in the PR for the developer. + +Extra tests asked by the coordinator: NaN is not null; a nullable tag narrowed and then +joined; a Spiral-style source with nullable key columns narrowed to `T`, enabling a `Join` +with a source whose keys are `T`. + +## Files + +- Create: `src/orcapod/core/operators/drop_nulls.py` (the operator) +- Modify: `src/orcapod/core/operators/__init__.py` (export), `src/orcapod/operators/...` + (public re-export, wherever `PolarsFilter` is listed) +- Modify: `src/orcapod/core/streams/base.py` (`drop_nulls` method), + `src/orcapod/protocols/core_protocols/streams.py` (protocol method) +- Modify: `src/orcapod/pipeline/serialization.py` (operator registry) +- Create: `tests/test_core/operators/test_drop_nulls.py` +- Create: `tests/test_pipeline/test_drop_nulls_job.py` (Delta store, cache hits) +- Modify: `tests/test_core/operators/test_async_sync_equivalence.py` (two cases) +- Modify: `tests/test_core/operators/test_operator_config.py` (round trip) +- Modify: `tests/test_hashing/golden/cases.py` (new case `pipeline/drop_nulls`), then + regenerate; the diff of `tests/test_hashing/golden/data/pipeline.json` must only add lines +- Modify: `docs/concepts/operators.md`, `CHANGELOG.md`, `DESIGN_ISSUES.md`, `CLAUDE.md`, + `.zed/rules` + +## Operator sketch + +```python +class DropNulls(UnaryOperator): + def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: + names = (columns,) if isinstance(columns, str) else tuple(columns) + if not names: + raise ValueError("DropNulls needs at least one column name.") + self.columns: tuple[str, ...] = tuple(sorted(set(names))) + super().__init__(**kwargs) + + def _narrow_table(self, table: pa.Table) -> pa.Table: + """Drop rows with a null in a named column present in ``table``; flag those + columns non-nullable; normalize every column to its large type.""" + named = [n for n in self.columns if n in table.column_names] + mask = None + for name in named: + valid = pc.is_valid(table.column(name)) + mask = valid if mask is None else pc.and_(mask, valid) + if mask is not None: + table = table.filter(mask) + table = arrow_utils.normalize_table_to_large_types(table) # keep_null behaviour checked + schema = table.schema + for name in named: + i = schema.get_field_index(name) + schema = schema.set(i, schema.field(i).with_nullable(False)) + return pa.Table.from_arrays(table.columns, schema=schema) # same arrays, new flags +``` + +Sync: `_narrow_table(stream.as_table(columns={"source": True, "system_tags": True, +"sort_by_tags": False}))` wrapped in `ArrowTableStream(table, tag_columns=stream.keys()[0])`. + +Async: per `(tag, data)`: narrow `tag.as_table(columns={"system_tags": True})` and +`data.as_table(columns={"source": True, "meta": True})`; if either comes back with zero rows +the row is dropped; otherwise emit `Tag(table, data_context=tag.data_context_key)` and +`Data(table, data_context=data.data_context_key)`. If row-by-row equivalence cannot be had, +fall back to the inherited barrier path and report it. + +## Tasks + +- [ ] **1. Tests first** (`tests/test_core/operators/test_drop_nulls.py`): rows with nulls + dropped and order preserved; `output_schema` and `as_table` narrowed; other columns' types + and nullability untouched; multiple columns; tag column; mixed tag and data; nested (list + with null items, struct with null children survive; null list / struct dropped; nested + flags untouched); NaN survives; already non-nullable (hashes untouched); missing column + (`InputValidationError`), system/source column names refused; empty `columns` + (`ValueError`); zero surviving rows keep the narrowed schema; logical types (`Path | None`, + `np.ndarray | None`); identity (str/list/tuple/set/duplicates/order equal; different names + differ); narrowed row hashes equal those of a source declared non-nullable; system tags and + source info unchanged; schema prediction vs `unary_static_process(...).output_schema(...)` + (comment at the non-public use); pod with a non-optional parameter refused before, runs + after; `Join` after it; nullable tag narrowed then joined with a `T`-keyed source; + `stream.drop_nulls` equals the operator. Run: all fail with `ImportError`. +- [ ] **2. More tests:** two `Case`s in `test_async_sync_equivalence.py` (`drop_nulls` + on `nul`, `drop_nulls_tag_and_data`), config round trip in `test_operator_config.py`, + `tests/test_pipeline/test_drop_nulls_job.py` (PipelineJob with `DeltaTableDatabase`: + run, results; second job on the same store makes no new pod calls). Commit (WIP, failing). +- [ ] **3. Implement** the operator, exports, stream method, protocol method, registry. + Run the new tests until green. Commit. +- [ ] **4. Golden:** add `@case("pipeline", "drop_nulls")` to `cases.py` (a source with a + nullable data column; summaries of the operator on one and on two columns), regenerate, + check `git diff --stat` and that the diff has no removed or changed line. Commit. +- [ ] **5. Docs:** `docs/concepts/operators.md` section with a runnable sample (drop null rows, + then a pod with a non-optional parameter); convenience-method list; `CHANGELOG.md` + `[0.3.0] — unreleased` Added; `DESIGN_ISSUES.md` entry appended; `CLAUDE.md` + `.zed/rules` + (project layout line, operator boundary line). Run `tests/test_docs`. Commit. +- [ ] **6. Verify:** `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (one Python + version at a time, no xdist). Merge `origin/main` if it moved and rerun. +- [ ] **7. PR** against `main` (`Fixes ITL-943`), link with `orca linear attach` and + `orca worktree set`, CodeRabbit round, `worker_done`. + +## Notes / progress log + +(Updated as work proceeds, for a successor.) + +- November 11: tasks 1-5 done and pushed. Operator in + `src/orcapod/core/operators/drop_nulls.py`; tests in + `tests/test_core/operators/test_drop_nulls.py`, + `tests/test_pipeline/test_drop_nulls_job.py`, two `Case`s plus a tagless test in + `test_async_sync_equivalence.py`, `TestDropNullsConfig` in `test_operator_config.py`. +- Golden: cases `pipeline/drop_nulls` and `pipeline/drop_nulls_job` appended to + `cases.py`; `pipeline.json` gained 128 lines, none removed or changed. +- The streaming async path is exactly equivalent row by row (including Data meta columns, so + no xfail was needed); no fallback to the barrier path. +- Large normalization uses `normalize_to_large_types(..., keep_null=True)` per field with + `restore_column_types` (the table-level helper maps `null` to `large_string`). +- NAME decided (developer, November 11, 2026): `DropNulls` / `drop_nulls`. Renamed class, + module (`drop_nulls.py`), stream method, registry key, export, docs, CHANGELOG, DESIGN_ISSUES, + CLAUDE.md, `.zed/rules`, tests (files too), golden case names and this plan file. The class + name is in `identity_structure`, so the two golden cases this PR added were regenerated with + it (additions only against main; no existing entry moved). +- Observation, not fixed (out of scope): a `PipelineJob` saved with an in-memory + `ArrowTableSource` and loaded again gives operator nodes whose pipeline hash differs from + the saved job's (same for `MapData`), because the source is not reconstructable. + diff --git a/superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md b/superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md new file mode 100644 index 000000000..7321c7255 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md @@ -0,0 +1,115 @@ +# DropNulls No-Argument Form Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `DropNulls()` / `stream.drop_nulls()` with no columns means every user tag column and +every user data column of the input stream (developer, November 12, 2026: "plain no argument = +drop all is fair and as applied to both tags and data columns"). + +**Architecture:** `columns` becomes `str | Collection[str] | None = None`. `self.columns` is +`None` for the no-argument form and is never replaced by resolved names: the names are resolved +from the input (`stream.keys()` on the sync and schema paths, `tag.keys()` + `data.keys()` per +row on the async path) each time, so one operator object works on any stream. The table rule +(`_narrow_table`) takes the names as an argument. Identity for the no-argument form is the +fixed value `("DropNulls", None)`; the named form stays `("DropNulls", (sorted names))`, so no +existing hash moves. + +**Tech Stack:** PyArrow, pytest, uv. + +**PR:** #311, branch `eywalker/itl-943-operator-that-drops-rows-where-given`. One commit, one push. + +--- + +## Decisions + +1. `columns=None` (the default) = all user tag and data columns. An explicitly **empty** + collection still raises `ValueError`: "no argument" and "an empty list" are different + intents (an empty list is usually a computed list that came out empty, and silently turning + it into "all" would drop rows nobody asked to drop). Stated in the docstring and tested. +2. Identity: `identity_structure()` is `("DropNulls", None)` for the no-argument form. `None` + cannot collide with any tuple of names. `to_config()` gives `{"columns": None}`, and + `from_config` round-trips through `cls(columns=None)`. `DropNulls()` and + `DropNulls([every column])` give the same output rows and schema on a stream; their + operator identities (content hash, pipeline hash, and so the output stream's hashes) differ. +3. "All" is `stream.keys()` (default column config): user tag and data columns only. System + tags, source-info, context and meta columns are never tested and never narrowed. +4. Schema prediction resolves the names from `stream.keys()`, not from the schema it was asked + for (which may include system tag columns). +5. Async path: names per row are `(*tag.keys(), *data.keys())`. + +## Files + +- `src/orcapod/core/operators/drop_nulls.py` — the operator. +- `src/orcapod/core/streams/base.py`, `src/orcapod/protocols/core_protocols/streams.py` — + `drop_nulls(columns=None, label=None)`. +- `tests/test_core/operators/test_drop_nulls.py` — new section "no argument: every column". +- `tests/test_core/operators/test_async_sync_equivalence.py` — `Case("drop_nulls_all", ...)` + and the tagless test extended. +- `tests/test_core/operators/test_operator_config.py` — config round trip of the no-argument form. +- `tests/test_hashing/golden/cases.py` + `data/pipeline.json` — new case + `pipeline/drop_nulls_all` (additions only). +- `docs/concepts/operators.md`, `changelog.d/ITL-943.md`, `design_issues/ITL-943-DN1.md`, + `CLAUDE.md`, `.zed/rules`. + +## Tasks + +- [ ] **1. Failing tests** in `tests/test_core/operators/test_drop_nulls.py` (public + interface: `DropNulls()(stream)` and `stream.drop_nulls()`): + - rows: `DropNulls()(readings())` keeps only id 1 (x, y, note all present); + - schema: every tag and data column `T`, Arrow fields non-nullable; system tag and + source-info fields unchanged, including when a source-info value is null; + - equals `DropNulls([all names])` in table (`ALL` config), `output_schema`, row hashes; + operator and stream hashes differ; + - one object on two streams with different columns; + - identity: `DropNulls().columns is None`, `DropNulls(None)` same hash, differs from any + named form; + - `DropNulls([])`, `DropNulls(())`, `DropNulls(set())` still raise `ValueError`; + `stream.drop_nulls([])` too; + - schema prediction vs `static_process(...).output_schema(...)` for `{}`, `system_tags`, + `source+system_tags` (non-public on purpose, commented); + - zero rows keep the types; an all-non-nullable stream is unchanged; idempotent; + - tagless stream; stream method equals operator. + Run `uv run pytest tests/test_core/operators/test_drop_nulls.py -q` and see them fail + (`TypeError: missing 1 required positional argument`). +- [ ] **2. Implement** in `drop_nulls.py`: + - `__init__(self, columns: str | Collection[str] | None = None, **kwargs)`; + `self.columns: tuple[str, ...] | None`; + - `_target_columns(tag_columns, data_columns) -> tuple[str, ...]`: `self.columns` or + `(*tag_columns, *data_columns)`; + - `_narrow_schema(schema, names)`, `_narrow_fields(table, names)`, + `_narrow_table(table, names)` (static rule functions of the names); + - `validate_unary_input`: nothing to check when `columns is None`; + - `unary_output_schema`, `unary_static_process`: names from `stream.keys()`; + - `async_execute`: names from `tag.keys()` and `data.keys()`; + - `identity_structure`, `to_config`, docstrings. + Stream method and protocol: `columns: str | Collection[str] | None = None`. +- [ ] **3. Async equivalence:** add `Case("drop_nulls_all", lambda: DropNulls(), ...)` and a + no-argument assertion in `test_drop_nulls_without_tag_columns`. +- [ ] **4. Config test:** `DropNulls().to_config()["config"] == {"columns": None}`, round trip + through `from_config` and `resolve_operator_from_config`. +- [ ] **5. Golden:** new case `pipeline/drop_nulls_all` in `cases.py`: + `operator[*].content_hash`, `stream_summary(DropNulls()(_nullable_readings()))`, and the + method form. Regenerate; `git diff --stat` on `data/` must show insertions only. +- [ ] **6. Docs:** `docs/concepts/operators.md` executed sample of the no-argument form with a + `notes` column that is legitimately empty, the plain warning (drops every row with a null in + ANY tag or data column; SpiralDB stores every column nullable; naming the columns is the + precise form). Update `changelog.d/ITL-943.md`, `design_issues/ITL-943-DN1.md`, `CLAUDE.md` + and `.zed/rules` (in step). +- [ ] **7. Verify:** `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), operator + docs + golden tests, full + suite once on 3.12 (`uv run --python 3.12 pytest tests/`). +- [ ] **8. Ship:** merge `origin/main` if it moved, one commit, one push, CI green, CodeRabbit + review of the new head, every review thread (GraphQL `reviewThreads`) and review body + answered, `worker_done` with the reviewed sha. + +## Notes / progress log + +- November 12: plan saved; work started. +- Tasks 1-6 done. Finding while extending the async matrix: a stream built straight from a + table lists its `__` meta columns among `keys()` data columns (ITL-846), so the sync path's + "all" tested and narrowed meta columns and the async path did not + (`test_async_meta_equals_sync[drop_nulls_all]` failed). `_target_columns` now leaves out + names with `constants.META_PREFIX`; `test_no_argument_leaves_meta_columns_alone` pins it. +- Golden: case `pipeline/drop_nulls_all` appended; `pipeline.json` gained 20 lines, none + removed or changed. Identity of the no-argument form: `("DropNulls", None)`; its content + hash and pipeline hash are equal (`operator[all].*` in the golden file). diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 631c13f68..0e1a61c1a 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -31,6 +31,7 @@ from orcapod.core.operators import ( Batch, DropDataColumns, + DropNulls, DropTagColumns, GroupBy, Index, @@ -312,6 +313,15 @@ class Case: ), Case("index", lambda: Index("l8", 0), lambda: (source_a(),)), Case("pick", lambda: Pick("d", "q"), lambda: (source_a(),)), + Case("drop_nulls", lambda: DropNulls("nul"), lambda: (source_a(),)), + Case( + "drop_nulls_many", + # A nullable column with a null, one without, a tag, a non-nullable list. + lambda: DropNulls(["nul", "s", "ta", "l8"]), + lambda: (source_a(),), + ), + # No names: every tag and data column, among them struct, logical and list ones. + Case("drop_nulls_all", lambda: DropNulls(), lambda: (source_a(),)), ] CASE = pytest.mark.parametrize("case", CASES, ids=[c.label for c in CASES]) @@ -325,6 +335,57 @@ def test_async_rows_equal_sync_rows(case: Case) -> None: ) +def test_drop_nulls_without_tag_columns() -> None: + """A stream without tag columns: the async path drops the same rows, in order. + + Outside the matrix, whose pairing needs unique tag rows. + """ + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", "c"], pa.string()), + }, + tags=(), + source_id="tagless", + nullable=("x", "s"), + ) + expected = sync_rows(DropNulls("x"), source) + actual = async_rows(DropNulls("x"), source) + assert [data.as_dict() for _, data in actual] == [ + {"x": 1, "s": "a"}, + {"x": 3, "s": "c"}, + ] + assert len(actual) == len(expected) + for (tag, data), (sync_tag, sync_data) in zip(actual, expected): + assert data.arrow_schema() == sync_data.arrow_schema() + assert data.content_hash() == sync_data.content_hash() + assert data.source_info() == sync_data.source_info() + assert tag.arrow_schema(columns={"system_tags": True}) == sync_tag.arrow_schema( + columns={"system_tags": True} + ) + assert list(tag.system_tags().items()) == list(sync_tag.system_tags().items()) + + # No names: both data columns are tested and narrowed on both paths. + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", None], pa.string()), + }, + tags=(), + source_id="tagless", + nullable=("x", "s"), + ) + expected = sync_rows(DropNulls(), source) + actual = async_rows(DropNulls(), source) + assert [data.as_dict() for _, data in actual] == [{"x": 1, "s": "a"}] + assert len(actual) == len(expected) == 1 + for (_, data), (_, sync_data) in zip(actual, expected): + assert data.arrow_schema() == sync_data.arrow_schema() + assert [f.nullable for f in data.arrow_schema()] == [False, False] + assert data.content_hash() == sync_data.content_hash() + assert data.source_info() == sync_data.source_info() + + # Pre-existing provenance defects, not type re-inference: out of scope for # ITL-837 and tracked elsewhere. Strict, so a fix flags them. _DROPPED_SOURCE_INFO = ( diff --git a/tests/test_core/operators/test_drop_nulls.py b/tests/test_core/operators/test_drop_nulls.py new file mode 100644 index 000000000..654f4be06 --- /dev/null +++ b/tests/test_core/operators/test_drop_nulls.py @@ -0,0 +1,827 @@ +"""``DropNulls``: drop rows with a null in a named column, narrow ``T | None`` to ``T`` (ITL-943). + +The operator keeps the rows where every named column is non-null and reports each +named column as non-nullable afterwards, in ``output_schema()`` and in the Arrow +table. Everything else passes through: other columns' types and nullability, +system tags, source info and row order. +""" + +from __future__ import annotations + +import math +import typing +from collections.abc import Sequence +from pathlib import Path +from typing import Any, TypeAlias + +import numpy as np +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.sources import ArrowTableSource, DictSource +from orcapod.core.streams import ArrowTableStream +from orcapod.errors import InputValidationError +from orcapod.operators import DropNulls, Join +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.system_constants import constants + +Column: TypeAlias = "pa.Array[Any] | pa.ChunkedArray[Any]" + +ALL = {"source": True, "system_tags": True} + + +def _source( + columns: dict[str, Column], + tags: Sequence[str] = ("id",), + nullable: Sequence[str] = (), + source_id: str = "src", +) -> ArrowTableSource: + """Build a source; every column not listed in ``nullable`` is non-nullable.""" + table = pa.table(list(columns.values()), names=list(columns)) + schema = pa.schema([f.with_nullable(f.name in nullable) for f in table.schema]) + return ArrowTableSource( + table.cast(schema), tag_columns=list(tags), source_id=source_id + ) + + +def readings(nullable: Sequence[str] = ("x", "y")) -> ArrowTableSource: + """Four rows; ``x`` is null in row 2, ``y`` in row 3; ``note`` is nullable too.""" + return _source( + { + "id": pa.array([1, 2, 3, 4], pa.int16()), + "x": pa.array([10, None, 30, 40], pa.int8()), + "y": pa.array([1.5, 2.5, None, 4.5], pa.float32()), + "note": pa.array(["a", None, "c", None], pa.string()), + }, + nullable=[*nullable, "note"], + ) + + +def _ids(stream: StreamProtocol) -> list[Any]: + """Return the ``id`` column of ``stream`` as a list, in row order.""" + return stream.as_table().column("id").to_pylist() + + +# ── rows ──────────────────────────────────────────────────────────────────── + + +def test_drops_rows_where_the_column_is_null() -> None: + """Rows whose named column is null are dropped; the others pass through.""" + result = DropNulls("x")(readings()) + assert result.as_table().to_pylist() == [ + {"id": 1, "x": 10, "y": 1.5, "note": "a"}, + {"id": 3, "x": 30, "y": None, "note": "c"}, + {"id": 4, "x": 40, "y": 4.5, "note": None}, + ] + + +def test_every_named_column_must_be_non_null() -> None: + """With several names, a row survives only if all of them are non-null.""" + assert _ids(DropNulls(["x", "y"])(readings())) == [1, 4] + + +def test_row_order_is_preserved() -> None: + """Surviving rows keep the input order; nothing is sorted.""" + source = _source( + { + "id": pa.array([4, 1, 3, 2], pa.int16()), + "x": pa.array([40, 10, None, 20], pa.int8()), + }, + nullable=["x"], + ) + assert [tag["id"] for tag, _ in DropNulls("x")(source).iter_data()] == [4, 1, 2] + + +def test_nan_is_not_null() -> None: + """A NaN is a value: its row survives and the column still becomes non-nullable.""" + source = _source( + { + "id": pa.array([1, 2, 3], pa.int16()), + "v": pa.array([1.0, float("nan"), None], pa.float64()), + }, + nullable=["v"], + ) + result = DropNulls("v")(source) + first, second = result.as_table().column("v").to_pylist() + assert first == 1.0 + assert isinstance(second, float) and math.isnan(second) + assert result.as_table().schema.field("v").nullable is False + assert result.output_schema()[1]["v"] is float + + +# ── types ─────────────────────────────────────────────────────────────────── + + +def test_output_schema_narrows_only_the_named_columns() -> None: + """``output_schema()`` reports ``T`` for the named column and leaves the rest.""" + source = readings() + assert source.output_schema()[1] == { + "x": int | None, + "y": float | None, + "note": str | None, + } + tag_schema, data_schema = DropNulls("x")(source).output_schema() + assert tag_schema == {"id": int} + assert data_schema == {"x": int, "y": float | None, "note": str | None} + + +def test_table_narrows_only_the_named_columns() -> None: + """Only the named field loses its nullable flag; widths and other flags are kept.""" + source = readings() + before = source.as_table(columns=ALL).schema + after = DropNulls("x")(source).as_table(columns=ALL).schema + assert after.names == before.names + assert after.field("x") == pa.field("x", pa.int8(), nullable=False) + # Width and nullability are kept; ``string`` takes its large variant. + assert after.field("y") == pa.field("y", pa.float32(), nullable=True) + assert after.field("note") == pa.field("note", pa.large_string(), nullable=True) + for name in before.names: + if name not in ("x", "note"): + assert after.field(name) == before.field(name) + + +def test_schema_prediction_matches_the_computed_stream() -> None: + """The predicted schema equals the schema of the computed stream.""" + source = readings() + op = DropNulls(["x", "id"]) + for config in ({}, {"system_tags": True}, {"source": True, "system_tags": True}): + # Non-public on purpose: a prediction is verified against the static result. + computed = op.static_process(source).output_schema(columns=config) + assert op(source).output_schema(columns=config) == computed + assert op.output_schema(source, columns=config) == computed + + +def test_zero_surviving_rows_keep_the_narrowed_types() -> None: + """A result with no rows has the schema of a non-empty result.""" + source = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([None, None], pa.int8()), + "s": pa.array(["a", "b"], pa.string()), + }, + nullable=["x", "s"], + ) + non_empty = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([5, None], pa.int8()), + "s": pa.array(["a", "b"], pa.string()), + }, + nullable=["x", "s"], + ) + result = DropNulls("x")(source) + table = result.as_table(columns=ALL) + assert table.num_rows == 0 + assert list(result.iter_data()) == [] + assert table.schema.field("x") == pa.field("x", pa.int8(), nullable=False) + assert result.output_schema() == ({"id": int}, {"x": int, "s": str | None}) + expected = DropNulls("x")(non_empty).as_table(columns=ALL).schema + assert table.schema == expected + + +def test_already_non_nullable_column_is_a_no_op() -> None: + """Naming a non-nullable column changes no schema, row or hash.""" + source = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([10, 20], pa.int8()), + } + ) + result = DropNulls("x")(source) + assert result.output_schema() == source.output_schema() + assert result.as_table(columns=ALL) == source.as_table(columns=ALL) + for (tag, data), (in_tag, in_data) in zip(result.iter_data(), source.iter_data()): + assert tag.content_hash() == in_tag.content_hash() + assert data.content_hash() == in_data.content_hash() + + +def test_applying_twice_equals_applying_once() -> None: + """The operator is idempotent.""" + once = DropNulls("x")(readings()) + twice = DropNulls("x")(once) + assert twice.as_table(columns=ALL) == once.as_table(columns=ALL) + assert twice.output_schema() == once.output_schema() + + +# ── tag columns ───────────────────────────────────────────────────────────── + + +def keyed(nullable: Sequence[str] = ("site", "x")) -> ArrowTableSource: + """Tags ``site`` (null in row 2) and ``id``; ``x`` is null in row 3.""" + return _source( + { + "site": pa.array(["a", None, "c"], pa.large_string()), + "id": pa.array([1, 2, 3], pa.int16()), + "x": pa.array([10, 20, None], pa.int8()), + }, + tags=("site", "id"), + nullable=nullable, + ) + + +def test_tag_column_is_narrowed() -> None: + """A named tag column is tested and narrowed like a data column.""" + source = keyed() + assert source.output_schema()[0] == {"site": str | None, "id": int} + result = DropNulls("site")(source) + assert _ids(result) == [1, 3] + assert result.keys() == source.keys() + assert result.output_schema() == ( + {"site": str, "id": int}, + {"x": int | None}, + ) + assert result.as_table().schema.field("site").nullable is False + assert result.as_table().schema.field("x").nullable is True + + +def test_tag_and_data_columns_in_one_call() -> None: + """Tag and data columns can be named together.""" + result = DropNulls(["x", "site"])(keyed()) + assert _ids(result) == [1] + assert result.output_schema() == ({"site": str, "id": int}, {"x": int}) + + +def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: + """A source whose key columns are nullable by specification (as a Spiral table's + are) cannot join a source whose keys are ``T``; after ``DropNulls`` it can.""" + events = _source( + { + "session": pa.array(["s1", None, "s3"], pa.large_string()), + "start": pa.array([100, 200, 300], pa.int64()), + }, + tags=("session",), + nullable=["session", "start"], + source_id="events", + ) + sessions = _source( + { + "session": pa.array(["s1", "s3", "s4"], pa.large_string()), + "animal": pa.array(["m1", "m3", "m4"], pa.large_string()), + }, + tags=("session",), + source_id="sessions", + ) + with pytest.raises(InputValidationError, match="not compatible"): + Join()(events, sessions) + + joined = Join()(DropNulls("session")(events), sessions) + assert joined.output_schema() == ( + {"session": str}, + {"start": int | None, "animal": str}, + ) + rows = sorted(joined.as_table().to_pylist(), key=lambda r: r["session"]) + assert rows == [ + {"session": "s1", "start": 100, "animal": "m1"}, + {"session": "s3", "start": 300, "animal": "m3"}, + ] + + +def test_stream_without_tag_columns() -> None: + """A stream with no tag columns is filtered and narrowed the same way.""" + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", "c"], pa.string()), + }, + tags=(), + nullable=["x", "s"], + ) + result = DropNulls("x")(source) + assert result.as_table().to_pylist() == [{"x": 1, "s": "a"}, {"x": 3, "s": "c"}] + assert result.output_schema() == ({}, {"x": int, "s": str | None}) + + +# ── nested and logical columns ────────────────────────────────────────────── + +ITEMS = pa.large_list(pa.field("item", pa.int8(), nullable=True)) +POINT = pa.struct([pa.field("a", pa.int8()), pa.field("b", pa.large_string())]) + + +def nested() -> ArrowTableSource: + """A source with a nullable list column and a nullable struct column.""" + return _source( + { + "id": pa.array([1, 2, 3, 4], pa.int16()), + "items": pa.array([[1, None], None, [], [4]], ITEMS), + "point": pa.array( + [{"a": 1, "b": None}, {"a": 2, "b": "x"}, None, {"a": None, "b": None}], + POINT, + ), + }, + nullable=["items", "point"], + ) + + +def test_only_the_top_level_value_is_tested_for_a_list() -> None: + """A null list is null; an empty list and a list holding nulls are not.""" + result = DropNulls("items")(nested()) + # A list holding a null and an empty list are not null lists. + assert result.as_table().column("items").to_pylist() == [[1, None], [], [4]] + field = result.as_table().schema.field("items") + assert field.nullable is False + assert field.type == ITEMS # the item field keeps its name and nullability + # The Python schema does not express item nullability; the Arrow type above does. + assert nested().output_schema()[1]["items"] == list[int] | None + assert result.output_schema()[1]["items"] == list[int] + + +def test_only_the_top_level_value_is_tested_for_a_struct() -> None: + """A null struct is null; a struct whose fields are null is not.""" + result = DropNulls("point")(nested()) + # A struct whose children are all null is not a null struct. + assert _ids(result) == [1, 2, 4] + field = result.as_table().schema.field("point") + assert field.nullable is False + assert field.type == POINT # children stay nullable + + +def logical() -> DictSource: + """A source with nullable ``Path`` and ``numpy.ndarray`` columns.""" + return DictSource( + data=[ + {"k": "a", "p": Path("/a"), "arr": np.arange(3)}, + {"k": "b", "p": None, "arr": np.arange(2)}, + {"k": "c", "p": Path("/c"), "arr": None}, + ], + tag_columns=["k"], + source_id="logical", + ) + + +def test_logical_columns_keep_their_logical_type() -> None: + """A logical column keeps its extension type and becomes non-nullable.""" + source = logical() + before = source.as_table().schema + assert source.output_schema()[1] == { + "p": Path | None, + "arr": np.ndarray | None, # pyright: ignore[reportMissingTypeArgument] # the schema holds the bare class + } + + result = DropNulls(["p", "arr"])(source) + assert result.output_schema()[1] == {"p": Path, "arr": np.ndarray} + after = result.as_table().schema + for name in ("p", "arr"): + assert isinstance(after.field(name).type, pa.BaseExtensionType) + assert after.field(name).type == before.field(name).type + assert after.field(name).nullable is False + ((tag, data),) = list(result.iter_data()) + assert tag["k"] == "a" + assert data["p"] == Path("/a") + array = data["arr"] + assert isinstance(array, np.ndarray) + assert array.tolist() == [0, 1, 2] + + +# ── arguments ─────────────────────────────────────────────────────────────── + + +def test_missing_column_is_an_input_validation_error() -> None: + """A name that is not a column of the stream is refused at call time.""" + with pytest.raises(InputValidationError, match="nope"): + DropNulls(["x", "nope"])(readings()) + + +def test_system_and_source_columns_cannot_be_named() -> None: + """System-tag and source-info columns are not tag or data columns.""" + source = readings() + hidden = [ + n + for n in source.as_table(columns=ALL).column_names + if n.startswith((constants.SYSTEM_TAG_PREFIX, constants.SOURCE_PREFIX)) + ] + assert len(hidden) >= 2 + for name in hidden: + with pytest.raises(InputValidationError): + DropNulls(name)(source) + + +def test_no_column_is_an_error_at_construction() -> None: + """An empty collection of names is refused when the operator is built.""" + with pytest.raises(ValueError, match="at least one"): + DropNulls([]) + + +# ── identity ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "columns", + [["x", "y"], ("x", "y"), {"x", "y"}, ["y", "x"], ["x", "y", "x"], frozenset("xy")], + ids=["list", "tuple", "set", "reversed", "duplicate", "frozenset"], +) +def test_identity_is_the_set_of_names(columns: Any) -> None: + """Container type, order and duplicates do not change the operator's hashes.""" + reference = DropNulls(["x", "y"]) + op = DropNulls(columns) + assert op.columns == ("x", "y") + assert op.content_hash() == reference.content_hash() + assert op.pipeline_hash() == reference.pipeline_hash() + source = readings() + assert op(source).content_hash() == reference(source).content_hash() + assert op(source).pipeline_hash() == reference(source).pipeline_hash() + + +def test_a_bare_name_is_one_column() -> None: + """A string is one column name, never a collection of characters.""" + assert DropNulls("note").columns == ("note",) + assert DropNulls("x").content_hash() == DropNulls(["x"]).content_hash() + + +def test_different_names_are_different_identities() -> None: + """Operators over different names hash differently, as do their outputs.""" + source = readings() + assert DropNulls("x").content_hash() != DropNulls("y").content_hash() + assert DropNulls("x").content_hash() != DropNulls(["x", "y"]).content_hash() + assert ( + DropNulls("x")(source).pipeline_hash() != DropNulls("y")(source).pipeline_hash() + ) + assert DropNulls("x")(source).pipeline_hash() != source.pipeline_hash() + + +def test_the_caller_s_collection_is_not_kept_or_mutated() -> None: + """The operator copies the names; later changes to the list do not reach it.""" + names = ["y", "x"] + op = DropNulls(names) + names.append("z") + assert names == ["y", "x", "z"] + assert op.columns == ("x", "y") + + +# ── hashes and provenance of surviving rows ───────────────────────────────── + + +def test_narrowed_rows_hash_as_rows_declared_non_nullable() -> None: + """Nullability is part of the Arrow schema hash, so narrowing a data column + changes the row's ``Data`` hash, to the hash the same row has in a stream + that declared the column non-nullable from the start. Tags are untouched.""" + columns: dict[str, Column] = { + "id": pa.array([1, 3], pa.int16()), + "x": pa.array([10, 30], pa.int8()), + } + declared = _source(columns) + nullable = _source( + { + "id": pa.array([1, 2, 3], pa.int16()), + "x": pa.array([10, None, 30], pa.int8()), + }, + nullable=["x"], + ) + input_rows = {tag["id"]: (tag, data) for tag, data in nullable.iter_data()} + narrowed = list(DropNulls("x")(nullable).iter_data()) + assert len(narrowed) == 2 + for (tag, data), (ref_tag, ref_data) in zip(narrowed, declared.iter_data()): + in_tag, in_data = input_rows[tag["id"]] + assert data.content_hash() == ref_data.content_hash() + assert data.content_hash() != in_data.content_hash() + assert tag.content_hash() == in_tag.content_hash() == ref_tag.content_hash() + + +def test_narrowing_a_tag_changes_the_tag_hash_only() -> None: + """Narrowing a tag column changes the row's tag hash and not its data hash.""" + source = keyed() + input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} + for tag, data in DropNulls("site")(source).iter_data(): + in_tag, in_data = input_rows[tag["id"]] + assert tag.content_hash() != in_tag.content_hash() + assert data.content_hash() == in_data.content_hash() + + +def test_system_tags_and_source_info_pass_through() -> None: + """System-tag and source-info columns keep their names, types and values.""" + source = readings() + before = source.as_table(columns=ALL) + after = DropNulls(["x", "y"])(source).as_table(columns=ALL) + hidden = [ + n + for n in before.column_names + if n.startswith((constants.SYSTEM_TAG_PREFIX, constants.SOURCE_PREFIX)) + ] + assert hidden and after.column_names == before.column_names + kept = before.take(pa.array([0, 3])) + for name in hidden: + assert after.schema.field(name) == before.schema.field(name) + assert after.column(name).to_pylist() == kept.column(name).to_pylist() + + input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} + for tag, data in DropNulls(["x", "y"])(source).iter_data(): + in_tag, in_data = input_rows[tag["id"]] + assert tag.system_tags() == in_tag.system_tags() + assert data.source_info() == in_data.source_info() + + +def test_the_input_stream_is_not_changed() -> None: + """Running the operator leaves the input stream's schema and rows as they were.""" + source = readings() + schema = source.as_table(columns=ALL).schema + rows = source.as_table(columns=ALL).to_pylist() + DropNulls(["x", "y"])(source).as_table() + assert source.as_table(columns=ALL).schema == schema + assert source.as_table(columns=ALL).to_pylist() == rows + + +# ── stream method ─────────────────────────────────────────────────────────── + + +def test_stream_method_equals_the_operator() -> None: + """``stream.drop_nulls`` gives the operator's rows and hashes.""" + source = readings() + by_method = source.drop_nulls(["x", "y"]) + by_operator = DropNulls(["x", "y"])(source) + assert by_method.as_table(columns=ALL) == by_operator.as_table(columns=ALL) + assert by_method.content_hash() == by_operator.content_hash() + assert by_method.pipeline_hash() == by_operator.pipeline_hash() + assert _ids(source.drop_nulls("x")) == [1, 3, 4] + + +# ── downstream ────────────────────────────────────────────────────────────── + + +@function_pod(output_keys="doubled") +def double_x(x: int) -> int: + """A pod with a non-optional parameter.""" + return x * 2 + + +def test_pod_with_a_non_optional_parameter_runs_only_after_the_operator() -> None: + """A pod taking a plain ``int`` refuses ``int | None`` and runs on the narrowed stream.""" + source = _source( + { + "id": pa.array([1, 2, 3], pa.int16()), + "x": pa.array([10, None, 30], pa.int64()), + }, + nullable=["x"], + ) + with pytest.raises(ValueError, match="not compatible"): + double_x.pod(source) + + result = double_x.pod(source.drop_nulls("x")) + assert result.as_table().to_pylist() == [ + {"id": 1, "doubled": 20}, + {"id": 3, "doubled": 60}, + ] + + +def test_join_after_the_operator() -> None: + """A narrowed stream joins, and the joined column stays non-nullable.""" + weights = _source( + { + "id": pa.array([1, 2, 3, 4], pa.int16()), + "w": pa.array([1.0, 2.0, 3.0, 4.0], pa.float64()), + }, + source_id="weights", + ) + joined = Join()(DropNulls("x")(readings()), weights) + assert joined.output_schema()[1]["x"] is int + assert sorted(_ids(joined)) == [1, 3, 4] + assert joined.as_table().schema.field("x").nullable is False + + +# ── no argument: every tag and data column ────────────────────────────────── + + +def _hidden(table: pa.Table) -> list[str]: + """Return the system-tag and source-info column names of ``table``.""" + return [ + n + for n in table.column_names + if n.startswith((constants.SYSTEM_TAG_PREFIX, constants.SOURCE_PREFIX)) + ] + + +def test_no_argument_drops_rows_with_a_null_in_any_column() -> None: + """Without names, a row survives only if every tag and data column has a value.""" + result = DropNulls()(readings()) + assert result.as_table().to_pylist() == [{"id": 1, "x": 10, "y": 1.5, "note": "a"}] + + +def test_no_argument_narrows_every_tag_and_data_column() -> None: + """Every tag and data column is ``T`` afterwards, in the schema and in Arrow.""" + source = keyed() + assert source.output_schema() == ( + {"site": str | None, "id": int}, + {"x": int | None}, + ) + result = DropNulls()(source) + assert _ids(result) == [1] + assert result.keys() == source.keys() + assert result.output_schema() == ({"site": str, "id": int}, {"x": int}) + schema = result.as_table().schema + assert [schema.field(n).nullable for n in ("site", "id", "x")] == [False] * 3 + + +def test_no_argument_equals_naming_every_column() -> None: + """``DropNulls()`` and ``DropNulls([every column])`` give the same rows and schema. + + The operators themselves are different identities, and so are their outputs' + stream hashes. + """ + source = keyed() + implicit = DropNulls() + named = DropNulls(["site", "id", "x"]) + by_all, by_name = implicit(source), named(source) + assert by_all.as_table(columns=ALL) == by_name.as_table(columns=ALL) + assert by_all.as_table(columns=ALL).schema == by_name.as_table(columns=ALL).schema + assert by_all.output_schema() == by_name.output_schema() + for (tag, data), (ref_tag, ref_data) in zip( + by_all.iter_data(), by_name.iter_data(), strict=True + ): + assert tag.content_hash() == ref_tag.content_hash() + assert data.content_hash() == ref_data.content_hash() + assert tag.system_tags() == ref_tag.system_tags() + + assert implicit.content_hash() != named.content_hash() + assert implicit.pipeline_hash() != named.pipeline_hash() + assert by_all.content_hash() != by_name.content_hash() + assert by_all.pipeline_hash() != by_name.pipeline_hash() + + +def test_no_argument_identity_is_one_fixed_value() -> None: + """The no-argument operator stores no names: its identity is the same on any stream.""" + op = DropNulls() + assert op.columns is None + before = (op.content_hash(), op.pipeline_hash()) + assert _ids(op(readings())) == [1] + assert _ids(op(keyed())) == [1] + assert op.columns is None + assert (op.content_hash(), op.pipeline_hash()) == before + assert DropNulls(None).content_hash() == op.content_hash() + assert DropNulls().content_hash() == op.content_hash() + for names in ("x", ["id", "x", "y", "note"], ["site", "id", "x"]): + assert DropNulls(names).content_hash() != op.content_hash() + + +@pytest.mark.parametrize("empty", [[], (), set(), frozenset()]) +def test_an_empty_collection_is_still_an_error(empty: Any) -> None: + """An empty collection is not "no argument": it is refused, never read as "all".""" + with pytest.raises(ValueError, match="at least one"): + DropNulls(empty) + with pytest.raises(ValueError, match="at least one"): + readings().drop_nulls(empty) + + +def test_no_argument_leaves_system_and_source_columns_alone() -> None: + """Only user tag and data columns are tested and narrowed. + + Row 1's source-info value for ``x`` is null: the row still survives, and the + system-tag and source-info fields keep their types, flags and values. + """ + source = readings() + before = source.as_table(columns=ALL) + hidden = _hidden(before) + assert len(hidden) >= 2 + source_x = f"{constants.SOURCE_PREFIX}x" + position = before.schema.get_field_index(source_x) + nullable_token = before.schema.field(position).with_nullable(True) + stream = ArrowTableStream( + before.set_column( + position, + nullable_token, + pa.array([None, "t2", "t3", "t4"], nullable_token.type), + ), + tag_columns=["id"], + ) + given = stream.as_table(columns=ALL) + assert given.schema.field(source_x).nullable is True + result = DropNulls()(stream) + after = result.as_table(columns=ALL) + assert _ids(result) == [1] + assert after.column(source_x).to_pylist() == [None] + assert after.column_names == given.column_names + for name in _hidden(given): + assert after.schema.field(name) == given.schema.field(name) + assert after.column(name).to_pylist() == given.column(name).to_pylist()[:1] + + +def test_no_argument_schema_prediction_matches_the_computed_stream() -> None: + """The predicted schema equals the computed stream's, with and without system tags.""" + op = DropNulls() + for source in (readings(), keyed()): + for config in ( + {}, + {"system_tags": True}, + {"source": True, "system_tags": True}, + ): + # Non-public on purpose: a prediction is verified against the static result. + computed = op.static_process(source).output_schema(columns=config) + assert op(source).output_schema(columns=config) == computed + assert op.output_schema(source, columns=config) == computed + assert all( + type(None) not in typing.get_args(python_type) + for name, python_type in {**computed[0], **computed[1]}.items() + if not name.startswith(constants.SYSTEM_TAG_PREFIX) + ) + + +def test_no_argument_zero_surviving_rows_keep_the_narrowed_types() -> None: + """A result with no rows has the schema of a non-empty result.""" + empty = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([None, 7], pa.int8()), + "s": pa.array(["a", None], pa.string()), + }, + nullable=["id", "x", "s"], + ) + non_empty = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([5, 7], pa.int8()), + "s": pa.array(["a", None], pa.string()), + }, + nullable=["id", "x", "s"], + ) + result = DropNulls()(empty) + table = result.as_table(columns=ALL) + assert table.num_rows == 0 + assert list(result.iter_data()) == [] + assert result.output_schema() == ({"id": int}, {"x": int, "s": str}) + assert table.schema.field("id") == pa.field("id", pa.int16(), nullable=False) + assert table.schema.field("x") == pa.field("x", pa.int8(), nullable=False) + assert table.schema.field("s") == pa.field("s", pa.large_string(), nullable=False) + assert table.schema == DropNulls()(non_empty).as_table(columns=ALL).schema + + +def test_no_argument_on_a_non_nullable_stream_is_a_no_op() -> None: + """A stream without a nullable column is unchanged, and twice equals once.""" + source = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([10, 20], pa.int8()), + } + ) + result = DropNulls()(source) + assert result.output_schema() == source.output_schema() + assert result.as_table(columns=ALL) == source.as_table(columns=ALL) + for (tag, data), (in_tag, in_data) in zip( + result.iter_data(), source.iter_data(), strict=True + ): + assert tag.content_hash() == in_tag.content_hash() + assert data.content_hash() == in_data.content_hash() + + once = DropNulls()(readings()) + twice = DropNulls()(once) + assert twice.as_table(columns=ALL) == once.as_table(columns=ALL) + assert twice.output_schema() == once.output_schema() + + +def test_no_argument_on_a_stream_without_tag_columns() -> None: + """A stream with no tag columns: every data column is tested and narrowed.""" + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", None], pa.string()), + }, + tags=(), + nullable=["x", "s"], + ) + result = DropNulls()(source) + assert result.as_table().to_pylist() == [{"x": 1, "s": "a"}] + assert result.output_schema() == ({}, {"x": int, "s": str}) + + +def test_no_argument_stream_method_equals_the_operator() -> None: + """``stream.drop_nulls()`` gives the no-argument operator's rows and hashes.""" + source = readings() + by_method = source.drop_nulls() + by_operator = DropNulls()(source) + assert by_method.as_table(columns=ALL) == by_operator.as_table(columns=ALL) + assert by_method.content_hash() == by_operator.content_hash() + assert by_method.pipeline_hash() == by_operator.pipeline_hash() + assert _ids(source.drop_nulls(label="complete")) == [1] + + +def test_no_argument_does_not_change_the_input_stream() -> None: + """Running the no-argument operator leaves the input stream as it was.""" + source = readings() + schema = source.as_table(columns=ALL).schema + rows = source.as_table(columns=ALL).to_pylist() + DropNulls()(source).as_table() + assert source.as_table(columns=ALL).schema == schema + assert source.as_table(columns=ALL).to_pylist() == rows + + +def test_no_argument_leaves_meta_columns_alone() -> None: + """A meta (``__``) column is not a data column for "every column". + + A stream built straight from a table lists its meta columns among its data + keys (ITL-846); a null there still drops no row and the field stays nullable. + """ + meta = f"{constants.META_PREFIX}note" + table = readings().as_table(columns=ALL) + table = table.append_column( + pa.field(meta, pa.large_string(), nullable=True), + pa.array([None, "m2", "m3", "m4"], pa.large_string()), + ) + stream = ArrowTableStream(table, tag_columns=["id"]) + result = DropNulls()(stream) + rows = list(result.iter_data()) + assert [tag["id"] for tag, _ in rows] == [1] + data = rows[0][1] + assert data.as_dict() == {"x": 10, "y": 1.5, "note": "a"} + meta_field = data.arrow_schema(columns={"meta": True}).field(meta) + assert meta_field == pa.field(meta, pa.large_string(), nullable=True) + assert data.as_table(columns={"meta": True}).column(meta).to_pylist() == [None] + # Non-public on purpose: a prediction is verified against the static result. + computed = DropNulls().static_process(stream).output_schema() + assert DropNulls().output_schema(stream) == computed diff --git a/tests/test_core/operators/test_operator_config.py b/tests/test_core/operators/test_operator_config.py index 414e9769d..7df983b33 100644 --- a/tests/test_core/operators/test_operator_config.py +++ b/tests/test_core/operators/test_operator_config.py @@ -3,6 +3,7 @@ from orcapod.core.operators import ( Batch, DropDataColumns, + DropNulls, DropTagColumns, Join, MapData, @@ -161,3 +162,46 @@ def test_non_serializable_predicates_marked(self): config = op.to_config() # Should either serialize the expression or mark as non-reconstructable assert "config" in config + + +class TestDropNullsConfig: + def test_to_config(self): + """The config holds the sorted unique names.""" + config = DropNulls(["y", "x", "y"]).to_config() + assert config["class_name"] == "DropNulls" + assert config["config"] == {"columns": ["x", "y"]} + + def test_round_trip(self): + """``from_config(to_config())`` gives an operator with the same hashes.""" + op = DropNulls(["y", "x"]) + restored = DropNulls.from_config(op.to_config()) + assert isinstance(restored, DropNulls) + assert restored.columns == ("x", "y") + assert restored.content_hash() == op.content_hash() + assert restored.pipeline_hash() == op.pipeline_hash() + + def test_registered_for_pipeline_loading(self): + """Pipeline loading resolves the operator by its class name.""" + from orcapod.pipeline.serialization import resolve_operator_from_config + + op = DropNulls(["y", "x"]) + restored = resolve_operator_from_config(op.to_config()) + assert isinstance(restored, DropNulls) + assert restored.content_hash() == op.content_hash() + + def test_no_argument_round_trip(self): + """The no-argument form serializes ``columns`` as ``None`` and round-trips.""" + from orcapod.pipeline.serialization import resolve_operator_from_config + + op = DropNulls() + config = op.to_config() + assert config["config"] == {"columns": None} + for restored in ( + DropNulls.from_config(config), + resolve_operator_from_config(config), + ): + assert isinstance(restored, DropNulls) + assert restored.columns is None + assert restored.content_hash() == op.content_hash() + assert restored.pipeline_hash() == op.pipeline_hash() + assert op.content_hash() != DropNulls(["x", "y"]).content_hash() diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index ca74208e9..a1b98c960 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -974,3 +974,99 @@ def pipeline_polars_filter_expr_job() -> dict[str, Any]: _jsonable(v) for v in records.column("__golden_record_id").to_pylist() ) return out + + +def _nullable_readings() -> Any: + """A source with a nullable tag and two nullable data columns, each with a null.""" + from orcapod.sources import ArrowTableSource + + table = pa.table( + { + "k": pa.array(["r1", "r2", "r3", "r4"], pa.large_string()), + "site": pa.array(["a", "b", "c", None], pa.large_string()), + "x": pa.array([1, None, 3, 4], pa.int64()), + "y": pa.array([1.5, 2.5, None, 4.5], pa.float64()), + }, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("site", pa.large_string(), nullable=True), + pa.field("x", pa.int64(), nullable=True), + pa.field("y", pa.float64(), nullable=True), + ] + ), + ) + return ArrowTableSource( + table, tag_columns=["k", "site"], source_id="golden_nullable" + ) + + +@case("pipeline", "drop_nulls") +def pipeline_drop_nulls() -> dict[str, Any]: + """Pin ``DropNulls``: the operator's own identity and the narrowed rows.""" + from orcapod.operators import DropNulls + + out = _prefixed("source", stream_summary(_nullable_readings())) + out["operator[x].content_hash"] = DropNulls("x").content_hash().to_string() + out["operator[x,y].content_hash"] = DropNulls(["y", "x"]).content_hash().to_string() + out.update(_prefixed("data", stream_summary(DropNulls("x")(_nullable_readings())))) + out.update( + _prefixed( + "data_many", stream_summary(DropNulls(["x", "y"])(_nullable_readings())) + ) + ) + out.update( + _prefixed("tag", stream_summary(_nullable_readings().drop_nulls("site"))) + ) + return out + + +@case("pipeline", "drop_nulls_all") +def pipeline_drop_nulls_all() -> dict[str, Any]: + """Pin the no-argument ``DropNulls``: its fixed identity and the narrowed rows.""" + from orcapod.operators import DropNulls + + out: dict[str, Any] = {} + out["operator[all].content_hash"] = DropNulls().content_hash().to_string() + out["operator[all].pipeline_hash"] = DropNulls().pipeline_hash().to_string() + out.update(_prefixed("all", stream_summary(DropNulls()(_nullable_readings())))) + out.update(_prefixed("method", stream_summary(_nullable_readings().drop_nulls()))) + return out + + +@case("pipeline", "drop_nulls_job") +def pipeline_drop_nulls_job() -> dict[str, Any]: + """Record a job with ``DropNulls`` upstream of a pod with non-optional parameters.""" + from orcapod import PipelineJob + from orcapod.databases import InMemoryArrowDatabase + + db = InMemoryArrowDatabase() + job = PipelineJob(name="golden_drop_null_job", store=db) + with job: + present = ( + _nullable_readings() + .drop_nulls(["x", "y"], label="present") + .map_data({"x": "age", "y": "weight"}, label="renamed") + ) + golden_ratio.pod(present, label="ratio") + job.run() + + out: dict[str, Any] = {} + for label in sorted(job.nodes): + out.update(_prefixed(f"node[{label}]", stream_summary(job.nodes[label]))) + paths = sorted(db.list_sources()) + out["table_paths"] = ["/".join(p) for p in paths] + for path in paths: + name = "/".join(path) + records = db.get_all_records(path, record_id_column="__golden_record_id") + assert records is not None + if "__input_data_hash" in records.column_names and "_result" in path: + # Result-table record ids are UUIDv7 (volatile); pin the lookup key instead. + out[f"table[{name}].input_data_hashes"] = sorted( + _jsonable(v) for v in records.column("__input_data_hash").to_pylist() + ) + else: + out[f"table[{name}].record_ids"] = sorted( + _jsonable(v) for v in records.column("__golden_record_id").to_pylist() + ) + return out diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index 396884e78..4e9642eed 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -58,6 +58,154 @@ "_tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4", "_tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4" ], + "drop_nulls.data.content_hash": "semantic_v0.1:56d87bbf372f115b88df8c734cd8a9fa6e31d37f57e4d26a01cf2b6836b94d32", + "drop_nulls.data.pipeline_hash": "semantic_v0.1:d2cfd2dce8882ab924b86a26b9f26f627dd2b293c1054a113dce3025e1587da3", + "drop_nulls.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001eb8bb21cea56694f30415a726a190021869b1839977483a983afccc228e3fa7d", + "drop_nulls.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:000001eeb56ac3c1727cfe75fca3334244e3f9d6afbf624a67662511dbd6abff6ecec5", + "drop_nulls.data.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls.data.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018b789706fed0bc06c83ab3049aacb9e7b228aef5d112d80ba59bc45b6981a5a9", + "drop_nulls.data.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls.data_many.content_hash": "semantic_v0.1:6e7edbd1167d450b18cae53f7daeeac3e489c6dde1921131f09f8880c7032e70", + "drop_nulls.data_many.pipeline_hash": "semantic_v0.1:b388b7f3dbab1086d9a2c96f82e2f957dd21bb74b1f68ddc57ee70174c1dac96", + "drop_nulls.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data_many.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", + "drop_nulls.data_many.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls.operator[x,y].content_hash": "semantic_v0.1:76689e842097817302c55b7a02dbcdbe22b980710489d83c3334a73b68dd9975", + "drop_nulls.operator[x].content_hash": "semantic_v0.1:e0b67b2eab584c0ee1a0a79af479684da3944f66c57d4bd9c0c758642179f358", + "drop_nulls.source.content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", + "drop_nulls.source.pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "drop_nulls.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_nulls.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_nulls.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_nulls.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_nulls.source.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls.source.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", + "drop_nulls.source.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls.tag.content_hash": "semantic_v0.1:4189d845c46ae389f3fd59578da62aa8f545fc5c3186bc91479ba7cec81f46e0", + "drop_nulls.tag.pipeline_hash": "semantic_v0.1:483f0e59fc6d4c5c4403b8a676ac20e863c123785fac25d113538c0f3081c093", + "drop_nulls.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.tag.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_nulls.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_nulls.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.tag.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_nulls.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.tag.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_nulls.tag.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_all.all.content_hash": "semantic_v0.1:91519c5bfe375ac6ab2979f02c3962f95782163690566a3cd4354e46aaab1bab", + "drop_nulls_all.all.pipeline_hash": "semantic_v0.1:47eff5b311b99e1ba76853aa1a681ff43c0b6be0b54c5b4cf4a8ceea07ff449a", + "drop_nulls_all.all.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_all.all.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_all.all.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls_all.all.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_all.method.content_hash": "semantic_v0.1:91519c5bfe375ac6ab2979f02c3962f95782163690566a3cd4354e46aaab1bab", + "drop_nulls_all.method.pipeline_hash": "semantic_v0.1:47eff5b311b99e1ba76853aa1a681ff43c0b6be0b54c5b4cf4a8ceea07ff449a", + "drop_nulls_all.method.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_all.method.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_all.method.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls_all.method.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_all.operator[all].content_hash": "semantic_v0.1:171e408bad76d85d0f84ab7ad461af9eaf131e34b6c8f8c482cd8bb1a141ce9a", + "drop_nulls_all.operator[all].pipeline_hash": "semantic_v0.1:171e408bad76d85d0f84ab7ad461af9eaf131e34b6c8f8c482cd8bb1a141ce9a", + "drop_nulls_job.node[golden_nullable].content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", + "drop_nulls_job.node[golden_nullable].pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", + "drop_nulls_job.node[golden_nullable].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_job.node[present].content_hash": "semantic_v0.1:6e7edbd1167d450b18cae53f7daeeac3e489c6dde1921131f09f8880c7032e70", + "drop_nulls_job.node[present].pipeline_hash": "semantic_v0.1:b388b7f3dbab1086d9a2c96f82e2f957dd21bb74b1f68ddc57ee70174c1dac96", + "drop_nulls_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[present].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", + "drop_nulls_job.node[present].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_job.node[ratio].content_hash": "semantic_v0.1:2bd33335d606563899ff2e7e2b45f8ab73dc22265d752eb98670f93a62613c35", + "drop_nulls_job.node[ratio].pipeline_hash": "semantic_v0.1:b4bf7581adb16db07b8a3c5042e92ed721ba44dbf109da6650a0289c3e7b42b0", + "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:00000150467e204fcd64131e2c5242f04cb75405ad27b0effab95b7024cac05989ce7b", + "drop_nulls_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000012f80ae46902b8c07a040d40c77b237701ab557207263bd934400b66bbfbd7daf", + "drop_nulls_job.node[ratio].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_job.node[renamed].content_hash": "semantic_v0.1:a5ac89746ac4e6b0e251185ea47d330f670d84dcba8b090202bfa1f71f6cc165", + "drop_nulls_job.node[renamed].pipeline_hash": "semantic_v0.1:9d0a308a0c2a3c9f2497291a86d0f7ff9c3d8fbf683cda11bb09d4833170df97", + "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3", + "drop_nulls_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", + "drop_nulls_job.node[renamed].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_job.table[golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1].input_data_hashes": [ + "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", + "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3" + ], + "drop_nulls_job.table[golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:b4bf7581adb16db07b8a3c5042e92ed721ba44dbf109da6650a0289c3e7b42b0/pdb_v1].record_ids": [ + "arrow_v0.1:0000012447639a486f05322571bf7908f2e5b1a551e430dc4eec49e54a22657f0a49aa", + "arrow_v0.1:00000124f29b7254f3a496da7cd95db7728fc265f082ae8091cd094bac96a0631c3563" + ], + "drop_nulls_job.table_paths": [ + "golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1", + "golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:b4bf7581adb16db07b8a3c5042e92ed721ba44dbf109da6650a0289c3e7b42b0/pdb_v1" + ], "group_by.content_hash": "semantic_v0.1:ee6478a83847e5e9d6bea916138170139aba8c5b43c6992a22d048a07b9b4526", "group_by.pipeline_hash": "semantic_v0.1:5ab1cb5d5ce9b641d19045bae0d94bbb9e15703bebfe37ec57fd13b141d2a0f0", "group_by.row[{\"cohort\":\"a\",\"subject\":[\"s1\",\"s2\"]}]._tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "7d212ee5c358519c9e4729f4a0898aad", diff --git a/tests/test_pipeline/test_drop_nulls_job.py b/tests/test_pipeline/test_drop_nulls_job.py new file mode 100644 index 000000000..47bdf8dc6 --- /dev/null +++ b/tests/test_pipeline/test_drop_nulls_job.py @@ -0,0 +1,148 @@ +"""``DropNulls`` in a ``PipelineJob`` against a Delta Lake store (ITL-943). + +The workshop shape: a source whose value column is nullable by specification, +``drop_nulls``, then a pod with a non-optional parameter. A second run on +the same store must reuse the stored results. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.function_pod import FunctionPod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import PipelineJob + +# Module level so the function's identity is the same across the runs of a test. +_CALLS: list[int] = [] + + +def to_seconds(recording_start_timestamp: int) -> float: + """A pod function with a non-optional parameter; records each call.""" + _CALLS.append(recording_start_timestamp) + return recording_start_timestamp / 1000 + + +@pytest.fixture(autouse=True) +def _reset_calls() -> Iterator[None]: + """Clear the shared invocation log before every test.""" + _CALLS.clear() + yield + + +@pytest.fixture +def store(tmp_path: Path) -> DeltaTableDatabase: + """A Delta Lake store in the test's temporary directory.""" + return DeltaTableDatabase(base_path=tmp_path / "store") + + +def _recordings(timestamps: list[int | None]) -> ArrowTableSource: + """A source whose ``recording_start_timestamp`` column is nullable.""" + table = pa.table( + { + "session": pa.array( + [f"s{i}" for i in range(len(timestamps))], pa.large_string() + ), + "recording_start_timestamp": pa.array(timestamps, pa.int64()), + }, + schema=pa.schema( + [ + pa.field("session", pa.large_string(), nullable=False), + pa.field("recording_start_timestamp", pa.int64(), nullable=True), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["session"], source_id="recordings") + + +def _run(store: DeltaTableDatabase, timestamps: list[int | None]) -> PipelineJob: + """Record and run ``source -> drop_nulls -> to_seconds`` as a ``PipelineJob``.""" + pod = FunctionPod(PythonDataFunction(to_seconds, output_keys="seconds")) # pyright: ignore[reportArgumentType] # PythonDataFunction is not yet typed as matching DataFunctionProtocol (baselined in other tests); it works at runtime + job = PipelineJob(name="recordings", store=store) + with job: + present = _recordings(timestamps).drop_nulls( + "recording_start_timestamp", label="present" + ) + pod(present, label="seconds") + return job.run() + + +def _seconds(job: PipelineJob) -> dict[str, float]: + """Return the pod's output as ``{session: seconds}``.""" + table = job.nodes["seconds"].as_table() + return dict( + zip(table.column("session").to_pylist(), table.column("seconds").to_pylist()) + ) + + +def test_job_drops_null_rows_and_runs_the_pod(store: DeltaTableDatabase) -> None: + """The pod runs once per non-null row and the operator node reports ``int``.""" + job = _run(store, [1000, None, 3000]) + assert sorted(_CALLS) == [1000, 3000] + assert _seconds(job) == {"s0": 1.0, "s2": 3.0} + present = job.nodes["present"] + assert present.output_schema()[1] == {"recording_start_timestamp": int} + field = present.as_table().schema.field("recording_start_timestamp") + assert field.nullable is False + + +def test_identical_rerun_hits_the_cache(store: DeltaTableDatabase) -> None: + """A second identical run on the same store makes no pod call.""" + _run(store, [1000, None, 3000]) + assert sorted(_CALLS) == [1000, 3000] + + _CALLS.clear() + job = _run(store, [1000, None, 3000]) + assert _CALLS == [], "second identical run must not recompute" + assert _seconds(job) == {"s0": 1.0, "s2": 3.0} + + +def test_fresh_store_recomputes(tmp_path: Path) -> None: + """Control for the cache-hit test: a different store has nothing to reuse.""" + _run(DeltaTableDatabase(base_path=tmp_path / "a"), [1000, None, 3000]) + _CALLS.clear() + _run(DeltaTableDatabase(base_path=tmp_path / "b"), [1000, None, 3000]) + assert sorted(_CALLS) == [1000, 3000] + + +def test_a_newly_non_null_row_is_the_only_one_computed( + store: DeltaTableDatabase, +) -> None: + """Rows computed before stay cached when a null row gains a value.""" + _run(store, [1000, None, 3000]) + _CALLS.clear() + job = _run(store, [1000, 2000, 3000]) + assert _CALLS == [2000] + assert _seconds(job) == {"s0": 1.0, "s1": 2.0, "s2": 3.0} + + +def test_job_save_and_load_round_trips_the_operator( + store: DeltaTableDatabase, tmp_path: Path +) -> None: + """A saved job records the operator's config and loads it back.""" + import json + + job = _run(store, [1000, None, 3000]) + path = tmp_path / "job.json" + job.save(str(path)) + + saved = [ + node["operator_config"] + for node in json.loads(path.read_text())["pipeline"]["nodes"].values() + if node.get("operator_config") + ] + assert [c["class_name"] for c in saved] == ["DropNulls"] + assert saved[0]["config"] == {"columns": ["recording_start_timestamp"]} + + loaded = PipelineJob.load(str(path), store=store) + present = loaded.nodes["present"] + assert present.output_schema() == ( + {"session": str}, + {"recording_start_timestamp": int}, + )