From 4f99b128ed3b7b6ac5d03ae5522411de3dba4145 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 17:33:36 -0800 Subject: [PATCH 01/11] docs(plans): ITL-943 DropNullRows operator plan Co-Authored-By: Claude Fable 5.2 --- .../2026-11-11-itl-943-drop-null-rows.md | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 superpowers/plans/2026-11-11-itl-943-drop-null-rows.md diff --git a/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md b/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md new file mode 100644 index 000000000..632267931 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md @@ -0,0 +1,151 @@ +# DropNullRows Operator Implementation Plan (ITL-943) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A unary operator `DropNullRows(columns)` and `stream.drop_null_rows(columns)` that keeps +only rows where every named column is non-null and narrows each named column from `T | None` +to `T` (schema and Arrow field). + +**Architecture:** One operator class in `src/orcapod/core/operators/drop_null_rows.py`. Pure +Arrow (validity mask, `Table.filter`, re-wrap the same arrays under a schema whose named +fields are non-nullable); no Polars. A streaming `async_execute` applies the same +table-level narrowing to each datagram's Arrow table. + +**Tech Stack:** pyarrow, pytest, `uv run`. + +--- + +## Design (approved by the coordinator, November 11, 2026) + +Coordinator's answer to the STEP 1 `ask`: (b)-(j) approved as proposed. (a) The NAME is the +developer's call, still open between `DropNulls`/`drop_nulls` and +`DropNullRows`/`drop_null_rows`; **use `DropNullRows` / `drop_null_rows` meanwhile and keep +the name easy to switch with one rename** (class, stream method, registry key, module, docs). + +- **Argument:** `columns: str | Collection[str]`; a bare `str` is one name. Empty collection is + `ValueError` at construction. Stored as `self.columns = tuple(sorted(set(names)))`. +- **Row rule:** a row survives only if every named column is non-null. +- **Missing column:** `InputValidationError` at call time (`validate_unary_input`), naming the + missing names. No `strict` flag. System-tag, source-info, meta and context columns cannot be + named (reported as missing). +- **Already non-nullable column:** accepted, no-op for that column (idempotent; postcondition + semantics; a pipeline's validity must not depend on declared upstream nullability). +- **Tag vs data:** one operator; names may be tag or data columns, mixed. +- **Nested:** only top-level validity is tested (null list / null struct; not an empty list, + list items, struct children, or NaN). Only the top-level field becomes non-nullable. +- **Identity:** `identity_structure() == ("DropNullRows", self.columns)`; same for content and + pipeline hash. `"x"`, `["x"]`, `("x",)`, `{"x"}`, `["x", "x"]` are one identity; order is + irrelevant. +- **Hash effects (measured):** nullability is in the Arrow schema hash. A surviving row's + `Data` content hash changes iff a named data column was nullable; its `Tag` content hash + changes iff a named tag column was nullable. The new hash equals the hash the same row has in + a stream that declared the column non-nullable from the start. System-tag names and values + and source-info tokens are unchanged. No existing golden value moves. +- **System tags:** name-preserving. Source-info columns untouched. Row order preserved. +- **Schema prediction:** upstream schema with `None` removed from each named column's type. + Verified against `unary_static_process(stream).output_schema(...)`. Zero-row results keep the + narrowed types. +- **Async:** own streaming `async_execute`; nullness read from each datagram's Arrow table, + surviving rows rebuilt from that table. Added to `test_async_sync_equivalence.py`. +- **Arrow:** every column large-normalized (`keep_null=True`), nothing else changed. +- **Save/load:** `to_config()["config"] == {"columns": [...]}`; registered in + `pipeline/serialization.py`. +- **Boundary note:** the operator reads validity only, never values. Add one line to the + operator table in `CLAUDE.md` and `.zed/rules`; put the wording in the PR for the developer. + +Extra tests asked by the coordinator: NaN is not null; a nullable tag narrowed and then +joined; a Spiral-style source with nullable key columns narrowed to `T`, enabling a `Join` +with a source whose keys are `T`. + +## Files + +- Create: `src/orcapod/core/operators/drop_null_rows.py` (the operator) +- Modify: `src/orcapod/core/operators/__init__.py` (export), `src/orcapod/operators/...` + (public re-export, wherever `PolarsFilter` is listed) +- Modify: `src/orcapod/core/streams/base.py` (`drop_null_rows` method), + `src/orcapod/protocols/core_protocols/streams.py` (protocol method) +- Modify: `src/orcapod/pipeline/serialization.py` (operator registry) +- Create: `tests/test_core/operators/test_drop_null_rows.py` +- Create: `tests/test_pipeline/test_drop_null_rows_job.py` (Delta store, cache hits) +- Modify: `tests/test_core/operators/test_async_sync_equivalence.py` (two cases) +- Modify: `tests/test_core/operators/test_operator_config.py` (round trip) +- Modify: `tests/test_hashing/golden/cases.py` (new case `pipeline/drop_null_rows`), then + regenerate; the diff of `tests/test_hashing/golden/data/pipeline.json` must only add lines +- Modify: `docs/concepts/operators.md`, `CHANGELOG.md`, `DESIGN_ISSUES.md`, `CLAUDE.md`, + `.zed/rules` + +## Operator sketch + +```python +class DropNullRows(UnaryOperator): + def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: + names = (columns,) if isinstance(columns, str) else tuple(columns) + if not names: + raise ValueError("DropNullRows needs at least one column name.") + self.columns: tuple[str, ...] = tuple(sorted(set(names))) + super().__init__(**kwargs) + + def _narrow_table(self, table: pa.Table) -> pa.Table: + """Drop rows with a null in a named column present in ``table``; flag those + columns non-nullable; normalize every column to its large type.""" + named = [n for n in self.columns if n in table.column_names] + mask = None + for name in named: + valid = pc.is_valid(table.column(name)) + mask = valid if mask is None else pc.and_(mask, valid) + if mask is not None: + table = table.filter(mask) + table = arrow_utils.normalize_table_to_large_types(table) # keep_null behaviour checked + schema = table.schema + for name in named: + i = schema.get_field_index(name) + schema = schema.set(i, schema.field(i).with_nullable(False)) + return pa.Table.from_arrays(table.columns, schema=schema) # same arrays, new flags +``` + +Sync: `_narrow_table(stream.as_table(columns={"source": True, "system_tags": True, +"sort_by_tags": False}))` wrapped in `ArrowTableStream(table, tag_columns=stream.keys()[0])`. + +Async: per `(tag, data)`: narrow `tag.as_table(columns={"system_tags": True})` and +`data.as_table(columns={"source": True, "meta": True})`; if either comes back with zero rows +the row is dropped; otherwise emit `Tag(table, data_context=tag.data_context_key)` and +`Data(table, data_context=data.data_context_key)`. If row-by-row equivalence cannot be had, +fall back to the inherited barrier path and report it. + +## Tasks + +- [ ] **1. Tests first** (`tests/test_core/operators/test_drop_null_rows.py`): rows with nulls + dropped and order preserved; `output_schema` and `as_table` narrowed; other columns' types + and nullability untouched; multiple columns; tag column; mixed tag and data; nested (list + with null items, struct with null children survive; null list / struct dropped; nested + flags untouched); NaN survives; already non-nullable (hashes untouched); missing column + (`InputValidationError`), system/source column names refused; empty `columns` + (`ValueError`); zero surviving rows keep the narrowed schema; logical types (`Path | None`, + `np.ndarray | None`); identity (str/list/tuple/set/duplicates/order equal; different names + differ); narrowed row hashes equal those of a source declared non-nullable; system tags and + source info unchanged; schema prediction vs `unary_static_process(...).output_schema(...)` + (comment at the non-public use); pod with a non-optional parameter refused before, runs + after; `Join` after it; nullable tag narrowed then joined with a `T`-keyed source; + `stream.drop_null_rows` equals the operator. Run: all fail with `ImportError`. +- [ ] **2. More tests:** two `Case`s in `test_async_sync_equivalence.py` (`drop_null_rows` + on `nul`, `drop_null_rows_tag_and_data`), config round trip in `test_operator_config.py`, + `tests/test_pipeline/test_drop_null_rows_job.py` (PipelineJob with `DeltaTableDatabase`: + run, results; second job on the same store makes no new pod calls). Commit (WIP, failing). +- [ ] **3. Implement** the operator, exports, stream method, protocol method, registry. + Run the new tests until green. Commit. +- [ ] **4. Golden:** add `@case("pipeline", "drop_null_rows")` to `cases.py` (a source with a + nullable data column; summaries of the operator on one and on two columns), regenerate, + check `git diff --stat` and that the diff has no removed or changed line. Commit. +- [ ] **5. Docs:** `docs/concepts/operators.md` section with a runnable sample (drop null rows, + then a pod with a non-optional parameter); convenience-method list; `CHANGELOG.md` + `[0.3.0] — unreleased` Added; `DESIGN_ISSUES.md` entry appended; `CLAUDE.md` + `.zed/rules` + (project layout line, operator boundary line). Run `tests/test_docs`. Commit. +- [ ] **6. Verify:** `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (one Python + version at a time, no xdist). Merge `origin/main` if it moved and rerun. +- [ ] **7. PR** against `main` (`Fixes ITL-943`), link with `orca linear attach` and + `orca worktree set`, CodeRabbit round, `worker_done`. + +## Notes / progress log + +(Updated as work proceeds, for a successor.) From 36cf709b05b0747b16922eaa4700dad65c0152bf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 17:37:18 -0800 Subject: [PATCH 02/11] feat(operators): add DropNullRows, which drops rows with a null in named columns and narrows them to T (ITL-943) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/operators/__init__.py | 2 + src/orcapod/core/operators/drop_null_rows.py | 252 +++++++++ src/orcapod/core/streams/base.py | 22 + src/orcapod/pipeline/serialization.py | 2 + .../protocols/core_protocols/streams.py | 11 + .../operators/test_async_sync_equivalence.py | 8 + .../operators/test_drop_null_rows.py | 525 ++++++++++++++++++ .../operators/test_operator_config.py | 24 + .../test_pipeline/test_drop_null_rows_job.py | 139 +++++ 9 files changed, 985 insertions(+) create mode 100644 src/orcapod/core/operators/drop_null_rows.py create mode 100644 tests/test_core/operators/test_drop_null_rows.py create mode 100644 tests/test_pipeline/test_drop_null_rows_job.py diff --git a/src/orcapod/core/operators/__init__.py b/src/orcapod/core/operators/__init__.py index b8e846d5f..8329d291d 100644 --- a/src/orcapod/core/operators/__init__.py +++ b/src/orcapod/core/operators/__init__.py @@ -5,6 +5,7 @@ SelectDataColumns, SelectTagColumns, ) +from .drop_null_rows import DropNullRows from .filters import PolarsFilter from .group_by import GroupBy from .index import Index @@ -27,6 +28,7 @@ "DropTagColumns", "DropDataColumns", "PolarsFilter", + "DropNullRows", "Pick", "Index", ] diff --git a/src/orcapod/core/operators/drop_null_rows.py b/src/orcapod/core/operators/drop_null_rows.py new file mode 100644 index 000000000..6530f5465 --- /dev/null +++ b/src/orcapod/core/operators/drop_null_rows.py @@ -0,0 +1,252 @@ +"""DropNullRows operator: drop rows with a null in named columns, narrow ``T | None`` to ``T``.""" + +from __future__ import annotations + +import types +import typing +from collections.abc import Collection, Sequence +from typing import TYPE_CHECKING, Any + +from orcapod.channels import ReadableChannel, WritableChannel +from orcapod.core.datagrams import Data, Tag +from orcapod.core.operators.base import UnaryOperator +from orcapod.core.streams.arrow_table_stream import ArrowTableStream +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol +from orcapod.types import ColumnConfig, DataType, Schema +from orcapod.utils import arrow_utils +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import pyarrow as pa + import pyarrow.compute as pc +else: + pa = LazyModule("pyarrow") + pc = LazyModule("pyarrow.compute") + + +def _without_none(python_type: DataType) -> DataType: + """Return ``python_type`` with ``None`` removed from a union. + + ``int | None`` gives ``int`` and ``int | str | None`` gives ``int | str``. + A type that does not admit ``None`` is returned unchanged. + + Args: + python_type: A column's Python type. + + Returns: + The type without its ``None`` member. + """ + if typing.get_origin(python_type) not in (typing.Union, types.UnionType): + return python_type + members = tuple(m for m in typing.get_args(python_type) if m is not type(None)) + if len(members) == len(typing.get_args(python_type)) or not members: + return python_type + return typing.Union[members] # noqa: UP007 # built from a runtime tuple of members + + +class DropNullRows(UnaryOperator): + """Keep the rows where every named column is non-null; narrow those columns to ``T``. + + A row survives only if all the named columns hold a value. Afterwards each + named column is non-nullable: ``T`` instead of ``T | None`` in + ``output_schema()``, and a non-nullable field in the Arrow table. A pod + whose parameter is a plain ``T`` can then consume the column. + + The names may be tag columns, data columns, or both. Only the validity of + the top-level value is read, never the value itself: a null list or a null + struct is null, while an empty list, a list holding nulls, a struct whose + fields are null and a floating-point NaN are not. Nested fields keep their + nullability. + + A named column that is already non-nullable is accepted and left as it is, + so applying the operator twice equals applying it once. + + Everything else passes through unchanged: the other columns' types and + nullability, system tags (names and values), source info and row order. + Because nullability is part of a row's Arrow schema, a surviving row's + ``Data`` content hash changes when a named data column was nullable, and + its ``Tag`` content hash changes when a named tag column was nullable. The + new hash is the one the row has in a stream that declared the column + non-nullable from the start. + + Args: + columns: One column name, or a collection of names. Duplicates and + order are ignored: the operator's identity is the set of names. + + Raises: + ValueError: If ``columns`` is empty. + """ + + def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: + names = (columns,) if isinstance(columns, str) else tuple(columns) + if not names: + raise ValueError( + "DropNullRows needs at least one column name; got an empty collection." + ) + # A sorted tuple of unique names, so a list, a tuple and a set of the + # same names (in any order) are one identity. + self.columns: tuple[str, ...] = tuple(sorted(set(names))) + super().__init__(**kwargs) + + def identity_structure(self) -> Any: + return (self.__class__.__name__, self.columns) + + def to_config(self) -> dict[str, Any]: + """Serialize this operator to a config dict. + + Returns: + A dict with ``class_name``, ``module_path``, and ``config`` keys, + where ``config`` contains ``columns``. + """ + config = super().to_config() + config["config"] = {"columns": list(self.columns)} + return config + + # ------------------------------------------------------------------ + # Validation and schema prediction + # ------------------------------------------------------------------ + + def validate_unary_input(self, stream: StreamProtocol) -> None: + """Check that every named column is a tag or data column of ``stream``. + + Args: + stream: The upstream stream. + + Raises: + InputValidationError: If a named column is not a tag or data column. + """ + tag_columns, data_columns = stream.keys() + missing = [ + name + for name in self.columns + if name not in tag_columns and name not in data_columns + ] + if missing: + raise InputValidationError( + f"DropNullRows: columns {missing} are not tag or data columns of the" + + f" input stream. Tag columns: {list(tag_columns)}; data columns:" + + f" {list(data_columns)}." + ) + + def _narrow_schema(self, schema: Schema) -> Schema: + """Return ``schema`` with ``None`` removed from each named column's type.""" + return Schema( + { + name: _without_none(python_type) + if name in self.columns + else python_type + for name, python_type in schema.items() + }, + optional_fields=schema.optional_fields, + ) + + def unary_output_schema( + self, + stream: StreamProtocol, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """Return the input schemas with the named columns narrowed to ``T``. + + Args: + stream: The upstream stream. + columns: Column inclusion config passed through to ``output_schema``. + all_info: Whether to include all info columns. + + Returns: + A ``(tag_schema, data_schema)`` tuple. + """ + tag_schema, data_schema = stream.output_schema( + columns=columns, all_info=all_info + ) + return self._narrow_schema(tag_schema), self._narrow_schema(data_schema) + + # ------------------------------------------------------------------ + # Execution + # ------------------------------------------------------------------ + + def _narrow_table(self, table: pa.Table) -> pa.Table: + """Drop the rows with a null in a named column and flag those columns non-nullable. + + Only the named columns present in ``table`` are considered, so the same + rule serves a whole stream table and a single row's tag or data table. + Every column takes the large-normalized form of its input type; no + other type, flag or value changes. + + Args: + table: A table holding some or all of the named columns. + + Returns: + The surviving rows, in order, with the named fields non-nullable. + """ + named = [name for name in self.columns if name in table.column_names] + mask = None + for name in named: + valid = pc.is_valid(table.column(name)) + mask = valid if mask is None else pc.and_(mask, valid) + if mask is not None: + table = table.filter(mask) + table = arrow_utils.normalize_table_to_large_types(table) + schema = table.schema + for name in named: + position = schema.get_field_index(name) + schema = schema.set(position, schema.field(position).with_nullable(False)) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + # Same arrays under new field flags: no cast, so no value or type can change. + return pa.Table.from_arrays(table.columns, schema=schema) + + def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: + """Return the stream's rows whose named columns are all non-null. + + Args: + stream: The upstream stream. + + Returns: + A stream over the surviving rows, in input order, with the named + columns non-nullable. A result with no rows has the same schema. + """ + table = stream.as_table( + columns={"source": True, "system_tags": True, "sort_by_tags": False} + ) + return ArrowTableStream( + self._narrow_table(table), + tag_columns=stream.keys()[0], + data_context=stream.data_context, # pyright: ignore[reportAttributeAccessIssue] # StreamProtocol omits data_context, which every concrete stream provides (ITL-844) + ) + + async def async_execute( + self, + inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + **kwargs: Any, + ) -> None: + """Streaming: test and narrow each row as it arrives, without materializing. + + Each row's tag and data Arrow tables go through the rule of the sync + path (``_narrow_table``), and a surviving row is rebuilt from those + tables with its own data context. + + Args: + inputs: Single-element sequence of readable channels. + output: Channel to send surviving rows to. + **kwargs: Ignored; present for protocol compliance. + """ + try: + async for tag, data in inputs[0]: + tag_table = self._narrow_table( + tag.as_table(columns={"system_tags": True}) + ) + data_table = self._narrow_table( + data.as_table(columns={"source": True, "meta": True}) + ) + if tag_table.num_rows == 0 or data_table.num_rows == 0: + continue + await output.send( + ( + Tag(tag_table, data_context=tag.data_context_key), + Data(data_table, data_context=data.data_context_key), + ) + ) + finally: + await output.close() diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 2629401e6..6253d6193 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -186,6 +186,28 @@ def polars_filter( self, label=label ) + def drop_null_rows( + self, + columns: str | Collection[str], + label: str | None = None, + ) -> StreamBase: + """Keep the rows where every named column is non-null; narrow those columns to ``T``. + + Each named column, tag or data, is non-nullable afterwards (``T`` + instead of ``T | None``), so a pod whose parameter is a plain ``T`` + can consume it. See ``orcapod.core.operators.DropNullRows``. + + Args: + columns: One column name, or a collection of names. + label: Optional node label for the pipeline graph. + + Returns: + A stream over the surviving rows, in input order. + """ + from orcapod.core.operators import DropNullRows + + return DropNullRows(columns)(self, label=label) + def select_tag_columns( self, tag_columns: str | Collection[str], diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 6800502da..70ad0d7ac 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -166,6 +166,7 @@ def _build_operator_registry() -> dict[str, type]: from orcapod.core.operators import ( Batch, DropDataColumns, + DropNullRows, DropTagColumns, GroupBy, Join, @@ -191,6 +192,7 @@ def _build_operator_registry() -> dict[str, type]: "MapTags": MapTags, "MapData": MapData, "PolarsFilter": PolarsFilter, + "DropNullRows": DropNullRows, } diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index b5000e7ef..02963a096 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -285,6 +285,17 @@ def polars_filter( **constraints: Any, ) -> "StreamProtocol": ... + def drop_null_rows( + self, + columns: str | Collection[str], + label: str | None = None, + ) -> "StreamProtocol": + """ + Keep the rows where every named column (tag or data) is non-null, and + narrow each named column from ``T | None`` to ``T``. + """ + ... + def select_tag_columns( self, tag_columns: str | Collection[str], diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 809af1813..68644ff1d 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -31,6 +31,7 @@ from orcapod.core.operators import ( Batch, DropDataColumns, + DropNullRows, DropTagColumns, GroupBy, Index, @@ -312,6 +313,13 @@ class Case: ), Case("index", lambda: Index("l8", 0), lambda: (source_a(),)), Case("pick", lambda: Pick("d", "q"), lambda: (source_a(),)), + Case("drop_null_rows", lambda: DropNullRows("nul"), lambda: (source_a(),)), + Case( + "drop_null_rows_many", + # A nullable column with a null, one without, a tag, a non-nullable list. + lambda: DropNullRows(["nul", "s", "ta", "l8"]), + lambda: (source_a(),), + ), ] CASE = pytest.mark.parametrize("case", CASES, ids=[c.label for c in CASES]) diff --git a/tests/test_core/operators/test_drop_null_rows.py b/tests/test_core/operators/test_drop_null_rows.py new file mode 100644 index 000000000..956ffa695 --- /dev/null +++ b/tests/test_core/operators/test_drop_null_rows.py @@ -0,0 +1,525 @@ +"""``DropNullRows``: drop rows with a null in a named column, narrow ``T | None`` to ``T`` (ITL-943). + +The operator keeps the rows where every named column is non-null and reports each +named column as non-nullable afterwards, in ``output_schema()`` and in the Arrow +table. Everything else passes through: other columns' types and nullability, +system tags, source info and row order. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from pathlib import Path +from typing import Any, TypeAlias + +import numpy as np +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.sources import ArrowTableSource, DictSource +from orcapod.errors import InputValidationError +from orcapod.operators import DropNullRows, Join +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.system_constants import constants + +Column: TypeAlias = "pa.Array[Any] | pa.ChunkedArray[Any]" + +ALL = {"source": True, "system_tags": True} + + +def _source( + columns: dict[str, Column], + tags: Sequence[str] = ("id",), + nullable: Sequence[str] = (), + source_id: str = "src", +) -> ArrowTableSource: + """Build a source; every column not listed in ``nullable`` is non-nullable.""" + table = pa.table(list(columns.values()), names=list(columns)) + schema = pa.schema([f.with_nullable(f.name in nullable) for f in table.schema]) + return ArrowTableSource( + table.cast(schema), tag_columns=list(tags), source_id=source_id + ) + + +def readings(nullable: Sequence[str] = ("x", "y")) -> ArrowTableSource: + """Four rows; ``x`` is null in row 2, ``y`` in row 3; ``note`` is nullable too.""" + return _source( + { + "id": pa.array([1, 2, 3, 4], pa.int16()), + "x": pa.array([10, None, 30, 40], pa.int8()), + "y": pa.array([1.5, 2.5, None, 4.5], pa.float32()), + "note": pa.array(["a", None, "c", None], pa.string()), + }, + nullable=[*nullable, "note"], + ) + + +def _ids(stream: StreamProtocol) -> list[Any]: + return stream.as_table().column("id").to_pylist() + + +# ── rows ──────────────────────────────────────────────────────────────────── + + +def test_drops_rows_where_the_column_is_null() -> None: + result = DropNullRows("x")(readings()) + assert result.as_table().to_pylist() == [ + {"id": 1, "x": 10, "y": 1.5, "note": "a"}, + {"id": 3, "x": 30, "y": None, "note": "c"}, + {"id": 4, "x": 40, "y": 4.5, "note": None}, + ] + + +def test_every_named_column_must_be_non_null() -> None: + assert _ids(DropNullRows(["x", "y"])(readings())) == [1, 4] + + +def test_row_order_is_preserved() -> None: + source = _source( + { + "id": pa.array([4, 1, 3, 2], pa.int16()), + "x": pa.array([40, 10, None, 20], pa.int8()), + }, + nullable=["x"], + ) + assert [tag["id"] for tag, _ in DropNullRows("x")(source).iter_data()] == [4, 1, 2] + + +def test_nan_is_not_null() -> None: + source = _source( + { + "id": pa.array([1, 2, 3], pa.int16()), + "v": pa.array([1.0, float("nan"), None], pa.float64()), + }, + nullable=["v"], + ) + result = DropNullRows("v")(source) + values = result.as_table().column("v").to_pylist() + assert values[0] == 1.0 and math.isnan(values[1]) and len(values) == 2 + assert result.as_table().schema.field("v").nullable is False + assert result.output_schema()[1]["v"] is float + + +# ── types ─────────────────────────────────────────────────────────────────── + + +def test_output_schema_narrows_only_the_named_columns() -> None: + source = readings() + assert source.output_schema()[1] == { + "x": int | None, + "y": float | None, + "note": str | None, + } + tag_schema, data_schema = DropNullRows("x")(source).output_schema() + assert tag_schema == {"id": int} + assert data_schema == {"x": int, "y": float | None, "note": str | None} + + +def test_table_narrows_only_the_named_columns() -> None: + source = readings() + before = source.as_table(columns=ALL).schema + after = DropNullRows("x")(source).as_table(columns=ALL).schema + assert after.names == before.names + assert after.field("x") == pa.field("x", pa.int8(), nullable=False) + # Width and nullability are kept; ``string`` takes its large variant. + assert after.field("y") == pa.field("y", pa.float32(), nullable=True) + assert after.field("note") == pa.field("note", pa.large_string(), nullable=True) + for name in before.names: + if name not in ("x", "note"): + assert after.field(name) == before.field(name) + + +def test_schema_prediction_matches_the_computed_stream() -> None: + source = readings() + op = DropNullRows(["x", "id"]) + for config in ({}, {"system_tags": True}, {"source": True, "system_tags": True}): + # Non-public on purpose: a prediction is verified against the static result. + computed = op.unary_static_process(source).output_schema(columns=config) + assert op(source).output_schema(columns=config) == computed + assert op.output_schema(source, columns=config) == computed + + +def test_zero_surviving_rows_keep_the_narrowed_types() -> None: + source = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([None, None], pa.int8()), + "s": pa.array(["a", "b"], pa.string()), + }, + nullable=["x", "s"], + ) + non_empty = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([5, None], pa.int8()), + "s": pa.array(["a", "b"], pa.string()), + }, + nullable=["x", "s"], + ) + result = DropNullRows("x")(source) + table = result.as_table(columns=ALL) + assert table.num_rows == 0 + assert list(result.iter_data()) == [] + assert table.schema.field("x") == pa.field("x", pa.int8(), nullable=False) + assert result.output_schema() == ({"id": int}, {"x": int, "s": str | None}) + expected = DropNullRows("x")(non_empty).as_table(columns=ALL).schema + assert table.schema == expected + + +def test_already_non_nullable_column_is_a_no_op() -> None: + source = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([10, 20], pa.int8()), + } + ) + result = DropNullRows("x")(source) + assert result.output_schema() == source.output_schema() + assert result.as_table(columns=ALL) == source.as_table(columns=ALL) + for (tag, data), (in_tag, in_data) in zip(result.iter_data(), source.iter_data()): + assert tag.content_hash() == in_tag.content_hash() + assert data.content_hash() == in_data.content_hash() + + +def test_applying_twice_equals_applying_once() -> None: + once = DropNullRows("x")(readings()) + twice = DropNullRows("x")(once) + assert twice.as_table(columns=ALL) == once.as_table(columns=ALL) + assert twice.output_schema() == once.output_schema() + + +# ── tag columns ───────────────────────────────────────────────────────────── + + +def keyed(nullable: Sequence[str] = ("site", "x")) -> ArrowTableSource: + """Tags ``site`` (null in row 2) and ``id``; ``x`` is null in row 3.""" + return _source( + { + "site": pa.array(["a", None, "c"], pa.large_string()), + "id": pa.array([1, 2, 3], pa.int16()), + "x": pa.array([10, 20, None], pa.int8()), + }, + tags=("site", "id"), + nullable=nullable, + ) + + +def test_tag_column_is_narrowed() -> None: + source = keyed() + assert source.output_schema()[0] == {"site": str | None, "id": int} + result = DropNullRows("site")(source) + assert _ids(result) == [1, 3] + assert result.keys() == source.keys() + assert result.output_schema() == ( + {"site": str, "id": int}, + {"x": int | None}, + ) + assert result.as_table().schema.field("site").nullable is False + assert result.as_table().schema.field("x").nullable is True + + +def test_tag_and_data_columns_in_one_call() -> None: + result = DropNullRows(["x", "site"])(keyed()) + assert _ids(result) == [1] + assert result.output_schema() == ({"site": str, "id": int}, {"x": int}) + + +def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: + """A source whose key columns are nullable by specification (as a Spiral table's + are) cannot join a source whose keys are ``T``; after ``DropNullRows`` it can.""" + events = _source( + { + "session": pa.array(["s1", None, "s3"], pa.large_string()), + "start": pa.array([100, 200, 300], pa.int64()), + }, + tags=("session",), + nullable=["session", "start"], + source_id="events", + ) + sessions = _source( + { + "session": pa.array(["s1", "s3", "s4"], pa.large_string()), + "animal": pa.array(["m1", "m3", "m4"], pa.large_string()), + }, + tags=("session",), + source_id="sessions", + ) + with pytest.raises(InputValidationError, match="not compatible"): + Join()(events, sessions) + + joined = Join()(DropNullRows("session")(events), sessions) + assert joined.output_schema() == ( + {"session": str}, + {"start": int | None, "animal": str}, + ) + rows = sorted(joined.as_table().to_pylist(), key=lambda r: r["session"]) + assert rows == [ + {"session": "s1", "start": 100, "animal": "m1"}, + {"session": "s3", "start": 300, "animal": "m3"}, + ] + + +# ── nested and logical columns ────────────────────────────────────────────── + +ITEMS = pa.large_list(pa.field("item", pa.int8(), nullable=True)) +POINT = pa.struct([pa.field("a", pa.int8()), pa.field("b", pa.large_string())]) + + +def nested() -> ArrowTableSource: + return _source( + { + "id": pa.array([1, 2, 3, 4], pa.int16()), + "items": pa.array([[1, None], None, [], [4]], ITEMS), + "point": pa.array( + [{"a": 1, "b": None}, {"a": 2, "b": "x"}, None, {"a": None, "b": None}], + POINT, + ), + }, + nullable=["items", "point"], + ) + + +def test_only_the_top_level_value_is_tested_for_a_list() -> None: + result = DropNullRows("items")(nested()) + # A list holding a null and an empty list are not null lists. + assert result.as_table().column("items").to_pylist() == [[1, None], [], [4]] + field = result.as_table().schema.field("items") + assert field.nullable is False + assert field.type == ITEMS # the item field keeps its name and nullability + # The Python schema does not express item nullability; the Arrow type above does. + assert nested().output_schema()[1]["items"] == list[int] | None + assert result.output_schema()[1]["items"] == list[int] + + +def test_only_the_top_level_value_is_tested_for_a_struct() -> None: + result = DropNullRows("point")(nested()) + # A struct whose children are all null is not a null struct. + assert _ids(result) == [1, 2, 4] + field = result.as_table().schema.field("point") + assert field.nullable is False + assert field.type == POINT # children stay nullable + + +def logical() -> DictSource: + return DictSource( + data=[ + {"k": "a", "p": Path("/a"), "arr": np.arange(3)}, + {"k": "b", "p": None, "arr": np.arange(2)}, + {"k": "c", "p": Path("/c"), "arr": None}, + ], + tag_columns=["k"], + source_id="logical", + ) + + +def test_logical_columns_keep_their_logical_type() -> None: + source = logical() + before = source.as_table().schema + assert source.output_schema()[1] == {"p": Path | None, "arr": np.ndarray | None} + + result = DropNullRows(["p", "arr"])(source) + assert result.output_schema()[1] == {"p": Path, "arr": np.ndarray} + after = result.as_table().schema + for name in ("p", "arr"): + assert isinstance(after.field(name).type, pa.BaseExtensionType) + assert after.field(name).type == before.field(name).type + assert after.field(name).nullable is False + ((tag, data),) = list(result.iter_data()) + assert tag["k"] == "a" + assert data["p"] == Path("/a") + assert isinstance(data["arr"], np.ndarray) + assert data["arr"].tolist() == [0, 1, 2] + + +# ── arguments ─────────────────────────────────────────────────────────────── + + +def test_missing_column_is_an_input_validation_error() -> None: + with pytest.raises(InputValidationError, match="nope"): + DropNullRows(["x", "nope"])(readings()) + + +def test_system_and_source_columns_cannot_be_named() -> None: + source = readings() + hidden = [ + n + for n in source.as_table(columns=ALL).column_names + if n.startswith((constants.SYSTEM_TAG_PREFIX, constants.SOURCE_PREFIX)) + ] + assert len(hidden) >= 2 + for name in hidden: + with pytest.raises(InputValidationError): + DropNullRows(name)(source) + + +def test_no_column_is_an_error_at_construction() -> None: + with pytest.raises(ValueError, match="at least one"): + DropNullRows([]) + + +# ── identity ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "columns", + [["x", "y"], ("x", "y"), {"x", "y"}, ["y", "x"], ["x", "y", "x"], frozenset("xy")], + ids=["list", "tuple", "set", "reversed", "duplicate", "frozenset"], +) +def test_identity_is_the_set_of_names(columns: Any) -> None: + reference = DropNullRows(["x", "y"]) + op = DropNullRows(columns) + assert op.columns == ("x", "y") + assert op.content_hash() == reference.content_hash() + assert op.pipeline_hash() == reference.pipeline_hash() + source = readings() + assert op(source).content_hash() == reference(source).content_hash() + assert op(source).pipeline_hash() == reference(source).pipeline_hash() + + +def test_a_bare_name_is_one_column() -> None: + assert DropNullRows("note").columns == ("note",) + assert DropNullRows("x").content_hash() == DropNullRows(["x"]).content_hash() + + +def test_different_names_are_different_identities() -> None: + source = readings() + assert DropNullRows("x").content_hash() != DropNullRows("y").content_hash() + assert DropNullRows("x").content_hash() != DropNullRows(["x", "y"]).content_hash() + assert ( + DropNullRows("x")(source).pipeline_hash() + != DropNullRows("y")(source).pipeline_hash() + ) + assert DropNullRows("x")(source).pipeline_hash() != source.pipeline_hash() + + +def test_the_caller_s_collection_is_not_kept_or_mutated() -> None: + names = ["y", "x"] + op = DropNullRows(names) + names.append("z") + assert names == ["y", "x", "z"] + assert op.columns == ("x", "y") + + +# ── hashes and provenance of surviving rows ───────────────────────────────── + + +def test_narrowed_rows_hash_as_rows_declared_non_nullable() -> None: + """Nullability is part of the Arrow schema hash, so narrowing a data column + changes the row's ``Data`` hash, to the hash the same row has in a stream + that declared the column non-nullable from the start. Tags are untouched.""" + columns: dict[str, Column] = { + "id": pa.array([1, 3], pa.int16()), + "x": pa.array([10, 30], pa.int8()), + } + declared = _source(columns) + nullable = _source( + { + "id": pa.array([1, 2, 3], pa.int16()), + "x": pa.array([10, None, 30], pa.int8()), + }, + nullable=["x"], + ) + input_rows = {tag["id"]: (tag, data) for tag, data in nullable.iter_data()} + narrowed = list(DropNullRows("x")(nullable).iter_data()) + assert len(narrowed) == 2 + for (tag, data), (ref_tag, ref_data) in zip(narrowed, declared.iter_data()): + in_tag, in_data = input_rows[tag["id"]] + assert data.content_hash() == ref_data.content_hash() + assert data.content_hash() != in_data.content_hash() + assert tag.content_hash() == in_tag.content_hash() == ref_tag.content_hash() + + +def test_narrowing_a_tag_changes_the_tag_hash_only() -> None: + source = keyed() + input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} + for tag, data in DropNullRows("site")(source).iter_data(): + in_tag, in_data = input_rows[tag["id"]] + assert tag.content_hash() != in_tag.content_hash() + assert data.content_hash() == in_data.content_hash() + + +def test_system_tags_and_source_info_pass_through() -> None: + source = readings() + before = source.as_table(columns=ALL) + after = DropNullRows(["x", "y"])(source).as_table(columns=ALL) + hidden = [ + n + for n in before.column_names + if n.startswith((constants.SYSTEM_TAG_PREFIX, constants.SOURCE_PREFIX)) + ] + assert hidden and after.column_names == before.column_names + kept = before.take(pa.array([0, 3])) + for name in hidden: + assert after.schema.field(name) == before.schema.field(name) + assert after.column(name).to_pylist() == kept.column(name).to_pylist() + + input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} + for tag, data in DropNullRows(["x", "y"])(source).iter_data(): + in_tag, in_data = input_rows[tag["id"]] + assert tag.system_tags() == in_tag.system_tags() + assert data.source_info() == in_data.source_info() + + +def test_the_input_stream_is_not_changed() -> None: + source = readings() + schema = source.as_table(columns=ALL).schema + rows = source.as_table(columns=ALL).to_pylist() + DropNullRows(["x", "y"])(source).as_table() + assert source.as_table(columns=ALL).schema == schema + assert source.as_table(columns=ALL).to_pylist() == rows + + +# ── stream method ─────────────────────────────────────────────────────────── + + +def test_stream_method_equals_the_operator() -> None: + source = readings() + by_method = source.drop_null_rows(["x", "y"]) + by_operator = DropNullRows(["x", "y"])(source) + assert by_method.as_table(columns=ALL) == by_operator.as_table(columns=ALL) + assert by_method.content_hash() == by_operator.content_hash() + assert by_method.pipeline_hash() == by_operator.pipeline_hash() + assert _ids(source.drop_null_rows("x")) == [1, 3, 4] + + +# ── downstream ────────────────────────────────────────────────────────────── + + +@function_pod(output_keys="doubled") +def double_x(x: int) -> int: + """A pod with a non-optional parameter.""" + return x * 2 + + +def test_pod_with_a_non_optional_parameter_runs_only_after_the_operator() -> None: + source = _source( + { + "id": pa.array([1, 2, 3], pa.int16()), + "x": pa.array([10, None, 30], pa.int64()), + }, + nullable=["x"], + ) + with pytest.raises(ValueError, match="not compatible"): + double_x.pod(source) + + result = double_x.pod(source.drop_null_rows("x")) + assert result.as_table().to_pylist() == [ + {"id": 1, "doubled": 20}, + {"id": 3, "doubled": 60}, + ] + + +def test_join_after_the_operator() -> None: + weights = _source( + { + "id": pa.array([1, 2, 3, 4], pa.int16()), + "w": pa.array([1.0, 2.0, 3.0, 4.0], pa.float64()), + }, + source_id="weights", + ) + joined = Join()(DropNullRows("x")(readings()), weights) + assert joined.output_schema()[1]["x"] is int + assert sorted(joined.as_table().column("id").to_pylist()) == [1, 3, 4] + assert joined.as_table().schema.field("x").nullable is False diff --git a/tests/test_core/operators/test_operator_config.py b/tests/test_core/operators/test_operator_config.py index 414e9769d..c676c786d 100644 --- a/tests/test_core/operators/test_operator_config.py +++ b/tests/test_core/operators/test_operator_config.py @@ -3,6 +3,7 @@ from orcapod.core.operators import ( Batch, DropDataColumns, + DropNullRows, DropTagColumns, Join, MapData, @@ -161,3 +162,26 @@ def test_non_serializable_predicates_marked(self): config = op.to_config() # Should either serialize the expression or mark as non-reconstructable assert "config" in config + + +class TestDropNullRowsConfig: + def test_to_config(self): + config = DropNullRows(["y", "x", "y"]).to_config() + assert config["class_name"] == "DropNullRows" + assert config["config"] == {"columns": ["x", "y"]} + + def test_round_trip(self): + op = DropNullRows(["y", "x"]) + restored = DropNullRows.from_config(op.to_config()) + assert isinstance(restored, DropNullRows) + assert restored.columns == ("x", "y") + assert restored.content_hash() == op.content_hash() + assert restored.pipeline_hash() == op.pipeline_hash() + + def test_registered_for_pipeline_loading(self): + from orcapod.pipeline.serialization import resolve_operator_from_config + + op = DropNullRows(["y", "x"]) + restored = resolve_operator_from_config(op.to_config()) + assert isinstance(restored, DropNullRows) + assert restored.content_hash() == op.content_hash() diff --git a/tests/test_pipeline/test_drop_null_rows_job.py b/tests/test_pipeline/test_drop_null_rows_job.py new file mode 100644 index 000000000..cf4164c81 --- /dev/null +++ b/tests/test_pipeline/test_drop_null_rows_job.py @@ -0,0 +1,139 @@ +"""``DropNullRows`` in a ``PipelineJob`` against a Delta Lake store (ITL-943). + +The workshop shape: a source whose value column is nullable by specification, +``drop_null_rows``, then a pod with a non-optional parameter. A second run on +the same store must reuse the stored results. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.function_pod import FunctionPod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import PipelineJob + +# Module level so the function's identity is the same across the runs of a test. +_CALLS: list[int] = [] + + +def to_seconds(recording_start_timestamp: int) -> float: + """A pod function with a non-optional parameter; records each call.""" + _CALLS.append(recording_start_timestamp) + return recording_start_timestamp / 1000 + + +@pytest.fixture(autouse=True) +def _reset_calls() -> Iterator[None]: + _CALLS.clear() + yield + + +@pytest.fixture +def store(tmp_path: Path) -> DeltaTableDatabase: + return DeltaTableDatabase(base_path=tmp_path / "store") + + +def _recordings(timestamps: list[int | None]) -> ArrowTableSource: + table = pa.table( + { + "session": pa.array( + [f"s{i}" for i in range(len(timestamps))], pa.large_string() + ), + "recording_start_timestamp": pa.array(timestamps, pa.int64()), + }, + schema=pa.schema( + [ + pa.field("session", pa.large_string(), nullable=False), + pa.field("recording_start_timestamp", pa.int64(), nullable=True), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["session"], source_id="recordings") + + +def _run(store: DeltaTableDatabase, timestamps: list[int | None]) -> PipelineJob: + pod = FunctionPod(PythonDataFunction(to_seconds, output_keys="seconds")) # pyright: ignore[reportArgumentType] # PythonDataFunction is not yet typed as matching DataFunctionProtocol (baselined in other tests); it works at runtime + job = PipelineJob(name="recordings", store=store) + with job: + present = _recordings(timestamps).drop_null_rows( + "recording_start_timestamp", label="present" + ) + pod(present, label="seconds") + return job.run() + + +def _seconds(job: PipelineJob) -> dict[str, float]: + table = job.nodes["seconds"].as_table() + return dict( + zip(table.column("session").to_pylist(), table.column("seconds").to_pylist()) + ) + + +def test_job_drops_null_rows_and_runs_the_pod(store: DeltaTableDatabase) -> None: + job = _run(store, [1000, None, 3000]) + assert sorted(_CALLS) == [1000, 3000] + assert _seconds(job) == {"s0": 1.0, "s2": 3.0} + present = job.nodes["present"] + assert present.output_schema()[1] == {"recording_start_timestamp": int} + field = present.as_table().schema.field("recording_start_timestamp") + assert field.nullable is False + + +def test_identical_rerun_hits_the_cache(store: DeltaTableDatabase) -> None: + _run(store, [1000, None, 3000]) + assert sorted(_CALLS) == [1000, 3000] + + _CALLS.clear() + job = _run(store, [1000, None, 3000]) + assert _CALLS == [], "second identical run must not recompute" + assert _seconds(job) == {"s0": 1.0, "s2": 3.0} + + +def test_fresh_store_recomputes(tmp_path: Path) -> None: + """Control for the cache-hit test: a different store has nothing to reuse.""" + _run(DeltaTableDatabase(base_path=tmp_path / "a"), [1000, None, 3000]) + _CALLS.clear() + _run(DeltaTableDatabase(base_path=tmp_path / "b"), [1000, None, 3000]) + assert sorted(_CALLS) == [1000, 3000] + + +def test_a_newly_non_null_row_is_the_only_one_computed( + store: DeltaTableDatabase, +) -> None: + _run(store, [1000, None, 3000]) + _CALLS.clear() + job = _run(store, [1000, 2000, 3000]) + assert _CALLS == [2000] + assert _seconds(job) == {"s0": 1.0, "s1": 2.0, "s2": 3.0} + + +def test_job_save_and_load_round_trips_the_operator( + store: DeltaTableDatabase, tmp_path: Path +) -> None: + import json + + job = _run(store, [1000, None, 3000]) + path = tmp_path / "job.json" + job.save(str(path)) + + saved = [ + node["operator_config"] + for node in json.loads(path.read_text())["pipeline"]["nodes"].values() + if node.get("operator_config") + ] + assert [c["class_name"] for c in saved] == ["DropNullRows"] + assert saved[0]["config"] == {"columns": ["recording_start_timestamp"]} + + loaded = PipelineJob.load(str(path), store=store) + present = loaded.nodes["present"] + assert present.output_schema() == ( + {"session": str}, + {"recording_start_timestamp": int}, + ) From 630b7cffc25bf99fac84e253726f372b1f1c773d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 17:39:15 -0800 Subject: [PATCH 03/11] docs(operators): document DropNullRows; add golden entries, changelog and design log (ITL-943) Co-Authored-By: Claude Fable 5.2 --- .zed/rules | 12 ++ CHANGELOG.md | 12 ++ CLAUDE.md | 14 +- DESIGN_ISSUES.md | 27 ++++ docs/concepts/operators.md | 90 ++++++++++++- tests/test_hashing/golden/cases.py | 87 +++++++++++++ tests/test_hashing/golden/data/pipeline.json | 128 +++++++++++++++++++ 7 files changed, 367 insertions(+), 3 deletions(-) diff --git a/.zed/rules b/.zed/rules index 0c226b8ac..855afd7a5 100644 --- a/.zed/rules +++ b/.zed/rules @@ -282,6 +282,7 @@ src/orcapod/ column_selection.py — Select/Drop Tag/Data columns mappers.py — MapTags, MapData (rename columns) filters.py — PolarsFilter + drop_null_rows.py — DropNullRows (drop rows with a null in named columns, T | None → T) hashing/ semantic_hashing/ — BaseSemanticHasher, type handlers semantic_types/ — Type conversion (Python ↔ Arrow) @@ -351,6 +352,11 @@ FunctionNode. Operators: inspect tags (never data content), can rename columns, cannot synthesize values. Function Pods: inspect data content (never tags), synthesize new values, cached by content. +Filters are the operators that read data columns (ITL-943): PolarsFilter evaluates its +predicate on values, and DropNullRows reads only whether a value is present (its validity), +never the value. A filter decides which rows exist and never synthesizes a value; no other +operator reads data content. + ### Two identity chains Every pipeline element has two parallel hashes: @@ -476,6 +482,12 @@ data. The rules: (circular — it delegates back to the pod) and NOT against as_table() (legitimately differs: output_schema ignores columns.source, so _source_*, _content_hash and _context_key appear in the table but never in the schema). +- DropNullRows keeps the rows where every named column (tag or data) is non-null and clears + the nullable flag of each named field, so T | None becomes T. Only top-level validity is + read (not list items, struct children or NaN). Its identity is the sorted tuple of unique + names. Both paths apply one rule, DropNullRows._narrow_table: an Arrow validity mask and + the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast + other than large normalization. A named column that is already non-nullable is a no-op. - GroupBy requires every column in `by` to be a scalar tag column; raises InputValidationError for unknown, data, or list-valued columns (list-valued usually means the stream came from Batch — group before batching, not after). Members sort by non-key tag values with system diff --git a/CHANGELOG.md b/CHANGELOG.md index dfdc26d3b..b12e7cd4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,18 @@ stay queued for the next flush. A flush that fails only for other reasons still ### Added +- **`DropNullRows` operator and `stream.drop_null_rows(columns)`** — keeps the rows where + every named column is non-null and narrows each named column from `T | None` to `T`, in + `output_schema()` and in the Arrow table, so a pod whose parameter is a plain `T` can + consume a column that is nullable by specification. The names may be tag columns, data + columns or both; a name that is not a column of the stream raises `InputValidationError`, + and a column that is already non-nullable is left as it is. Only the top-level value is + tested (a null list or struct; not list items, struct fields or NaN). Other columns, system + tags, source info and row order are unchanged. The operator's identity is the set of names. + A surviving row's data hash (or tag hash, for a tag column) changes when the named column + was nullable, because nullability is part of the row's Arrow schema; it becomes the hash + the row has in a stream that declared the column non-nullable. No existing hash changes. + (ITL-943) - **`orcapod.errors.StoreWriteError`** — raised when a node's results or records cannot be written to the store. It names the node, the row's tag and the table, and chains the database's own exception as its cause. (ITL-878 #287) diff --git a/CLAUDE.md b/CLAUDE.md index 1908ec144..37e5e0905 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -293,7 +293,8 @@ src/orcapod/ │ ├── group_by.py # GroupBy (many→one reduction keyed on tag values) │ ├── column_selection.py # Select/Drop Tag/Data columns │ ├── mappers.py # MapTags, MapData (rename columns) -│ └── filters.py # PolarsFilter +│ ├── filters.py # PolarsFilter +│ └── drop_null_rows.py # DropNullRows (drop rows with a null in named columns, T | None → T) ├── hashing/ │ └── semantic_hashing/ # BaseSemanticHasher, type handlers ├── semantic_types/ # Type conversion (Python ↔ Arrow) @@ -375,6 +376,11 @@ FunctionNode. tag tuple into one row with list-valued members. It still synthesizes no new data values — every emitted element came from an input row — and it keys only on tags, never on data. +Filters are the operators that read data columns (ITL-943): `PolarsFilter` evaluates its +predicate on values, and `DropNullRows` reads only whether a value is present (its validity), +never the value. A filter decides which rows exist and never synthesizes a value; no other +operator reads data content. + ### Two identity chains Every pipeline element has two parallel hashes: @@ -524,6 +530,12 @@ data. The rules: `as_table()` (which legitimately differs: `ArrowTableStream.output_schema` ignores `columns.source`, so `_source_*`, `_content_hash`, and `_context_key` appear in the table but never in the schema). +- `DropNullRows` keeps the rows where every named column (tag or data) is non-null and clears + the nullable flag of each named field, so `T | None` becomes `T`. Only top-level validity is + read (not list items, struct children or NaN). Its identity is the sorted tuple of unique + names. Both paths apply one rule, `DropNullRows._narrow_table`: an Arrow validity mask and + the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast + other than large normalization. A named column that is already non-nullable is a no-op. - `GroupBy` requires every column in `by` to be a scalar tag column; it raises `InputValidationError` for unknown, data, or list-valued columns (the last usually means the stream came from `Batch` — group before batching, not after). Members are sorted by non-key diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 695708a20..bbe59cfa7 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2442,3 +2442,30 @@ results. Known compromise: the struct's children are nullable at the Arrow level, because Polars drops nested non-null flags on its way back to Arrow (ITL-842). The logical type enforces non-null instead. Tighten under a new storage version once nested nullability survives Polars. + +--- + +## `src/orcapod/core/operators/drop_null_rows.py` — null rows and nullable columns (ITL-943) + +### DN1 — No way to turn a `T | None` column into `T` +**Status:** resolved +**Severity:** medium +**Issue:** ITL-943 + +Many value columns are nullable by specification (EDI, Spiral tables), and a pod written as +`def f(x: int)` is refused on an `int | None` column. `PolarsFilter` could drop the null rows, +but the column stayed `T | None`, so the pod was still refused; a nullable key column likewise +could not be joined with a non-nullable one. + +**Fix:** `DropNullRows(columns)` and `stream.drop_null_rows(columns)` keep the rows where every +named column (tag or data) is non-null and clear the nullable flag of each named field. +Decisions (coordinator, November 11, 2026): one operator for tag and data columns; only +top-level validity is read; a missing column is `InputValidationError`; an already non-nullable +column is a no-op; identity is the sorted tuple of unique names; sync and async paths share one +table rule (`_narrow_table`), pure Arrow. A surviving row's `Data` (or `Tag`) content hash +changes when a named column was nullable, to the hash of the same row declared non-nullable; +no existing golden value moved. + +Open: the class and method names (`DropNullRows` / `drop_null_rows` versus `DropNulls` / +`drop_nulls`) await the developer's decision. Possible follow-up: a variant that raises on a +null instead of dropping the row. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 6dc1f9487..67eafe3ab 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -28,6 +28,10 @@ a tag tuple into a single row with list-valued members. It stays on the operator boundary because it synthesizes no new data values -- every element of every emitted list came from an input row -- and it keys only on tags, never on data content. +The filters are the operators that read data columns: `PolarsFilter` evaluates its predicate +on values, and `DropNullRows` reads only whether a value is present (its validity), never the +value. Both decide which rows exist; neither synthesizes a value. + ## Operator categories ### `UnaryOperator` -- single input @@ -350,6 +354,88 @@ except ValueError as exc: # Cannot hash the Polars expression 'col("age").python_udf()' for an Orcapod identity ``` +### DropNullRows + +`DropNullRows(columns)` keeps the rows where every named column is non-null, and narrows +each named column from `T | None` to `T`. Use it when a column is nullable by specification +but a function needs a value: a pod whose parameter is a plain `int` refuses an `int | None` +column, and `PolarsFilter` can drop the null rows but leaves the type as it was. + +```python +import pyarrow as pa +from orcapod import function_pod +from orcapod.operators import DropNullRows +from orcapod.sources import ArrowTableSource + +recordings = ArrowTableSource( + pa.table( + { + "session": pa.array(["s1", "s2", "s3"], pa.large_string()), + "start_ms": pa.array([1000, None, 3000], pa.int64()), + }, + schema=pa.schema( + [ + pa.field("session", pa.large_string(), nullable=False), + pa.field("start_ms", pa.int64(), nullable=True), + ] + ), + ), + tag_columns=["session"], +) +print(recordings.output_schema()[1]) +# Schema({'start_ms': int | None}) + + +@function_pod(output_keys="start_s") +def to_seconds(start_ms: int) -> float: + return start_ms / 1000 + + +try: + to_seconds.pod(recordings) +except ValueError as exc: + print(str(exc).rsplit(": ", 1)[1]) +# column 'start_ms' has type int | None, the function expects int + +present = DropNullRows("start_ms")(recordings) +print(present.output_schema()[1]) +# Schema({'start_ms': int}) +print(to_seconds.pod(present).as_table().to_pylist()) +# [{'session': 's1', 'start_s': 1.0}, {'session': 's3', 'start_s': 3.0}] +``` + +The rules: + +- **Columns.** Pass one name or a collection of names; a row survives only if every named + column is non-null. The names may be tag columns, data columns, or both. A name that is + not a tag or data column of the stream raises `InputValidationError`, and an empty + collection raises `ValueError`. +- **What counts as null.** Only the top-level value: a null list or a null struct is null. An + empty list, a list that holds nulls, a struct whose fields are null and a floating-point + NaN are not. Nested fields keep their nullability. +- **Already non-nullable columns** are accepted and left as they are, so applying the + operator twice is the same as applying it once. +- **Everything else is unchanged**: other columns' types and nullability, logical types + (a `Path | None` column becomes `Path`), system tags, source info and row order. A result + with no rows has the same narrowed schema. +- **Identity** is the set of names: `DropNullRows("a")`, `DropNullRows(["a"])` and + `DropNullRows({"a"})` are one operator, and the order of names does not matter. +- **Row hashes.** Nullability is part of a row's Arrow schema, so a surviving row's data + hash changes when a named data column was nullable (its tag hash, when a named tag column + was). The new hash is the one the row has in a stream that declared the column + non-nullable from the start. + +Tag columns work the same way. A stream whose key column is `str | None` cannot be joined +with one whose key is `str` (`Join` never reconciles types), so narrow the key first. The +stream method is `drop_null_rows`: + +```python +by_method = recordings.drop_null_rows(["session", "start_ms"]) +by_operator = DropNullRows(["start_ms", "session"])(recordings) +print(by_method.pipeline_hash() == by_operator.pipeline_hash()) +# True +``` + ### Pick and Index `Pick(column, key)` extracts one entry from a struct- or dict-typed data column, and @@ -444,8 +530,8 @@ operators or function pods. Every stream also exposes the operators as methods, which reads well when chaining: `join`, `semi_join`, `batch`, `group_by`, `select_tag_columns`, `select_data_columns`, -`drop_tag_columns`, `drop_data_columns`, `map_tags`, `map_data`, `polars_filter`, `pick` and -`index`. +`drop_tag_columns`, `drop_data_columns`, `map_tags`, `map_data`, `polars_filter`, +`drop_null_rows`, `pick` and `index`. ```python import polars as pl diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index ada9870a7..5b2d2620d 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -941,3 +941,90 @@ def pipeline_polars_filter_expr_job() -> dict[str, Any]: _jsonable(v) for v in records.column("__golden_record_id").to_pylist() ) return out + + +def _nullable_readings() -> Any: + """A source with a nullable tag and two nullable data columns, each with a null.""" + from orcapod.sources import ArrowTableSource + + table = pa.table( + { + "k": pa.array(["r1", "r2", "r3", "r4"], pa.large_string()), + "site": pa.array(["a", "b", "c", None], pa.large_string()), + "x": pa.array([1, None, 3, 4], pa.int64()), + "y": pa.array([1.5, 2.5, None, 4.5], pa.float64()), + }, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("site", pa.large_string(), nullable=True), + pa.field("x", pa.int64(), nullable=True), + pa.field("y", pa.float64(), nullable=True), + ] + ), + ) + return ArrowTableSource( + table, tag_columns=["k", "site"], source_id="golden_nullable" + ) + + +@case("pipeline", "drop_null_rows") +def pipeline_drop_null_rows() -> dict[str, Any]: + """Pin ``DropNullRows``: the operator's own identity and the narrowed rows.""" + from orcapod.operators import DropNullRows + + out = _prefixed("source", stream_summary(_nullable_readings())) + out["operator[x].content_hash"] = DropNullRows("x").content_hash().to_string() + out["operator[x,y].content_hash"] = ( + DropNullRows(["y", "x"]).content_hash().to_string() + ) + out.update( + _prefixed("data", stream_summary(DropNullRows("x")(_nullable_readings()))) + ) + out.update( + _prefixed( + "data_many", stream_summary(DropNullRows(["x", "y"])(_nullable_readings())) + ) + ) + out.update( + _prefixed("tag", stream_summary(_nullable_readings().drop_null_rows("site"))) + ) + return out + + +@case("pipeline", "drop_null_rows_job") +def pipeline_drop_null_rows_job() -> dict[str, Any]: + """Record a job with ``DropNullRows`` upstream of a pod with non-optional parameters.""" + from orcapod import PipelineJob + from orcapod.databases import InMemoryArrowDatabase + + db = InMemoryArrowDatabase() + job = PipelineJob(name="golden_drop_null_job", store=db) + with job: + present = ( + _nullable_readings() + .drop_null_rows(["x", "y"], label="present") + .map_data({"x": "age", "y": "weight"}, label="renamed") + ) + golden_ratio.pod(present, label="ratio") + job.run() + + out: dict[str, Any] = {} + for label in sorted(job.nodes): + out.update(_prefixed(f"node[{label}]", stream_summary(job.nodes[label]))) + paths = sorted(db.list_sources()) + out["table_paths"] = ["/".join(p) for p in paths] + for path in paths: + name = "/".join(path) + records = db.get_all_records(path, record_id_column="__golden_record_id") + assert records is not None + if "__input_data_hash" in records.column_names and "_result" in path: + # Result-table record ids are UUIDv7 (volatile); pin the lookup key instead. + out[f"table[{name}].input_data_hashes"] = sorted( + _jsonable(v) for v in records.column("__input_data_hash").to_pylist() + ) + else: + out[f"table[{name}].record_ids"] = sorted( + _jsonable(v) for v in records.column("__golden_record_id").to_pylist() + ) + return out diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index d5b5257ce..3220c05f4 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -58,6 +58,134 @@ "_tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4", "_tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4" ], + "drop_null_rows.data.content_hash": "semantic_v0.1:bdc2fa3420424910e6d063e8e6a3068153c13c039fde0759ae487981a0237570", + "drop_null_rows.data.pipeline_hash": "semantic_v0.1:272051f71278ae99b6ac0e254ceea47c7e34f45c88c1842ed764b2ac57d6863b", + "drop_null_rows.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.data.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001eb8bb21cea56694f30415a726a190021869b1839977483a983afccc228e3fa7d", + "drop_null_rows.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_null_rows.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.data.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:000001eeb56ac3c1727cfe75fca3334244e3f9d6afbf624a67662511dbd6abff6ecec5", + "drop_null_rows.data.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows.data.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.data.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018b789706fed0bc06c83ab3049aacb9e7b228aef5d112d80ba59bc45b6981a5a9", + "drop_null_rows.data.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows.data_many.content_hash": "semantic_v0.1:0e68d2042ef50caa781bbe3cc0106fd5dd1793b35cbc3c6a880fb9b1c8185d10", + "drop_null_rows.data_many.pipeline_hash": "semantic_v0.1:cd2bfc56cd87c29a5ee5f706af29dc23e6ef6a846873be9a87ca6cd84b7d3f81", + "drop_null_rows.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_null_rows.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.data_many.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", + "drop_null_rows.data_many.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows.operator[x,y].content_hash": "semantic_v0.1:a63930534fb25a43c4aedfbf4c82f68c427e19573e0de84f8c54d731dbb6278a", + "drop_null_rows.operator[x].content_hash": "semantic_v0.1:e656ec6b37a716f1acbcce2f7166e1db2c756d910cafd00b91562221e2925c22", + "drop_null_rows.source.content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", + "drop_null_rows.source.pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "drop_null_rows.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.source.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_null_rows.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_null_rows.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.source.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_null_rows.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_null_rows.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.source.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_null_rows.source.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows.source.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.source.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", + "drop_null_rows.source.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows.tag.content_hash": "semantic_v0.1:598883c5d524994fb1931a5f3734a674004dc38483059c4a11d8b799969aefff", + "drop_null_rows.tag.pipeline_hash": "semantic_v0.1:0d006d549eb02dfce4fab969506fd229359deac589a2654e5e670ee22ffc94a0", + "drop_null_rows.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.tag.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_null_rows.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_null_rows.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.tag.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_null_rows.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_null_rows.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows.tag.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_null_rows.tag.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows_job.node[golden_nullable].content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", + "drop_null_rows_job.node[golden_nullable].pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", + "drop_null_rows_job.node[golden_nullable].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows_job.node[present].content_hash": "semantic_v0.1:0e68d2042ef50caa781bbe3cc0106fd5dd1793b35cbc3c6a880fb9b1c8185d10", + "drop_null_rows_job.node[present].pipeline_hash": "semantic_v0.1:cd2bfc56cd87c29a5ee5f706af29dc23e6ef6a846873be9a87ca6cd84b7d3f81", + "drop_null_rows_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_null_rows_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[present].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", + "drop_null_rows_job.node[present].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows_job.node[ratio].content_hash": "semantic_v0.1:ee34576f6923e09b72b7a46fc59575cd5323deaa73d27430a114644663b69976", + "drop_null_rows_job.node[ratio].pipeline_hash": "semantic_v0.1:c422b770e44ff13a0a3b76ebe04437d5365b66ada44f47de514f87ac6d4d8419", + "drop_null_rows_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:00000150467e204fcd64131e2c5242f04cb75405ad27b0effab95b7024cac05989ce7b", + "drop_null_rows_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000012f80ae46902b8c07a040d40c77b237701ab557207263bd934400b66bbfbd7daf", + "drop_null_rows_job.node[ratio].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows_job.node[renamed].content_hash": "semantic_v0.1:4122fb621cb9a122bd6a09e6525f7957b10974441fac2ae820222ab60e25b7fd", + "drop_null_rows_job.node[renamed].pipeline_hash": "semantic_v0.1:f3c1f000bc2042acbe3fbb43217c8910f5443f018ff74117bd9f33d058771725", + "drop_null_rows_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_null_rows_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3", + "drop_null_rows_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_null_rows_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_null_rows_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", + "drop_null_rows_job.node[renamed].system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_null_rows_job.table[golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1].input_data_hashes": [ + "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", + "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3" + ], + "drop_null_rows_job.table[golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:c422b770e44ff13a0a3b76ebe04437d5365b66ada44f47de514f87ac6d4d8419/pdb_v1].record_ids": [ + "arrow_v0.1:0000012447639a486f05322571bf7908f2e5b1a551e430dc4eec49e54a22657f0a49aa", + "arrow_v0.1:00000124f29b7254f3a496da7cd95db7728fc265f082ae8091cd094bac96a0631c3563" + ], + "drop_null_rows_job.table_paths": [ + "golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1", + "golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:c422b770e44ff13a0a3b76ebe04437d5365b66ada44f47de514f87ac6d4d8419/pdb_v1" + ], "group_by.content_hash": "semantic_v0.1:ee6478a83847e5e9d6bea916138170139aba8c5b43c6992a22d048a07b9b4526", "group_by.pipeline_hash": "semantic_v0.1:5ab1cb5d5ce9b641d19045bae0d94bbb9e15703bebfe37ec57fd13b141d2a0f0", "group_by.row[{\"cohort\":\"a\"}]._tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "7d212ee5c358519c9e4729f4a0898aad", From 2a2f8983f0be686e0256eb97fb6f74ffe10f651d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 17:40:40 -0800 Subject: [PATCH 04/11] test(operators): type-clean DropNullRows tests (ITL-943) Co-Authored-By: Claude Fable 5.2 --- .../test_core/operators/test_drop_null_rows.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_core/operators/test_drop_null_rows.py b/tests/test_core/operators/test_drop_null_rows.py index 956ffa695..162f8d9ca 100644 --- a/tests/test_core/operators/test_drop_null_rows.py +++ b/tests/test_core/operators/test_drop_null_rows.py @@ -96,8 +96,9 @@ def test_nan_is_not_null() -> None: nullable=["v"], ) result = DropNullRows("v")(source) - values = result.as_table().column("v").to_pylist() - assert values[0] == 1.0 and math.isnan(values[1]) and len(values) == 2 + first, second = result.as_table().column("v").to_pylist() + assert first == 1.0 + assert isinstance(second, float) and math.isnan(second) assert result.as_table().schema.field("v").nullable is False assert result.output_schema()[1]["v"] is float @@ -317,7 +318,10 @@ def logical() -> DictSource: def test_logical_columns_keep_their_logical_type() -> None: source = logical() before = source.as_table().schema - assert source.output_schema()[1] == {"p": Path | None, "arr": np.ndarray | None} + assert source.output_schema()[1] == { + "p": Path | None, + "arr": np.ndarray | None, # pyright: ignore[reportMissingTypeArgument] # the schema holds the bare class + } result = DropNullRows(["p", "arr"])(source) assert result.output_schema()[1] == {"p": Path, "arr": np.ndarray} @@ -329,8 +333,9 @@ def test_logical_columns_keep_their_logical_type() -> None: ((tag, data),) = list(result.iter_data()) assert tag["k"] == "a" assert data["p"] == Path("/a") - assert isinstance(data["arr"], np.ndarray) - assert data["arr"].tolist() == [0, 1, 2] + array = data["arr"] + assert isinstance(array, np.ndarray) + assert array.tolist() == [0, 1, 2] # ── arguments ─────────────────────────────────────────────────────────────── @@ -521,5 +526,5 @@ def test_join_after_the_operator() -> None: ) joined = Join()(DropNullRows("x")(readings()), weights) assert joined.output_schema()[1]["x"] is int - assert sorted(joined.as_table().column("id").to_pylist()) == [1, 3, 4] + assert sorted(_ids(joined)) == [1, 3, 4] assert joined.as_table().schema.field("x").nullable is False From 2bef2daca8841d35512977a64eb25a4614f6660a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 17:44:05 -0800 Subject: [PATCH 05/11] fix(operators): DropNullRows keeps null-typed columns and handles streams without tags (ITL-943) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/operators/drop_null_rows.py | 80 ++++++++++++++----- .../2026-11-11-itl-943-drop-null-rows.md | 22 +++++ .../operators/test_async_sync_equivalence.py | 31 +++++++ .../operators/test_drop_null_rows.py | 14 ++++ 4 files changed, 128 insertions(+), 19 deletions(-) diff --git a/src/orcapod/core/operators/drop_null_rows.py b/src/orcapod/core/operators/drop_null_rows.py index 6530f5465..b336ffd45 100644 --- a/src/orcapod/core/operators/drop_null_rows.py +++ b/src/orcapod/core/operators/drop_null_rows.py @@ -167,13 +167,59 @@ def unary_output_schema( # Execution # ------------------------------------------------------------------ + def _valid_rows(self, table: pa.Table) -> pa.ChunkedArray[Any] | None: + """Return the mask of rows whose named columns present in ``table`` are all non-null. + + Args: + table: A table holding some or all of the named columns. + + Returns: + A boolean column, one value per row, or ``None`` when ``table`` + holds none of the named columns (every row is kept). + """ + mask = None + for name in self.columns: + if name in table.column_names: + valid = pc.is_valid(table.column(name)) + mask = valid if mask is None else pc.and_(mask, valid) + return mask + + def _narrow_fields(self, table: pa.Table) -> pa.Table: + """Flag the named columns present in ``table`` non-nullable. + + Every column takes the large-normalized form of its input type; no + other type, flag or value changes. The named columns must hold no null. + + Args: + table: A table whose named columns hold no null. + + Returns: + The same rows with the named fields non-nullable. + """ + table = arrow_utils.restore_column_types( + table, + { + field.name: arrow_utils.normalize_to_large_types( + field.type, keep_null=True + ) + for field in table.schema + }, + ) + schema = table.schema + for name in self.columns: + position = schema.get_field_index(name) + if position >= 0: + schema = schema.set( # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + position, schema.field(position).with_nullable(False) + ) + # Same arrays under new field flags: no cast, so no value or type can change. + return pa.Table.from_arrays(table.columns, schema=schema) + def _narrow_table(self, table: pa.Table) -> pa.Table: """Drop the rows with a null in a named column and flag those columns non-nullable. Only the named columns present in ``table`` are considered, so the same rule serves a whole stream table and a single row's tag or data table. - Every column takes the large-normalized form of its input type; no - other type, flag or value changes. Args: table: A table holding some or all of the named columns. @@ -181,20 +227,10 @@ def _narrow_table(self, table: pa.Table) -> pa.Table: Returns: The surviving rows, in order, with the named fields non-nullable. """ - named = [name for name in self.columns if name in table.column_names] - mask = None - for name in named: - valid = pc.is_valid(table.column(name)) - mask = valid if mask is None else pc.and_(mask, valid) + mask = self._valid_rows(table) if mask is not None: table = table.filter(mask) - table = arrow_utils.normalize_table_to_large_types(table) - schema = table.schema - for name in named: - position = schema.get_field_index(name) - schema = schema.set(position, schema.field(position).with_nullable(False)) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized - # Same arrays under new field flags: no cast, so no value or type can change. - return pa.Table.from_arrays(table.columns, schema=schema) + return self._narrow_fields(table) def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: """Return the stream's rows whose named columns are all non-null. @@ -240,13 +276,19 @@ async def async_execute( data_table = self._narrow_table( data.as_table(columns={"source": True, "meta": True}) ) - if tag_table.num_rows == 0 or data_table.num_rows == 0: + # A table with no columns (a stream without tags) has no rows + # to lose, so only a table that held a named column can drop the row. + if (tag_table.num_columns and tag_table.num_rows == 0) or ( + data_table.num_columns and data_table.num_rows == 0 + ): continue + new_tag = ( + Tag(tag_table, data_context=tag.data_context_key) + if tag_table.num_columns + else tag + ) await output.send( - ( - Tag(tag_table, data_context=tag.data_context_key), - Data(data_table, data_context=data.data_context_key), - ) + (new_tag, Data(data_table, data_context=data.data_context_key)) ) finally: await output.close() diff --git a/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md b/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md index 632267931..035e373f7 100644 --- a/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md +++ b/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md @@ -149,3 +149,25 @@ fall back to the inherited barrier path and report it. ## Notes / progress log (Updated as work proceeds, for a successor.) + +- November 11: tasks 1-5 done and pushed. Operator in + `src/orcapod/core/operators/drop_null_rows.py`; tests in + `tests/test_core/operators/test_drop_null_rows.py`, + `tests/test_pipeline/test_drop_null_rows_job.py`, two `Case`s plus a tagless test in + `test_async_sync_equivalence.py`, `TestDropNullRowsConfig` in `test_operator_config.py`. +- Golden: cases `pipeline/drop_null_rows` and `pipeline/drop_null_rows_job` appended to + `cases.py`; `pipeline.json` gained 128 lines, none removed or changed. +- The streaming async path is exactly equivalent row by row (including Data meta columns, so + no xfail was needed); no fallback to the barrier path. +- Large normalization uses `normalize_to_large_types(..., keep_null=True)` per field with + `restore_column_types` (the table-level helper maps `null` to `large_string`). +- Open: the NAME (developer's call). To rename: class and module `drop_null_rows.py`, stream + method in `core/streams/base.py` and `protocols/core_protocols/streams.py`, registry key in + `pipeline/serialization.py`, export in `core/operators/__init__.py`, docs, CHANGELOG, + DESIGN_ISSUES, CLAUDE.md, `.zed/rules`, tests and golden case names. A rename changes the + operator's identity hash (the class name is in `identity_structure`), so the two golden + cases must be regenerated with it (new entries only; nothing released yet). +- Observation, not fixed (out of scope): a `PipelineJob` saved with an in-memory + `ArrowTableSource` and loaded again gives operator nodes whose pipeline hash differs from + the saved job's (same for `MapData`), because the source is not reconstructable. + diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 68644ff1d..e7874c3c5 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -333,6 +333,37 @@ def test_async_rows_equal_sync_rows(case: Case) -> None: ) +def test_drop_null_rows_without_tag_columns() -> None: + """A stream without tag columns: the async path drops the same rows, in order. + + Outside the matrix, whose pairing needs unique tag rows. + """ + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", "c"], pa.string()), + }, + tags=(), + source_id="tagless", + nullable=("x", "s"), + ) + expected = sync_rows(DropNullRows("x"), source) + actual = async_rows(DropNullRows("x"), source) + assert [data.as_dict() for _, data in actual] == [ + {"x": 1, "s": "a"}, + {"x": 3, "s": "c"}, + ] + assert len(actual) == len(expected) + for (tag, data), (sync_tag, sync_data) in zip(actual, expected): + assert data.arrow_schema() == sync_data.arrow_schema() + assert data.content_hash() == sync_data.content_hash() + assert data.source_info() == sync_data.source_info() + assert tag.arrow_schema(columns={"system_tags": True}) == sync_tag.arrow_schema( + columns={"system_tags": True} + ) + assert list(tag.system_tags().items()) == list(sync_tag.system_tags().items()) + + # Pre-existing provenance defects, not type re-inference: out of scope for # ITL-837 and tracked elsewhere. Strict, so a fix flags them. _DROPPED_SOURCE_INFO = ( diff --git a/tests/test_core/operators/test_drop_null_rows.py b/tests/test_core/operators/test_drop_null_rows.py index 162f8d9ca..cdea01bc9 100644 --- a/tests/test_core/operators/test_drop_null_rows.py +++ b/tests/test_core/operators/test_drop_null_rows.py @@ -262,6 +262,20 @@ def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: ] +def test_stream_without_tag_columns() -> None: + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", "c"], pa.string()), + }, + tags=(), + nullable=["x", "s"], + ) + result = DropNullRows("x")(source) + assert result.as_table().to_pylist() == [{"x": 1, "s": "a"}, {"x": 3, "s": "c"}] + assert result.output_schema() == ({}, {"x": int, "s": str | None}) + + # ── nested and logical columns ────────────────────────────────────────────── ITEMS = pa.large_list(pa.field("item", pa.int8(), nullable=True)) From 4c1df1baa9a0ff31afb37fac9775f8aeb3ed549d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 17:44:48 -0800 Subject: [PATCH 06/11] refactor(operators): inline DropNullRows validity mask (ITL-943) Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/operators/drop_null_rows.py | 23 +++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/orcapod/core/operators/drop_null_rows.py b/src/orcapod/core/operators/drop_null_rows.py index b336ffd45..b95621378 100644 --- a/src/orcapod/core/operators/drop_null_rows.py +++ b/src/orcapod/core/operators/drop_null_rows.py @@ -167,23 +167,6 @@ def unary_output_schema( # Execution # ------------------------------------------------------------------ - def _valid_rows(self, table: pa.Table) -> pa.ChunkedArray[Any] | None: - """Return the mask of rows whose named columns present in ``table`` are all non-null. - - Args: - table: A table holding some or all of the named columns. - - Returns: - A boolean column, one value per row, or ``None`` when ``table`` - holds none of the named columns (every row is kept). - """ - mask = None - for name in self.columns: - if name in table.column_names: - valid = pc.is_valid(table.column(name)) - mask = valid if mask is None else pc.and_(mask, valid) - return mask - def _narrow_fields(self, table: pa.Table) -> pa.Table: """Flag the named columns present in ``table`` non-nullable. @@ -227,7 +210,11 @@ def _narrow_table(self, table: pa.Table) -> pa.Table: Returns: The surviving rows, in order, with the named fields non-nullable. """ - mask = self._valid_rows(table) + mask = None + for name in self.columns: + if name in table.column_names: + valid = pc.is_valid(table.column(name)) + mask = valid if mask is None else pc.and_(mask, valid) if mask is not None: table = table.filter(mask) return self._narrow_fields(table) From 4d43c9b1c52df5141fc80baa68981f1b3998a4ea Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:07:24 -0800 Subject: [PATCH 07/11] docs(operators): address CodeRabbit review of DropNullRows (ITL-943) Qualify the operator boundary table for filters, document identity_structure, verify the schema prediction against static_process, add test docstrings. Co-Authored-By: Claude Fable 5.2 --- .zed/rules | 3 +- CLAUDE.md | 2 +- docs/concepts/operators.md | 2 +- src/orcapod/core/operators/drop_null_rows.py | 6 ++++ .../operators/test_drop_null_rows.py | 34 ++++++++++++++++++- .../operators/test_operator_config.py | 3 ++ .../test_pipeline/test_drop_null_rows_job.py | 9 +++++ 7 files changed, 55 insertions(+), 4 deletions(-) diff --git a/.zed/rules b/.zed/rules index 855afd7a5..712f22203 100644 --- a/.zed/rules +++ b/.zed/rules @@ -349,7 +349,8 @@ FunctionNode. ### Strict operator / function pod boundary -Operators: inspect tags (never data content), can rename columns, cannot synthesize values. +Operators: inspect tags (never data content, except filters; see below), can rename columns, +cannot synthesize values. Function Pods: inspect data content (never tags), synthesize new values, cached by content. Filters are the operators that read data columns (ITL-943): PolarsFilter evaluates its diff --git a/CLAUDE.md b/CLAUDE.md index 37e5e0905..ba7a259c4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -366,7 +366,7 @@ FunctionNode. | | Operator | Function Pod | |---|---|---| -| Inspects data content | Never | Yes | +| Inspects data content | Never, except filters (see below) | Yes | | Inspects / uses tags | Yes | No | | Can rename columns | Yes | No | | Synthesizes new values | No | Yes | diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 67eafe3ab..e8b79ee12 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -13,7 +13,7 @@ This separation is a core Orcapod design principle: | | Operator | Function Pod | |---|---|---| -| Inspects data content | Never | Yes | +| Inspects data content | Never, except filters (see below) | Yes | | Inspects / uses tags | Yes | No | | Can rename columns | Yes | No | | Synthesizes new values | No | Yes | diff --git a/src/orcapod/core/operators/drop_null_rows.py b/src/orcapod/core/operators/drop_null_rows.py index b95621378..817d5d856 100644 --- a/src/orcapod/core/operators/drop_null_rows.py +++ b/src/orcapod/core/operators/drop_null_rows.py @@ -90,6 +90,12 @@ def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: super().__init__(**kwargs) def identity_structure(self) -> Any: + """Return the operator's identity: its class name and the set of column names. + + Returns: + A ``(class_name, columns)`` tuple, where ``columns`` is the sorted + tuple of unique names. + """ return (self.__class__.__name__, self.columns) def to_config(self) -> dict[str, Any]: diff --git a/tests/test_core/operators/test_drop_null_rows.py b/tests/test_core/operators/test_drop_null_rows.py index cdea01bc9..56df1f2ce 100644 --- a/tests/test_core/operators/test_drop_null_rows.py +++ b/tests/test_core/operators/test_drop_null_rows.py @@ -57,6 +57,7 @@ def readings(nullable: Sequence[str] = ("x", "y")) -> ArrowTableSource: def _ids(stream: StreamProtocol) -> list[Any]: + """Return the ``id`` column of ``stream`` as a list, in row order.""" return stream.as_table().column("id").to_pylist() @@ -64,6 +65,7 @@ def _ids(stream: StreamProtocol) -> list[Any]: def test_drops_rows_where_the_column_is_null() -> None: + """Rows whose named column is null are dropped; the others pass through.""" result = DropNullRows("x")(readings()) assert result.as_table().to_pylist() == [ {"id": 1, "x": 10, "y": 1.5, "note": "a"}, @@ -73,10 +75,12 @@ def test_drops_rows_where_the_column_is_null() -> None: def test_every_named_column_must_be_non_null() -> None: + """With several names, a row survives only if all of them are non-null.""" assert _ids(DropNullRows(["x", "y"])(readings())) == [1, 4] def test_row_order_is_preserved() -> None: + """Surviving rows keep the input order; nothing is sorted.""" source = _source( { "id": pa.array([4, 1, 3, 2], pa.int16()), @@ -88,6 +92,7 @@ def test_row_order_is_preserved() -> None: def test_nan_is_not_null() -> None: + """A NaN is a value: its row survives and the column still becomes non-nullable.""" source = _source( { "id": pa.array([1, 2, 3], pa.int16()), @@ -107,6 +112,7 @@ def test_nan_is_not_null() -> None: def test_output_schema_narrows_only_the_named_columns() -> None: + """``output_schema()`` reports ``T`` for the named column and leaves the rest.""" source = readings() assert source.output_schema()[1] == { "x": int | None, @@ -119,6 +125,7 @@ def test_output_schema_narrows_only_the_named_columns() -> None: def test_table_narrows_only_the_named_columns() -> None: + """Only the named field loses its nullable flag; widths and other flags are kept.""" source = readings() before = source.as_table(columns=ALL).schema after = DropNullRows("x")(source).as_table(columns=ALL).schema @@ -133,16 +140,18 @@ def test_table_narrows_only_the_named_columns() -> None: def test_schema_prediction_matches_the_computed_stream() -> None: + """The predicted schema equals the schema of the computed stream.""" source = readings() op = DropNullRows(["x", "id"]) for config in ({}, {"system_tags": True}, {"source": True, "system_tags": True}): # Non-public on purpose: a prediction is verified against the static result. - computed = op.unary_static_process(source).output_schema(columns=config) + computed = op.static_process(source).output_schema(columns=config) assert op(source).output_schema(columns=config) == computed assert op.output_schema(source, columns=config) == computed def test_zero_surviving_rows_keep_the_narrowed_types() -> None: + """A result with no rows has the schema of a non-empty result.""" source = _source( { "id": pa.array([1, 2], pa.int16()), @@ -170,6 +179,7 @@ def test_zero_surviving_rows_keep_the_narrowed_types() -> None: def test_already_non_nullable_column_is_a_no_op() -> None: + """Naming a non-nullable column changes no schema, row or hash.""" source = _source( { "id": pa.array([1, 2], pa.int16()), @@ -185,6 +195,7 @@ def test_already_non_nullable_column_is_a_no_op() -> None: def test_applying_twice_equals_applying_once() -> None: + """The operator is idempotent.""" once = DropNullRows("x")(readings()) twice = DropNullRows("x")(once) assert twice.as_table(columns=ALL) == once.as_table(columns=ALL) @@ -208,6 +219,7 @@ def keyed(nullable: Sequence[str] = ("site", "x")) -> ArrowTableSource: def test_tag_column_is_narrowed() -> None: + """A named tag column is tested and narrowed like a data column.""" source = keyed() assert source.output_schema()[0] == {"site": str | None, "id": int} result = DropNullRows("site")(source) @@ -222,6 +234,7 @@ def test_tag_column_is_narrowed() -> None: def test_tag_and_data_columns_in_one_call() -> None: + """Tag and data columns can be named together.""" result = DropNullRows(["x", "site"])(keyed()) assert _ids(result) == [1] assert result.output_schema() == ({"site": str, "id": int}, {"x": int}) @@ -263,6 +276,7 @@ def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: def test_stream_without_tag_columns() -> None: + """A stream with no tag columns is filtered and narrowed the same way.""" source = _source( { "x": pa.array([1, None, 3], pa.int8()), @@ -283,6 +297,7 @@ def test_stream_without_tag_columns() -> None: def nested() -> ArrowTableSource: + """A source with a nullable list column and a nullable struct column.""" return _source( { "id": pa.array([1, 2, 3, 4], pa.int16()), @@ -297,6 +312,7 @@ def nested() -> ArrowTableSource: def test_only_the_top_level_value_is_tested_for_a_list() -> None: + """A null list is null; an empty list and a list holding nulls are not.""" result = DropNullRows("items")(nested()) # A list holding a null and an empty list are not null lists. assert result.as_table().column("items").to_pylist() == [[1, None], [], [4]] @@ -309,6 +325,7 @@ def test_only_the_top_level_value_is_tested_for_a_list() -> None: def test_only_the_top_level_value_is_tested_for_a_struct() -> None: + """A null struct is null; a struct whose fields are null is not.""" result = DropNullRows("point")(nested()) # A struct whose children are all null is not a null struct. assert _ids(result) == [1, 2, 4] @@ -318,6 +335,7 @@ def test_only_the_top_level_value_is_tested_for_a_struct() -> None: def logical() -> DictSource: + """A source with nullable ``Path`` and ``numpy.ndarray`` columns.""" return DictSource( data=[ {"k": "a", "p": Path("/a"), "arr": np.arange(3)}, @@ -330,6 +348,7 @@ def logical() -> DictSource: def test_logical_columns_keep_their_logical_type() -> None: + """A logical column keeps its extension type and becomes non-nullable.""" source = logical() before = source.as_table().schema assert source.output_schema()[1] == { @@ -356,11 +375,13 @@ def test_logical_columns_keep_their_logical_type() -> None: def test_missing_column_is_an_input_validation_error() -> None: + """A name that is not a column of the stream is refused at call time.""" with pytest.raises(InputValidationError, match="nope"): DropNullRows(["x", "nope"])(readings()) def test_system_and_source_columns_cannot_be_named() -> None: + """System-tag and source-info columns are not tag or data columns.""" source = readings() hidden = [ n @@ -374,6 +395,7 @@ def test_system_and_source_columns_cannot_be_named() -> None: def test_no_column_is_an_error_at_construction() -> None: + """An empty collection of names is refused when the operator is built.""" with pytest.raises(ValueError, match="at least one"): DropNullRows([]) @@ -387,6 +409,7 @@ def test_no_column_is_an_error_at_construction() -> None: ids=["list", "tuple", "set", "reversed", "duplicate", "frozenset"], ) def test_identity_is_the_set_of_names(columns: Any) -> None: + """Container type, order and duplicates do not change the operator's hashes.""" reference = DropNullRows(["x", "y"]) op = DropNullRows(columns) assert op.columns == ("x", "y") @@ -398,11 +421,13 @@ def test_identity_is_the_set_of_names(columns: Any) -> None: def test_a_bare_name_is_one_column() -> None: + """A string is one column name, never a collection of characters.""" assert DropNullRows("note").columns == ("note",) assert DropNullRows("x").content_hash() == DropNullRows(["x"]).content_hash() def test_different_names_are_different_identities() -> None: + """Operators over different names hash differently, as do their outputs.""" source = readings() assert DropNullRows("x").content_hash() != DropNullRows("y").content_hash() assert DropNullRows("x").content_hash() != DropNullRows(["x", "y"]).content_hash() @@ -414,6 +439,7 @@ def test_different_names_are_different_identities() -> None: def test_the_caller_s_collection_is_not_kept_or_mutated() -> None: + """The operator copies the names; later changes to the list do not reach it.""" names = ["y", "x"] op = DropNullRows(names) names.append("z") @@ -451,6 +477,7 @@ def test_narrowed_rows_hash_as_rows_declared_non_nullable() -> None: def test_narrowing_a_tag_changes_the_tag_hash_only() -> None: + """Narrowing a tag column changes the row's tag hash and not its data hash.""" source = keyed() input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} for tag, data in DropNullRows("site")(source).iter_data(): @@ -460,6 +487,7 @@ def test_narrowing_a_tag_changes_the_tag_hash_only() -> None: def test_system_tags_and_source_info_pass_through() -> None: + """System-tag and source-info columns keep their names, types and values.""" source = readings() before = source.as_table(columns=ALL) after = DropNullRows(["x", "y"])(source).as_table(columns=ALL) @@ -482,6 +510,7 @@ def test_system_tags_and_source_info_pass_through() -> None: def test_the_input_stream_is_not_changed() -> None: + """Running the operator leaves the input stream's schema and rows as they were.""" source = readings() schema = source.as_table(columns=ALL).schema rows = source.as_table(columns=ALL).to_pylist() @@ -494,6 +523,7 @@ def test_the_input_stream_is_not_changed() -> None: def test_stream_method_equals_the_operator() -> None: + """``stream.drop_null_rows`` gives the operator's rows and hashes.""" source = readings() by_method = source.drop_null_rows(["x", "y"]) by_operator = DropNullRows(["x", "y"])(source) @@ -513,6 +543,7 @@ def double_x(x: int) -> int: def test_pod_with_a_non_optional_parameter_runs_only_after_the_operator() -> None: + """A pod taking a plain ``int`` refuses ``int | None`` and runs on the narrowed stream.""" source = _source( { "id": pa.array([1, 2, 3], pa.int16()), @@ -531,6 +562,7 @@ def test_pod_with_a_non_optional_parameter_runs_only_after_the_operator() -> Non def test_join_after_the_operator() -> None: + """A narrowed stream joins, and the joined column stays non-nullable.""" weights = _source( { "id": pa.array([1, 2, 3, 4], pa.int16()), diff --git a/tests/test_core/operators/test_operator_config.py b/tests/test_core/operators/test_operator_config.py index c676c786d..a7836b05f 100644 --- a/tests/test_core/operators/test_operator_config.py +++ b/tests/test_core/operators/test_operator_config.py @@ -166,11 +166,13 @@ def test_non_serializable_predicates_marked(self): class TestDropNullRowsConfig: def test_to_config(self): + """The config holds the sorted unique names.""" config = DropNullRows(["y", "x", "y"]).to_config() assert config["class_name"] == "DropNullRows" assert config["config"] == {"columns": ["x", "y"]} def test_round_trip(self): + """``from_config(to_config())`` gives an operator with the same hashes.""" op = DropNullRows(["y", "x"]) restored = DropNullRows.from_config(op.to_config()) assert isinstance(restored, DropNullRows) @@ -179,6 +181,7 @@ def test_round_trip(self): assert restored.pipeline_hash() == op.pipeline_hash() def test_registered_for_pipeline_loading(self): + """Pipeline loading resolves the operator by its class name.""" from orcapod.pipeline.serialization import resolve_operator_from_config op = DropNullRows(["y", "x"]) diff --git a/tests/test_pipeline/test_drop_null_rows_job.py b/tests/test_pipeline/test_drop_null_rows_job.py index cf4164c81..ee0a18d59 100644 --- a/tests/test_pipeline/test_drop_null_rows_job.py +++ b/tests/test_pipeline/test_drop_null_rows_job.py @@ -31,16 +31,19 @@ def to_seconds(recording_start_timestamp: int) -> float: @pytest.fixture(autouse=True) def _reset_calls() -> Iterator[None]: + """Clear the shared invocation log before every test.""" _CALLS.clear() yield @pytest.fixture def store(tmp_path: Path) -> DeltaTableDatabase: + """A Delta Lake store in the test's temporary directory.""" return DeltaTableDatabase(base_path=tmp_path / "store") def _recordings(timestamps: list[int | None]) -> ArrowTableSource: + """A source whose ``recording_start_timestamp`` column is nullable.""" table = pa.table( { "session": pa.array( @@ -59,6 +62,7 @@ def _recordings(timestamps: list[int | None]) -> ArrowTableSource: def _run(store: DeltaTableDatabase, timestamps: list[int | None]) -> PipelineJob: + """Record and run ``source -> drop_null_rows -> to_seconds`` as a ``PipelineJob``.""" pod = FunctionPod(PythonDataFunction(to_seconds, output_keys="seconds")) # pyright: ignore[reportArgumentType] # PythonDataFunction is not yet typed as matching DataFunctionProtocol (baselined in other tests); it works at runtime job = PipelineJob(name="recordings", store=store) with job: @@ -70,6 +74,7 @@ def _run(store: DeltaTableDatabase, timestamps: list[int | None]) -> PipelineJob def _seconds(job: PipelineJob) -> dict[str, float]: + """Return the pod's output as ``{session: seconds}``.""" table = job.nodes["seconds"].as_table() return dict( zip(table.column("session").to_pylist(), table.column("seconds").to_pylist()) @@ -77,6 +82,7 @@ def _seconds(job: PipelineJob) -> dict[str, float]: def test_job_drops_null_rows_and_runs_the_pod(store: DeltaTableDatabase) -> None: + """The pod runs once per non-null row and the operator node reports ``int``.""" job = _run(store, [1000, None, 3000]) assert sorted(_CALLS) == [1000, 3000] assert _seconds(job) == {"s0": 1.0, "s2": 3.0} @@ -87,6 +93,7 @@ def test_job_drops_null_rows_and_runs_the_pod(store: DeltaTableDatabase) -> None def test_identical_rerun_hits_the_cache(store: DeltaTableDatabase) -> None: + """A second identical run on the same store makes no pod call.""" _run(store, [1000, None, 3000]) assert sorted(_CALLS) == [1000, 3000] @@ -107,6 +114,7 @@ def test_fresh_store_recomputes(tmp_path: Path) -> None: def test_a_newly_non_null_row_is_the_only_one_computed( store: DeltaTableDatabase, ) -> None: + """Rows computed before stay cached when a null row gains a value.""" _run(store, [1000, None, 3000]) _CALLS.clear() job = _run(store, [1000, 2000, 3000]) @@ -117,6 +125,7 @@ def test_a_newly_non_null_row_is_the_only_one_computed( def test_job_save_and_load_round_trips_the_operator( store: DeltaTableDatabase, tmp_path: Path ) -> None: + """A saved job records the operator's config and loads it back.""" import json job = _run(store, [1000, None, 3000]) From 7479bc53eb534d64ea218b2e36dd787d89b1e362 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 19:59:14 -0800 Subject: [PATCH 08/11] refactor(operators)!: rename DropNullRows to DropNulls and drop_null_rows to drop_nulls (ITL-943) The developer chose the name on November 11, 2026. Renames the class, module, stream method, registry key, export, tests, docs and golden case names; no alias. The class name is part of the operator identity, so the two golden cases added by this PR are regenerated (additions only against main). Co-Authored-By: Claude Fable 5.2 --- .zed/rules | 8 +- CHANGELOG.md | 2 +- CLAUDE.md | 8 +- DESIGN_ISSUES.md | 6 +- docs/concepts/operators.md | 22 +- src/orcapod/core/operators/__init__.py | 4 +- .../{drop_null_rows.py => drop_nulls.py} | 8 +- src/orcapod/core/streams/base.py | 8 +- src/orcapod/pipeline/serialization.py | 4 +- .../protocols/core_protocols/streams.py | 2 +- ...ws.md => 2026-11-11-itl-943-drop-nulls.md} | 62 +++--- .../operators/test_async_sync_equivalence.py | 14 +- ...t_drop_null_rows.py => test_drop_nulls.py} | 91 ++++----- .../operators/test_operator_config.py | 18 +- tests/test_hashing/golden/cases.py | 30 ++- tests/test_hashing/golden/data/pipeline.json | 192 +++++++++--------- ...ull_rows_job.py => test_drop_nulls_job.py} | 10 +- 17 files changed, 241 insertions(+), 248 deletions(-) rename src/orcapod/core/operators/{drop_null_rows.py => drop_nulls.py} (97%) rename superpowers/plans/{2026-11-11-itl-943-drop-null-rows.md => 2026-11-11-itl-943-drop-nulls.md} (79%) rename tests/test_core/operators/{test_drop_null_rows.py => test_drop_nulls.py} (89%) rename tests/test_pipeline/{test_drop_null_rows_job.py => test_drop_nulls_job.py} (93%) diff --git a/.zed/rules b/.zed/rules index 712f22203..52b80fd55 100644 --- a/.zed/rules +++ b/.zed/rules @@ -282,7 +282,7 @@ src/orcapod/ column_selection.py — Select/Drop Tag/Data columns mappers.py — MapTags, MapData (rename columns) filters.py — PolarsFilter - drop_null_rows.py — DropNullRows (drop rows with a null in named columns, T | None → T) + drop_nulls.py — DropNulls (drop rows with a null in named columns, T | None → T) hashing/ semantic_hashing/ — BaseSemanticHasher, type handlers semantic_types/ — Type conversion (Python ↔ Arrow) @@ -354,7 +354,7 @@ cannot synthesize values. Function Pods: inspect data content (never tags), synthesize new values, cached by content. Filters are the operators that read data columns (ITL-943): PolarsFilter evaluates its -predicate on values, and DropNullRows reads only whether a value is present (its validity), +predicate on values, and DropNulls reads only whether a value is present (its validity), never the value. A filter decides which rows exist and never synthesizes a value; no other operator reads data content. @@ -483,10 +483,10 @@ data. The rules: (circular — it delegates back to the pod) and NOT against as_table() (legitimately differs: output_schema ignores columns.source, so _source_*, _content_hash and _context_key appear in the table but never in the schema). -- DropNullRows keeps the rows where every named column (tag or data) is non-null and clears +- DropNulls keeps the rows where every named column (tag or data) is non-null and clears the nullable flag of each named field, so T | None becomes T. Only top-level validity is read (not list items, struct children or NaN). Its identity is the sorted tuple of unique - names. Both paths apply one rule, DropNullRows._narrow_table: an Arrow validity mask and + names. Both paths apply one rule, DropNulls._narrow_table: an Arrow validity mask and the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast other than large normalization. A named column that is already non-nullable is a no-op. - GroupBy requires every column in `by` to be a scalar tag column; raises InputValidationError diff --git a/CHANGELOG.md b/CHANGELOG.md index b12e7cd4d..31fc40c64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,7 +72,7 @@ stay queued for the next flush. A flush that fails only for other reasons still ### Added -- **`DropNullRows` operator and `stream.drop_null_rows(columns)`** — keeps the rows where +- **`DropNulls` operator and `stream.drop_nulls(columns)`** — keeps the rows where every named column is non-null and narrows each named column from `T | None` to `T`, in `output_schema()` and in the Arrow table, so a pod whose parameter is a plain `T` can consume a column that is nullable by specification. The names may be tag columns, data diff --git a/CLAUDE.md b/CLAUDE.md index ba7a259c4..2d9d14578 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -294,7 +294,7 @@ src/orcapod/ │ ├── column_selection.py # Select/Drop Tag/Data columns │ ├── mappers.py # MapTags, MapData (rename columns) │ ├── filters.py # PolarsFilter -│ └── drop_null_rows.py # DropNullRows (drop rows with a null in named columns, T | None → T) +│ └── drop_nulls.py # DropNulls (drop rows with a null in named columns, T | None → T) ├── hashing/ │ └── semantic_hashing/ # BaseSemanticHasher, type handlers ├── semantic_types/ # Type conversion (Python ↔ Arrow) @@ -377,7 +377,7 @@ tag tuple into one row with list-valued members. It still synthesizes no new dat every emitted element came from an input row — and it keys only on tags, never on data. Filters are the operators that read data columns (ITL-943): `PolarsFilter` evaluates its -predicate on values, and `DropNullRows` reads only whether a value is present (its validity), +predicate on values, and `DropNulls` reads only whether a value is present (its validity), never the value. A filter decides which rows exist and never synthesizes a value; no other operator reads data content. @@ -530,10 +530,10 @@ data. The rules: `as_table()` (which legitimately differs: `ArrowTableStream.output_schema` ignores `columns.source`, so `_source_*`, `_content_hash`, and `_context_key` appear in the table but never in the schema). -- `DropNullRows` keeps the rows where every named column (tag or data) is non-null and clears +- `DropNulls` keeps the rows where every named column (tag or data) is non-null and clears the nullable flag of each named field, so `T | None` becomes `T`. Only top-level validity is read (not list items, struct children or NaN). Its identity is the sorted tuple of unique - names. Both paths apply one rule, `DropNullRows._narrow_table`: an Arrow validity mask and + names. Both paths apply one rule, `DropNulls._narrow_table`: an Arrow validity mask and the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast other than large normalization. A named column that is already non-nullable is a no-op. - `GroupBy` requires every column in `by` to be a scalar tag column; it raises diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index bbe59cfa7..7fa76ef74 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2445,7 +2445,7 @@ instead. Tighten under a new storage version once nested nullability survives Po --- -## `src/orcapod/core/operators/drop_null_rows.py` — null rows and nullable columns (ITL-943) +## `src/orcapod/core/operators/drop_nulls.py` — null rows and nullable columns (ITL-943) ### DN1 — No way to turn a `T | None` column into `T` **Status:** resolved @@ -2457,7 +2457,7 @@ Many value columns are nullable by specification (EDI, Spiral tables), and a pod but the column stayed `T | None`, so the pod was still refused; a nullable key column likewise could not be joined with a non-nullable one. -**Fix:** `DropNullRows(columns)` and `stream.drop_null_rows(columns)` keep the rows where every +**Fix:** `DropNulls(columns)` and `stream.drop_nulls(columns)` keep the rows where every named column (tag or data) is non-null and clear the nullable flag of each named field. Decisions (coordinator, November 11, 2026): one operator for tag and data columns; only top-level validity is read; a missing column is `InputValidationError`; an already non-nullable @@ -2466,6 +2466,6 @@ table rule (`_narrow_table`), pure Arrow. A surviving row's `Data` (or `Tag`) co changes when a named column was nullable, to the hash of the same row declared non-nullable; no existing golden value moved. -Open: the class and method names (`DropNullRows` / `drop_null_rows` versus `DropNulls` / +Open: the class and method names (`DropNulls` / `drop_nulls` versus `DropNulls` / `drop_nulls`) await the developer's decision. Possible follow-up: a variant that raises on a null instead of dropping the row. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index e8b79ee12..378f7ed78 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -29,7 +29,7 @@ boundary because it synthesizes no new data values -- every element of every emi from an input row -- and it keys only on tags, never on data content. The filters are the operators that read data columns: `PolarsFilter` evaluates its predicate -on values, and `DropNullRows` reads only whether a value is present (its validity), never the +on values, and `DropNulls` reads only whether a value is present (its validity), never the value. Both decide which rows exist; neither synthesizes a value. ## Operator categories @@ -354,9 +354,9 @@ except ValueError as exc: # Cannot hash the Polars expression 'col("age").python_udf()' for an Orcapod identity ``` -### DropNullRows +### DropNulls -`DropNullRows(columns)` keeps the rows where every named column is non-null, and narrows +`DropNulls(columns)` keeps the rows where every named column is non-null, and narrows each named column from `T | None` to `T`. Use it when a column is nullable by specification but a function needs a value: a pod whose parameter is a plain `int` refuses an `int | None` column, and `PolarsFilter` can drop the null rows but leaves the type as it was. @@ -364,7 +364,7 @@ column, and `PolarsFilter` can drop the null rows but leaves the type as it was. ```python import pyarrow as pa from orcapod import function_pod -from orcapod.operators import DropNullRows +from orcapod.operators import DropNulls from orcapod.sources import ArrowTableSource recordings = ArrowTableSource( @@ -397,7 +397,7 @@ except ValueError as exc: print(str(exc).rsplit(": ", 1)[1]) # column 'start_ms' has type int | None, the function expects int -present = DropNullRows("start_ms")(recordings) +present = DropNulls("start_ms")(recordings) print(present.output_schema()[1]) # Schema({'start_ms': int}) print(to_seconds.pod(present).as_table().to_pylist()) @@ -418,8 +418,8 @@ The rules: - **Everything else is unchanged**: other columns' types and nullability, logical types (a `Path | None` column becomes `Path`), system tags, source info and row order. A result with no rows has the same narrowed schema. -- **Identity** is the set of names: `DropNullRows("a")`, `DropNullRows(["a"])` and - `DropNullRows({"a"})` are one operator, and the order of names does not matter. +- **Identity** is the set of names: `DropNulls("a")`, `DropNulls(["a"])` and + `DropNulls({"a"})` are one operator, and the order of names does not matter. - **Row hashes.** Nullability is part of a row's Arrow schema, so a surviving row's data hash changes when a named data column was nullable (its tag hash, when a named tag column was). The new hash is the one the row has in a stream that declared the column @@ -427,11 +427,11 @@ The rules: Tag columns work the same way. A stream whose key column is `str | None` cannot be joined with one whose key is `str` (`Join` never reconciles types), so narrow the key first. The -stream method is `drop_null_rows`: +stream method is `drop_nulls`: ```python -by_method = recordings.drop_null_rows(["session", "start_ms"]) -by_operator = DropNullRows(["start_ms", "session"])(recordings) +by_method = recordings.drop_nulls(["session", "start_ms"]) +by_operator = DropNulls(["start_ms", "session"])(recordings) print(by_method.pipeline_hash() == by_operator.pipeline_hash()) # True ``` @@ -531,7 +531,7 @@ operators or function pods. Every stream also exposes the operators as methods, which reads well when chaining: `join`, `semi_join`, `batch`, `group_by`, `select_tag_columns`, `select_data_columns`, `drop_tag_columns`, `drop_data_columns`, `map_tags`, `map_data`, `polars_filter`, -`drop_null_rows`, `pick` and `index`. +`drop_nulls`, `pick` and `index`. ```python import polars as pl diff --git a/src/orcapod/core/operators/__init__.py b/src/orcapod/core/operators/__init__.py index 8329d291d..c80f6b21f 100644 --- a/src/orcapod/core/operators/__init__.py +++ b/src/orcapod/core/operators/__init__.py @@ -5,7 +5,7 @@ SelectDataColumns, SelectTagColumns, ) -from .drop_null_rows import DropNullRows +from .drop_nulls import DropNulls from .filters import PolarsFilter from .group_by import GroupBy from .index import Index @@ -28,7 +28,7 @@ "DropTagColumns", "DropDataColumns", "PolarsFilter", - "DropNullRows", + "DropNulls", "Pick", "Index", ] diff --git a/src/orcapod/core/operators/drop_null_rows.py b/src/orcapod/core/operators/drop_nulls.py similarity index 97% rename from src/orcapod/core/operators/drop_null_rows.py rename to src/orcapod/core/operators/drop_nulls.py index 817d5d856..cfe248ad9 100644 --- a/src/orcapod/core/operators/drop_null_rows.py +++ b/src/orcapod/core/operators/drop_nulls.py @@ -1,4 +1,4 @@ -"""DropNullRows operator: drop rows with a null in named columns, narrow ``T | None`` to ``T``.""" +"""DropNulls operator: drop rows with a null in named columns, narrow ``T | None`` to ``T``.""" from __future__ import annotations @@ -45,7 +45,7 @@ def _without_none(python_type: DataType) -> DataType: return typing.Union[members] # noqa: UP007 # built from a runtime tuple of members -class DropNullRows(UnaryOperator): +class DropNulls(UnaryOperator): """Keep the rows where every named column is non-null; narrow those columns to ``T``. A row survives only if all the named columns hold a value. Afterwards each @@ -82,7 +82,7 @@ def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: names = (columns,) if isinstance(columns, str) else tuple(columns) if not names: raise ValueError( - "DropNullRows needs at least one column name; got an empty collection." + "DropNulls needs at least one column name; got an empty collection." ) # A sorted tuple of unique names, so a list, a tuple and a set of the # same names (in any order) are one identity. @@ -130,7 +130,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: ] if missing: raise InputValidationError( - f"DropNullRows: columns {missing} are not tag or data columns of the" + f"DropNulls: columns {missing} are not tag or data columns of the" + f" input stream. Tag columns: {list(tag_columns)}; data columns:" + f" {list(data_columns)}." ) diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 6253d6193..d34e71914 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -186,7 +186,7 @@ def polars_filter( self, label=label ) - def drop_null_rows( + def drop_nulls( self, columns: str | Collection[str], label: str | None = None, @@ -195,7 +195,7 @@ def drop_null_rows( Each named column, tag or data, is non-nullable afterwards (``T`` instead of ``T | None``), so a pod whose parameter is a plain ``T`` - can consume it. See ``orcapod.core.operators.DropNullRows``. + can consume it. See ``orcapod.core.operators.DropNulls``. Args: columns: One column name, or a collection of names. @@ -204,9 +204,9 @@ def drop_null_rows( Returns: A stream over the surviving rows, in input order. """ - from orcapod.core.operators import DropNullRows + from orcapod.core.operators import DropNulls - return DropNullRows(columns)(self, label=label) + return DropNulls(columns)(self, label=label) def select_tag_columns( self, diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 70ad0d7ac..d9619d064 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -166,7 +166,7 @@ def _build_operator_registry() -> dict[str, type]: from orcapod.core.operators import ( Batch, DropDataColumns, - DropNullRows, + DropNulls, DropTagColumns, GroupBy, Join, @@ -192,7 +192,7 @@ def _build_operator_registry() -> dict[str, type]: "MapTags": MapTags, "MapData": MapData, "PolarsFilter": PolarsFilter, - "DropNullRows": DropNullRows, + "DropNulls": DropNulls, } diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index 02963a096..f33029eda 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -285,7 +285,7 @@ def polars_filter( **constraints: Any, ) -> "StreamProtocol": ... - def drop_null_rows( + def drop_nulls( self, columns: str | Collection[str], label: str | None = None, diff --git a/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md b/superpowers/plans/2026-11-11-itl-943-drop-nulls.md similarity index 79% rename from superpowers/plans/2026-11-11-itl-943-drop-null-rows.md rename to superpowers/plans/2026-11-11-itl-943-drop-nulls.md index 035e373f7..b19df3578 100644 --- a/superpowers/plans/2026-11-11-itl-943-drop-null-rows.md +++ b/superpowers/plans/2026-11-11-itl-943-drop-nulls.md @@ -1,12 +1,12 @@ -# DropNullRows Operator Implementation Plan (ITL-943) +# DropNulls Operator Implementation Plan (ITL-943) > **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** A unary operator `DropNullRows(columns)` and `stream.drop_null_rows(columns)` that keeps +**Goal:** A unary operator `DropNulls(columns)` and `stream.drop_nulls(columns)` that keeps only rows where every named column is non-null and narrows each named column from `T | None` to `T` (schema and Arrow field). -**Architecture:** One operator class in `src/orcapod/core/operators/drop_null_rows.py`. Pure +**Architecture:** One operator class in `src/orcapod/core/operators/drop_nulls.py`. Pure Arrow (validity mask, `Table.filter`, re-wrap the same arrays under a schema whose named fields are non-nullable); no Polars. A streaming `async_execute` applies the same table-level narrowing to each datagram's Arrow table. @@ -17,10 +17,9 @@ table-level narrowing to each datagram's Arrow table. ## Design (approved by the coordinator, November 11, 2026) -Coordinator's answer to the STEP 1 `ask`: (b)-(j) approved as proposed. (a) The NAME is the -developer's call, still open between `DropNulls`/`drop_nulls` and -`DropNullRows`/`drop_null_rows`; **use `DropNullRows` / `drop_null_rows` meanwhile and keep -the name easy to switch with one rename** (class, stream method, registry key, module, docs). +Coordinator's answer to the STEP 1 `ask`: (b)-(j) approved as proposed. (a) The NAME was the +developer's call; the developer chose **`DropNulls` / `drop_nulls`** on November 11, 2026, and +the operator (first built under a longer working name) was renamed everywhere, with no alias. - **Argument:** `columns: str | Collection[str]`; a bare `str` is one name. Empty collection is `ValueError` at construction. Stored as `self.columns = tuple(sorted(set(names)))`. @@ -33,7 +32,7 @@ the name easy to switch with one rename** (class, stream method, registry key, m - **Tag vs data:** one operator; names may be tag or data columns, mixed. - **Nested:** only top-level validity is tested (null list / null struct; not an empty list, list items, struct children, or NaN). Only the top-level field becomes non-nullable. -- **Identity:** `identity_structure() == ("DropNullRows", self.columns)`; same for content and +- **Identity:** `identity_structure() == ("DropNulls", self.columns)`; same for content and pipeline hash. `"x"`, `["x"]`, `("x",)`, `{"x"}`, `["x", "x"]` are one identity; order is irrelevant. - **Hash effects (measured):** nullability is in the Arrow schema hash. A surviving row's @@ -59,17 +58,17 @@ with a source whose keys are `T`. ## Files -- Create: `src/orcapod/core/operators/drop_null_rows.py` (the operator) +- Create: `src/orcapod/core/operators/drop_nulls.py` (the operator) - Modify: `src/orcapod/core/operators/__init__.py` (export), `src/orcapod/operators/...` (public re-export, wherever `PolarsFilter` is listed) -- Modify: `src/orcapod/core/streams/base.py` (`drop_null_rows` method), +- Modify: `src/orcapod/core/streams/base.py` (`drop_nulls` method), `src/orcapod/protocols/core_protocols/streams.py` (protocol method) - Modify: `src/orcapod/pipeline/serialization.py` (operator registry) -- Create: `tests/test_core/operators/test_drop_null_rows.py` -- Create: `tests/test_pipeline/test_drop_null_rows_job.py` (Delta store, cache hits) +- Create: `tests/test_core/operators/test_drop_nulls.py` +- Create: `tests/test_pipeline/test_drop_nulls_job.py` (Delta store, cache hits) - Modify: `tests/test_core/operators/test_async_sync_equivalence.py` (two cases) - Modify: `tests/test_core/operators/test_operator_config.py` (round trip) -- Modify: `tests/test_hashing/golden/cases.py` (new case `pipeline/drop_null_rows`), then +- Modify: `tests/test_hashing/golden/cases.py` (new case `pipeline/drop_nulls`), then regenerate; the diff of `tests/test_hashing/golden/data/pipeline.json` must only add lines - Modify: `docs/concepts/operators.md`, `CHANGELOG.md`, `DESIGN_ISSUES.md`, `CLAUDE.md`, `.zed/rules` @@ -77,11 +76,11 @@ with a source whose keys are `T`. ## Operator sketch ```python -class DropNullRows(UnaryOperator): +class DropNulls(UnaryOperator): def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: names = (columns,) if isinstance(columns, str) else tuple(columns) if not names: - raise ValueError("DropNullRows needs at least one column name.") + raise ValueError("DropNulls needs at least one column name.") self.columns: tuple[str, ...] = tuple(sorted(set(names))) super().__init__(**kwargs) @@ -114,7 +113,7 @@ fall back to the inherited barrier path and report it. ## Tasks -- [ ] **1. Tests first** (`tests/test_core/operators/test_drop_null_rows.py`): rows with nulls +- [ ] **1. Tests first** (`tests/test_core/operators/test_drop_nulls.py`): rows with nulls dropped and order preserved; `output_schema` and `as_table` narrowed; other columns' types and nullability untouched; multiple columns; tag column; mixed tag and data; nested (list with null items, struct with null children survive; null list / struct dropped; nested @@ -126,14 +125,14 @@ fall back to the inherited barrier path and report it. source info unchanged; schema prediction vs `unary_static_process(...).output_schema(...)` (comment at the non-public use); pod with a non-optional parameter refused before, runs after; `Join` after it; nullable tag narrowed then joined with a `T`-keyed source; - `stream.drop_null_rows` equals the operator. Run: all fail with `ImportError`. -- [ ] **2. More tests:** two `Case`s in `test_async_sync_equivalence.py` (`drop_null_rows` - on `nul`, `drop_null_rows_tag_and_data`), config round trip in `test_operator_config.py`, - `tests/test_pipeline/test_drop_null_rows_job.py` (PipelineJob with `DeltaTableDatabase`: + `stream.drop_nulls` equals the operator. Run: all fail with `ImportError`. +- [ ] **2. More tests:** two `Case`s in `test_async_sync_equivalence.py` (`drop_nulls` + on `nul`, `drop_nulls_tag_and_data`), config round trip in `test_operator_config.py`, + `tests/test_pipeline/test_drop_nulls_job.py` (PipelineJob with `DeltaTableDatabase`: run, results; second job on the same store makes no new pod calls). Commit (WIP, failing). - [ ] **3. Implement** the operator, exports, stream method, protocol method, registry. Run the new tests until green. Commit. -- [ ] **4. Golden:** add `@case("pipeline", "drop_null_rows")` to `cases.py` (a source with a +- [ ] **4. Golden:** add `@case("pipeline", "drop_nulls")` to `cases.py` (a source with a nullable data column; summaries of the operator on one and on two columns), regenerate, check `git diff --stat` and that the diff has no removed or changed line. Commit. - [ ] **5. Docs:** `docs/concepts/operators.md` section with a runnable sample (drop null rows, @@ -151,22 +150,21 @@ fall back to the inherited barrier path and report it. (Updated as work proceeds, for a successor.) - November 11: tasks 1-5 done and pushed. Operator in - `src/orcapod/core/operators/drop_null_rows.py`; tests in - `tests/test_core/operators/test_drop_null_rows.py`, - `tests/test_pipeline/test_drop_null_rows_job.py`, two `Case`s plus a tagless test in - `test_async_sync_equivalence.py`, `TestDropNullRowsConfig` in `test_operator_config.py`. -- Golden: cases `pipeline/drop_null_rows` and `pipeline/drop_null_rows_job` appended to + `src/orcapod/core/operators/drop_nulls.py`; tests in + `tests/test_core/operators/test_drop_nulls.py`, + `tests/test_pipeline/test_drop_nulls_job.py`, two `Case`s plus a tagless test in + `test_async_sync_equivalence.py`, `TestDropNullsConfig` in `test_operator_config.py`. +- Golden: cases `pipeline/drop_nulls` and `pipeline/drop_nulls_job` appended to `cases.py`; `pipeline.json` gained 128 lines, none removed or changed. - The streaming async path is exactly equivalent row by row (including Data meta columns, so no xfail was needed); no fallback to the barrier path. - Large normalization uses `normalize_to_large_types(..., keep_null=True)` per field with `restore_column_types` (the table-level helper maps `null` to `large_string`). -- Open: the NAME (developer's call). To rename: class and module `drop_null_rows.py`, stream - method in `core/streams/base.py` and `protocols/core_protocols/streams.py`, registry key in - `pipeline/serialization.py`, export in `core/operators/__init__.py`, docs, CHANGELOG, - DESIGN_ISSUES, CLAUDE.md, `.zed/rules`, tests and golden case names. A rename changes the - operator's identity hash (the class name is in `identity_structure`), so the two golden - cases must be regenerated with it (new entries only; nothing released yet). +- NAME decided (developer, November 11, 2026): `DropNulls` / `drop_nulls`. Renamed class, + module (`drop_nulls.py`), stream method, registry key, export, docs, CHANGELOG, DESIGN_ISSUES, + CLAUDE.md, `.zed/rules`, tests (files too), golden case names and this plan file. The class + name is in `identity_structure`, so the two golden cases this PR added were regenerated with + it (additions only against main; no existing entry moved). - Observation, not fixed (out of scope): a `PipelineJob` saved with an in-memory `ArrowTableSource` and loaded again gives operator nodes whose pipeline hash differs from the saved job's (same for `MapData`), because the source is not reconstructable. diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index e7874c3c5..a4cf6aa22 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -31,7 +31,7 @@ from orcapod.core.operators import ( Batch, DropDataColumns, - DropNullRows, + DropNulls, DropTagColumns, GroupBy, Index, @@ -313,11 +313,11 @@ class Case: ), Case("index", lambda: Index("l8", 0), lambda: (source_a(),)), Case("pick", lambda: Pick("d", "q"), lambda: (source_a(),)), - Case("drop_null_rows", lambda: DropNullRows("nul"), lambda: (source_a(),)), + Case("drop_nulls", lambda: DropNulls("nul"), lambda: (source_a(),)), Case( - "drop_null_rows_many", + "drop_nulls_many", # A nullable column with a null, one without, a tag, a non-nullable list. - lambda: DropNullRows(["nul", "s", "ta", "l8"]), + lambda: DropNulls(["nul", "s", "ta", "l8"]), lambda: (source_a(),), ), ] @@ -333,7 +333,7 @@ def test_async_rows_equal_sync_rows(case: Case) -> None: ) -def test_drop_null_rows_without_tag_columns() -> None: +def test_drop_nulls_without_tag_columns() -> None: """A stream without tag columns: the async path drops the same rows, in order. Outside the matrix, whose pairing needs unique tag rows. @@ -347,8 +347,8 @@ def test_drop_null_rows_without_tag_columns() -> None: source_id="tagless", nullable=("x", "s"), ) - expected = sync_rows(DropNullRows("x"), source) - actual = async_rows(DropNullRows("x"), source) + expected = sync_rows(DropNulls("x"), source) + actual = async_rows(DropNulls("x"), source) assert [data.as_dict() for _, data in actual] == [ {"x": 1, "s": "a"}, {"x": 3, "s": "c"}, diff --git a/tests/test_core/operators/test_drop_null_rows.py b/tests/test_core/operators/test_drop_nulls.py similarity index 89% rename from tests/test_core/operators/test_drop_null_rows.py rename to tests/test_core/operators/test_drop_nulls.py index 56df1f2ce..cd0396de4 100644 --- a/tests/test_core/operators/test_drop_null_rows.py +++ b/tests/test_core/operators/test_drop_nulls.py @@ -1,4 +1,4 @@ -"""``DropNullRows``: drop rows with a null in a named column, narrow ``T | None`` to ``T`` (ITL-943). +"""``DropNulls``: drop rows with a null in a named column, narrow ``T | None`` to ``T`` (ITL-943). The operator keeps the rows where every named column is non-null and reports each named column as non-nullable afterwards, in ``output_schema()`` and in the Arrow @@ -20,7 +20,7 @@ from orcapod import function_pod from orcapod.core.sources import ArrowTableSource, DictSource from orcapod.errors import InputValidationError -from orcapod.operators import DropNullRows, Join +from orcapod.operators import DropNulls, Join from orcapod.protocols.core_protocols import StreamProtocol from orcapod.system_constants import constants @@ -66,7 +66,7 @@ def _ids(stream: StreamProtocol) -> list[Any]: def test_drops_rows_where_the_column_is_null() -> None: """Rows whose named column is null are dropped; the others pass through.""" - result = DropNullRows("x")(readings()) + result = DropNulls("x")(readings()) assert result.as_table().to_pylist() == [ {"id": 1, "x": 10, "y": 1.5, "note": "a"}, {"id": 3, "x": 30, "y": None, "note": "c"}, @@ -76,7 +76,7 @@ def test_drops_rows_where_the_column_is_null() -> None: def test_every_named_column_must_be_non_null() -> None: """With several names, a row survives only if all of them are non-null.""" - assert _ids(DropNullRows(["x", "y"])(readings())) == [1, 4] + assert _ids(DropNulls(["x", "y"])(readings())) == [1, 4] def test_row_order_is_preserved() -> None: @@ -88,7 +88,7 @@ def test_row_order_is_preserved() -> None: }, nullable=["x"], ) - assert [tag["id"] for tag, _ in DropNullRows("x")(source).iter_data()] == [4, 1, 2] + assert [tag["id"] for tag, _ in DropNulls("x")(source).iter_data()] == [4, 1, 2] def test_nan_is_not_null() -> None: @@ -100,7 +100,7 @@ def test_nan_is_not_null() -> None: }, nullable=["v"], ) - result = DropNullRows("v")(source) + result = DropNulls("v")(source) first, second = result.as_table().column("v").to_pylist() assert first == 1.0 assert isinstance(second, float) and math.isnan(second) @@ -119,7 +119,7 @@ def test_output_schema_narrows_only_the_named_columns() -> None: "y": float | None, "note": str | None, } - tag_schema, data_schema = DropNullRows("x")(source).output_schema() + tag_schema, data_schema = DropNulls("x")(source).output_schema() assert tag_schema == {"id": int} assert data_schema == {"x": int, "y": float | None, "note": str | None} @@ -128,7 +128,7 @@ def test_table_narrows_only_the_named_columns() -> None: """Only the named field loses its nullable flag; widths and other flags are kept.""" source = readings() before = source.as_table(columns=ALL).schema - after = DropNullRows("x")(source).as_table(columns=ALL).schema + after = DropNulls("x")(source).as_table(columns=ALL).schema assert after.names == before.names assert after.field("x") == pa.field("x", pa.int8(), nullable=False) # Width and nullability are kept; ``string`` takes its large variant. @@ -142,7 +142,7 @@ def test_table_narrows_only_the_named_columns() -> None: def test_schema_prediction_matches_the_computed_stream() -> None: """The predicted schema equals the schema of the computed stream.""" source = readings() - op = DropNullRows(["x", "id"]) + op = DropNulls(["x", "id"]) for config in ({}, {"system_tags": True}, {"source": True, "system_tags": True}): # Non-public on purpose: a prediction is verified against the static result. computed = op.static_process(source).output_schema(columns=config) @@ -168,13 +168,13 @@ def test_zero_surviving_rows_keep_the_narrowed_types() -> None: }, nullable=["x", "s"], ) - result = DropNullRows("x")(source) + result = DropNulls("x")(source) table = result.as_table(columns=ALL) assert table.num_rows == 0 assert list(result.iter_data()) == [] assert table.schema.field("x") == pa.field("x", pa.int8(), nullable=False) assert result.output_schema() == ({"id": int}, {"x": int, "s": str | None}) - expected = DropNullRows("x")(non_empty).as_table(columns=ALL).schema + expected = DropNulls("x")(non_empty).as_table(columns=ALL).schema assert table.schema == expected @@ -186,7 +186,7 @@ def test_already_non_nullable_column_is_a_no_op() -> None: "x": pa.array([10, 20], pa.int8()), } ) - result = DropNullRows("x")(source) + result = DropNulls("x")(source) assert result.output_schema() == source.output_schema() assert result.as_table(columns=ALL) == source.as_table(columns=ALL) for (tag, data), (in_tag, in_data) in zip(result.iter_data(), source.iter_data()): @@ -196,8 +196,8 @@ def test_already_non_nullable_column_is_a_no_op() -> None: def test_applying_twice_equals_applying_once() -> None: """The operator is idempotent.""" - once = DropNullRows("x")(readings()) - twice = DropNullRows("x")(once) + once = DropNulls("x")(readings()) + twice = DropNulls("x")(once) assert twice.as_table(columns=ALL) == once.as_table(columns=ALL) assert twice.output_schema() == once.output_schema() @@ -222,7 +222,7 @@ def test_tag_column_is_narrowed() -> None: """A named tag column is tested and narrowed like a data column.""" source = keyed() assert source.output_schema()[0] == {"site": str | None, "id": int} - result = DropNullRows("site")(source) + result = DropNulls("site")(source) assert _ids(result) == [1, 3] assert result.keys() == source.keys() assert result.output_schema() == ( @@ -235,14 +235,14 @@ def test_tag_column_is_narrowed() -> None: def test_tag_and_data_columns_in_one_call() -> None: """Tag and data columns can be named together.""" - result = DropNullRows(["x", "site"])(keyed()) + result = DropNulls(["x", "site"])(keyed()) assert _ids(result) == [1] assert result.output_schema() == ({"site": str, "id": int}, {"x": int}) def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: """A source whose key columns are nullable by specification (as a Spiral table's - are) cannot join a source whose keys are ``T``; after ``DropNullRows`` it can.""" + are) cannot join a source whose keys are ``T``; after ``DropNulls`` it can.""" events = _source( { "session": pa.array(["s1", None, "s3"], pa.large_string()), @@ -263,7 +263,7 @@ def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: with pytest.raises(InputValidationError, match="not compatible"): Join()(events, sessions) - joined = Join()(DropNullRows("session")(events), sessions) + joined = Join()(DropNulls("session")(events), sessions) assert joined.output_schema() == ( {"session": str}, {"start": int | None, "animal": str}, @@ -285,7 +285,7 @@ def test_stream_without_tag_columns() -> None: tags=(), nullable=["x", "s"], ) - result = DropNullRows("x")(source) + result = DropNulls("x")(source) assert result.as_table().to_pylist() == [{"x": 1, "s": "a"}, {"x": 3, "s": "c"}] assert result.output_schema() == ({}, {"x": int, "s": str | None}) @@ -313,7 +313,7 @@ def nested() -> ArrowTableSource: def test_only_the_top_level_value_is_tested_for_a_list() -> None: """A null list is null; an empty list and a list holding nulls are not.""" - result = DropNullRows("items")(nested()) + result = DropNulls("items")(nested()) # A list holding a null and an empty list are not null lists. assert result.as_table().column("items").to_pylist() == [[1, None], [], [4]] field = result.as_table().schema.field("items") @@ -326,7 +326,7 @@ def test_only_the_top_level_value_is_tested_for_a_list() -> None: def test_only_the_top_level_value_is_tested_for_a_struct() -> None: """A null struct is null; a struct whose fields are null is not.""" - result = DropNullRows("point")(nested()) + result = DropNulls("point")(nested()) # A struct whose children are all null is not a null struct. assert _ids(result) == [1, 2, 4] field = result.as_table().schema.field("point") @@ -356,7 +356,7 @@ def test_logical_columns_keep_their_logical_type() -> None: "arr": np.ndarray | None, # pyright: ignore[reportMissingTypeArgument] # the schema holds the bare class } - result = DropNullRows(["p", "arr"])(source) + result = DropNulls(["p", "arr"])(source) assert result.output_schema()[1] == {"p": Path, "arr": np.ndarray} after = result.as_table().schema for name in ("p", "arr"): @@ -377,7 +377,7 @@ def test_logical_columns_keep_their_logical_type() -> None: def test_missing_column_is_an_input_validation_error() -> None: """A name that is not a column of the stream is refused at call time.""" with pytest.raises(InputValidationError, match="nope"): - DropNullRows(["x", "nope"])(readings()) + DropNulls(["x", "nope"])(readings()) def test_system_and_source_columns_cannot_be_named() -> None: @@ -391,13 +391,13 @@ def test_system_and_source_columns_cannot_be_named() -> None: assert len(hidden) >= 2 for name in hidden: with pytest.raises(InputValidationError): - DropNullRows(name)(source) + DropNulls(name)(source) def test_no_column_is_an_error_at_construction() -> None: """An empty collection of names is refused when the operator is built.""" with pytest.raises(ValueError, match="at least one"): - DropNullRows([]) + DropNulls([]) # ── identity ──────────────────────────────────────────────────────────────── @@ -410,8 +410,8 @@ def test_no_column_is_an_error_at_construction() -> None: ) def test_identity_is_the_set_of_names(columns: Any) -> None: """Container type, order and duplicates do not change the operator's hashes.""" - reference = DropNullRows(["x", "y"]) - op = DropNullRows(columns) + reference = DropNulls(["x", "y"]) + op = DropNulls(columns) assert op.columns == ("x", "y") assert op.content_hash() == reference.content_hash() assert op.pipeline_hash() == reference.pipeline_hash() @@ -422,26 +422,25 @@ def test_identity_is_the_set_of_names(columns: Any) -> None: def test_a_bare_name_is_one_column() -> None: """A string is one column name, never a collection of characters.""" - assert DropNullRows("note").columns == ("note",) - assert DropNullRows("x").content_hash() == DropNullRows(["x"]).content_hash() + assert DropNulls("note").columns == ("note",) + assert DropNulls("x").content_hash() == DropNulls(["x"]).content_hash() def test_different_names_are_different_identities() -> None: """Operators over different names hash differently, as do their outputs.""" source = readings() - assert DropNullRows("x").content_hash() != DropNullRows("y").content_hash() - assert DropNullRows("x").content_hash() != DropNullRows(["x", "y"]).content_hash() + assert DropNulls("x").content_hash() != DropNulls("y").content_hash() + assert DropNulls("x").content_hash() != DropNulls(["x", "y"]).content_hash() assert ( - DropNullRows("x")(source).pipeline_hash() - != DropNullRows("y")(source).pipeline_hash() + DropNulls("x")(source).pipeline_hash() != DropNulls("y")(source).pipeline_hash() ) - assert DropNullRows("x")(source).pipeline_hash() != source.pipeline_hash() + assert DropNulls("x")(source).pipeline_hash() != source.pipeline_hash() def test_the_caller_s_collection_is_not_kept_or_mutated() -> None: """The operator copies the names; later changes to the list do not reach it.""" names = ["y", "x"] - op = DropNullRows(names) + op = DropNulls(names) names.append("z") assert names == ["y", "x", "z"] assert op.columns == ("x", "y") @@ -467,7 +466,7 @@ def test_narrowed_rows_hash_as_rows_declared_non_nullable() -> None: nullable=["x"], ) input_rows = {tag["id"]: (tag, data) for tag, data in nullable.iter_data()} - narrowed = list(DropNullRows("x")(nullable).iter_data()) + narrowed = list(DropNulls("x")(nullable).iter_data()) assert len(narrowed) == 2 for (tag, data), (ref_tag, ref_data) in zip(narrowed, declared.iter_data()): in_tag, in_data = input_rows[tag["id"]] @@ -480,7 +479,7 @@ def test_narrowing_a_tag_changes_the_tag_hash_only() -> None: """Narrowing a tag column changes the row's tag hash and not its data hash.""" source = keyed() input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} - for tag, data in DropNullRows("site")(source).iter_data(): + for tag, data in DropNulls("site")(source).iter_data(): in_tag, in_data = input_rows[tag["id"]] assert tag.content_hash() != in_tag.content_hash() assert data.content_hash() == in_data.content_hash() @@ -490,7 +489,7 @@ def test_system_tags_and_source_info_pass_through() -> None: """System-tag and source-info columns keep their names, types and values.""" source = readings() before = source.as_table(columns=ALL) - after = DropNullRows(["x", "y"])(source).as_table(columns=ALL) + after = DropNulls(["x", "y"])(source).as_table(columns=ALL) hidden = [ n for n in before.column_names @@ -503,7 +502,7 @@ def test_system_tags_and_source_info_pass_through() -> None: assert after.column(name).to_pylist() == kept.column(name).to_pylist() input_rows = {tag["id"]: (tag, data) for tag, data in source.iter_data()} - for tag, data in DropNullRows(["x", "y"])(source).iter_data(): + for tag, data in DropNulls(["x", "y"])(source).iter_data(): in_tag, in_data = input_rows[tag["id"]] assert tag.system_tags() == in_tag.system_tags() assert data.source_info() == in_data.source_info() @@ -514,7 +513,7 @@ def test_the_input_stream_is_not_changed() -> None: source = readings() schema = source.as_table(columns=ALL).schema rows = source.as_table(columns=ALL).to_pylist() - DropNullRows(["x", "y"])(source).as_table() + DropNulls(["x", "y"])(source).as_table() assert source.as_table(columns=ALL).schema == schema assert source.as_table(columns=ALL).to_pylist() == rows @@ -523,14 +522,14 @@ def test_the_input_stream_is_not_changed() -> None: def test_stream_method_equals_the_operator() -> None: - """``stream.drop_null_rows`` gives the operator's rows and hashes.""" + """``stream.drop_nulls`` gives the operator's rows and hashes.""" source = readings() - by_method = source.drop_null_rows(["x", "y"]) - by_operator = DropNullRows(["x", "y"])(source) + by_method = source.drop_nulls(["x", "y"]) + by_operator = DropNulls(["x", "y"])(source) assert by_method.as_table(columns=ALL) == by_operator.as_table(columns=ALL) assert by_method.content_hash() == by_operator.content_hash() assert by_method.pipeline_hash() == by_operator.pipeline_hash() - assert _ids(source.drop_null_rows("x")) == [1, 3, 4] + assert _ids(source.drop_nulls("x")) == [1, 3, 4] # ── downstream ────────────────────────────────────────────────────────────── @@ -554,7 +553,7 @@ def test_pod_with_a_non_optional_parameter_runs_only_after_the_operator() -> Non with pytest.raises(ValueError, match="not compatible"): double_x.pod(source) - result = double_x.pod(source.drop_null_rows("x")) + result = double_x.pod(source.drop_nulls("x")) assert result.as_table().to_pylist() == [ {"id": 1, "doubled": 20}, {"id": 3, "doubled": 60}, @@ -570,7 +569,7 @@ def test_join_after_the_operator() -> None: }, source_id="weights", ) - joined = Join()(DropNullRows("x")(readings()), weights) + joined = Join()(DropNulls("x")(readings()), weights) assert joined.output_schema()[1]["x"] is int assert sorted(_ids(joined)) == [1, 3, 4] assert joined.as_table().schema.field("x").nullable is False diff --git a/tests/test_core/operators/test_operator_config.py b/tests/test_core/operators/test_operator_config.py index a7836b05f..df8902030 100644 --- a/tests/test_core/operators/test_operator_config.py +++ b/tests/test_core/operators/test_operator_config.py @@ -3,7 +3,7 @@ from orcapod.core.operators import ( Batch, DropDataColumns, - DropNullRows, + DropNulls, DropTagColumns, Join, MapData, @@ -164,18 +164,18 @@ def test_non_serializable_predicates_marked(self): assert "config" in config -class TestDropNullRowsConfig: +class TestDropNullsConfig: def test_to_config(self): """The config holds the sorted unique names.""" - config = DropNullRows(["y", "x", "y"]).to_config() - assert config["class_name"] == "DropNullRows" + config = DropNulls(["y", "x", "y"]).to_config() + assert config["class_name"] == "DropNulls" assert config["config"] == {"columns": ["x", "y"]} def test_round_trip(self): """``from_config(to_config())`` gives an operator with the same hashes.""" - op = DropNullRows(["y", "x"]) - restored = DropNullRows.from_config(op.to_config()) - assert isinstance(restored, DropNullRows) + op = DropNulls(["y", "x"]) + restored = DropNulls.from_config(op.to_config()) + assert isinstance(restored, DropNulls) assert restored.columns == ("x", "y") assert restored.content_hash() == op.content_hash() assert restored.pipeline_hash() == op.pipeline_hash() @@ -184,7 +184,7 @@ def test_registered_for_pipeline_loading(self): """Pipeline loading resolves the operator by its class name.""" from orcapod.pipeline.serialization import resolve_operator_from_config - op = DropNullRows(["y", "x"]) + op = DropNulls(["y", "x"]) restored = resolve_operator_from_config(op.to_config()) - assert isinstance(restored, DropNullRows) + assert isinstance(restored, DropNulls) assert restored.content_hash() == op.content_hash() diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 5b2d2620d..0dba31082 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -968,33 +968,29 @@ def _nullable_readings() -> Any: ) -@case("pipeline", "drop_null_rows") -def pipeline_drop_null_rows() -> dict[str, Any]: - """Pin ``DropNullRows``: the operator's own identity and the narrowed rows.""" - from orcapod.operators import DropNullRows +@case("pipeline", "drop_nulls") +def pipeline_drop_nulls() -> dict[str, Any]: + """Pin ``DropNulls``: the operator's own identity and the narrowed rows.""" + from orcapod.operators import DropNulls out = _prefixed("source", stream_summary(_nullable_readings())) - out["operator[x].content_hash"] = DropNullRows("x").content_hash().to_string() - out["operator[x,y].content_hash"] = ( - DropNullRows(["y", "x"]).content_hash().to_string() - ) - out.update( - _prefixed("data", stream_summary(DropNullRows("x")(_nullable_readings()))) - ) + out["operator[x].content_hash"] = DropNulls("x").content_hash().to_string() + out["operator[x,y].content_hash"] = DropNulls(["y", "x"]).content_hash().to_string() + out.update(_prefixed("data", stream_summary(DropNulls("x")(_nullable_readings())))) out.update( _prefixed( - "data_many", stream_summary(DropNullRows(["x", "y"])(_nullable_readings())) + "data_many", stream_summary(DropNulls(["x", "y"])(_nullable_readings())) ) ) out.update( - _prefixed("tag", stream_summary(_nullable_readings().drop_null_rows("site"))) + _prefixed("tag", stream_summary(_nullable_readings().drop_nulls("site"))) ) return out -@case("pipeline", "drop_null_rows_job") -def pipeline_drop_null_rows_job() -> dict[str, Any]: - """Record a job with ``DropNullRows`` upstream of a pod with non-optional parameters.""" +@case("pipeline", "drop_nulls_job") +def pipeline_drop_nulls_job() -> dict[str, Any]: + """Record a job with ``DropNulls`` upstream of a pod with non-optional parameters.""" from orcapod import PipelineJob from orcapod.databases import InMemoryArrowDatabase @@ -1003,7 +999,7 @@ def pipeline_drop_null_rows_job() -> dict[str, Any]: with job: present = ( _nullable_readings() - .drop_null_rows(["x", "y"], label="present") + .drop_nulls(["x", "y"], label="present") .map_data({"x": "age", "y": "weight"}, label="renamed") ) golden_ratio.pod(present, label="ratio") diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index 3220c05f4..27e4fbcd5 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -58,133 +58,133 @@ "_tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4", "_tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4" ], - "drop_null_rows.data.content_hash": "semantic_v0.1:bdc2fa3420424910e6d063e8e6a3068153c13c039fde0759ae487981a0237570", - "drop_null_rows.data.pipeline_hash": "semantic_v0.1:272051f71278ae99b6ac0e254ceea47c7e34f45c88c1842ed764b2ac57d6863b", - "drop_null_rows.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.data.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001eb8bb21cea56694f30415a726a190021869b1839977483a983afccc228e3fa7d", - "drop_null_rows.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", - "drop_null_rows.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.data.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:000001eeb56ac3c1727cfe75fca3334244e3f9d6afbf624a67662511dbd6abff6ecec5", - "drop_null_rows.data.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows.data.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.data.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018b789706fed0bc06c83ab3049aacb9e7b228aef5d112d80ba59bc45b6981a5a9", - "drop_null_rows.data.system_tag_columns": [ + "drop_nulls.data.content_hash": "semantic_v0.1:56d87bbf372f115b88df8c734cd8a9fa6e31d37f57e4d26a01cf2b6836b94d32", + "drop_nulls.data.pipeline_hash": "semantic_v0.1:d2cfd2dce8882ab924b86a26b9f26f627dd2b293c1054a113dce3025e1587da3", + "drop_nulls.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.data.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001eb8bb21cea56694f30415a726a190021869b1839977483a983afccc228e3fa7d", + "drop_nulls.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls.data.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:000001eeb56ac3c1727cfe75fca3334244e3f9d6afbf624a67662511dbd6abff6ecec5", + "drop_nulls.data.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls.data.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018b789706fed0bc06c83ab3049aacb9e7b228aef5d112d80ba59bc45b6981a5a9", + "drop_nulls.data.system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows.data_many.content_hash": "semantic_v0.1:0e68d2042ef50caa781bbe3cc0106fd5dd1793b35cbc3c6a880fb9b1c8185d10", - "drop_null_rows.data_many.pipeline_hash": "semantic_v0.1:cd2bfc56cd87c29a5ee5f706af29dc23e6ef6a846873be9a87ca6cd84b7d3f81", - "drop_null_rows.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", - "drop_null_rows.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.data_many.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", - "drop_null_rows.data_many.system_tag_columns": [ + "drop_nulls.data_many.content_hash": "semantic_v0.1:6e7edbd1167d450b18cae53f7daeeac3e489c6dde1921131f09f8880c7032e70", + "drop_nulls.data_many.pipeline_hash": "semantic_v0.1:b388b7f3dbab1086d9a2c96f82e2f957dd21bb74b1f68ddc57ee70174c1dac96", + "drop_nulls.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data_many.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls.data_many.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.data_many.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", + "drop_nulls.data_many.system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows.operator[x,y].content_hash": "semantic_v0.1:a63930534fb25a43c4aedfbf4c82f68c427e19573e0de84f8c54d731dbb6278a", - "drop_null_rows.operator[x].content_hash": "semantic_v0.1:e656ec6b37a716f1acbcce2f7166e1db2c756d910cafd00b91562221e2925c22", - "drop_null_rows.source.content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", - "drop_null_rows.source.pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", - "drop_null_rows.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.source.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", - "drop_null_rows.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", - "drop_null_rows.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.source.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", - "drop_null_rows.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", - "drop_null_rows.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.source.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", - "drop_null_rows.source.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows.source.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.source.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", - "drop_null_rows.source.system_tag_columns": [ + "drop_nulls.operator[x,y].content_hash": "semantic_v0.1:76689e842097817302c55b7a02dbcdbe22b980710489d83c3334a73b68dd9975", + "drop_nulls.operator[x].content_hash": "semantic_v0.1:e0b67b2eab584c0ee1a0a79af479684da3944f66c57d4bd9c0c758642179f358", + "drop_nulls.source.content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", + "drop_nulls.source.pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "drop_nulls.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.source.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_nulls.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_nulls.source.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_nulls.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls.source.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_nulls.source.row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls.source.row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.source.row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", + "drop_nulls.source.system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows.tag.content_hash": "semantic_v0.1:598883c5d524994fb1931a5f3734a674004dc38483059c4a11d8b799969aefff", - "drop_null_rows.tag.pipeline_hash": "semantic_v0.1:0d006d549eb02dfce4fab969506fd229359deac589a2654e5e670ee22ffc94a0", - "drop_null_rows.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.tag.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", - "drop_null_rows.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", - "drop_null_rows.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.tag.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", - "drop_null_rows.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", - "drop_null_rows.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows.tag.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", - "drop_null_rows.tag.system_tag_columns": [ + "drop_nulls.tag.content_hash": "semantic_v0.1:4189d845c46ae389f3fd59578da62aa8f545fc5c3186bc91479ba7cec81f46e0", + "drop_nulls.tag.pipeline_hash": "semantic_v0.1:483f0e59fc6d4c5c4403b8a676ac20e863c123785fac25d113538c0f3081c093", + "drop_nulls.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls.tag.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.tag.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_nulls.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_nulls.tag.row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.tag.row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_nulls.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls.tag.row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls.tag.row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_nulls.tag.system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows_job.node[golden_nullable].content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", - "drop_null_rows_job.node[golden_nullable].pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", - "drop_null_rows_job.node[golden_nullable].system_tag_columns": [ + "drop_nulls_job.node[golden_nullable].content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", + "drop_nulls_job.node[golden_nullable].pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000015b0250945bfa748ea5e0acb4fcc5460287542abafb493d74a5d284768f69f87b", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "ba63ce278a505051ad97c7cc16b2d03a", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r2\",\"site\":\"b\"}].data_hash": "arrow_v0.1:00000183b36c7d2fe7bc531397fa15b8dc1fcbcab652726be1efb883d10306b82080b5", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "8137e0d5f5cd5a3ab0e7354b7c28916c", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r3\",\"site\":\"c\"}].data_hash": "arrow_v0.1:00000177b42d12787b7375fdd55be973d3cefd48e5bb4f34b3b5be65e8ff7a7e90a8cc", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d09d83607e71ffd272982a6a6b3d671f475d320f46783805aa126a53f8263e8f", + "drop_nulls_job.node[golden_nullable].system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows_job.node[present].content_hash": "semantic_v0.1:0e68d2042ef50caa781bbe3cc0106fd5dd1793b35cbc3c6a880fb9b1c8185d10", - "drop_null_rows_job.node[present].pipeline_hash": "semantic_v0.1:cd2bfc56cd87c29a5ee5f706af29dc23e6ef6a846873be9a87ca6cd84b7d3f81", - "drop_null_rows_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", - "drop_null_rows_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[present].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", - "drop_null_rows_job.node[present].system_tag_columns": [ + "drop_nulls_job.node[present].content_hash": "semantic_v0.1:6e7edbd1167d450b18cae53f7daeeac3e489c6dde1921131f09f8880c7032e70", + "drop_nulls_job.node[present].pipeline_hash": "semantic_v0.1:b388b7f3dbab1086d9a2c96f82e2f957dd21bb74b1f68ddc57ee70174c1dac96", + "drop_nulls_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[present].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[present].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[present].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:000001d3a51be60e62f2177ae9011b1aff722b935b88458df036821f383bca39534103", + "drop_nulls_job.node[present].system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows_job.node[ratio].content_hash": "semantic_v0.1:ee34576f6923e09b72b7a46fc59575cd5323deaa73d27430a114644663b69976", - "drop_null_rows_job.node[ratio].pipeline_hash": "semantic_v0.1:c422b770e44ff13a0a3b76ebe04437d5365b66ada44f47de514f87ac6d4d8419", - "drop_null_rows_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:00000150467e204fcd64131e2c5242f04cb75405ad27b0effab95b7024cac05989ce7b", - "drop_null_rows_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000012f80ae46902b8c07a040d40c77b237701ab557207263bd934400b66bbfbd7daf", - "drop_null_rows_job.node[ratio].system_tag_columns": [ + "drop_nulls_job.node[ratio].content_hash": "semantic_v0.1:2bd33335d606563899ff2e7e2b45f8ab73dc22265d752eb98670f93a62613c35", + "drop_nulls_job.node[ratio].pipeline_hash": "semantic_v0.1:b4bf7581adb16db07b8a3c5042e92ed721ba44dbf109da6650a0289c3e7b42b0", + "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:00000150467e204fcd64131e2c5242f04cb75405ad27b0effab95b7024cac05989ce7b", + "drop_nulls_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[ratio].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000012f80ae46902b8c07a040d40c77b237701ab557207263bd934400b66bbfbd7daf", + "drop_nulls_job.node[ratio].system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows_job.node[renamed].content_hash": "semantic_v0.1:4122fb621cb9a122bd6a09e6525f7957b10974441fac2ae820222ab60e25b7fd", - "drop_null_rows_job.node[renamed].pipeline_hash": "semantic_v0.1:f3c1f000bc2042acbe3fbb43217c8910f5443f018ff74117bd9f33d058771725", - "drop_null_rows_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", - "drop_null_rows_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3", - "drop_null_rows_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", - "drop_null_rows_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", - "drop_null_rows_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", - "drop_null_rows_job.node[renamed].system_tag_columns": [ + "drop_nulls_job.node[renamed].content_hash": "semantic_v0.1:a5ac89746ac4e6b0e251185ea47d330f670d84dcba8b090202bfa1f71f6cc165", + "drop_nulls_job.node[renamed].pipeline_hash": "semantic_v0.1:9d0a308a0c2a3c9f2497291a86d0f7ff9c3d8fbf683cda11bb09d4833170df97", + "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3", + "drop_nulls_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "7e0a96359315576aa855218763d31ad3", + "drop_nulls_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_job.node[renamed].row[{\"k\":\"r4\",\"site\":null}].data_hash": "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", + "drop_nulls_job.node[renamed].system_tag_columns": [ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], - "drop_null_rows_job.table[golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1].input_data_hashes": [ + "drop_nulls_job.table[golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1].input_data_hashes": [ "arrow_v0.1:0000018191385eb4792cd677726cf2dc80c904ae931d6dea6cb2879fb34c456a4721e6", "arrow_v0.1:000001fc3f74cf0c504d17e9a3fa1c58762105b401863d4c5f07a333fbb49612ed43b3" ], - "drop_null_rows_job.table[golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:c422b770e44ff13a0a3b76ebe04437d5365b66ada44f47de514f87ac6d4d8419/pdb_v1].record_ids": [ + "drop_nulls_job.table[golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:b4bf7581adb16db07b8a3c5042e92ed721ba44dbf109da6650a0289c3e7b42b0/pdb_v1].record_ids": [ "arrow_v0.1:0000012447639a486f05322571bf7908f2e5b1a551e430dc4eec49e54a22657f0a49aa", "arrow_v0.1:00000124f29b7254f3a496da7cd95db7728fc265f082ae8091cd094bac96a0631c3563" ], - "drop_null_rows_job.table_paths": [ + "drop_nulls_job.table_paths": [ "golden_drop_null_job/_result/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/rdb_v1", - "golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:c422b770e44ff13a0a3b76ebe04437d5365b66ada44f47de514f87ac6d4d8419/pdb_v1" + "golden_drop_null_job/golden_ratio/semantic_v0.1:6e5510afd046261efb0286a51b6c3ce3146ff457fe06c21498eaa8193cc224e5/v0/python.function.v0/schema:semantic_v0.1:b4bf7581adb16db07b8a3c5042e92ed721ba44dbf109da6650a0289c3e7b42b0/pdb_v1" ], "group_by.content_hash": "semantic_v0.1:ee6478a83847e5e9d6bea916138170139aba8c5b43c6992a22d048a07b9b4526", "group_by.pipeline_hash": "semantic_v0.1:5ab1cb5d5ce9b641d19045bae0d94bbb9e15703bebfe37ec57fd13b141d2a0f0", diff --git a/tests/test_pipeline/test_drop_null_rows_job.py b/tests/test_pipeline/test_drop_nulls_job.py similarity index 93% rename from tests/test_pipeline/test_drop_null_rows_job.py rename to tests/test_pipeline/test_drop_nulls_job.py index ee0a18d59..47bdf8dc6 100644 --- a/tests/test_pipeline/test_drop_null_rows_job.py +++ b/tests/test_pipeline/test_drop_nulls_job.py @@ -1,7 +1,7 @@ -"""``DropNullRows`` in a ``PipelineJob`` against a Delta Lake store (ITL-943). +"""``DropNulls`` in a ``PipelineJob`` against a Delta Lake store (ITL-943). The workshop shape: a source whose value column is nullable by specification, -``drop_null_rows``, then a pod with a non-optional parameter. A second run on +``drop_nulls``, then a pod with a non-optional parameter. A second run on the same store must reuse the stored results. """ @@ -62,11 +62,11 @@ def _recordings(timestamps: list[int | None]) -> ArrowTableSource: def _run(store: DeltaTableDatabase, timestamps: list[int | None]) -> PipelineJob: - """Record and run ``source -> drop_null_rows -> to_seconds`` as a ``PipelineJob``.""" + """Record and run ``source -> drop_nulls -> to_seconds`` as a ``PipelineJob``.""" pod = FunctionPod(PythonDataFunction(to_seconds, output_keys="seconds")) # pyright: ignore[reportArgumentType] # PythonDataFunction is not yet typed as matching DataFunctionProtocol (baselined in other tests); it works at runtime job = PipelineJob(name="recordings", store=store) with job: - present = _recordings(timestamps).drop_null_rows( + present = _recordings(timestamps).drop_nulls( "recording_start_timestamp", label="present" ) pod(present, label="seconds") @@ -137,7 +137,7 @@ def test_job_save_and_load_round_trips_the_operator( for node in json.loads(path.read_text())["pipeline"]["nodes"].values() if node.get("operator_config") ] - assert [c["class_name"] for c in saved] == ["DropNullRows"] + assert [c["class_name"] for c in saved] == ["DropNulls"] assert saved[0]["config"] == {"columns": ["recording_start_timestamp"]} loaded = PipelineJob.load(str(path), store=store) From be11eb6fb983e502ffc76d13e2810255084a3b1a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 22:08:35 -0800 Subject: [PATCH 09/11] docs(changelog): move the ITL-943 entry to a changelog fragment (ITL-961) Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 12 ------------ changelog.d/ITL-943.md | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 changelog.d/ITL-943.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 31fc40c64..dfdc26d3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,18 +72,6 @@ stay queued for the next flush. A flush that fails only for other reasons still ### Added -- **`DropNulls` operator and `stream.drop_nulls(columns)`** — keeps the rows where - every named column is non-null and narrows each named column from `T | None` to `T`, in - `output_schema()` and in the Arrow table, so a pod whose parameter is a plain `T` can - consume a column that is nullable by specification. The names may be tag columns, data - columns or both; a name that is not a column of the stream raises `InputValidationError`, - and a column that is already non-nullable is left as it is. Only the top-level value is - tested (a null list or struct; not list items, struct fields or NaN). Other columns, system - tags, source info and row order are unchanged. The operator's identity is the set of names. - A surviving row's data hash (or tag hash, for a tag column) changes when the named column - was nullable, because nullability is part of the row's Arrow schema; it becomes the hash - the row has in a stream that declared the column non-nullable. No existing hash changes. - (ITL-943) - **`orcapod.errors.StoreWriteError`** — raised when a node's results or records cannot be written to the store. It names the node, the row's tag and the table, and chains the database's own exception as its cause. (ITL-878 #287) diff --git a/changelog.d/ITL-943.md b/changelog.d/ITL-943.md new file mode 100644 index 000000000..4e4298689 --- /dev/null +++ b/changelog.d/ITL-943.md @@ -0,0 +1,14 @@ +### Added + +- **`DropNulls` operator and `stream.drop_nulls(columns)`** — keeps the rows where + every named column is non-null and narrows each named column from `T | None` to `T`, in + `output_schema()` and in the Arrow table, so a pod whose parameter is a plain `T` can + consume a column that is nullable by specification. The names may be tag columns, data + columns or both; a name that is not a column of the stream raises `InputValidationError`, + and a column that is already non-nullable is left as it is. Only the top-level value is + tested (a null list or struct; not list items, struct fields or NaN). Other columns, system + tags, source info and row order are unchanged. The operator's identity is the set of names. + A surviving row's data hash (or tag hash, for a tag column) changes when the named column + was nullable, because nullability is part of the row's Arrow schema; it becomes the hash + the row has in a stream that declared the column non-nullable. No existing hash changes. + (ITL-943) From fad6832308433777b2838fb72007ab804b6a113d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 23:14:59 -0800 Subject: [PATCH 10/11] docs(design-issues): move the ITL-943 entries to design_issues/ files (ITL-968) Co-Authored-By: Claude Fable 5.2 --- DESIGN_ISSUES.md | 26 -------------------------- design_issues/ITL-943-DN1.md | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 26 deletions(-) create mode 100644 design_issues/ITL-943-DN1.md diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index e6cd94930..e322a9507 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2710,29 +2710,3 @@ never hidden). Follow-up idea, unverified: a manifest-only row count (`spiral.core.manifest_stats(snapshot).key_space_stats.total_rows`, see the ITL-186 design) could replace the shortcut cheaply once confirmed live. ---- - -## `src/orcapod/core/operators/drop_nulls.py` — null rows and nullable columns (ITL-943) - -### DN1 — No way to turn a `T | None` column into `T` -**Status:** resolved -**Severity:** medium -**Issue:** ITL-943 - -Many value columns are nullable by specification (EDI, Spiral tables), and a pod written as -`def f(x: int)` is refused on an `int | None` column. `PolarsFilter` could drop the null rows, -but the column stayed `T | None`, so the pod was still refused; a nullable key column likewise -could not be joined with a non-nullable one. - -**Fix:** `DropNulls(columns)` and `stream.drop_nulls(columns)` keep the rows where every -named column (tag or data) is non-null and clear the nullable flag of each named field. -Decisions (coordinator, November 11, 2026): one operator for tag and data columns; only -top-level validity is read; a missing column is `InputValidationError`; an already non-nullable -column is a no-op; identity is the sorted tuple of unique names; sync and async paths share one -table rule (`_narrow_table`), pure Arrow. A surviving row's `Data` (or `Tag`) content hash -changes when a named column was nullable, to the hash of the same row declared non-nullable; -no existing golden value moved. - -Open: the class and method names (`DropNulls` / `drop_nulls` versus `DropNulls` / -`drop_nulls`) await the developer's decision. Possible follow-up: a variant that raises on a -null instead of dropping the row. diff --git a/design_issues/ITL-943-DN1.md b/design_issues/ITL-943-DN1.md new file mode 100644 index 000000000..dd159b880 --- /dev/null +++ b/design_issues/ITL-943-DN1.md @@ -0,0 +1,24 @@ +## `src/orcapod/core/operators/drop_nulls.py` — null rows and nullable columns (ITL-943) + +### DN1 — No way to turn a `T | None` column into `T` +**Status:** resolved +**Severity:** medium +**Issue:** ITL-943 + +Many value columns are nullable by specification (EDI, Spiral tables), and a pod written as +`def f(x: int)` is refused on an `int | None` column. `PolarsFilter` could drop the null rows, +but the column stayed `T | None`, so the pod was still refused; a nullable key column likewise +could not be joined with a non-nullable one. + +**Fix:** `DropNulls(columns)` and `stream.drop_nulls(columns)` keep the rows where every +named column (tag or data) is non-null and clear the nullable flag of each named field. +Decisions (coordinator, November 11, 2026): one operator for tag and data columns; only +top-level validity is read; a missing column is `InputValidationError`; an already non-nullable +column is a no-op; identity is the sorted tuple of unique names; sync and async paths share one +table rule (`_narrow_table`), pure Arrow. A surviving row's `Data` (or `Tag`) content hash +changes when a named column was nullable, to the hash of the same row declared non-nullable; +no existing golden value moved. + +Open: the class and method names (`DropNulls` / `drop_nulls` versus `DropNulls` / +`drop_nulls`) await the developer's decision. Possible follow-up: a variant that raises on a +null instead of dropping the row. From 8cffc5acf064008c84137a5d39fcd1853ac23d94 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 00:42:26 -0800 Subject: [PATCH 11/11] feat(drop_nulls): no argument means every tag and data column (ITL-943) DropNulls() and stream.drop_nulls() without names test and narrow every user tag and data column of the input stream. The names are resolved from each input (stream.keys(), or the row's tag and data keys on the async path) and never stored, so the no-argument form has one fixed identity, ("DropNulls", None), and one operator object works on any stream. System-tag, source-info, context and meta columns are never included. An explicitly empty collection still raises ValueError. Golden: case pipeline/drop_nulls_all added; no existing entry moved. Co-Authored-By: Claude Fable 5.2 --- .zed/rules | 8 +- CLAUDE.md | 8 +- changelog.d/ITL-943.md | 11 +- design_issues/ITL-943-DN1.md | 11 + docs/concepts/operators.md | 63 ++++- src/orcapod/core/operators/drop_nulls.py | 137 +++++++--- src/orcapod/core/streams/base.py | 10 +- .../protocols/core_protocols/streams.py | 5 +- ...26-11-12-itl-943-drop-nulls-all-columns.md | 115 ++++++++ .../operators/test_async_sync_equivalence.py | 22 ++ tests/test_core/operators/test_drop_nulls.py | 252 ++++++++++++++++++ .../operators/test_operator_config.py | 17 ++ tests/test_hashing/golden/cases.py | 13 + tests/test_hashing/golden/data/pipeline.json | 20 ++ 14 files changed, 645 insertions(+), 47 deletions(-) create mode 100644 superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md diff --git a/.zed/rules b/.zed/rules index 7b54e50e7..8e4381b93 100644 --- a/.zed/rules +++ b/.zed/rules @@ -301,7 +301,7 @@ src/orcapod/ column_selection.py — Select/Drop Tag/Data columns mappers.py — MapTags, MapData (rename columns) filters.py — PolarsFilter - drop_nulls.py — DropNulls (drop rows with a null in named columns, T | None → T) + drop_nulls.py — DropNulls (drop rows with a null in named or all columns, T | None → T) hashing/ semantic_hashing/ — BaseSemanticHasher, type handlers semantic_types/ — Type conversion (Python ↔ Arrow) @@ -532,6 +532,12 @@ data. The rules: names. Both paths apply one rule, DropNulls._narrow_table: an Arrow validity mask and the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast other than large normalization. A named column that is already non-nullable is a no-op. +- DropNulls() with no argument (columns=None) means every user tag and data column of + the input (developer, November 12, 2026). The names are resolved from each input + (DropNulls._target_columns: stream.keys(), or the row's tag and data keys on the + async path) and never stored, so its identity is the fixed ("DropNulls", None), + different from any list of names. System-tag, source-info, context and meta columns are + never included. An explicitly empty collection is not "no argument": it raises ValueError. - GroupBy requires every column in `by` to be a scalar tag column; raises InputValidationError for unknown, data, or list-valued columns (list-valued usually means the stream came from Batch — group before batching, not after). Members sort by non-key tag values with system diff --git a/CLAUDE.md b/CLAUDE.md index 763c8588c..e4d26e1e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -313,7 +313,7 @@ src/orcapod/ │ ├── column_selection.py # Select/Drop Tag/Data columns │ ├── mappers.py # MapTags, MapData (rename columns) │ ├── filters.py # PolarsFilter -│ └── drop_nulls.py # DropNulls (drop rows with a null in named columns, T | None → T) +│ └── drop_nulls.py # DropNulls (drop rows with a null in named or all columns, T | None → T) ├── hashing/ │ └── semantic_hashing/ # BaseSemanticHasher, type handlers ├── semantic_types/ # Type conversion (Python ↔ Arrow) @@ -575,6 +575,12 @@ data. The rules: names. Both paths apply one rule, `DropNulls._narrow_table`: an Arrow validity mask and the same arrays re-wrapped under the new field flags, with no Polars round-trip and no cast other than large normalization. A named column that is already non-nullable is a no-op. +- `DropNulls()` with no argument (`columns=None`) means every user tag and data column of + the input (developer, November 12, 2026). The names are resolved from each input + (`DropNulls._target_columns`: `stream.keys()`, or the row's tag and data keys on the + async path) and never stored, so its identity is the fixed `("DropNulls", None)`, + different from any list of names. System-tag, source-info, context and meta columns are + never included. An explicitly empty collection is not "no argument": it raises `ValueError`. - `GroupBy` requires every column in `by` to be a scalar tag column; it raises `InputValidationError` for unknown, data, or list-valued columns (the last usually means the stream came from `Batch` — group before batching, not after). Members are sorted by non-key diff --git a/changelog.d/ITL-943.md b/changelog.d/ITL-943.md index 4e4298689..705db8913 100644 --- a/changelog.d/ITL-943.md +++ b/changelog.d/ITL-943.md @@ -5,9 +5,16 @@ `output_schema()` and in the Arrow table, so a pod whose parameter is a plain `T` can consume a column that is nullable by specification. The names may be tag columns, data columns or both; a name that is not a column of the stream raises `InputValidationError`, - and a column that is already non-nullable is left as it is. Only the top-level value is + and a column that is already non-nullable is left as it is. With no argument + (`DropNulls()`, `stream.drop_nulls()`) the columns are every tag and data column of the + input stream: a row with a null in any of them is dropped and all of them become + non-nullable. System-tag, source-info, context and meta columns are never included. On a + source whose columns are all nullable (SpiralDB stores every column nullable) this form can + drop most rows; naming the columns is the precise form. An empty collection is not "no + argument" and raises `ValueError`. Only the top-level value is tested (a null list or struct; not list items, struct fields or NaN). Other columns, system - tags, source info and row order are unchanged. The operator's identity is the set of names. + tags, source info and row order are unchanged. The operator's identity is the set of names; + the no-argument form has one fixed identity, different from that of any list of names. A surviving row's data hash (or tag hash, for a tag column) changes when the named column was nullable, because nullability is part of the row's Arrow schema; it becomes the hash the row has in a stream that declared the column non-nullable. No existing hash changes. diff --git a/design_issues/ITL-943-DN1.md b/design_issues/ITL-943-DN1.md index dd159b880..c5c1f03ac 100644 --- a/design_issues/ITL-943-DN1.md +++ b/design_issues/ITL-943-DN1.md @@ -19,6 +19,17 @@ table rule (`_narrow_table`), pure Arrow. A surviving row's `Data` (or `Tag`) co changes when a named column was nullable, to the hash of the same row declared non-nullable; no existing golden value moved. +No-argument form (developer, November 12, 2026: "plain no argument = drop all is fair and as +applied to both tags and data columns"): `DropNulls()` / `stream.drop_nulls()` tests and +narrows every user tag and data column of the input. The names are resolved from each input +(`stream.keys()`; per row on the async path) and never stored, so the identity is the fixed +`("DropNulls", None)` and one operator object works on any stream. `DropNulls()` and +`DropNulls([every column])` give the same rows and schema but are different identities. +System-tag, source-info, context and meta columns are never included (meta columns are +filtered by prefix, because a stream built straight from a table lists them among its data +keys, ITL-846). An explicitly empty collection still raises `ValueError`: an empty list is a +different intent from no argument, and a computed list that came out empty must not drop rows. + Open: the class and method names (`DropNulls` / `drop_nulls` versus `DropNulls` / `drop_nulls`) await the developer's decision. Possible follow-up: a variant that raises on a null instead of dropping the row. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index fad381db1..945ba9ad5 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -377,7 +377,8 @@ except ValueError as exc: ### DropNulls `DropNulls(columns)` keeps the rows where every named column is non-null, and narrows -each named column from `T | None` to `T`. Use it when a column is nullable by specification +each named column from `T | None` to `T`. `DropNulls()`, with no names, does this for every +tag and data column (see [Every column](#every-column) below). Use it when a column is nullable by specification but a function needs a value: a pod whose parameter is a plain `int` refuses an `int | None` column, and `PolarsFilter` can drop the null rows but leaves the type as it was. @@ -428,8 +429,10 @@ The rules: - **Columns.** Pass one name or a collection of names; a row survives only if every named column is non-null. The names may be tag columns, data columns, or both. A name that is - not a tag or data column of the stream raises `InputValidationError`, and an empty - collection raises `ValueError`. + not a tag or data column of the stream raises `InputValidationError`. With no argument + the columns are every tag and data column of the input. An empty collection is not "no + argument": it raises `ValueError`, so a list of names that came out empty never drops + rows by surprise. - **What counts as null.** Only the top-level value: a null list or a null struct is null. An empty list, a list that holds nulls, a struct whose fields are null and a floating-point NaN are not. Nested fields keep their nullability. @@ -440,6 +443,7 @@ The rules: with no rows has the same narrowed schema. - **Identity** is the set of names: `DropNulls("a")`, `DropNulls(["a"])` and `DropNulls({"a"})` are one operator, and the order of names does not matter. + `DropNulls()` has one fixed identity of its own, whatever stream it is applied to. - **Row hashes.** Nullability is part of a row's Arrow schema, so a surviving row's data hash changes when a named data column was nullable (its tag hash, when a named tag column was). The new hash is the one the row has in a stream that declared the column @@ -456,6 +460,59 @@ print(by_method.pipeline_hash() == by_operator.pipeline_hash()) # True ``` +#### Every column + +`DropNulls()` and `stream.drop_nulls()`, with no names, drop every row that has a null in +**any** tag or data column, and make every tag and data column non-nullable. System-tag, +source-info, context and meta columns are never tested or changed. + +```python +visits = ArrowTableSource( + pa.table( + { + "session": pa.array(["s1", "s2", "s3", "s4"], pa.large_string()), + "start_ms": pa.array([1000, None, 3000, 4000], pa.int64()), + "notes": pa.array([None, "noisy", None, "ok"], pa.large_string()), + } + ), + tag_columns=["session"], +) +print(visits.output_schema()) +# (Schema({'session': str | None}), Schema({'start_ms': int | None, 'notes': str | None})) + +complete = visits.drop_nulls() +print(complete.output_schema()) +# (Schema({'session': str}), Schema({'start_ms': int, 'notes': str})) +print(complete.as_table().to_pylist()) +# [{'session': 's4', 'start_ms': 4000, 'notes': 'ok'}] + +timed = visits.drop_nulls("start_ms") +print(timed.as_table().to_pylist()) +# [{'session': 's1', 'start_ms': 1000, 'notes': None}, {'session': 's3', 'start_ms': 3000, 'notes': None}, {'session': 's4', 'start_ms': 4000, 'notes': 'ok'}] +``` + +!!! warning "The no-argument form can drop most of a table" + + A null in any tag or data column drops the row, including a column that is empty for a + good reason. Above, `notes` is empty for most sessions, so three of the four rows are + gone although only one of them lacks `start_ms`. Sources whose columns are all nullable + are common: SpiralDB stores every column as nullable. For such a source, name the columns + a downstream step needs (`drop_nulls("start_ms")`); that is the precise form. Use the + no-argument form only when every column must be present. + +The column names are read from each input stream when the operator is applied, so one +`DropNulls()` object works on any stream. On the same stream it gives the rows and schema +of `DropNulls([every tag and data column])`, but the two are different operators: their +identities, and so the pipeline and content hashes of their output streams, differ. + +```python +by_name = DropNulls(["session", "start_ms", "notes"])(visits) +print(complete.as_table() == by_name.as_table()) +# True +print(complete.pipeline_hash() == by_name.pipeline_hash()) +# False +``` + ### Pick and Index `Pick(column, key)` extracts one entry from a struct- or dict-typed data column, and diff --git a/src/orcapod/core/operators/drop_nulls.py b/src/orcapod/core/operators/drop_nulls.py index cfe248ad9..b938653f1 100644 --- a/src/orcapod/core/operators/drop_nulls.py +++ b/src/orcapod/core/operators/drop_nulls.py @@ -1,4 +1,4 @@ -"""DropNulls operator: drop rows with a null in named columns, narrow ``T | None`` to ``T``.""" +"""DropNulls operator: drop rows with a null in the chosen columns, narrow ``T | None`` to ``T``.""" from __future__ import annotations @@ -13,6 +13,7 @@ from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol +from orcapod.system_constants import constants from orcapod.types import ColumnConfig, DataType, Schema from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -53,6 +54,15 @@ class DropNulls(UnaryOperator): ``output_schema()``, and a non-nullable field in the Arrow table. A pod whose parameter is a plain ``T`` can then consume the column. + Without names (``DropNulls()``), the columns are every tag column and every + data column of the input stream: a row with a null in any of them is + dropped, and all of them are non-nullable afterwards. The names are taken + from each input stream when the operator is applied and are never stored, + so one ``DropNulls()`` works on any stream. System-tag, source-info, + context and meta columns are never among them. On a stream whose columns + are all nullable and legitimately empty in places, this form can drop most + rows; naming the columns is the precise form. + The names may be tag columns, data columns, or both. Only the validity of the top-level value is read, never the value itself: a null list or a null struct is null, while an empty list, a list holding nulls, a struct whose @@ -71,22 +81,36 @@ class DropNulls(UnaryOperator): non-nullable from the start. Args: - columns: One column name, or a collection of names. Duplicates and - order are ignored: the operator's identity is the set of names. + columns: One column name, a collection of names, or ``None`` (the + default) for every tag and data column of the input stream. + Duplicates and order are ignored: the operator's identity is the + set of names, or one fixed value when no names are given. An empty + collection is not "no argument" and is refused: a list of names + that came out empty is never read as "every column". Raises: - ValueError: If ``columns`` is empty. + ValueError: If ``columns`` is an empty collection. """ - def __init__(self, columns: str | Collection[str], **kwargs: Any) -> None: - names = (columns,) if isinstance(columns, str) else tuple(columns) - if not names: - raise ValueError( - "DropNulls needs at least one column name; got an empty collection." - ) - # A sorted tuple of unique names, so a list, a tuple and a set of the - # same names (in any order) are one identity. - self.columns: tuple[str, ...] = tuple(sorted(set(names))) + def __init__( + self, columns: str | Collection[str] | None = None, **kwargs: Any + ) -> None: + self.columns: tuple[str, ...] | None + if columns is None: + # Every tag and data column of whichever stream the operator is + # applied to; resolved per input, never stored. + self.columns = None + else: + names = (columns,) if isinstance(columns, str) else tuple(columns) + if not names: + raise ValueError( + "DropNulls needs at least one column name; got an empty" + + " collection. Pass no argument to mean every tag and data" + + " column." + ) + # A sorted tuple of unique names, so a list, a tuple and a set of the + # same names (in any order) are one identity. + self.columns = tuple(sorted(set(names))) super().__init__(**kwargs) def identity_structure(self) -> Any: @@ -94,7 +118,8 @@ def identity_structure(self) -> Any: Returns: A ``(class_name, columns)`` tuple, where ``columns`` is the sorted - tuple of unique names. + tuple of unique names, or ``None`` for the no-argument form (every + tag and data column), whatever stream it is applied to. """ return (self.__class__.__name__, self.columns) @@ -103,12 +128,38 @@ def to_config(self) -> dict[str, Any]: Returns: A dict with ``class_name``, ``module_path``, and ``config`` keys, - where ``config`` contains ``columns``. + where ``config`` contains ``columns``: the list of names, or + ``None`` for the no-argument form. """ config = super().to_config() - config["config"] = {"columns": list(self.columns)} + config["config"] = { + "columns": None if self.columns is None else list(self.columns) + } return config + def _target_columns( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> tuple[str, ...]: + """Return the columns to test and narrow for an input with these columns. + + Args: + tag_columns: The input's tag column names. + data_columns: The input's data column names. + + Returns: + The operator's names, or every given tag and data column when the + operator was built without names. Meta columns are left out: a + stream built straight from a table lists them among its data + columns (ITL-846), and they are not user columns. + """ + if self.columns is not None: + return self.columns + return tuple( + name + for name in (*tag_columns, *data_columns) + if not name.startswith(constants.META_PREFIX) + ) + # ------------------------------------------------------------------ # Validation and schema prediction # ------------------------------------------------------------------ @@ -122,6 +173,8 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: Raises: InputValidationError: If a named column is not a tag or data column. """ + if self.columns is None: + return tag_columns, data_columns = stream.keys() missing = [ name @@ -135,13 +188,12 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: + f" {list(data_columns)}." ) - def _narrow_schema(self, schema: Schema) -> Schema: - """Return ``schema`` with ``None`` removed from each named column's type.""" + @staticmethod + def _narrow_schema(schema: Schema, names: Collection[str]) -> Schema: + """Return ``schema`` with ``None`` removed from the type of each column in ``names``.""" return Schema( { - name: _without_none(python_type) - if name in self.columns - else python_type + name: _without_none(python_type) if name in names else python_type for name, python_type in schema.items() }, optional_fields=schema.optional_fields, @@ -156,6 +208,9 @@ def unary_output_schema( ) -> tuple[Schema, Schema]: """Return the input schemas with the named columns narrowed to ``T``. + Without names, every tag and data column of ``stream`` is narrowed; + system-tag columns included by ``columns`` are not. + Args: stream: The upstream stream. columns: Column inclusion config passed through to ``output_schema``. @@ -167,20 +222,26 @@ def unary_output_schema( tag_schema, data_schema = stream.output_schema( columns=columns, all_info=all_info ) - return self._narrow_schema(tag_schema), self._narrow_schema(data_schema) + names = self._target_columns(*stream.keys()) + return ( + self._narrow_schema(tag_schema, names), + self._narrow_schema(data_schema, names), + ) # ------------------------------------------------------------------ # Execution # ------------------------------------------------------------------ - def _narrow_fields(self, table: pa.Table) -> pa.Table: - """Flag the named columns present in ``table`` non-nullable. + @staticmethod + def _narrow_fields(table: pa.Table, names: Collection[str]) -> pa.Table: + """Flag the columns in ``names`` that are present in ``table`` non-nullable. Every column takes the large-normalized form of its input type; no other type, flag or value changes. The named columns must hold no null. Args: table: A table whose named columns hold no null. + names: The columns to flag. Returns: The same rows with the named fields non-nullable. @@ -195,7 +256,7 @@ def _narrow_fields(self, table: pa.Table) -> pa.Table: }, ) schema = table.schema - for name in self.columns: + for name in names: position = schema.get_field_index(name) if position >= 0: schema = schema.set( # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized @@ -204,26 +265,29 @@ def _narrow_fields(self, table: pa.Table) -> pa.Table: # Same arrays under new field flags: no cast, so no value or type can change. return pa.Table.from_arrays(table.columns, schema=schema) - def _narrow_table(self, table: pa.Table) -> pa.Table: + @staticmethod + def _narrow_table(table: pa.Table, names: Collection[str]) -> pa.Table: """Drop the rows with a null in a named column and flag those columns non-nullable. - Only the named columns present in ``table`` are considered, so the same - rule serves a whole stream table and a single row's tag or data table. + Only the columns in ``names`` that are present in ``table`` are + considered, so the same rule serves a whole stream table and a single + row's tag or data table. Args: table: A table holding some or all of the named columns. + names: The columns to test and narrow. Returns: The surviving rows, in order, with the named fields non-nullable. """ mask = None - for name in self.columns: + for name in names: if name in table.column_names: valid = pc.is_valid(table.column(name)) mask = valid if mask is None else pc.and_(mask, valid) if mask is not None: table = table.filter(mask) - return self._narrow_fields(table) + return DropNulls._narrow_fields(table, names) def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: """Return the stream's rows whose named columns are all non-null. @@ -238,9 +302,10 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: table = stream.as_table( columns={"source": True, "system_tags": True, "sort_by_tags": False} ) + tag_columns, data_columns = stream.keys() return ArrowTableStream( - self._narrow_table(table), - tag_columns=stream.keys()[0], + self._narrow_table(table, self._target_columns(tag_columns, data_columns)), + tag_columns=tag_columns, data_context=stream.data_context, # pyright: ignore[reportAttributeAccessIssue] # StreamProtocol omits data_context, which every concrete stream provides (ITL-844) ) @@ -254,7 +319,8 @@ async def async_execute( Each row's tag and data Arrow tables go through the rule of the sync path (``_narrow_table``), and a surviving row is rebuilt from those - tables with its own data context. + tables with its own data context. Without names, the columns are the + row's own tag and data keys. Args: inputs: Single-element sequence of readable channels. @@ -263,11 +329,12 @@ async def async_execute( """ try: async for tag, data in inputs[0]: + names = self._target_columns(tag.keys(), data.keys()) tag_table = self._narrow_table( - tag.as_table(columns={"system_tags": True}) + tag.as_table(columns={"system_tags": True}), names ) data_table = self._narrow_table( - data.as_table(columns={"source": True, "meta": True}) + data.as_table(columns={"source": True, "meta": True}), names ) # A table with no columns (a stream without tags) has no rows # to lose, so only a table that held a named column can drop the row. diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 33bcf63e2..e6d05cebf 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -189,17 +189,21 @@ def polars_filter( def drop_nulls( self, - columns: str | Collection[str], + columns: str | Collection[str] | None = None, label: str | None = None, ) -> StreamBase: """Keep the rows where every named column is non-null; narrow those columns to ``T``. Each named column, tag or data, is non-nullable afterwards (``T`` instead of ``T | None``), so a pod whose parameter is a plain ``T`` - can consume it. See ``orcapod.core.operators.DropNulls``. + can consume it. Without names, every tag and data column of this + stream is tested and narrowed: a row with a null in any of them is + dropped. See ``orcapod.core.operators.DropNulls``. Args: - columns: One column name, or a collection of names. + columns: One column name, a collection of names, or ``None`` (the + default) for every tag and data column. An empty collection + raises ``ValueError``. label: Optional node label for the pipeline graph. Returns: diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index f33029eda..894e437a3 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -287,12 +287,13 @@ def polars_filter( def drop_nulls( self, - columns: str | Collection[str], + columns: str | Collection[str] | None = None, label: str | None = None, ) -> "StreamProtocol": """ Keep the rows where every named column (tag or data) is non-null, and - narrow each named column from ``T | None`` to ``T``. + narrow each named column from ``T | None`` to ``T``. Without names, + every tag and data column is tested and narrowed. """ ... diff --git a/superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md b/superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md new file mode 100644 index 000000000..7321c7255 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-943-drop-nulls-all-columns.md @@ -0,0 +1,115 @@ +# DropNulls No-Argument Form Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `DropNulls()` / `stream.drop_nulls()` with no columns means every user tag column and +every user data column of the input stream (developer, November 12, 2026: "plain no argument = +drop all is fair and as applied to both tags and data columns"). + +**Architecture:** `columns` becomes `str | Collection[str] | None = None`. `self.columns` is +`None` for the no-argument form and is never replaced by resolved names: the names are resolved +from the input (`stream.keys()` on the sync and schema paths, `tag.keys()` + `data.keys()` per +row on the async path) each time, so one operator object works on any stream. The table rule +(`_narrow_table`) takes the names as an argument. Identity for the no-argument form is the +fixed value `("DropNulls", None)`; the named form stays `("DropNulls", (sorted names))`, so no +existing hash moves. + +**Tech Stack:** PyArrow, pytest, uv. + +**PR:** #311, branch `eywalker/itl-943-operator-that-drops-rows-where-given`. One commit, one push. + +--- + +## Decisions + +1. `columns=None` (the default) = all user tag and data columns. An explicitly **empty** + collection still raises `ValueError`: "no argument" and "an empty list" are different + intents (an empty list is usually a computed list that came out empty, and silently turning + it into "all" would drop rows nobody asked to drop). Stated in the docstring and tested. +2. Identity: `identity_structure()` is `("DropNulls", None)` for the no-argument form. `None` + cannot collide with any tuple of names. `to_config()` gives `{"columns": None}`, and + `from_config` round-trips through `cls(columns=None)`. `DropNulls()` and + `DropNulls([every column])` give the same output rows and schema on a stream; their + operator identities (content hash, pipeline hash, and so the output stream's hashes) differ. +3. "All" is `stream.keys()` (default column config): user tag and data columns only. System + tags, source-info, context and meta columns are never tested and never narrowed. +4. Schema prediction resolves the names from `stream.keys()`, not from the schema it was asked + for (which may include system tag columns). +5. Async path: names per row are `(*tag.keys(), *data.keys())`. + +## Files + +- `src/orcapod/core/operators/drop_nulls.py` — the operator. +- `src/orcapod/core/streams/base.py`, `src/orcapod/protocols/core_protocols/streams.py` — + `drop_nulls(columns=None, label=None)`. +- `tests/test_core/operators/test_drop_nulls.py` — new section "no argument: every column". +- `tests/test_core/operators/test_async_sync_equivalence.py` — `Case("drop_nulls_all", ...)` + and the tagless test extended. +- `tests/test_core/operators/test_operator_config.py` — config round trip of the no-argument form. +- `tests/test_hashing/golden/cases.py` + `data/pipeline.json` — new case + `pipeline/drop_nulls_all` (additions only). +- `docs/concepts/operators.md`, `changelog.d/ITL-943.md`, `design_issues/ITL-943-DN1.md`, + `CLAUDE.md`, `.zed/rules`. + +## Tasks + +- [ ] **1. Failing tests** in `tests/test_core/operators/test_drop_nulls.py` (public + interface: `DropNulls()(stream)` and `stream.drop_nulls()`): + - rows: `DropNulls()(readings())` keeps only id 1 (x, y, note all present); + - schema: every tag and data column `T`, Arrow fields non-nullable; system tag and + source-info fields unchanged, including when a source-info value is null; + - equals `DropNulls([all names])` in table (`ALL` config), `output_schema`, row hashes; + operator and stream hashes differ; + - one object on two streams with different columns; + - identity: `DropNulls().columns is None`, `DropNulls(None)` same hash, differs from any + named form; + - `DropNulls([])`, `DropNulls(())`, `DropNulls(set())` still raise `ValueError`; + `stream.drop_nulls([])` too; + - schema prediction vs `static_process(...).output_schema(...)` for `{}`, `system_tags`, + `source+system_tags` (non-public on purpose, commented); + - zero rows keep the types; an all-non-nullable stream is unchanged; idempotent; + - tagless stream; stream method equals operator. + Run `uv run pytest tests/test_core/operators/test_drop_nulls.py -q` and see them fail + (`TypeError: missing 1 required positional argument`). +- [ ] **2. Implement** in `drop_nulls.py`: + - `__init__(self, columns: str | Collection[str] | None = None, **kwargs)`; + `self.columns: tuple[str, ...] | None`; + - `_target_columns(tag_columns, data_columns) -> tuple[str, ...]`: `self.columns` or + `(*tag_columns, *data_columns)`; + - `_narrow_schema(schema, names)`, `_narrow_fields(table, names)`, + `_narrow_table(table, names)` (static rule functions of the names); + - `validate_unary_input`: nothing to check when `columns is None`; + - `unary_output_schema`, `unary_static_process`: names from `stream.keys()`; + - `async_execute`: names from `tag.keys()` and `data.keys()`; + - `identity_structure`, `to_config`, docstrings. + Stream method and protocol: `columns: str | Collection[str] | None = None`. +- [ ] **3. Async equivalence:** add `Case("drop_nulls_all", lambda: DropNulls(), ...)` and a + no-argument assertion in `test_drop_nulls_without_tag_columns`. +- [ ] **4. Config test:** `DropNulls().to_config()["config"] == {"columns": None}`, round trip + through `from_config` and `resolve_operator_from_config`. +- [ ] **5. Golden:** new case `pipeline/drop_nulls_all` in `cases.py`: + `operator[*].content_hash`, `stream_summary(DropNulls()(_nullable_readings()))`, and the + method form. Regenerate; `git diff --stat` on `data/` must show insertions only. +- [ ] **6. Docs:** `docs/concepts/operators.md` executed sample of the no-argument form with a + `notes` column that is legitimately empty, the plain warning (drops every row with a null in + ANY tag or data column; SpiralDB stores every column nullable; naming the columns is the + precise form). Update `changelog.d/ITL-943.md`, `design_issues/ITL-943-DN1.md`, `CLAUDE.md` + and `.zed/rules` (in step). +- [ ] **7. Verify:** `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), operator + docs + golden tests, full + suite once on 3.12 (`uv run --python 3.12 pytest tests/`). +- [ ] **8. Ship:** merge `origin/main` if it moved, one commit, one push, CI green, CodeRabbit + review of the new head, every review thread (GraphQL `reviewThreads`) and review body + answered, `worker_done` with the reviewed sha. + +## Notes / progress log + +- November 12: plan saved; work started. +- Tasks 1-6 done. Finding while extending the async matrix: a stream built straight from a + table lists its `__` meta columns among `keys()` data columns (ITL-846), so the sync path's + "all" tested and narrowed meta columns and the async path did not + (`test_async_meta_equals_sync[drop_nulls_all]` failed). `_target_columns` now leaves out + names with `constants.META_PREFIX`; `test_no_argument_leaves_meta_columns_alone` pins it. +- Golden: case `pipeline/drop_nulls_all` appended; `pipeline.json` gained 20 lines, none + removed or changed. Identity of the no-argument form: `("DropNulls", None)`; its content + hash and pipeline hash are equal (`operator[all].*` in the golden file). diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 69e369ae4..0e1a61c1a 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -320,6 +320,8 @@ class Case: lambda: DropNulls(["nul", "s", "ta", "l8"]), lambda: (source_a(),), ), + # No names: every tag and data column, among them struct, logical and list ones. + Case("drop_nulls_all", lambda: DropNulls(), lambda: (source_a(),)), ] CASE = pytest.mark.parametrize("case", CASES, ids=[c.label for c in CASES]) @@ -363,6 +365,26 @@ def test_drop_nulls_without_tag_columns() -> None: ) assert list(tag.system_tags().items()) == list(sync_tag.system_tags().items()) + # No names: both data columns are tested and narrowed on both paths. + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", None], pa.string()), + }, + tags=(), + source_id="tagless", + nullable=("x", "s"), + ) + expected = sync_rows(DropNulls(), source) + actual = async_rows(DropNulls(), source) + assert [data.as_dict() for _, data in actual] == [{"x": 1, "s": "a"}] + assert len(actual) == len(expected) == 1 + for (_, data), (_, sync_data) in zip(actual, expected): + assert data.arrow_schema() == sync_data.arrow_schema() + assert [f.nullable for f in data.arrow_schema()] == [False, False] + assert data.content_hash() == sync_data.content_hash() + assert data.source_info() == sync_data.source_info() + # Pre-existing provenance defects, not type re-inference: out of scope for # ITL-837 and tracked elsewhere. Strict, so a fix flags them. diff --git a/tests/test_core/operators/test_drop_nulls.py b/tests/test_core/operators/test_drop_nulls.py index cd0396de4..654f4be06 100644 --- a/tests/test_core/operators/test_drop_nulls.py +++ b/tests/test_core/operators/test_drop_nulls.py @@ -9,6 +9,7 @@ from __future__ import annotations import math +import typing from collections.abc import Sequence from pathlib import Path from typing import Any, TypeAlias @@ -19,6 +20,7 @@ from orcapod import function_pod from orcapod.core.sources import ArrowTableSource, DictSource +from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.operators import DropNulls, Join from orcapod.protocols.core_protocols import StreamProtocol @@ -573,3 +575,253 @@ def test_join_after_the_operator() -> None: assert joined.output_schema()[1]["x"] is int assert sorted(_ids(joined)) == [1, 3, 4] assert joined.as_table().schema.field("x").nullable is False + + +# ── no argument: every tag and data column ────────────────────────────────── + + +def _hidden(table: pa.Table) -> list[str]: + """Return the system-tag and source-info column names of ``table``.""" + return [ + n + for n in table.column_names + if n.startswith((constants.SYSTEM_TAG_PREFIX, constants.SOURCE_PREFIX)) + ] + + +def test_no_argument_drops_rows_with_a_null_in_any_column() -> None: + """Without names, a row survives only if every tag and data column has a value.""" + result = DropNulls()(readings()) + assert result.as_table().to_pylist() == [{"id": 1, "x": 10, "y": 1.5, "note": "a"}] + + +def test_no_argument_narrows_every_tag_and_data_column() -> None: + """Every tag and data column is ``T`` afterwards, in the schema and in Arrow.""" + source = keyed() + assert source.output_schema() == ( + {"site": str | None, "id": int}, + {"x": int | None}, + ) + result = DropNulls()(source) + assert _ids(result) == [1] + assert result.keys() == source.keys() + assert result.output_schema() == ({"site": str, "id": int}, {"x": int}) + schema = result.as_table().schema + assert [schema.field(n).nullable for n in ("site", "id", "x")] == [False] * 3 + + +def test_no_argument_equals_naming_every_column() -> None: + """``DropNulls()`` and ``DropNulls([every column])`` give the same rows and schema. + + The operators themselves are different identities, and so are their outputs' + stream hashes. + """ + source = keyed() + implicit = DropNulls() + named = DropNulls(["site", "id", "x"]) + by_all, by_name = implicit(source), named(source) + assert by_all.as_table(columns=ALL) == by_name.as_table(columns=ALL) + assert by_all.as_table(columns=ALL).schema == by_name.as_table(columns=ALL).schema + assert by_all.output_schema() == by_name.output_schema() + for (tag, data), (ref_tag, ref_data) in zip( + by_all.iter_data(), by_name.iter_data(), strict=True + ): + assert tag.content_hash() == ref_tag.content_hash() + assert data.content_hash() == ref_data.content_hash() + assert tag.system_tags() == ref_tag.system_tags() + + assert implicit.content_hash() != named.content_hash() + assert implicit.pipeline_hash() != named.pipeline_hash() + assert by_all.content_hash() != by_name.content_hash() + assert by_all.pipeline_hash() != by_name.pipeline_hash() + + +def test_no_argument_identity_is_one_fixed_value() -> None: + """The no-argument operator stores no names: its identity is the same on any stream.""" + op = DropNulls() + assert op.columns is None + before = (op.content_hash(), op.pipeline_hash()) + assert _ids(op(readings())) == [1] + assert _ids(op(keyed())) == [1] + assert op.columns is None + assert (op.content_hash(), op.pipeline_hash()) == before + assert DropNulls(None).content_hash() == op.content_hash() + assert DropNulls().content_hash() == op.content_hash() + for names in ("x", ["id", "x", "y", "note"], ["site", "id", "x"]): + assert DropNulls(names).content_hash() != op.content_hash() + + +@pytest.mark.parametrize("empty", [[], (), set(), frozenset()]) +def test_an_empty_collection_is_still_an_error(empty: Any) -> None: + """An empty collection is not "no argument": it is refused, never read as "all".""" + with pytest.raises(ValueError, match="at least one"): + DropNulls(empty) + with pytest.raises(ValueError, match="at least one"): + readings().drop_nulls(empty) + + +def test_no_argument_leaves_system_and_source_columns_alone() -> None: + """Only user tag and data columns are tested and narrowed. + + Row 1's source-info value for ``x`` is null: the row still survives, and the + system-tag and source-info fields keep their types, flags and values. + """ + source = readings() + before = source.as_table(columns=ALL) + hidden = _hidden(before) + assert len(hidden) >= 2 + source_x = f"{constants.SOURCE_PREFIX}x" + position = before.schema.get_field_index(source_x) + nullable_token = before.schema.field(position).with_nullable(True) + stream = ArrowTableStream( + before.set_column( + position, + nullable_token, + pa.array([None, "t2", "t3", "t4"], nullable_token.type), + ), + tag_columns=["id"], + ) + given = stream.as_table(columns=ALL) + assert given.schema.field(source_x).nullable is True + result = DropNulls()(stream) + after = result.as_table(columns=ALL) + assert _ids(result) == [1] + assert after.column(source_x).to_pylist() == [None] + assert after.column_names == given.column_names + for name in _hidden(given): + assert after.schema.field(name) == given.schema.field(name) + assert after.column(name).to_pylist() == given.column(name).to_pylist()[:1] + + +def test_no_argument_schema_prediction_matches_the_computed_stream() -> None: + """The predicted schema equals the computed stream's, with and without system tags.""" + op = DropNulls() + for source in (readings(), keyed()): + for config in ( + {}, + {"system_tags": True}, + {"source": True, "system_tags": True}, + ): + # Non-public on purpose: a prediction is verified against the static result. + computed = op.static_process(source).output_schema(columns=config) + assert op(source).output_schema(columns=config) == computed + assert op.output_schema(source, columns=config) == computed + assert all( + type(None) not in typing.get_args(python_type) + for name, python_type in {**computed[0], **computed[1]}.items() + if not name.startswith(constants.SYSTEM_TAG_PREFIX) + ) + + +def test_no_argument_zero_surviving_rows_keep_the_narrowed_types() -> None: + """A result with no rows has the schema of a non-empty result.""" + empty = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([None, 7], pa.int8()), + "s": pa.array(["a", None], pa.string()), + }, + nullable=["id", "x", "s"], + ) + non_empty = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([5, 7], pa.int8()), + "s": pa.array(["a", None], pa.string()), + }, + nullable=["id", "x", "s"], + ) + result = DropNulls()(empty) + table = result.as_table(columns=ALL) + assert table.num_rows == 0 + assert list(result.iter_data()) == [] + assert result.output_schema() == ({"id": int}, {"x": int, "s": str}) + assert table.schema.field("id") == pa.field("id", pa.int16(), nullable=False) + assert table.schema.field("x") == pa.field("x", pa.int8(), nullable=False) + assert table.schema.field("s") == pa.field("s", pa.large_string(), nullable=False) + assert table.schema == DropNulls()(non_empty).as_table(columns=ALL).schema + + +def test_no_argument_on_a_non_nullable_stream_is_a_no_op() -> None: + """A stream without a nullable column is unchanged, and twice equals once.""" + source = _source( + { + "id": pa.array([1, 2], pa.int16()), + "x": pa.array([10, 20], pa.int8()), + } + ) + result = DropNulls()(source) + assert result.output_schema() == source.output_schema() + assert result.as_table(columns=ALL) == source.as_table(columns=ALL) + for (tag, data), (in_tag, in_data) in zip( + result.iter_data(), source.iter_data(), strict=True + ): + assert tag.content_hash() == in_tag.content_hash() + assert data.content_hash() == in_data.content_hash() + + once = DropNulls()(readings()) + twice = DropNulls()(once) + assert twice.as_table(columns=ALL) == once.as_table(columns=ALL) + assert twice.output_schema() == once.output_schema() + + +def test_no_argument_on_a_stream_without_tag_columns() -> None: + """A stream with no tag columns: every data column is tested and narrowed.""" + source = _source( + { + "x": pa.array([1, None, 3], pa.int8()), + "s": pa.array(["a", "b", None], pa.string()), + }, + tags=(), + nullable=["x", "s"], + ) + result = DropNulls()(source) + assert result.as_table().to_pylist() == [{"x": 1, "s": "a"}] + assert result.output_schema() == ({}, {"x": int, "s": str}) + + +def test_no_argument_stream_method_equals_the_operator() -> None: + """``stream.drop_nulls()`` gives the no-argument operator's rows and hashes.""" + source = readings() + by_method = source.drop_nulls() + by_operator = DropNulls()(source) + assert by_method.as_table(columns=ALL) == by_operator.as_table(columns=ALL) + assert by_method.content_hash() == by_operator.content_hash() + assert by_method.pipeline_hash() == by_operator.pipeline_hash() + assert _ids(source.drop_nulls(label="complete")) == [1] + + +def test_no_argument_does_not_change_the_input_stream() -> None: + """Running the no-argument operator leaves the input stream as it was.""" + source = readings() + schema = source.as_table(columns=ALL).schema + rows = source.as_table(columns=ALL).to_pylist() + DropNulls()(source).as_table() + assert source.as_table(columns=ALL).schema == schema + assert source.as_table(columns=ALL).to_pylist() == rows + + +def test_no_argument_leaves_meta_columns_alone() -> None: + """A meta (``__``) column is not a data column for "every column". + + A stream built straight from a table lists its meta columns among its data + keys (ITL-846); a null there still drops no row and the field stays nullable. + """ + meta = f"{constants.META_PREFIX}note" + table = readings().as_table(columns=ALL) + table = table.append_column( + pa.field(meta, pa.large_string(), nullable=True), + pa.array([None, "m2", "m3", "m4"], pa.large_string()), + ) + stream = ArrowTableStream(table, tag_columns=["id"]) + result = DropNulls()(stream) + rows = list(result.iter_data()) + assert [tag["id"] for tag, _ in rows] == [1] + data = rows[0][1] + assert data.as_dict() == {"x": 10, "y": 1.5, "note": "a"} + meta_field = data.arrow_schema(columns={"meta": True}).field(meta) + assert meta_field == pa.field(meta, pa.large_string(), nullable=True) + assert data.as_table(columns={"meta": True}).column(meta).to_pylist() == [None] + # Non-public on purpose: a prediction is verified against the static result. + computed = DropNulls().static_process(stream).output_schema() + assert DropNulls().output_schema(stream) == computed diff --git a/tests/test_core/operators/test_operator_config.py b/tests/test_core/operators/test_operator_config.py index df8902030..7df983b33 100644 --- a/tests/test_core/operators/test_operator_config.py +++ b/tests/test_core/operators/test_operator_config.py @@ -188,3 +188,20 @@ def test_registered_for_pipeline_loading(self): restored = resolve_operator_from_config(op.to_config()) assert isinstance(restored, DropNulls) assert restored.content_hash() == op.content_hash() + + def test_no_argument_round_trip(self): + """The no-argument form serializes ``columns`` as ``None`` and round-trips.""" + from orcapod.pipeline.serialization import resolve_operator_from_config + + op = DropNulls() + config = op.to_config() + assert config["config"] == {"columns": None} + for restored in ( + DropNulls.from_config(config), + resolve_operator_from_config(config), + ): + assert isinstance(restored, DropNulls) + assert restored.columns is None + assert restored.content_hash() == op.content_hash() + assert restored.pipeline_hash() == op.pipeline_hash() + assert op.content_hash() != DropNulls(["x", "y"]).content_hash() diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 0dba31082..b671d99c2 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -988,6 +988,19 @@ def pipeline_drop_nulls() -> dict[str, Any]: return out +@case("pipeline", "drop_nulls_all") +def pipeline_drop_nulls_all() -> dict[str, Any]: + """Pin the no-argument ``DropNulls``: its fixed identity and the narrowed rows.""" + from orcapod.operators import DropNulls + + out: dict[str, Any] = {} + out["operator[all].content_hash"] = DropNulls().content_hash().to_string() + out["operator[all].pipeline_hash"] = DropNulls().pipeline_hash().to_string() + out.update(_prefixed("all", stream_summary(DropNulls()(_nullable_readings())))) + out.update(_prefixed("method", stream_summary(_nullable_readings().drop_nulls()))) + return out + + @case("pipeline", "drop_nulls_job") def pipeline_drop_nulls_job() -> dict[str, Any]: """Record a job with ``DropNulls`` upstream of a pod with non-optional parameters.""" diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index 7b926fd9d..4e9642eed 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -120,6 +120,26 @@ "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" ], + "drop_nulls_all.all.content_hash": "semantic_v0.1:91519c5bfe375ac6ab2979f02c3962f95782163690566a3cd4354e46aaab1bab", + "drop_nulls_all.all.pipeline_hash": "semantic_v0.1:47eff5b311b99e1ba76853aa1a681ff43c0b6be0b54c5b4cf4a8ceea07ff449a", + "drop_nulls_all.all.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_all.all.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_all.all.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls_all.all.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_all.method.content_hash": "semantic_v0.1:91519c5bfe375ac6ab2979f02c3962f95782163690566a3cd4354e46aaab1bab", + "drop_nulls_all.method.pipeline_hash": "semantic_v0.1:47eff5b311b99e1ba76853aa1a681ff43c0b6be0b54c5b4cf4a8ceea07ff449a", + "drop_nulls_all.method.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21", + "drop_nulls_all.method.row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "golden_nullable", + "drop_nulls_all.method.row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v0.1:0000016046daed9824fbd1c65ca52abc266f9228cf63dbf5b1ebda0c6be32e0fe71d93", + "drop_nulls_all.method.system_tag_columns": [ + "_tag_source_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", + "_tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4" + ], + "drop_nulls_all.operator[all].content_hash": "semantic_v0.1:171e408bad76d85d0f84ab7ad461af9eaf131e34b6c8f8c482cd8bb1a141ce9a", + "drop_nulls_all.operator[all].pipeline_hash": "semantic_v0.1:171e408bad76d85d0f84ab7ad461af9eaf131e34b6c8f8c482cd8bb1a141ce9a", "drop_nulls_job.node[golden_nullable].content_hash": "semantic_v0.1:4c5d737323c9ee8bb4e85cd222fb3cd8c9df73f33386b58405bf324c2ab91993", "drop_nulls_job.node[golden_nullable].pipeline_hash": "semantic_v0.1:945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4", "drop_nulls_job.node[golden_nullable].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::945dba2cc44de551a809f36add80dc743194c3bb621fb0263debf3eda46368f4": "1224ec4c966e5248830dc52577126e21",