From 79a488f71c5cd8622ea8e4c717a761345df7c88f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 17:04:01 -0800 Subject: [PATCH 1/4] feat(operators): add DropColumns and SelectColumns, stream drop_columns and select_columns (WIP) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/__init__.py | 4 + .../core/operators/column_selection.py | 257 ++++++++++++++ src/orcapod/core/streams/base.py | 110 ++++++ src/orcapod/pipeline/serialization.py | 4 + .../protocols/core_protocols/streams.py | 26 ++ ...2026-11-13-itl-1084-drop-select-columns.md | 72 ++++ .../operators/test_tag_and_data_columns.py | 335 ++++++++++++++++++ 7 files changed, 808 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-1084-drop-select-columns.md create mode 100644 tests/test_core/operators/test_tag_and_data_columns.py diff --git a/src/orcapod/core/operators/__init__.py b/src/orcapod/core/operators/__init__.py index c80f6b21f..bdaa8adfa 100644 --- a/src/orcapod/core/operators/__init__.py +++ b/src/orcapod/core/operators/__init__.py @@ -1,7 +1,9 @@ from .batch import Batch from .column_selection import ( + DropColumns, DropDataColumns, DropTagColumns, + SelectColumns, SelectDataColumns, SelectTagColumns, ) @@ -27,6 +29,8 @@ "SelectDataColumns", "DropTagColumns", "DropDataColumns", + "SelectColumns", + "DropColumns", "PolarsFilter", "DropNulls", "Pick", diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index f3ed67c56..5348fe5b2 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -1,4 +1,5 @@ import logging +from abc import abstractmethod from collections.abc import Collection, Mapping, Sequence from typing import TYPE_CHECKING, Any @@ -540,6 +541,262 @@ def identity_structure(self) -> Any: ) +def _require_a_data_column(remaining_data_columns: Collection[str]) -> None: + """Raise unless at least one data column remains. + + A stream needs at least one data column (``ArrowTableStream`` refuses a table + without one), so a column operator whose result would have none is refused + when it is called, before any row is built. + + Args: + remaining_data_columns: The data columns the result would keep. + + Raises: + InputValidationError: When ``remaining_data_columns`` is empty. + """ + if not remaining_data_columns: + raise InputValidationError( + "no data column would remain: a stream needs at least one data column" + ) + + +def _is_meta(name: str) -> bool: + """Whether ``name`` is a meta column or a raw stream's source-info column for one.""" + return name.startswith( + (constants.META_PREFIX, f"{constants.SOURCE_PREFIX}{constants.META_PREFIX}") + ) + + +class _TagAndDataColumnsOperator(UnaryOperator): + """Base of the operators that act on named columns, tag or data alike (ITL-1084). + + A subclass says only which of a group's columns it removes (``_removed``); + everything else is shared. For names of one group the rows equal those of the + tag-only or data-only operator: system tags pass through, the source-info + column of a removed data column goes with it, and the stream's column order is + kept. Unlike those operators, no meta (``__``) column of the input is carried, + on either path, as ``Join`` does; and a result without a data column is refused + when the operator is called, on both paths. + """ + + _verb: str = "" + + def __init__(self, columns: str | Collection[str], strict: bool = True, **kwargs): + self.columns = _canonical_columns(columns) + self.strict = strict + super().__init__(**kwargs) + + @abstractmethod + def _removed(self, columns: Collection[str]) -> list[str]: + """Return the names among ``columns`` (one group, in order) this operator removes.""" + + def to_config(self) -> dict[str, Any]: + """Serialize this operator to a config dict. + + Returns: + A dict with ``class_name``, ``module_path``, and ``config`` keys, + where ``config`` contains ``columns`` and ``strict``. + """ + config = super().to_config() + config["config"] = { + "columns": list(self.columns), + "strict": self.strict, + } + return config + + def identity_structure(self) -> Any: + return ( + self.__class__.__name__, + list(self.columns), + self.strict, + ) + + def _removals( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> tuple[list[str], list[str]]: + """Return the tag columns and the user data columns this operator removes. + + Args: + tag_columns: The input's tag columns, in order. + data_columns: The input's user data columns, in order (no meta column). + """ + return self._removed(tag_columns), self._removed(data_columns) + + def _validate_columns( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> None: + """Check the names against the input's columns and the result's data columns. + + Args: + tag_columns: The input's tag columns. + data_columns: The input's user data columns (no meta column). + + Raises: + InputValidationError: When ``strict`` and a name is neither a tag nor a + data column, or when the result would have no data column. + """ + missing = set(self.columns) - set(tag_columns) - set(data_columns) + if missing and self.strict: + raise InputValidationError( + f"Missing columns: {sorted(missing)}. Make sure all specified columns " + f"to {self._verb} are tag or data columns of the stream, or use " + f"strict=False to ignore missing columns" + ) + _, data_removed = self._removals(tag_columns, data_columns) + try: + _require_a_data_column([c for c in data_columns if c not in data_removed]) + except InputValidationError as error: + raise InputValidationError( + f"{self.__class__.__name__}({list(self.columns)}) would leave no data " + f"column: {error} (it removes data columns {data_removed})" + ) from error + + @staticmethod + def _user_columns(stream: StreamProtocol) -> tuple[tuple[str, ...], list[str]]: + """The stream's tag columns and its data columns without meta columns. + + A raw ``ArrowTableStream`` lists its meta (``__``) columns as data columns. + """ + tag_columns, data_columns = stream.keys() + return tag_columns, [c for c in data_columns if not _is_meta(c)] + + def validate_unary_input(self, stream: StreamProtocol) -> None: + """Validate the names against the input stream. + + Raises: + InputValidationError: When ``strict`` and a name is neither a tag nor a + data column, or when the result would have no data column. + """ + self._validate_columns(*self._user_columns(stream)) + + def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: + tag_columns, data_columns = self._user_columns(stream) + tags_removed, data_removed = self._removals(tag_columns, data_columns) + table = stream.as_table( + columns={"source": True, "system_tags": True, "sort_by_tags": False} + ) + removed = { + *tags_removed, + *data_removed, + *(f"{constants.SOURCE_PREFIX}{c}" for c in data_removed), + } + return ArrowTableStream( + table.drop_columns( + [c for c in table.column_names if c in removed or _is_meta(c)] + ), + tag_columns=[c for c in tag_columns if c not in tags_removed], + data_context=stream.data_context_key, + ) + + def unary_output_schema( + self, + stream: StreamProtocol, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + tag_schema, data_schema = stream.output_schema( + columns=columns, all_info=all_info + ) + tags_removed, data_removed = self._removals(*self._user_columns(stream)) + # System tag columns are not among the removed names, so they are kept. + return ( + Schema({k: v for k, v in tag_schema.items() if k not in tags_removed}), + Schema( + { + k: v + for k, v in data_schema.items() + if k not in data_removed and not _is_meta(k) + } + ), + ) + + async def async_execute( + self, + inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + *, + input_pipeline_hashes: Sequence[ContentHash], + ) -> None: + """Streaming: remove the columns per row without materializing. + + The names are validated against the first row, as ``validate_unary_input`` + validates them against the stream. + + Args: + inputs: The one readable input channel. + output: Writable channel for downstream consumption. + input_pipeline_hashes: Not used: this operator keeps system-tag + names as they are, so no column is named after an input's + pipeline hash. + """ + try: + removals: tuple[list[str], list[str]] | None = None + async for tag, data in inputs[0]: + if removals is None: + tag_keys, data_keys = tag.keys(), data.keys() + self._validate_columns(tag_keys, data_keys) + removals = self._removals(tag_keys, data_keys) + tags_removed, data_removed = removals + if tags_removed: + tag = tag.drop(*tags_removed) + if data_removed: + data = data.drop(*data_removed) + if data.meta_columns: + data = data.drop_meta_columns(*data.meta_columns) + await output.send((tag, data)) + finally: + await output.close() + + +class DropColumns(_TagAndDataColumnsOperator): + """Drop the named columns, whether each is a tag or a data column (ITL-1084). + + For tag names only it gives the rows of ``DropTagColumns``, for data names only + those of ``DropDataColumns``. A dropped data column's source-info column goes + with it; system tags are kept. + + Args: + columns: One column name or a collection of names. + strict: When ``True`` (the default), a name that is neither a tag nor a data + column of the input raises ``InputValidationError``; when ``False`` it + is ignored. + + Raises: + InputValidationError: When called on a stream whose data columns would all + be dropped: a stream needs at least one data column. + """ + + _verb = "drop" + + def _removed(self, columns: Collection[str]) -> list[str]: + return [c for c in columns if c in self.columns] + + +class SelectColumns(_TagAndDataColumnsOperator): + """Keep exactly the named columns, tag or data; drop every other one (ITL-1084). + + Every user tag and data column not named is dropped, with the source-info column + of each dropped data column; system tags are kept. Naming no tag is allowed (the + result has no tag column). + + Args: + columns: One column name or a collection of names. + strict: When ``True`` (the default), a name that is neither a tag nor a data + column of the input raises ``InputValidationError``; when ``False`` it + is ignored. + + Raises: + InputValidationError: When called on a stream none of whose data columns is + named: a stream needs at least one data column. + """ + + _verb = "select" + + def _removed(self, columns: Collection[str]) -> list[str]: + return [c for c in columns if c not in self.columns] + + class MapTags(UnaryOperator): """ Operator that maps tags in a stream using a user-defined function. diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index eb1728f65..a5a81af4d 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -67,6 +67,47 @@ def check_head_size(n: int) -> None: raise ValueError(f"head() takes a non-negative number of rows n, got {n}") +def _column_names(columns: tuple[str | Collection[str], ...]) -> tuple[str, ...]: + """Normalize the positional names of ``drop_columns`` and ``select_columns``. + + The names are given as one or more strings, or as one collection of strings + that is the only positional argument. + + Args: + columns: The positional arguments as received. + + Returns: + The names, in the order given. + + Raises: + TypeError: When no name is given, a collection is mixed with further + positional arguments, or a name is not a string. + ValueError: When the one collection given is empty. + """ + if not columns: + raise TypeError("expected at least one column name") + first = columns[0] + if isinstance(first, str): + names: tuple[object, ...] = columns + elif len(columns) > 1: + raise TypeError( + "give the column names either as strings or as one collection of " + "strings, not both" + ) + elif isinstance(first, Collection): + names = tuple(first) + if not names: + raise ValueError( + "expected at least one column name, got an empty collection" + ) + else: + raise TypeError(f"expected column names as strings, got {type(first).__name__}") + not_strings = [name for name in names if not isinstance(name, str)] + if not_strings: + raise TypeError(f"column names must be strings, got {not_strings!r}") + return tuple(str(name) for name in names) + + class StreamBase(TraceableBase): @property @abstractmethod @@ -298,6 +339,75 @@ def drop_data_columns( return DropDataColumns(data_columns, strict=strict)(self, label=label) + def drop_columns( + self, + *columns: str | Collection[str], + strict: bool = True, + label: str | None = None, + ) -> StreamBase: + """Drop the named columns, whether each is a tag or a data column. + + ``s.drop_columns("a", "b")`` equals ``s.drop_columns(["a", "b"])``. A + dropped data column's source-info column goes with it; system tags are + kept. See ``orcapod.core.operators.DropColumns``. + + Args: + *columns: One or more column names, or one collection of names as the + only positional argument. + strict: When ``True`` (the default), a name that is neither a tag nor + a data column raises ``InputValidationError``; when ``False`` it + is ignored. + label: Optional node label for the pipeline graph. + + Returns: + The stream without the named columns. + + Raises: + TypeError: When no name is given, a collection is mixed with further + positional arguments, or a name is not a string. + ValueError: When the one collection given is empty. + InputValidationError: When a ``strict`` name is missing, or when no + data column would remain. + """ + from orcapod.core.operators import DropColumns + + return DropColumns(_column_names(columns), strict=strict)(self, label=label) + + def select_columns( + self, + *columns: str | Collection[str], + strict: bool = True, + label: str | None = None, + ) -> StreamBase: + """Keep exactly the named columns, tag or data; drop every other one. + + ``s.select_columns("a", "b")`` equals ``s.select_columns(["a", "b"])``. + Every user tag and data column not named is dropped, with the source-info + column of each dropped data column; system tags are kept. Naming no tag + is allowed. See ``orcapod.core.operators.SelectColumns``. + + Args: + *columns: One or more column names, or one collection of names as the + only positional argument. + strict: When ``True`` (the default), a name that is neither a tag nor + a data column raises ``InputValidationError``; when ``False`` it + is ignored. + label: Optional node label for the pipeline graph. + + Returns: + The stream with only the named columns. + + Raises: + TypeError: When no name is given, a collection is mixed with further + positional arguments, or a name is not a string. + ValueError: When the one collection given is empty. + InputValidationError: When a ``strict`` name is missing, or when no + data column is named. + """ + from orcapod.core.operators import SelectColumns + + return SelectColumns(_column_names(columns), strict=strict)(self, label=label) + def pick( self, column: str, diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 567ac0c0a..a7146b066 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -181,6 +181,7 @@ def _build_operator_registry() -> dict[str, type]: """ from orcapod.core.operators import ( Batch, + DropColumns, DropDataColumns, DropNulls, DropTagColumns, @@ -190,6 +191,7 @@ def _build_operator_registry() -> dict[str, type]: MapTags, MergeJoin, PolarsFilter, + SelectColumns, SelectDataColumns, SelectTagColumns, SemiJoin, @@ -205,6 +207,8 @@ def _build_operator_registry() -> dict[str, type]: "DropTagColumns": DropTagColumns, "SelectDataColumns": SelectDataColumns, "DropDataColumns": DropDataColumns, + "SelectColumns": SelectColumns, + "DropColumns": DropColumns, "MapTags": MapTags, "MapData": MapData, "PolarsFilter": PolarsFilter, diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index 1ae16dab4..4a7930df4 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -405,6 +405,32 @@ def drop_data_columns( """ ... + def drop_columns( + self, + *columns: str | Collection[str], + strict: bool = True, + label: str | None = None, + ) -> "StreamProtocol": + """Drop the named columns, whether each is a tag or a data column. + + Names are given as one or more strings, or as one collection of strings + as the only positional argument. + """ + ... + + def select_columns( + self, + *columns: str | Collection[str], + strict: bool = True, + label: str | None = None, + ) -> "StreamProtocol": + """Keep exactly the named columns, tag or data; drop every other one. + + Names are given as one or more strings, or as one collection of strings + as the only positional argument. + """ + ... + def batch( self, batch_size: int = 0, diff --git a/superpowers/plans/2026-11-13-itl-1084-drop-select-columns.md b/superpowers/plans/2026-11-13-itl-1084-drop-select-columns.md new file mode 100644 index 000000000..db34e71ce --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1084-drop-select-columns.md @@ -0,0 +1,72 @@ +# ITL-1084 drop_columns / select_columns Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `DropColumns` / `SelectColumns` operators and `StreamBase.drop_columns` / `select_columns` +methods that act on tag and data columns in one call, changing nothing that exists. + +**Architecture:** One private base class `_TagAndDataColumnsOperator(UnaryOperator)` in +`core/operators/column_selection.py` holds construction (`_canonical_columns`), config, identity, +validation, the sync table rebuild, the schema prediction and the async path; the two subclasses +only say which names of a column group they remove (`_removed`). One module helper +`_require_a_data_column(remaining)` holds the "a stream needs a data column" rule (coordinator: +ITL-1080 will apply it to the existing four). The stream methods normalize their arguments with +one private helper `_column_names(...)` in `core/streams/base.py`. + +**Tech Stack:** Python, PyArrow, pytest via `uv run`. + +## Decisions (coordinator, 2026-11-13) + +- Edge rule 3, option A: when the result would have no data column, both new operators raise + `InputValidationError` early: at `op(stream)` (`validate_unary_input`) and on the first row in + `async_execute`. Today's existing four raise late (`ValueError` at iteration) on sync and stay + silent on async; they are NOT changed here. +- Zero tags after `select_columns` is allowed (a tagless stream iterates fine today). +- Meta (`__`) columns are not carried, on either path (matches ITL-1080, which removes + `Join._without_meta`, so this branch keeps its own small helper; after ITL-1080 merges it can + be replaced by `meta_columns.without_meta_columns` / `without_data_meta`). + +## Behaviour matched from the existing four + +- Sync: `stream.as_table(columns={"source": True, "system_tags": True, "sort_by_tags": False})`, + drop the removed tag columns, the removed data columns and their `_source_` columns, + rebuild `ArrowTableStream(table, tag_columns=remaining tags)`. System tags pass through. +- Strict: a name that is neither a tag nor a data column raises `InputValidationError` + ("Missing columns: ... use strict=False ..."); non-strict ignores it. +- Identity: `(class name, sorted unique names list, strict)`; config `{"columns", "strict"}`. +- Async: own `async_execute`, per-row `tag.drop` / `data.drop`. + +## Files + +- Modify: `src/orcapod/core/operators/column_selection.py` (base class, two operators, helpers) +- Modify: `src/orcapod/core/operators/__init__.py`, `src/orcapod/pipeline/serialization.py` +- Modify: `src/orcapod/core/streams/base.py` (`_column_names`, two methods) +- Modify: `src/orcapod/protocols/core_protocols/streams.py` (two protocol methods) +- Create: `tests/test_core/operators/test_tag_and_data_columns.py` +- Modify: `tests/test_core/operators/test_async_sync_equivalence.py` (CASES), + `tests/test_core/operators/test_operator_config.py`, `tests/test_hashing/golden/cases.py` + (new case `pipeline/tag_and_data_columns`), golden `pipeline.json` (additions only) +- Docs: `docs/concepts/operators.md`, `docs/api/operators.md`, `docs/reference/hashing.md`, + `CLAUDE.md`, `.zed/rules`, `changelog.d/ITL-1084.md` + +## Tasks + +- [ ] 1. Write `tests/test_core/operators/test_tag_and_data_columns.py` (public interface): + argument forms (`'a','b'` == `['a','b']`), rejections (no name, mix, non-string, empty + collection), drop tag+data, select tag+data, rows equal DropTagColumns / DropDataColumns / + SelectTagColumns / SelectDataColumns for single-group names, strict/non-strict, no data column + left raises `InputValidationError` at call time, zero tags allowed, identity spellings, + prediction vs `unary_static_process(...).output_schema(...)`, source info of kept columns, + meta not carried (function-pod input). Run: fails (ImportError / AttributeError). +- [ ] 2. Implement operators + stream methods + exports + registry. Run the new test file: pass. +- [ ] 3. Add CASES to async equivalence; config round-trip; run those files. +- [ ] 4. Golden: add case, regenerate, `git diff --stat` must show additions only; run golden test. +- [ ] 5. Docs, CLAUDE.md, .zed/rules, changelog fragment; run doc sample test for operators.md, + `assemble_changelog.py --check`. +- [ ] 6. ruff format/check, basedpyright exit code; one pytest process on the touched test files. +- [ ] 7. Merge origin/main, push, PR ("Fixes ITL-1084", golden diffstat), Copilot reviewer, + orca linear attach / worktree set, answer threads, CI green on 3.11/3.12/3.13, worker_done. + +## Notes for a successor + +(update as work proceeds) diff --git a/tests/test_core/operators/test_tag_and_data_columns.py b/tests/test_core/operators/test_tag_and_data_columns.py new file mode 100644 index 000000000..1d7d94989 --- /dev/null +++ b/tests/test_core/operators/test_tag_and_data_columns.py @@ -0,0 +1,335 @@ +"""``drop_columns`` / ``select_columns``: one call across tag and data columns (ITL-1084). + +``DropColumns`` and ``SelectColumns`` act on named columns whether each is a tag or a +data column. For names of one group they give the rows of ``DropTagColumns``, +``DropDataColumns``, ``SelectTagColumns`` and ``SelectDataColumns``; their identities +differ by design. Tests drive the operators through ``op(stream)`` and the stream +methods, and read rows through the stream's public methods. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod.core.operators import ( + DropColumns, + DropDataColumns, + DropTagColumns, + SelectColumns, + SelectDataColumns, + SelectTagColumns, +) +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.sources import DictSource +from orcapod.system_constants import constants +from tests.test_core.operators.test_async_sync_equivalence import ( + async_rows, + cached_pod_stream, + double_a, + meta_stream, + sync_rows, +) + +META = constants.META_PREFIX + + +def _source() -> DictSource: + """Tags ``cohort``, ``subject``; data ``age``, ``weight``, ``name``.""" + return DictSource( + data=[ + {"cohort": "a", "subject": "m1", "age": 12, "weight": 25.3, "name": "x"}, + {"cohort": "a", "subject": "m2", "age": 8, "weight": 22.1, "name": "y"}, + {"cohort": "b", "subject": "m3", "age": 15, "weight": 27.8, "name": "z"}, + ], + tag_columns=["cohort", "subject"], + source_id="subjects", + ) + + +def _rows(stream: StreamProtocol) -> list[tuple[dict[str, Any], dict[str, Any]]]: + return [(tag.as_dict(), data.as_dict()) for tag, data in stream.iter_data()] + + +def _table(stream: StreamProtocol) -> pa.Table: + return stream.as_table(all_info=True) + + +# ── the stream methods' arguments ─────────────────────────────────────────── + + +@pytest.mark.parametrize("method", ["drop_columns", "select_columns"]) +def test_positional_names_equal_one_collection(method: str) -> None: + source = _source() + by_position = getattr(source, method)("subject", "age") + by_collection = getattr(source, method)(["subject", "age"]) + assert by_position.keys() == by_collection.keys() + assert _rows(by_position) == _rows(by_collection) + assert by_position.content_hash() == by_collection.content_hash() + assert by_position.pipeline_hash() == by_collection.pipeline_hash() + + +@pytest.mark.parametrize("method", ["drop_columns", "select_columns"]) +def test_one_positional_name(method: str) -> None: + source = _source() + assert ( + getattr(source, method)("age").pipeline_hash() + == getattr(source, method)(["age"]).pipeline_hash() + ) + + +@pytest.mark.parametrize("method", ["drop_columns", "select_columns"]) +@pytest.mark.parametrize( + ("args", "error"), + [ + ((), TypeError), + ((["age"], "weight"), TypeError), + (("age", ["weight"]), TypeError), + (("age", 3), TypeError), + (([("age",)],), TypeError), + ((3,), TypeError), + (([],), ValueError), + ], + ids=[ + "no_name", + "collection_then_name", + "name_then_collection", + "non_string_positional", + "non_string_in_collection", + "not_a_name", + "empty_collection", + ], +) +def test_bad_arguments_are_rejected( + method: str, args: tuple[Any, ...], error: type[Exception] +) -> None: + with pytest.raises(error): + getattr(_source(), method)(*args) + + +def test_strict_and_label_are_keyword_only() -> None: + stream = _source().drop_columns("age", "nope", strict=False, label="dropped") + assert stream.keys() == (("cohort", "subject"), ("weight", "name")) + assert stream.label == "dropped" + + +# ── what is kept ──────────────────────────────────────────────────────────── + + +def test_drop_columns_drops_tags_and_data() -> None: + stream = _source().drop_columns("subject", "age") + assert stream.keys() == (("cohort",), ("weight", "name")) + assert _rows(stream)[0] == ({"cohort": "a"}, {"weight": 25.3, "name": "x"}) + columns = _table(stream).column_names + assert f"{constants.SOURCE_PREFIX}age" not in columns + assert f"{constants.SOURCE_PREFIX}weight" in columns + assert any(name.startswith(constants.SYSTEM_TAG_PREFIX) for name in columns) + + +def test_select_columns_keeps_only_the_named_columns() -> None: + stream = _source().select_columns("weight", "cohort") + # The stream's column order, whatever order the names were given in. + assert stream.keys() == (("cohort",), ("weight",)) + assert _rows(stream)[2] == ({"cohort": "b"}, {"weight": 27.8}) + columns = _table(stream).column_names + assert f"{constants.SOURCE_PREFIX}weight" in columns + assert f"{constants.SOURCE_PREFIX}age" not in columns + assert f"{constants.SOURCE_PREFIX}name" not in columns + assert any(name.startswith(constants.SYSTEM_TAG_PREFIX) for name in columns) + + +def test_select_columns_may_keep_no_tag() -> None: + stream = _source().select_columns("age") + assert stream.keys() == ((), ("age",)) + assert [data for _, data in _rows(stream)] == [ + {"age": 12}, + {"age": 8}, + {"age": 15}, + ] + + +# ── one group: the rows of the existing operators ─────────────────────────── + + +GROUP_CASES: list[tuple[str, Callable[[], Any], Callable[[], Any]]] = [ + ("drop_tags", lambda: DropColumns(["subject"]), lambda: DropTagColumns(["subject"])), + ("drop_data", lambda: DropColumns(["age", "name"]), lambda: DropDataColumns(["age", "name"])), + ("select_tags", lambda: SelectColumns(["cohort", "age", "weight", "name"]), lambda: SelectTagColumns(["cohort"])), + ("select_data", lambda: SelectColumns(["cohort", "subject", "weight"]), lambda: SelectDataColumns(["weight"])), +] # fmt: skip + + +@pytest.mark.parametrize( + ("make_new", "make_old"), + [(new, old) for _, new, old in GROUP_CASES], + ids=[label for label, _, _ in GROUP_CASES], +) +def test_one_group_gives_the_rows_of_the_existing_operator( + make_new: Callable[[], Any], make_old: Callable[[], Any] +) -> None: + new = make_new()(_source()) + old = make_old()(_source()) + assert new.keys() == old.keys() + assert _table(new).equals(_table(old)) + for (tag, data), (old_tag, old_data) in zip(new.iter_data(), old.iter_data()): + assert tag.content_hash() == old_tag.content_hash() + assert data.content_hash() == old_data.content_hash() + assert data.source_info() == old_data.source_info() + # Identities differ by design: a new operator is a new pipeline step. + assert new.pipeline_hash() != old.pipeline_hash() + + +# ── strict ────────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("op_cls", [DropColumns, SelectColumns]) +def test_strict_rejects_an_unknown_name(op_cls: type) -> None: + with pytest.raises(InputValidationError, match="nope"): + op_cls(["age", "nope"])(_source()) + + +def test_non_strict_ignores_an_unknown_name() -> None: + assert DropColumns(["age", "nope"], strict=False)(_source()).keys() == ( + ("cohort", "subject"), + ("weight", "name"), + ) + assert SelectColumns(["age", "nope"], strict=False)(_source()).keys() == ( + (), + ("age",), + ) + + +# ── no data column left ───────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "make_op", + [ + lambda: DropColumns(["age", "weight", "name"]), + lambda: DropColumns(["subject", "age", "weight", "name"]), + lambda: SelectColumns(["cohort", "subject"]), + lambda: SelectColumns(["nope"], strict=False), + ], + ids=["drop_all_data", "drop_all_data_and_a_tag", "select_tags_only", "select_none"], +) +def test_no_data_column_left_raises_at_the_call(make_op: Callable[[], Any]) -> None: + with pytest.raises(InputValidationError, match="needs at least one data column"): + make_op()(_source()) + + +@pytest.mark.parametrize( + "make_op", + [lambda: DropColumns(["age", "weight", "name"]), lambda: SelectColumns(["cohort"])], + ids=["drop_all_data", "select_tags_only"], +) +def test_no_data_column_left_raises_on_the_async_path( + make_op: Callable[[], Any], +) -> None: + with pytest.raises(InputValidationError, match="needs at least one data column"): + async_rows(make_op(), _source()) + + +# ── identity ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("op_cls", [DropColumns, SelectColumns]) +def test_every_spelling_of_one_set_of_names_has_one_identity(op_cls: type) -> None: + hashes = { + op_cls(columns)(_source()).pipeline_hash() + for columns in ( + ["age", "subject"], + ["subject", "age"], + ("subject", "age", "age"), + {"age", "subject"}, + ) + } + assert len(hashes) == 1 + assert ( + op_cls(["age"])(_source()).pipeline_hash() + == op_cls("age")(_source()).pipeline_hash() + ) + + +@pytest.mark.parametrize("op_cls", [DropColumns, SelectColumns]) +def test_strict_enters_the_identity(op_cls: type) -> None: + assert ( + op_cls(["age"])(_source()).pipeline_hash() + != op_cls(["age"], strict=False)(_source()).pipeline_hash() + ) + + +def test_drop_and_select_of_one_name_differ() -> None: + assert ( + DropColumns(["age"])(_source()).pipeline_hash() + != SelectColumns(["age"])(_source()).pipeline_hash() + ) + + +# ── schema prediction ─────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "make_op", + [ + lambda: DropColumns(["subject", "age"]), + lambda: SelectColumns(["cohort", "weight"]), + lambda: SelectColumns(["weight"]), + lambda: DropColumns(["nope"], strict=False), + ], + ids=["drop", "select", "select_no_tag", "drop_nothing"], +) +@pytest.mark.parametrize( + "columns", + [None, {"system_tags": True}, {"source": True}], + ids=["default", "system_tags", "source"], +) +def test_prediction_equals_the_computed_schema( + make_op: Callable[[], Any], columns: dict[str, bool] | None +) -> None: + op = make_op() + source = _source() + predicted = op(source).output_schema(columns=columns) + # Verifying a schema prediction is the standing exception: compare against + # the computed stream, ``unary_static_process(...).output_schema(...)``. + computed = op.unary_static_process(source).output_schema(columns=columns) + assert predicted == computed + + +# ── meta columns are not carried ──────────────────────────────────────────── + + +@pytest.mark.parametrize( + "make_stream", + [ + lambda: meta_stream("s", "v", [f"{META}m"]), + lambda: cached_pod_stream("a", double_a), + ], + ids=["raw_meta_columns", "cached_pod"], +) +@pytest.mark.parametrize( + "make_op", + [ + lambda: DropColumns(["k"]), + lambda: SelectColumns(["k", "v", "out_a"], strict=False), + ], + ids=["drop", "select"], +) +def test_meta_columns_are_not_carried( + make_stream: Callable[[], StreamProtocol], make_op: Callable[[], Any] +) -> None: + stream = make_stream() + for rows in (sync_rows(make_op(), stream), async_rows(make_op(), stream)): + assert rows + for _, data in rows: + assert not data.meta_columns + assert not any(name.startswith(META) for name in data.keys()) + result = make_op()(stream) + assert not any(name.startswith(META) for name in result.keys()[1]) + assert not any( + name.startswith(META) + for name in result.output_schema(columns={"meta": True})[1] + ) From 31c42527997e7b9ee43f0cbfc06c11e8097bfe04 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 17:08:06 -0800 Subject: [PATCH 2/4] feat(operators): async cases, golden entries, docs and changelog for DropColumns and SelectColumns Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 2 +- CLAUDE.md | 2 +- changelog.d/ITL-1084.md | 3 ++ docs/api/operators.md | 4 +++ docs/concepts/operators.md | 33 +++++++++++++++++ docs/reference/hashing.md | 2 +- .../core/operators/column_selection.py | 16 ++++----- .../operators/test_async_sync_equivalence.py | 13 +++++++ .../test_core/operators/test_empty_inputs.py | 14 ++++++++ .../operators/test_operator_config.py | 17 +++++++++ tests/test_hashing/golden/cases.py | 23 ++++++++++++ tests/test_hashing/golden/data/pipeline.json | 36 +++++++++++++++++++ 12 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 changelog.d/ITL-1084.md diff --git a/.zed/rules b/.zed/rules index f14340ed6..6da886e96 100644 --- a/.zed/rules +++ b/.zed/rules @@ -450,7 +450,7 @@ src/orcapod/ semijoin.py — SemiJoin (binary, non-commutative) batch.py — Batch (group rows by count, types become list[T]) group_by.py — GroupBy (many→one reduction keyed on tag values) - column_selection.py — Select/Drop Tag/Data columns + column_selection.py — Select/Drop Tag/Data columns; SelectColumns/DropColumns (tag or data) mappers.py — MapTags, MapData (rename columns) filters.py — PolarsFilter drop_nulls.py — DropNulls (drop rows with a null in named or all columns, T | None → T) diff --git a/CLAUDE.md b/CLAUDE.md index 95deee60a..47af24f97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -462,7 +462,7 @@ src/orcapod/ │ ├── semijoin.py # SemiJoin (binary, non-commutative) │ ├── batch.py # Batch (group rows by count, types become list[T]) │ ├── group_by.py # GroupBy (many→one reduction keyed on tag values) -│ ├── column_selection.py # Select/Drop Tag/Data columns +│ ├── column_selection.py # Select/Drop Tag/Data columns; SelectColumns/DropColumns (tag or data) │ ├── mappers.py # MapTags, MapData (rename columns) │ ├── filters.py # PolarsFilter │ └── drop_nulls.py # DropNulls (drop rows with a null in named or all columns, T | None → T) diff --git a/changelog.d/ITL-1084.md b/changelog.d/ITL-1084.md new file mode 100644 index 000000000..931f580f9 --- /dev/null +++ b/changelog.d/ITL-1084.md @@ -0,0 +1,3 @@ +### Added + +- `stream.drop_columns(...)` and `stream.select_columns(...)`, with the operators `DropColumns` and `SelectColumns`, act on named columns whether each is a tag or a data column (ITL-1084). `drop_columns` removes the named columns; `select_columns` keeps exactly the named columns and removes every other tag and data column. The names are given as one or more strings or as one collection of strings: `stream.drop_columns("a", "b")` is `stream.drop_columns(["a", "b"])`; `strict` and `label` are keyword-only. For names of one group they give the rows of `DropTagColumns`, `DropDataColumns`, `SelectTagColumns` and `SelectDataColumns`, which are unchanged, as are their hashes. Unlike those four, a call whose result would have no data column raises `InputValidationError` at once, on the sync and async paths, and no meta (`__`) column of the input is carried, as with `Join`. diff --git a/docs/api/operators.md b/docs/api/operators.md index 2c1b2f3c1..096288499 100644 --- a/docs/api/operators.md +++ b/docs/api/operators.md @@ -20,6 +20,10 @@ Operators perform structural transformations on streams without inspecting or sy ::: orcapod.core.operators.DropDataColumns +::: orcapod.core.operators.DropColumns + +::: orcapod.core.operators.SelectColumns + ::: orcapod.core.operators.MapTags ::: orcapod.core.operators.MapData diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 60d6656e0..7fc3aeb71 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -320,6 +320,39 @@ print(result.keys()[1]) # ('weight',) Selecting a column that does not exist raises `InputValidationError`; pass `strict=False` to ignore missing columns instead. +Two more act on named columns whether each is a tag or a data column: + +- **`DropColumns(columns=["col1", "col2"])`** -- remove the named columns, tag or data +- **`SelectColumns(columns=["col1", "col2"])`** -- keep exactly the named columns, tag or + data, and remove every other tag and data column + +Streams offer them as `drop_columns` and `select_columns`, which take the names as one or +more strings or as one collection: `stream.drop_columns("a", "b")` is +`stream.drop_columns(["a", "b"])`. A removed data column's source-info column goes with it, +and system tags are kept, as with the four operators above. + +```python +sessions = DictSource( + data=[ + {"subject_id": "mouse_01", "session": 1, "age": 12, "weight": 25.3}, + {"subject_id": "mouse_02", "session": 2, "age": 8, "weight": 22.1}, + ], + tag_columns=["subject_id", "session"], +) + +dropped = sessions.drop_columns("session", "age") +print(dropped.keys()) # (('subject_id',), ('weight',)) +``` + +```python +selected = sessions.select_columns(["session", "weight"]) +print(selected.keys()) # (('session',), ('weight',)) +``` + +A stream needs at least one data column, so `drop_columns` of every data column, or +`select_columns` naming no data column, raises `InputValidationError` when it is called. +Selecting no tag column is allowed. + `SelectDataColumns` is also how you narrow a stream before a [function pod](function-pods.md), which accepts only the data columns its function declares. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 59a7a3d6e..afae9406b 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -330,7 +330,7 @@ The table below shows what each class returns from `identity_structure()`. | `DerivedSource` with `tags_as_data` | `(class_name, output_schema, source_id, ("tags_as_data", sorted items))`; the default `source_id` is `tags_as_data: