diff --git a/.zed/rules b/.zed/rules index f14340ed6..6da886e96 100644 --- a/.zed/rules +++ b/.zed/rules @@ -450,7 +450,7 @@ src/orcapod/ semijoin.py — SemiJoin (binary, non-commutative) batch.py — Batch (group rows by count, types become list[T]) group_by.py — GroupBy (many→one reduction keyed on tag values) - column_selection.py — Select/Drop Tag/Data columns + column_selection.py — Select/Drop Tag/Data columns; SelectColumns/DropColumns (tag or data) mappers.py — MapTags, MapData (rename columns) filters.py — PolarsFilter drop_nulls.py — DropNulls (drop rows with a null in named or all columns, T | None → T) diff --git a/CLAUDE.md b/CLAUDE.md index 95deee60a..47af24f97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -462,7 +462,7 @@ src/orcapod/ │ ├── semijoin.py # SemiJoin (binary, non-commutative) │ ├── batch.py # Batch (group rows by count, types become list[T]) │ ├── group_by.py # GroupBy (many→one reduction keyed on tag values) -│ ├── column_selection.py # Select/Drop Tag/Data columns +│ ├── column_selection.py # Select/Drop Tag/Data columns; SelectColumns/DropColumns (tag or data) │ ├── mappers.py # MapTags, MapData (rename columns) │ ├── filters.py # PolarsFilter │ └── drop_nulls.py # DropNulls (drop rows with a null in named or all columns, T | None → T) diff --git a/changelog.d/ITL-1084.md b/changelog.d/ITL-1084.md new file mode 100644 index 000000000..29e4867b2 --- /dev/null +++ b/changelog.d/ITL-1084.md @@ -0,0 +1,3 @@ +### Added + +- `stream.drop_columns(...)` and `stream.select_columns(...)`, with the operators `DropColumns` and `SelectColumns`, act on named columns whether each is a tag or a data column (ITL-1084). `drop_columns` removes the named columns; `select_columns` keeps exactly the named columns and removes every other tag and data column. The names are given as one or more strings or as one collection of strings: `stream.drop_columns("a", "b")` is `stream.drop_columns(["a", "b"])`; `strict` and `label` are keyword-only. For names of one group, `drop_columns` gives the rows of `DropTagColumns` or `DropDataColumns`; `select_columns` gives those of `SelectTagColumns` or `SelectDataColumns` when its names also include every column of the other group (named alone, one group's names remove the other group's columns). The four existing operators are unchanged, as are their hashes. Unlike them, a call whose result would have no data column raises `InputValidationError` at once, on the sync and async paths, and no data meta (`__`) column of the input is carried, as with `Join`. diff --git a/docs/api/operators.md b/docs/api/operators.md index 2c1b2f3c1..096288499 100644 --- a/docs/api/operators.md +++ b/docs/api/operators.md @@ -20,6 +20,10 @@ Operators perform structural transformations on streams without inspecting or sy ::: orcapod.core.operators.DropDataColumns +::: orcapod.core.operators.DropColumns + +::: orcapod.core.operators.SelectColumns + ::: orcapod.core.operators.MapTags ::: orcapod.core.operators.MapData diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 60d6656e0..7fc3aeb71 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -320,6 +320,39 @@ print(result.keys()[1]) # ('weight',) Selecting a column that does not exist raises `InputValidationError`; pass `strict=False` to ignore missing columns instead. +Two more act on named columns whether each is a tag or a data column: + +- **`DropColumns(columns=["col1", "col2"])`** -- remove the named columns, tag or data +- **`SelectColumns(columns=["col1", "col2"])`** -- keep exactly the named columns, tag or + data, and remove every other tag and data column + +Streams offer them as `drop_columns` and `select_columns`, which take the names as one or +more strings or as one collection: `stream.drop_columns("a", "b")` is +`stream.drop_columns(["a", "b"])`. A removed data column's source-info column goes with it, +and system tags are kept, as with the four operators above. + +```python +sessions = DictSource( + data=[ + {"subject_id": "mouse_01", "session": 1, "age": 12, "weight": 25.3}, + {"subject_id": "mouse_02", "session": 2, "age": 8, "weight": 22.1}, + ], + tag_columns=["subject_id", "session"], +) + +dropped = sessions.drop_columns("session", "age") +print(dropped.keys()) # (('subject_id',), ('weight',)) +``` + +```python +selected = sessions.select_columns(["session", "weight"]) +print(selected.keys()) # (('session',), ('weight',)) +``` + +A stream needs at least one data column, so `drop_columns` of every data column, or +`select_columns` naming no data column, raises `InputValidationError` when it is called. +Selecting no tag column is allowed. + `SelectDataColumns` is also how you narrow a stream before a [function pod](function-pods.md), which accepts only the data columns its function declares. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 59a7a3d6e..afae9406b 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -330,7 +330,7 @@ The table below shows what each class returns from `identity_structure()`. | `DerivedSource` with `tags_as_data` | `(class_name, output_schema, source_id, ("tags_as_data", sorted items))`; the default `source_id` is `tags_as_data: