From b03ba89996d67ef59f1cd6a3fb42464d39ae10a7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 11:11:44 -0800 Subject: [PATCH 1/7] feat(operators)!: map_tags / map_data refuse a name that is not a column of that kind; add map_columns (ITL-1160) MapTags and MapData (and the stream methods) refuse, by default, a mapping key that is not a column of the kind they rename, with an InputValidationError naming the missing names, the columns of that kind and, for a column of the other kind, the method that renames it. The check runs at validation (graph build) and on the async path against the first row. strict=False keeps the earlier behaviour. strict and label are keyword-only on the stream methods. Add MapColumns / stream.map_columns: tag and data columns renamed in one step; a tag stays a tag, data stays data. Remove the dead duplicate MapTags from column_selection.py. This commit moves no pinned hash: strict is not yet in the mapper identities (that change and its golden regeneration follow separately). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +- CLAUDE.md | 4 +- changelog.d/ITL-1160.md | 8 + docs/api/operators.md | 2 + docs/concepts/operators.md | 43 +- docs/reference/hashing.md | 1 + docs/tutorial/03-operators.md | 6 +- orcapod-design.md | 4 +- src/orcapod/core/operators/__init__.py | 3 +- .../core/operators/column_selection.py | 101 +--- src/orcapod/core/operators/mappers.py | 406 +++++++++++++++- src/orcapod/core/streams/base.py | 81 +++- src/orcapod/pipeline/serialization.py | 2 + .../protocols/core_protocols/streams.py | 24 +- .../test_native_async_operators.py | 4 +- .../operators/test_async_sync_equivalence.py | 32 +- .../operators/test_mappers_strict.py | 446 ++++++++++++++++++ .../test_pass_through_operator_staleness.py | 8 +- tests/test_hashing/golden/cases.py | 36 ++ 19 files changed, 1064 insertions(+), 151 deletions(-) create mode 100644 changelog.d/ITL-1160.md create mode 100644 tests/test_core/operators/test_mappers_strict.py diff --git a/.zed/rules b/.zed/rules index 22af2b746..77d5f8dfc 100644 --- a/.zed/rules +++ b/.zed/rules @@ -504,7 +504,7 @@ src/orcapod/ batch.py — Batch (group rows by count, types become list[T]) group_by.py — GroupBy (many→one reduction keyed on tag values) column_selection.py — Select/Drop Tag/Data columns; SelectColumns/DropColumns (tag or data) - mappers.py — MapTags, MapData (rename columns) + mappers.py — MapTags, MapData, MapColumns (rename columns; strict by default) filters.py — PolarsFilter drop_nulls.py — DropNulls (drop rows with a null in named or all columns, T | None → T) hashing/ @@ -629,7 +629,7 @@ ITL-1080): on the sync and the async path, in rows and in the predicted output s raw ArrowTableStream's `__` columns and for the meta a cached pod's or function node's rows carry. One rule, core/operators/meta_columns.py. A carry-forward mode is ITL-867. The meta prefix (`__`) is reserved: ArrowTableStream and the source builder refuse a tag named -with it, and MapTags / MapData (targets) and Pick / Index (out) refuse it as a new name +with it, and MapTags / MapData / MapColumns (targets) and Pick / Index (out) refuse it as a new name (schema_utils.require_names_not_meta, at construction). Operator or derived source? (developer, November 11, 2026) The test is: does any data diff --git a/CLAUDE.md b/CLAUDE.md index f7d9ad41c..8e5d3dc5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -516,7 +516,7 @@ src/orcapod/ │ ├── batch.py # Batch (group rows by count, types become list[T]) │ ├── group_by.py # GroupBy (many→one reduction keyed on tag values) │ ├── column_selection.py # Select/Drop Tag/Data columns; SelectColumns/DropColumns (tag or data) -│ ├── mappers.py # MapTags, MapData (rename columns) +│ ├── mappers.py # MapTags, MapData, MapColumns (rename columns; strict by default) │ ├── filters.py # PolarsFilter │ └── drop_nulls.py # DropNulls (drop rows with a null in named or all columns, T | None → T) ├── hashing/ @@ -649,7 +649,7 @@ raw `ArrowTableStream`'s `__` columns and for the meta a cached pod's or functio carry. The rule is in one place, `core/operators/meta_columns.py` (sync results through the operator base classes, async rows through `WithoutMetaWriter`). A carry-forward mode is ITL-867. The meta prefix (`__`) is reserved: `ArrowTableStream` and the source builder refuse a tag named -with it, and `MapTags` / `MapData` (targets) and `Pick` / `Index` (`out`) refuse it as a new name +with it, and `MapTags` / `MapData` / `MapColumns` (targets) and `Pick` / `Index` (`out`) refuse it as a new name (`schema_utils.require_names_not_meta`, at construction). **Operator or derived source?** (developer, November 11, 2026) The test is: does any data diff --git a/changelog.d/ITL-1160.md b/changelog.d/ITL-1160.md new file mode 100644 index 000000000..a0abd3a1f --- /dev/null +++ b/changelog.d/ITL-1160.md @@ -0,0 +1,8 @@ +### Breaking changes and migration + +- `map_tags` / `MapTags` and `map_data` / `MapData` refuse, by default, a mapping key that is not a column of the kind they rename (ITL-1160). Before, such a key was accepted and renamed nothing, which hid mistakes such as `map_tags` of a column that had become data after `group_by`. The call now raises `InputValidationError` naming the missing names and the columns of that kind; a name that is a column of the other kind says so (`'age' is a data column: use map_data`). The check runs when the operator is called on a stream, before any row is processed, and on the async path against the first row. A raw stream's meta (`__`) column is not a data column to rename. Migration: fix the name, use the other method (or `map_columns`), or pass `strict=False` to keep the old behaviour. `strict` is not part of the operator's identity, so no hash moves; it is saved in `to_config()`. +- The `strict` and `label` arguments of `map_tags` and `map_data` are keyword-only: `stream.map_tags(name_map, False, "label")` raises `TypeError`; write `label="label"`. + +### Added + +- `stream.map_columns(name_map)` and the operator `MapColumns` rename tag and data columns in one step (ITL-1160). A renamed tag stays a tag and a renamed data column stays data, with its source-info column; system tags pass through. `drop_unmapped=True` drops every tag and data column not in the mapping. As with `map_tags` and `map_data`, a name that is neither a tag nor a data column is refused unless `strict=False`, a new name taken by a column that is not renamed, or given to two columns, is refused, a `__` new name is refused when the operator is built, and a result with no data column is refused. Like `SelectColumns`, it carries no data meta (`__`) column of the input. Its identity is `("MapColumns", name_map, drop_unmapped)`. diff --git a/docs/api/operators.md b/docs/api/operators.md index 096288499..8421cfa9e 100644 --- a/docs/api/operators.md +++ b/docs/api/operators.md @@ -28,6 +28,8 @@ Operators perform structural transformations on streams without inspecting or sy ::: orcapod.core.operators.MapData +::: orcapod.core.operators.MapColumns + ::: orcapod.core.operators.PolarsFilter ::: orcapod.core.operators.Pick diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 4a16b1ed2..d6111ead0 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -42,8 +42,8 @@ function pod's `__computed`, that describe how an input row was produced. An ope output row is produced by the operator, so its `Data` holds none of them and its predicted output schema lists none, on the sync and the async path alike. A way to carry them forward across operators is an open design question (ITL-867). The `__` prefix is reserved for meta -columns: a stream and a source refuse a tag column named with it, and `MapTags` and `MapData` -refuse it as the new name of a tag or data column. +columns: a stream and a source refuse a tag column named with it, and `MapTags`, `MapData` +and `MapColumns` refuse it as the new name of a tag or data column. ## Operator categories @@ -385,8 +385,11 @@ Selecting no tag column is allowed. - **`MapTags(name_map={"old_name": "new_name"})`** -- rename tag columns - **`MapData(name_map={"old_name": "new_name"})`** -- rename data columns +- **`MapColumns(name_map={"old_name": "new_name"})`** -- rename columns, tag or data, in + one step -Columns that are not in `name_map` are kept unless you pass `drop_unmapped=True`. +Columns that are not in `name_map` are kept unless you pass `drop_unmapped=True`. A renamed +data column's source-info column is renamed with it, and system tags are kept. ```python from orcapod.operators import MapData @@ -395,6 +398,31 @@ renamed = MapData(name_map={"age": "age_weeks"})(source) print(renamed.keys()[1]) # ('age_weeks', 'weight') ``` +Streams offer them as `map_tags`, `map_data` and `map_columns`. `map_columns` renames tag and +data columns in one call; a renamed tag stays a tag and a renamed data column stays data: + +```python +renamed = sessions.map_columns({"session": "session_no", "age": "age_weeks"}) +print(renamed.keys()) # (('subject_id', 'session_no'), ('age_weeks', 'weight')) +``` + +A name that is not a column the operator renames raises `InputValidationError` when the +operator is called: a tag name for `map_data`, a data name for `map_tags`, and for all three a +name that is no column at all. The message names the columns that exist and, for a column of +the other kind, the method that renames it. Pass `strict=False` to ignore such names instead. + +```python +from orcapod.errors import InputValidationError + +try: + sessions.map_tags({"age": "age_weeks"}) +except InputValidationError as error: + print(error) +# MapTags: ['age'] not among the tag columns ['subject_id', 'session']; 'age' is a data column: use map_data; use strict=False to ignore names that are not tag columns +``` + +A new name already taken by a column that is not renamed is refused too. + ### PolarsFilter Filter rows using Polars expressions: @@ -703,8 +731,8 @@ operators or function pods. Every stream also exposes the operators as methods, which reads well when chaining: `join`, `semi_join`, `batch`, `group_by`, `select_tag_columns`, `select_data_columns`, -`drop_tag_columns`, `drop_data_columns`, `map_tags`, `map_data`, `polars_filter`, -`drop_nulls`, `pick` and `index`. +`drop_tag_columns`, `drop_data_columns`, `select_columns`, `drop_columns`, `map_tags`, +`map_data`, `map_columns`, `polars_filter`, `drop_nulls`, `pick` and `index`. ```python import polars as pl @@ -716,8 +744,9 @@ for tag, data in adults.iter_data(): ``` !!! note - The fluent `map_tags` and `map_data` methods and the `MapTags` and `MapData` operator - classes all keep the columns they do not rename. Pass `drop_unmapped=True` to drop them. + The fluent `map_tags`, `map_data` and `map_columns` methods and the `MapTags`, `MapData` + and `MapColumns` operator classes all keep the columns they do not rename. Pass + `drop_unmapped=True` to drop them. ## How it connects to other concepts diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 32934838e..eca8a4752 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -333,6 +333,7 @@ The table below shows what each class returns from `identity_structure()`. | Operators (unary) | `(operator_class_name, upstream_stream)` | Stream reference resolved via `content_resolver` | | Operators (binary/N-ary) | `(operator_class_name, argument_symmetry(streams))` | `frozenset` for commutative (Join, MergeJoin); `tuple` for ordered (SemiJoin) | | `SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, `DropDataColumns`, `SelectColumns`, `DropColumns` | `(operator_class_name, sorted unique column names as a list, strict)` | The operators act on a set of names: a list, a tuple, a set, a single string, a repeated or a reordered name list give one hash (ITL-942) | +| `MapTags`, `MapData`, `MapColumns` | `(operator_class_name, name_map, drop_unmapped)` | `strict` is not part of it: it decides only whether an input is refused, never what an accepted input gives (ITL-1160) | **Inputs of a commutative operator are a set (kept by decision, ITL-942).** A pipeline hash covers the structure of the pipeline, not only its schemas. `MergeJoin()(m1, m1)`, a stream diff --git a/docs/tutorial/03-operators.md b/docs/tutorial/03-operators.md index 64379fd0f..aad5bb24d 100644 --- a/docs/tutorial/03-operators.md +++ b/docs/tutorial/03-operators.md @@ -210,8 +210,10 @@ print(renamed.keys()) # (('subject_id',), ('sex', 'age', 'genotype')) ``` -Columns you do not rename are kept. Pass `drop_unmapped=True` to keep only the renamed -ones: +Columns you do not rename are kept. A name that is not a column of that kind is refused: a +`map_data` of a tag column raises `InputValidationError` and says to use `map_tags`. +`map_columns` renames tag and data columns in one call. Pass `drop_unmapped=True` to keep only +the renamed ones: ```python print(subjects.map_data({"age_weeks": "age"}, drop_unmapped=True).keys()) diff --git a/orcapod-design.md b/orcapod-design.md index 8a55fda7c..080d02065 100644 --- a/orcapod-design.md +++ b/orcapod-design.md @@ -86,8 +86,8 @@ Keep only specified tag or data columns. Optional `strict` mode raises on missin ### DropTagColumns / DropDataColumns Remove specified tag or data columns. `DropDataColumns` also removes associated source-info columns. -### MapTags / MapData -Rename tag or data columns via a name mapping. `MapData` automatically renames associated source-info columns. Optional `drop_unmapped` mode removes columns not in the mapping. +### MapTags / MapData / MapColumns +Rename tag or data columns via a name mapping; `MapColumns` renames both in one step, and a tag stays a tag. A renamed data column's source-info column is renamed with it. Optional `drop_unmapped` mode removes columns not in the mapping. A name that is not a column of the mapped kind is refused by default (`strict=True`, not part of the identity; ITL-1160). ### PolarsFilter Applies Polars filtering predicates to rows. Output schema is unchanged from input. diff --git a/src/orcapod/core/operators/__init__.py b/src/orcapod/core/operators/__init__.py index bdaa8adfa..032cd0a60 100644 --- a/src/orcapod/core/operators/__init__.py +++ b/src/orcapod/core/operators/__init__.py @@ -12,7 +12,7 @@ from .group_by import GroupBy from .index import Index from .join import Join -from .mappers import MapData, MapTags +from .mappers import MapColumns, MapData, MapTags from .merge_join import MergeJoin from .pick import Pick from .semijoin import SemiJoin @@ -23,6 +23,7 @@ "SemiJoin", "MapTags", "MapData", + "MapColumns", "Batch", "GroupBy", "SelectTagColumns", diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index ae6f1db48..d8004f24d 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from collections.abc import Collection, Mapping, Sequence +from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any, ClassVar from orcapod.core.operators.meta_columns import WithoutMetaWriter @@ -583,7 +583,7 @@ def identity_structure(self) -> Any: ) -def _is_meta(name: str) -> bool: +def is_meta_column(name: str) -> bool: """Whether ``name`` is a meta column or a raw stream's source-info column for one.""" return name.startswith( (constants.META_PREFIX, f"{constants.SOURCE_PREFIX}{constants.META_PREFIX}") @@ -683,7 +683,7 @@ def _user_columns(stream: StreamProtocol) -> tuple[tuple[str, ...], list[str]]: A raw ``ArrowTableStream`` lists its meta (``__``) columns as data columns. """ tag_columns, data_columns = stream.keys() - return tag_columns, [c for c in data_columns if not _is_meta(c)] + return tag_columns, [c for c in data_columns if not is_meta_column(c)] def validate_unary_input(self, stream: StreamProtocol) -> None: """Validate the names against the input stream. @@ -711,7 +711,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: c for c in table.column_names # Only data meta goes; a tag column is never stripped as meta. - if c in removed or (_is_meta(c) and c not in tag_columns) + if c in removed or (is_meta_column(c) and c not in tag_columns) ] ), tag_columns=[c for c in tag_columns if c not in tags_removed], @@ -736,7 +736,7 @@ def unary_output_schema( { k: v for k, v in data_schema.items() - if k not in data_removed and not _is_meta(k) + if k not in data_removed and not is_meta_column(k) } ), ) @@ -824,94 +824,3 @@ class SelectColumns(_TagAndDataColumnsOperator): def _removed(self, columns: Collection[str]) -> list[str]: return [c for c in columns if c not in self.columns] - - -class MapTags(UnaryOperator): - """ - Operator that maps tags in a stream using a user-defined function. - The function is applied to each tag in the stream, and the resulting tags - are returned as a new stream. - """ - - def __init__( - self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs - ): - self.name_map = dict(name_map) - self.drop_unmapped = drop_unmapped - super().__init__(**kwargs) - - def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: - tag_columns, data_columns = stream.keys() - missing_tags = set(tag_columns) - set(self.name_map.keys()) - - if not any(n in tag_columns for n in self.name_map): - # nothing to rename in the tags, return stream as is - return stream - - table = stream.as_table(columns={"source": True, "system_tags": True}) - - name_map = { - tc: self.name_map.get(tc, tc) for tc in tag_columns - } # rename the tag as necessary - new_tag_columns = [name_map[tc] for tc in tag_columns] - for c in data_columns: - name_map[c] = c # no renaming on data columns - - renamed_table = table.rename_columns(name_map) - - if missing_tags and self.drop_unmapped: - # drop any tags that are not in the name map - renamed_table = renamed_table.drop_columns(list(missing_tags)) - - return ArrowTableStream( - renamed_table, - tag_columns=new_tag_columns, - ) - - def validate_unary_input(self, stream: StreamProtocol) -> None: - """ - This method should be implemented by subclasses to validate the inputs to the operator. - It takes two streams as input and raises an error if the inputs are not valid. - """ - # verify that renamed value does NOT collide with other columns - tag_columns, data_columns = stream.keys() - relevant_source = [] - relevant_target = [] - for source, target in self.name_map.items(): - if source in tag_columns: - relevant_source.append(source) - relevant_target.append(target) - remaining_tag_columns = set(tag_columns) - set(relevant_source) - overlapping_tag_columns = remaining_tag_columns.intersection(relevant_target) - overlapping_data_columns = set(data_columns).intersection(relevant_target) - - if overlapping_tag_columns or overlapping_data_columns: - message = f"Renaming {self.name_map} would cause collisions with existing columns: " - if overlapping_tag_columns: - message += f"overlapping tag columns: {overlapping_tag_columns}." - if overlapping_data_columns: - message += f"overlapping data columns: {overlapping_data_columns}." - raise InputValidationError(message) - - def unary_output_schema( - self, - stream: StreamProtocol, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - tag_schema, data_schema = stream.output_schema( - columns=columns, all_info=all_info - ) - - # Create new data schema with renamed keys - new_tag_schema = {self.name_map.get(k, k): v for k, v in tag_schema.items()} - - return Schema(new_tag_schema), data_schema - - def identity_structure(self) -> Any: - return ( - self.__class__.__name__, - self.name_map, - self.drop_unmapped, - ) diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 2175d6ff3..7afbb822e 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -1,10 +1,13 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from typing import TYPE_CHECKING, Any from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator -from orcapod.core.operators.column_selection import require_a_data_column +from orcapod.core.operators.column_selection import ( + is_meta_column, + require_a_data_column, +) from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol @@ -19,34 +22,92 @@ pa = LazyModule("pyarrow") -class MapData(UnaryOperator): +def _other_group_note( + names: Collection[str], other_columns: Collection[str], group: str, method: str +) -> str: + """Name the missing names that are columns of the other group, for an error. + + Args: + names: The names that are not columns of the renamed group. + other_columns: The input's columns of the other group. + group: The other group, ``"tag"`` or ``"data"``. + method: The stream method that renames a column of the other group. + + Returns: + One ``"; 'x' is a data column: use map_data"`` clause per such name, or an + empty string. """ - Operator that maps data in a stream using a user-defined function. - The function is applied to each data in the stream, and the resulting data - are returned as a new stream. + return "".join( + f"; {n!r} is a {group} column: use {method}" + for n in names + if n in other_columns + ) + + +class MapData(UnaryOperator): + """Rename data columns. + + Args: + name_map: Each data column to rename, mapped to its new name. + drop_unmapped: When ``True``, the data columns not in ``name_map`` are + dropped. + strict: When ``True`` (the default), a key of ``name_map`` that is not a + data column of the input raises ``InputValidationError``; when + ``False`` it is ignored. Not part of the operator's identity: it + decides only whether an input is refused, never what an accepted + input gives. """ def __init__( - self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs + self, + name_map: Mapping[str, str], + drop_unmapped: bool = False, + strict: bool = True, + **kwargs, ): # A data column never gets a meta column's name (ITL-1080): refused here, # so no path (sync, async or the prediction) can produce one. schema_utils.require_names_not_meta(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped + self.strict = strict super().__init__(**kwargs) + def _require_data_columns( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> None: + """Refuse, when ``strict``, a name that is not a data column (ITL-1160). + + Args: + tag_columns: The input's tag columns. + data_columns: The input's data columns, without meta columns. + + Raises: + InputValidationError: When ``strict`` and a key of ``name_map`` is + not one of ``data_columns``. + """ + missing = [n for n in self.name_map if n not in data_columns] + if missing and self.strict: + raise InputValidationError( + f"MapData: {missing} not among the data columns " + f"{list(data_columns)}" + f"{_other_group_note(missing, tag_columns, 'tag', 'map_tags')}; " + f"use strict=False to ignore names that are not data columns" + ) + def to_config(self) -> dict[str, Any]: """Serialize this MapData operator to a config dict. Returns: A dict with ``class_name``, ``module_path``, and ``config`` keys, - where ``config`` contains ``name_map`` and ``drop_unmapped``. + where ``config`` contains ``name_map``, ``drop_unmapped`` and + ``strict``. """ config = super().to_config() config["config"] = { "name_map": dict(self.name_map), "drop_unmapped": self.drop_unmapped, + "strict": self.strict, } return config @@ -94,8 +155,11 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: return ArrowTableStream(renamed_table, tag_columns=tag_columns) def validate_unary_input(self, stream: StreamProtocol) -> None: - # verify that renamed value does NOT collide with other columns tag_columns, data_columns = stream.keys() + self._require_data_columns( + tag_columns, [c for c in data_columns if not is_meta_column(c)] + ) + # verify that renamed value does NOT collide with other columns relevant_source = [] relevant_target = [] for source, target in self.name_map.items(): @@ -162,6 +226,11 @@ async def async_execute( async for tag, data in inputs[0]: if rename_map is None: pkt_keys = data.keys() + # The names are checked against the first row, as + # ``validate_unary_input`` checks them against the stream. + self._require_data_columns( + tag.keys(), [k for k in pkt_keys if not is_meta_column(k)] + ) rename_map = { k: self.name_map[k] for k in pkt_keys if k in self.name_map } @@ -187,33 +256,68 @@ def identity_structure(self) -> Any: class MapTags(UnaryOperator): - """ - Operator that maps tags in a stream using a user-defined function. - The function is applied to each tag in the stream, and the resulting tags - are returned as a new stream. + """Rename tag columns. + + Args: + name_map: Each tag column to rename, mapped to its new name. + drop_unmapped: When ``True``, the tag columns not in ``name_map`` are + dropped. + strict: When ``True`` (the default), a key of ``name_map`` that is not a + tag column of the input raises ``InputValidationError``; when + ``False`` it is ignored. Not part of the operator's identity: it + decides only whether an input is refused, never what an accepted + input gives. """ def __init__( - self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs + self, + name_map: Mapping[str, str], + drop_unmapped: bool = False, + strict: bool = True, + **kwargs, ): # A tag never gets a meta column's name (ITL-1080): refused here, so no # path (sync, async or the prediction) can produce one. schema_utils.require_names_not_meta(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped + self.strict = strict super().__init__(**kwargs) + def _require_tag_columns( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> None: + """Refuse, when ``strict``, a name that is not a tag column (ITL-1160). + + Args: + tag_columns: The input's tag columns. + data_columns: The input's data columns, without meta columns. + + Raises: + InputValidationError: When ``strict`` and a key of ``name_map`` is + not one of ``tag_columns``. + """ + missing = [n for n in self.name_map if n not in tag_columns] + if missing and self.strict: + raise InputValidationError( + f"MapTags: {missing} not among the tag columns {list(tag_columns)}" + f"{_other_group_note(missing, data_columns, 'data', 'map_data')}; " + f"use strict=False to ignore names that are not tag columns" + ) + def to_config(self) -> dict[str, Any]: """Serialize this MapTags operator to a config dict. Returns: A dict with ``class_name``, ``module_path``, and ``config`` keys, - where ``config`` contains ``name_map`` and ``drop_unmapped``. + where ``config`` contains ``name_map``, ``drop_unmapped`` and + ``strict``. """ config = super().to_config() config["config"] = { "name_map": dict(self.name_map), "drop_unmapped": self.drop_unmapped, + "strict": self.strict, } return config @@ -256,8 +360,11 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: This method should be implemented by subclasses to validate the inputs to the operator. It takes two streams as input and raises an error if the inputs are not valid. """ - # verify that renamed value does NOT collide with other columns tag_columns, data_columns = stream.keys() + self._require_tag_columns( + tag_columns, [c for c in data_columns if not is_meta_column(c)] + ) + # verify that renamed value does NOT collide with other columns relevant_source = [] relevant_target = [] for source, target in self.name_map.items(): @@ -318,6 +425,11 @@ async def async_execute( async for tag, data in inputs[0]: if rename_map is None: tag_keys = tag.keys() + # The names are checked against the first row, as + # ``validate_unary_input`` checks them against the stream. + self._require_tag_columns( + tag_keys, [k for k in data.keys() if not is_meta_column(k)] + ) rename_map = { k: self.name_map[k] for k in tag_keys if k in self.name_map } @@ -341,3 +453,265 @@ def identity_structure(self) -> Any: self.name_map, self.drop_unmapped, ) + + +class MapColumns(UnaryOperator): + """Rename columns, whether each is a tag or a data column (ITL-1160). + + The counterpart of ``SelectColumns`` and ``DropColumns`` for renaming: one + step renames tag and data columns alike. A renamed tag stays a tag and a + renamed data column stays data; a renamed data column's source-info column + is renamed with it. System tags pass through, the stream's column order is + kept, and, as for ``SelectColumns``, no data meta (``__``) column of the + input is carried. + + Args: + name_map: Each column to rename, tag or data, mapped to its new name. + drop_unmapped: When ``True``, every user tag and data column not in + ``name_map`` is dropped (with the source-info column of a dropped + data column); system tags are kept. + strict: When ``True`` (the default), a key of ``name_map`` that is + neither a tag nor a data column of the input raises + ``InputValidationError``; when ``False`` it is ignored. Not part of + the operator's identity: it decides only whether an input is + refused, never what an accepted input gives. + + Raises: + InputValidationError: When called on a stream where a new name is taken + by a column that is not renamed, two columns get the same new name, + or (with ``drop_unmapped``) no data column would remain. + """ + + def __init__( + self, + name_map: Mapping[str, str], + drop_unmapped: bool = False, + strict: bool = True, + **kwargs, + ): + # No column gets a meta column's name (ITL-1080): refused here, so no + # path (sync, async or the prediction) can produce one. + schema_utils.require_names_not_meta(name_map.values()) + self.name_map = dict(name_map) + self.drop_unmapped = drop_unmapped + self.strict = strict + super().__init__(**kwargs) + + def to_config(self) -> dict[str, Any]: + """Serialize this MapColumns operator to a config dict. + + Returns: + A dict with ``class_name``, ``module_path``, and ``config`` keys, + where ``config`` contains ``name_map``, ``drop_unmapped`` and + ``strict``. + """ + config = super().to_config() + config["config"] = { + "name_map": dict(self.name_map), + "drop_unmapped": self.drop_unmapped, + "strict": self.strict, + } + return config + + def identity_structure(self) -> Any: + return ( + self.__class__.__name__, + self.name_map, + self.drop_unmapped, + ) + + @staticmethod + def _user_columns(stream: StreamProtocol) -> tuple[tuple[str, ...], list[str]]: + """The stream's tag columns and its data columns without meta columns. + + A raw ``ArrowTableStream`` lists its meta (``__``) columns as data columns. + """ + tag_columns, data_columns = stream.keys() + return tag_columns, [c for c in data_columns if not is_meta_column(c)] + + def _plan( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> tuple[dict[str, str], dict[str, str], list[str], list[str]]: + """Return the tag renames, data renames, dropped tags and dropped data. + + Args: + tag_columns: The input's tag columns, in order. + data_columns: The input's user data columns, in order (no meta column). + """ + tag_renames = {c: self.name_map[c] for c in tag_columns if c in self.name_map} + data_renames = {c: self.name_map[c] for c in data_columns if c in self.name_map} + if not self.drop_unmapped: + return tag_renames, data_renames, [], [] + return ( + tag_renames, + data_renames, + [c for c in tag_columns if c not in self.name_map], + [c for c in data_columns if c not in self.name_map], + ) + + def _validate_columns( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> None: + """Check the names against the input's columns and the result's columns. + + Args: + tag_columns: The input's tag columns. + data_columns: The input's user data columns (no meta column). + + Raises: + InputValidationError: When ``strict`` and a name is neither a tag nor + a data column; when a new name is taken by a column that is not + renamed, or given to two columns; or when the result would have + no data column. + """ + missing = [ + n for n in self.name_map if n not in tag_columns and n not in data_columns + ] + if missing and self.strict: + raise InputValidationError( + f"MapColumns: {missing} not among the tag columns " + f"{list(tag_columns)} or the data columns {list(data_columns)}; " + f"use strict=False to ignore names that are not columns" + ) + tag_renames, data_renames, _, data_dropped = self._plan( + tag_columns, data_columns + ) + renamed = {**tag_renames, **data_renames} + targets = list(renamed.values()) + not_renamed = [c for c in (*tag_columns, *data_columns) if c not in renamed] + taken = sorted(set(targets).intersection(not_renamed)) + doubled = sorted({t for t in targets if targets.count(t) > 1}) + if taken or doubled: + message = f"MapColumns: renaming {renamed} would cause collisions:" + if taken: + message += f" columns that are not renamed already have {taken}." + if doubled: + message += f" two columns would both be named {doubled}." + raise InputValidationError(message) + try: + require_a_data_column( + [data_renames.get(c, c) for c in data_columns if c not in data_dropped] + ) + except InputValidationError as error: + raise InputValidationError( + f"MapColumns({self.name_map}, drop_unmapped=True) would leave no " + f"data column: {error}" + ) from error + + def validate_unary_input(self, stream: StreamProtocol) -> None: + """Validate the names against the input stream. + + Raises: + InputValidationError: As ``_validate_columns`` describes. + """ + self._validate_columns(*self._user_columns(stream)) + + def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: + tag_columns, data_columns = self._user_columns(stream) + tag_renames, data_renames, tags_dropped, data_dropped = self._plan( + tag_columns, data_columns + ) + table = stream.as_table( + columns={"source": True, "system_tags": True, "sort_by_tags": False} + ) + dropped = { + *tags_dropped, + *data_dropped, + *(f"{constants.SOURCE_PREFIX}{c}" for c in data_dropped), + } + table = table.drop_columns( + [ + c + for c in table.column_names + # Only data meta goes; a tag column is never stripped as meta. + if c in dropped or (is_meta_column(c) and c not in tag_columns) + ] + ) + renames = { + **tag_renames, + **data_renames, + **{ + f"{constants.SOURCE_PREFIX}{c}": f"{constants.SOURCE_PREFIX}{t}" + for c, t in data_renames.items() + }, + } + return ArrowTableStream( + table.rename_columns([renames.get(c, c) for c in table.column_names]), + tag_columns=[ + tag_renames.get(c, c) for c in tag_columns if c not in tags_dropped + ], + data_context=stream.data_context_key, + ) + + def unary_output_schema( + self, + stream: StreamProtocol, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + tag_schema, data_schema = stream.output_schema( + columns=columns, all_info=all_info + ) + tag_renames, data_renames, tags_dropped, data_dropped = self._plan( + *self._user_columns(stream) + ) + # System tag columns are never renamed or dropped, so they are kept. + return ( + Schema( + { + tag_renames.get(k, k): v + for k, v in tag_schema.items() + if k not in tags_dropped + } + ), + Schema( + { + data_renames.get(k, k): v + for k, v in data_schema.items() + if k not in data_dropped and not is_meta_column(k) + } + ), + ) + + async def async_execute( + self, + inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + *, + input_pipeline_hashes: Sequence[ContentHash], + ) -> None: + """Streaming: rename (and drop) the columns per row without materializing. + + The names are validated against the first row, as ``validate_unary_input`` + validates them against the stream. + + Args: + inputs: The one readable input channel. + output: Writable channel for downstream consumption. + input_pipeline_hashes: Not used: this operator keeps system-tag + names as they are, so no column is named after an input's + pipeline hash. + """ + try: + output = WithoutMetaWriter(output) + plan: tuple[dict[str, str], dict[str, str], list[str], list[str]] | None + plan = None + async for tag, data in inputs[0]: + if plan is None: + tag_keys = tag.keys() + data_keys = [k for k in data.keys() if not is_meta_column(k)] + self._validate_columns(tag_keys, data_keys) + plan = self._plan(tag_keys, data_keys) + tag_renames, data_renames, tags_dropped, data_dropped = plan + if tags_dropped: + tag = tag.drop(*tags_dropped) + if data_dropped: + data = data.drop(*data_dropped) + if tag_renames: + tag = tag.rename(tag_renames) + if data_renames: + data = data.rename(data_renames) + await output.send((tag, data)) + finally: + await output.close() diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 2449511cf..603dc7f40 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -306,29 +306,94 @@ def map_tags( self, name_map: Mapping[str, str], drop_unmapped: bool = False, + *, + strict: bool = True, label: str | None = None, ) -> StreamBase: - """ - Maps the tags in this stream according to the provided name_map. - Tags not in the name_map are kept unless drop_unmapped is True. + """Rename tag columns. See ``orcapod.core.operators.MapTags``. + + Args: + name_map: Each tag column to rename, mapped to its new name. + drop_unmapped: When ``True``, the tags not in ``name_map`` are dropped. + strict: When ``True`` (the default), a key of ``name_map`` that is not + a tag column raises ``InputValidationError``; when ``False`` it is + ignored. + label: Optional node label for the pipeline graph. + + Returns: + The stream with the tag columns renamed. + + Raises: + InputValidationError: When a ``strict`` name is not a tag column, or a + new name collides with an existing column. """ from orcapod.core.operators import MapTags - return MapTags(name_map, drop_unmapped)(self, label=label) + return MapTags(name_map, drop_unmapped, strict=strict)(self, label=label) def map_data( self, name_map: Mapping[str, str], drop_unmapped: bool = False, + *, + strict: bool = True, label: str | None = None, ) -> StreamBase: - """ - Maps the data in this stream according to the provided data_map. - Data columns not in the name_map are kept unless drop_unmapped is True. + """Rename data columns. See ``orcapod.core.operators.MapData``. + + Args: + name_map: Each data column to rename, mapped to its new name. + drop_unmapped: When ``True``, the data columns not in ``name_map`` + are dropped. + strict: When ``True`` (the default), a key of ``name_map`` that is not + a data column raises ``InputValidationError``; when ``False`` it is + ignored. + label: Optional node label for the pipeline graph. + + Returns: + The stream with the data columns renamed. + + Raises: + InputValidationError: When a ``strict`` name is not a data column, a + new name collides with an existing column, or no data column + would remain. """ from orcapod.core.operators import MapData - return MapData(name_map, drop_unmapped)(self, label=label) + return MapData(name_map, drop_unmapped, strict=strict)(self, label=label) + + def map_columns( + self, + name_map: Mapping[str, str], + drop_unmapped: bool = False, + *, + strict: bool = True, + label: str | None = None, + ) -> StreamBase: + """Rename columns, tag or data, in one step. + + A renamed tag stays a tag and a renamed data column stays data. See + ``orcapod.core.operators.MapColumns``. + + Args: + name_map: Each column to rename, tag or data, mapped to its new name. + drop_unmapped: When ``True``, every tag and data column not in + ``name_map`` is dropped; system tags are kept. + strict: When ``True`` (the default), a key of ``name_map`` that is + neither a tag nor a data column raises ``InputValidationError``; + when ``False`` it is ignored. + label: Optional node label for the pipeline graph. + + Returns: + The stream with the columns renamed. + + Raises: + InputValidationError: When a ``strict`` name is missing, a new name + collides with another column, or no data column would remain. + """ + from orcapod.core.operators import MapColumns + + return MapColumns(name_map, drop_unmapped, strict=strict)(self, label=label) def batch( self, diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 5785266b3..ac7bce8c5 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -192,6 +192,7 @@ def _build_operator_registry() -> dict[str, type]: DropTagColumns, GroupBy, Join, + MapColumns, MapData, MapTags, MergeJoin, @@ -216,6 +217,7 @@ def _build_operator_registry() -> dict[str, type]: "DropColumns": DropColumns, "MapTags": MapTags, "MapData": MapData, + "MapColumns": MapColumns, "PolarsFilter": PolarsFilter, "DropNulls": DropNulls, } diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index b1967f56a..e806025fd 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -350,11 +350,14 @@ def map_tags( self, name_map: Mapping[str, str], drop_unmapped: bool = False, + *, + strict: bool = True, label: str | None = None, ) -> "StreamProtocol": """ Map tag names in this stream to new names based on the provided mapping. - Tags not in the mapping are kept unless ``drop_unmapped`` is True. + Tags not in the mapping are kept unless ``drop_unmapped`` is True. With + ``strict`` (the default), a name that is not a tag column is refused. """ ... @@ -362,11 +365,30 @@ def map_data( self, name_map: Mapping[str, str], drop_unmapped: bool = False, + *, + strict: bool = True, label: str | None = None, ) -> "StreamProtocol": """ Map data names in this stream to new names based on the provided mapping. Data columns not in the mapping are kept unless ``drop_unmapped`` is True. + With ``strict`` (the default), a name that is not a data column is refused. + """ + ... + + def map_columns( + self, + name_map: Mapping[str, str], + drop_unmapped: bool = False, + *, + strict: bool = True, + label: str | None = None, + ) -> "StreamProtocol": + """Rename columns, tag or data, in one step; a tag stays a tag. + + Columns not in the mapping are kept unless ``drop_unmapped`` is True. With + ``strict`` (the default), a name that is neither a tag nor a data column + is refused. """ ... diff --git a/tests/test_channels/test_native_async_operators.py b/tests/test_channels/test_native_async_operators.py index 8f47f3d12..82c774af7 100644 --- a/tests/test_channels/test_native_async_operators.py +++ b/tests/test_channels/test_native_async_operators.py @@ -598,7 +598,7 @@ async def test_drop_unmapped(self): @pytest.mark.asyncio async def test_no_matching_rename_passthrough(self): stream = make_two_tag_stream() - op = MapTags(name_map={"nonexistent": "nope"}) + op = MapTags(name_map={"nonexistent": "nope"}, strict=False) results = await run_unary(op, stream) assert len(results) == 3 @@ -716,7 +716,7 @@ async def test_source_info_renamed(self): @pytest.mark.asyncio async def test_no_matching_rename_passthrough(self): stream = make_simple_stream() - op = MapData(name_map={"nonexistent": "nope"}) + op = MapData(name_map={"nonexistent": "nope"}, strict=False) results = await run_unary(op, stream) assert len(results) == 3 diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 46679eb08..499462dce 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -37,6 +37,7 @@ GroupBy, Index, Join, + MapColumns, MapData, MapTags, MergeJoin, @@ -366,6 +367,17 @@ class Case: ), Case("map_tags", lambda: MapTags({"ta": "tb"}), lambda: (source_a(),)), Case("map_data", lambda: MapData({"i8": "j8"}), lambda: (source_a(),)), + # Tag and data columns in one call (ITL-1160); a logical column among them. + Case( + "map_columns", + lambda: MapColumns({"ta": "tb", "i8": "j8", "p": "q"}), + lambda: (source_a(),), + ), + Case( + "map_columns_drop_unmapped", + lambda: MapColumns({"k": "kk", "st": "su", "nul": "n"}, drop_unmapped=True), + lambda: (source_a(),), + ), Case( "select_tag_columns", lambda: SelectTagColumns(["k"]), @@ -1640,13 +1652,13 @@ def test_map_data_does_not_rename_a_meta_column_into_data() -> None: lists ``__`` columns, so the meta value reached a data column (CodeRabbit, #369). """ source = _with_meta(source_a(), _meta_columns("m", 3)) - expected = sync_rows(MapData({"__m_i8": "renamed"}), source) - actual = async_rows(MapData({"__m_i8": "renamed"}), source) + expected = sync_rows(MapData({"__m_i8": "renamed"}, strict=False), source) + actual = async_rows(MapData({"__m_i8": "renamed"}, strict=False), source) for rows in (expected, actual): for _, data in rows: assert "renamed" not in data.keys() assert_equivalent(expected, actual) - op = MapData({"__m_i8": "renamed"}) + op = MapData({"__m_i8": "renamed"}, strict=False) predicted = op.output_schema(source) assert "renamed" not in predicted[1] # Non-public on purpose: a prediction is verified against the static result. @@ -1656,9 +1668,9 @@ def test_map_data_does_not_rename_a_meta_column_into_data() -> None: def test_map_data_meta_source_does_not_collide() -> None: """A meta source is not a rename source, so it cannot collide with a data column.""" source = _with_meta(source_b(nested=False), _meta_columns("m", 3)) - op = MapData({"__m_i8": "i32", "u8": "v8"}) + op = MapData({"__m_i8": "i32", "u8": "v8"}, strict=False) expected = sync_rows(op, source) - actual = async_rows(MapData({"__m_i8": "i32", "u8": "v8"}), source) + actual = async_rows(MapData({"__m_i8": "i32", "u8": "v8"}, strict=False), source) for _, data in expected: assert set(data.keys()) == {"i32", "v8", "f32", "ls"} assert_equivalent(expected, actual) @@ -1678,9 +1690,9 @@ def test_map_data_dropping_every_data_column_raises_alike( """ source = _with_meta(source_b(nested=False), _meta_columns("m", 3)) with pytest.raises(InputValidationError, match="no data column") as sync_error: - _ = MapData(name_map, drop_unmapped=True)(source) + _ = MapData(name_map, drop_unmapped=True, strict=False)(source) with pytest.raises(InputValidationError, match="no data column") as async_error: - _ = async_rows(MapData(name_map, drop_unmapped=True), source) + _ = async_rows(MapData(name_map, drop_unmapped=True, strict=False), source) assert str(async_error.value) == str(sync_error.value) @@ -1691,9 +1703,11 @@ def test_map_tags_dropping_every_tag_agrees_on_both_paths() -> None: every tag while the sync path and the prediction dropped them. """ source = source_a() - op = MapTags({"absent": "a"}, drop_unmapped=True) + op = MapTags({"absent": "a"}, drop_unmapped=True, strict=False) expected = sync_rows(op, source) - actual = async_rows(MapTags({"absent": "a"}, drop_unmapped=True), source) + actual = async_rows( + MapTags({"absent": "a"}, drop_unmapped=True, strict=False), source + ) assert op.output_schema(source)[0] == {} for rows in (expected, actual): assert len(rows) == 3 diff --git a/tests/test_core/operators/test_mappers_strict.py b/tests/test_core/operators/test_mappers_strict.py new file mode 100644 index 000000000..a8f889b84 --- /dev/null +++ b/tests/test_core/operators/test_mappers_strict.py @@ -0,0 +1,446 @@ +"""``map_tags``, ``map_data`` and ``map_columns`` refuse a name they cannot rename (ITL-1160). + +A key of the mapping that is not a column of the mapped kind is refused by default +(``strict=True``) with an ``InputValidationError`` naming the missing names and the +columns of that kind; a name that is a column of the other kind says which method +renames it. The check runs when the operator is called on a stream, before any row +is processed, and on the async path against the first row. ``strict=False`` keeps +the earlier behaviour: such a name is ignored. ``strict`` is not part of the +operator's identity. ``MapColumns`` renames tag and data columns in one step; a tag +stays a tag and data stays data. Tests drive the operators through ``op(stream)`` +and the stream methods. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod.core.operators import MapColumns, MapData, MapTags +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.sources import DictSource +from orcapod.system_constants import constants +from tests.test_core.operators.test_async_sync_equivalence import ( + async_rows, + cached_pod_stream, + double_a, + meta_stream, + sync_rows, +) + +META = constants.META_PREFIX +SOURCE = constants.SOURCE_PREFIX + + +def _source() -> DictSource: + """Tags ``cohort``, ``subject``; data ``age``, ``weight``.""" + return DictSource( + data=[ + {"cohort": "a", "subject": "m1", "age": 12, "weight": 25.3}, + {"cohort": "a", "subject": "m2", "age": 8, "weight": 22.1}, + {"cohort": "b", "subject": "m3", "age": 15, "weight": 27.8}, + ], + tag_columns=["cohort", "subject"], + tag_only_columns=True, + source_id="subjects", + ) + + +def _rows(stream: StreamProtocol) -> list[tuple[dict[str, Any], dict[str, Any]]]: + return [(tag.as_dict(), data.as_dict()) for tag, data in stream.iter_data()] + + +def _async_rows_as_dicts( + op: Any, stream: StreamProtocol +) -> list[tuple[dict[str, Any], dict[str, Any]]]: + return [(tag.as_dict(), data.as_dict()) for tag, data in async_rows(op, stream)] + + +# ── refusal: a name that is not a column of the mapped kind ──────────────── + +# (operator class, stream method, name map, fragments the message must contain) +REFUSALS = [ + ( + MapTags, + "map_tags", + {"nope": "z"}, + ["['nope'] not among the tag columns ['cohort', 'subject']", "strict=False"], + ), + ( + MapTags, + "map_tags", + {"age": "z"}, + ["['age'] not among the tag columns", "'age' is a data column: use map_data"], + ), + ( + MapData, + "map_data", + {"nope": "z"}, + ["['nope'] not among the data columns ['age', 'weight']", "strict=False"], + ), + ( + MapData, + "map_data", + {"subject": "z"}, + [ + "['subject'] not among the data columns", + "'subject' is a tag column: use map_tags", + ], + ), + ( + MapColumns, + "map_columns", + {"nope": "z"}, + [ + "['nope'] not among the tag columns ['cohort', 'subject'] or the data " + "columns ['age', 'weight']", + "strict=False", + ], + ), +] +REFUSAL_IDS = [ + "map_tags_missing", + "map_tags_data_name", + "map_data_missing", + "map_data_tag_name", + "map_columns_missing", +] + + +def _assert_message( + error: pytest.ExceptionInfo[InputValidationError], parts: list[str] +): + for part in parts: + assert part in str(error.value) + + +@pytest.mark.parametrize( + ("op_cls", "method", "name_map", "parts"), REFUSALS, ids=REFUSAL_IDS +) +def test_the_stream_method_refuses_by_default( + op_cls: type, method: str, name_map: dict[str, str], parts: list[str] +) -> None: + with pytest.raises(InputValidationError) as error: + getattr(_source(), method)(name_map) + _assert_message(error, parts) + + +@pytest.mark.parametrize( + ("op_cls", "method", "name_map", "parts"), REFUSALS, ids=REFUSAL_IDS +) +def test_the_operator_refuses_when_called( + op_cls: type, method: str, name_map: dict[str, str], parts: list[str] +) -> None: + op = op_cls(name_map) # building it is fine: the input is not known yet + with pytest.raises(InputValidationError) as error: + op(_source()) + _assert_message(error, parts) + + +@pytest.mark.parametrize( + ("op_cls", "method", "name_map", "parts"), REFUSALS, ids=REFUSAL_IDS +) +def test_the_async_path_refuses( + op_cls: type, method: str, name_map: dict[str, str], parts: list[str] +) -> None: + with pytest.raises(InputValidationError) as error: + async_rows(op_cls(name_map), _source()) + _assert_message(error, parts) + + +@pytest.mark.parametrize( + ("method", "name_map"), + [ + ("map_tags", {"subject": "animal", "nope": "z"}), + ("map_data", {"age": "years", "nope": "z"}), + ("map_columns", {"subject": "animal", "age": "years", "nope": "z"}), + ], +) +def test_one_missing_name_refuses_the_whole_map( + method: str, name_map: dict[str, str] +) -> None: + with pytest.raises(InputValidationError, match=r"\['nope'\]"): + getattr(_source(), method)(name_map) + + +def test_every_missing_name_is_named() -> None: + with pytest.raises(InputValidationError) as error: + _source().map_tags({"age": "a", "nope": "n", "weight": "w"}) + message = str(error.value) + assert "['age', 'nope', 'weight']" in message + assert "'age' is a data column: use map_data" in message + assert "'weight' is a data column: use map_data" in message + assert "'nope' is a" not in message + + +@pytest.mark.parametrize( + ("op", "stream_factory"), + [ + ( + lambda: MapData({f"{META}m": "renamed"}), + lambda: meta_stream("s", "v", [f"{META}m"]), + ), + ( + lambda: MapColumns({f"{META}m": "renamed"}), + lambda: meta_stream("s", "v", [f"{META}m"]), + ), + ], + ids=["map_data", "map_columns"], +) +def test_a_meta_column_is_not_a_column_to_rename( + op: Callable[[], Any], stream_factory: Callable[[], StreamProtocol] +) -> None: + with pytest.raises(InputValidationError, match="not among the"): + op()(stream_factory()) + with pytest.raises(InputValidationError, match="not among the"): + async_rows(op(), stream_factory()) + + +# ── strict=False keeps the earlier behaviour ─────────────────────────────── + + +@pytest.mark.parametrize( + ("op_cls", "method", "name_map"), + [(op_cls, method, name_map) for op_cls, method, name_map, _ in REFUSALS], + ids=REFUSAL_IDS, +) +def test_non_strict_ignores_the_name( + op_cls: type, method: str, name_map: dict[str, str] +) -> None: + source = _source() + by_method = getattr(source, method)(name_map, strict=False) + assert by_method.keys() == source.keys() + assert _rows(by_method) == _rows(source) + assert _async_rows_as_dicts(op_cls(name_map, strict=False), source) == _rows(source) + + +@pytest.mark.parametrize( + ("method", "name_map", "keys"), + [ + ( + "map_tags", + {"subject": "animal", "age": "z"}, + (("cohort", "animal"), ("age", "weight")), + ), + ( + "map_data", + {"age": "years", "subject": "z"}, + (("cohort", "subject"), ("years", "weight")), + ), + ( + "map_columns", + {"subject": "animal", "nope": "z"}, + (("cohort", "animal"), ("age", "weight")), + ), + ], +) +def test_non_strict_renames_the_names_that_exist( + method: str, name_map: dict[str, str], keys: tuple[tuple[str, ...], ...] +) -> None: + assert getattr(_source(), method)(name_map, strict=False).keys() == keys + + +# ── strict is not part of the identity ───────────────────────────────────── + + +@pytest.mark.parametrize( + ("op_cls", "name_map"), + [ + (MapTags, {"subject": "animal"}), + (MapData, {"age": "years"}), + (MapColumns, {"subject": "animal", "age": "years"}), + ], + ids=["map_tags", "map_data", "map_columns"], +) +def test_strict_is_not_part_of_the_identity( + op_cls: type, name_map: dict[str, str] +) -> None: + strict = op_cls(name_map)(_source()) + lenient = op_cls(name_map, strict=False)(_source()) + assert strict.content_hash() == lenient.content_hash() + assert strict.pipeline_hash() == lenient.pipeline_hash() + + +@pytest.mark.parametrize("op_cls", [MapTags, MapData, MapColumns]) +@pytest.mark.parametrize("strict", [True, False]) +def test_strict_survives_a_config_round_trip(op_cls: type, strict: bool) -> None: + op = op_cls({"a": "b"}, drop_unmapped=True, strict=strict) + config = op.to_config() + assert config["config"]["strict"] is strict + restored = op_cls.from_config(config) + assert restored.strict is strict + assert restored.drop_unmapped is True + assert restored.name_map == {"a": "b"} + + +@pytest.mark.parametrize("method", ["map_tags", "map_data", "map_columns"]) +def test_strict_and_label_are_keyword_only(method: str) -> None: + with pytest.raises(TypeError): + getattr(_source(), method)({"age": "years"}, False, True) + + +# ── MapColumns: tag and data columns in one step ─────────────────────────── + + +def test_map_columns_renames_tags_and_data() -> None: + stream = _source().map_columns( + {"subject": "animal", "age": "years"}, label="renamed" + ) + assert stream.keys() == (("cohort", "animal"), ("years", "weight")) + assert _rows(stream)[0] == ( + {"cohort": "a", "animal": "m1"}, + {"years": 12, "weight": 25.3}, + ) + columns = stream.as_table(all_info=True).column_names + assert f"{SOURCE}years" in columns + assert f"{SOURCE}age" not in columns + assert stream.label == "renamed" + + +def test_map_columns_is_one_step() -> None: + source = _source() + stream = source.map_columns({"subject": "animal", "age": "years"}) + assert isinstance(stream.producer, MapColumns) + assert list(stream.upstreams) == [source] + + +def test_map_columns_gives_the_rows_of_map_tags_then_map_data() -> None: + source = _source() + one_step = source.map_columns({"subject": "animal", "age": "years"}) + two_steps = source.map_tags({"subject": "animal"}).map_data({"age": "years"}) + assert one_step.keys() == two_steps.keys() + assert _rows(one_step) == _rows(two_steps) + assert ( + one_step.as_table(all_info=True).drop_columns(["_content_hash"]).column_names + == two_steps.as_table(all_info=True) + .drop_columns(["_content_hash"]) + .column_names + ) + # A different pipeline: one operator step, not two. + assert one_step.pipeline_hash() != two_steps.pipeline_hash() + + +def test_map_columns_of_one_group_has_its_own_identity() -> None: + source = _source() + assert ( + source.map_columns({"age": "years"}).pipeline_hash() + != source.map_data({"age": "years"}).pipeline_hash() + ) + assert _rows(source.map_columns({"age": "years"})) == _rows( + source.map_data({"age": "years"}) + ) + + +def test_map_columns_swaps_two_names_within_a_group() -> None: + stream = _source().map_columns({"age": "weight", "weight": "age"}) + assert stream.keys() == (("cohort", "subject"), ("weight", "age")) + assert _rows(stream)[0][1] == {"weight": 12, "age": 25.3} + + +def test_map_columns_drop_unmapped_keeps_only_the_named_columns() -> None: + stream = _source().map_columns( + {"subject": "animal", "weight": "kg"}, drop_unmapped=True + ) + assert stream.keys() == (("animal",), ("kg",)) + columns = stream.as_table(all_info=True).column_names + assert f"{SOURCE}age" not in columns + assert any(c.startswith(constants.SYSTEM_TAG_PREFIX) for c in columns) + + +@pytest.mark.parametrize( + ("name_map", "match"), + [ + ({"subject": "age"}, r"not renamed already have \['age'\]"), + ({"age": "cohort"}, r"not renamed already have \['cohort'\]"), + ({"subject": "x", "age": "x"}, r"two columns would both be named \['x'\]"), + ], + ids=["tag_onto_data", "data_onto_tag", "two_onto_one"], +) +def test_map_columns_refuses_a_collision(name_map: dict[str, str], match: str) -> None: + with pytest.raises(InputValidationError, match=match): + MapColumns(name_map)(_source()) + with pytest.raises(InputValidationError, match=match): + async_rows(MapColumns(name_map), _source()) + + +def test_map_columns_refuses_leaving_no_data_column() -> None: + op = MapColumns({"subject": "animal"}, drop_unmapped=True) + with pytest.raises(InputValidationError, match="needs at least one data column"): + op(_source()) + with pytest.raises(InputValidationError, match="needs at least one data column"): + async_rows(op, _source()) + + +def test_map_columns_refuses_a_meta_named_target() -> None: + with pytest.raises(InputValidationError, match="'__x'.*reserved for meta"): + MapColumns({"age": "__x"}) + + +@pytest.mark.parametrize( + "make_op", + [ + lambda: MapColumns({"subject": "animal", "age": "years"}), + lambda: MapColumns({"subject": "animal", "weight": "kg"}, drop_unmapped=True), + lambda: MapColumns({"nope": "z"}, strict=False), + ], + ids=["rename", "drop_unmapped", "nothing"], +) +@pytest.mark.parametrize( + "columns", + [None, {"system_tags": True}, {"source": True}], + ids=["default", "system_tags", "source"], +) +def test_map_columns_prediction_equals_the_computed_schema( + make_op: Callable[[], Any], columns: dict[str, bool] | None +) -> None: + op = make_op() + source = _source() + predicted = op(source).output_schema(columns=columns) + # Verifying a schema prediction is the standing exception: compare against + # the computed stream, ``unary_static_process(...).output_schema(...)``. + computed = op.unary_static_process(source).output_schema(columns=columns) + assert predicted == computed + + +@pytest.mark.parametrize( + "make_stream", + [ + lambda: meta_stream("s", "v", [f"{META}m"]), + lambda: cached_pod_stream("a", double_a), + ], + ids=["raw_meta_columns", "cached_pod"], +) +def test_map_columns_does_not_carry_meta_columns( + make_stream: Callable[[], StreamProtocol], +) -> None: + stream = make_stream() + data_column = stream.keys()[1][0] + + def make_op() -> MapColumns: + return MapColumns({"k": "key", data_column: "value"}) + + for rows in (sync_rows(make_op(), stream), async_rows(make_op(), stream)): + assert rows + for tag, data in rows: + assert tag.keys() == ("key",) + assert data.keys() == ("value",) + assert not data.meta_columns + result = make_op()(stream) + assert not any(name.startswith(META) for name in result.keys()[1]) + assert not any( + name.startswith(META) + for name in result.output_schema(columns={"meta": True})[1] + ) + + +def test_map_columns_keeps_arrow_types() -> None: + stream = _source().map_columns({"subject": "animal", "age": "years"}) + source_schema = _source().as_table().schema + schema = stream.as_table().schema + assert schema.field("animal").type == source_schema.field("subject").type + assert schema.field("years").type == source_schema.field("age").type + assert schema.field("years").type == pa.int64() diff --git a/tests/test_core/streams/test_pass_through_operator_staleness.py b/tests/test_core/streams/test_pass_through_operator_staleness.py index de8652e91..cc9779885 100644 --- a/tests/test_core/streams/test_pass_through_operator_staleness.py +++ b/tests/test_core/streams/test_pass_through_operator_staleness.py @@ -146,8 +146,8 @@ def _double_pod() -> FunctionPod: "select_data": lambda s: SelectDataColumns(list(s.keys()[1]))(s), "drop_tags": lambda s: DropTagColumns(["absent"], strict=False)(s), "drop_data": lambda s: DropDataColumns(["absent"], strict=False)(s), - "map_tags": lambda s: MapTags({"absent": "renamed"})(s), - "map_data": lambda s: MapData({"absent": "renamed"})(s), + "map_tags": lambda s: MapTags({"absent": "renamed"}, strict=False)(s), + "map_data": lambda s: MapData({"absent": "renamed"}, strict=False)(s), "filter": lambda s: PolarsFilter()(s), "join_one": lambda s: Join()(s), "semi_join_disjoint": lambda s: SemiJoin()(s, _unrelated()), @@ -174,7 +174,9 @@ def _upstream(kind: str) -> tuple[StreamProtocol, StreamProtocol]: if kind == "pod_chain": return _double_pod().process(first), first if kind == "operator_chain": - return MapData({"absent": "renamed"})(SelectDataColumns(["y"])(first)), first + return MapData({"absent": "renamed"}, strict=False)( + SelectDataColumns(["y"])(first) + ), first raise AssertionError(kind) diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 6dd0c6f54..07bc400d1 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -1395,6 +1395,42 @@ def pipeline_tag_and_data_columns() -> dict[str, Any]: return out +@case("pipeline", "map_columns") +def pipeline_map_columns() -> dict[str, Any]: + """``MapColumns``: tag and data columns renamed in one step (ITL-1160). + + ``stream_summary`` keys rows by user tags, so every stream pinned here keeps a + tag whose values are unique per row. + """ + from orcapod.operators import MapColumns + from orcapod.sources import DictSource + + cohort_subjects = DictSource( + data=[ + {"cohort": "a", "subject": "s1", "age": 3, "weight": 20.5}, + {"cohort": "a", "subject": "s2", "age": 4, "weight": 22.25}, + {"cohort": "b", "subject": "s3", "age": 5, "weight": 19.0}, + ], + tag_columns=["cohort", "subject"], + tag_only_columns=True, + source_id="cohort_subjects", + ) + streams = { + "rename": MapColumns(name_map={"subject": "animal", "age": "age_years"})( + cohort_subjects + ), + "drop_unmapped": MapColumns( + name_map={"subject": "animal", "weight": "kg"}, drop_unmapped=True + )(cohort_subjects), + } + out: dict[str, Any] = {} + for name, stream in streams.items(): + row_keys = [_row_key(tag.as_dict()) for tag, _ in stream.iter_data()] + assert len(set(row_keys)) == len(row_keys) == 3, (name, row_keys) + out.update(_prefixed(name, stream_summary(stream))) + return out + + @case("pipeline", "polars_filter") def pipeline_polars_filter() -> dict[str, Any]: from orcapod.operators import PolarsFilter From 21b9b8a5aa6b351bb3fa2d87f5e74c4dfcad4ef6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 11:16:31 -0800 Subject: [PATCH 2/7] feat(operators)!: strict is part of the MapTags / MapData / MapColumns identity (ITL-1160) Developer ruling (2026-11-16): an operator's identity is its full configuration, so strict enters the identity of the three mappers exactly as it does for the column selection operators. This moves the pinned hashes of the golden map_data case and of the drop_nulls_job nodes downstream of its map_data step; the golden files are regenerated separately, on top of ITL-1154 (#406). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1160.md | 5 +++-- docs/reference/hashing.md | 2 +- orcapod-design.md | 2 +- src/orcapod/core/operators/mappers.py | 18 +++++++++--------- .../test_core/operators/test_mappers_strict.py | 16 ++++++++-------- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/changelog.d/ITL-1160.md b/changelog.d/ITL-1160.md index a0abd3a1f..040f10452 100644 --- a/changelog.d/ITL-1160.md +++ b/changelog.d/ITL-1160.md @@ -1,8 +1,9 @@ ### Breaking changes and migration -- `map_tags` / `MapTags` and `map_data` / `MapData` refuse, by default, a mapping key that is not a column of the kind they rename (ITL-1160). Before, such a key was accepted and renamed nothing, which hid mistakes such as `map_tags` of a column that had become data after `group_by`. The call now raises `InputValidationError` naming the missing names and the columns of that kind; a name that is a column of the other kind says so (`'age' is a data column: use map_data`). The check runs when the operator is called on a stream, before any row is processed, and on the async path against the first row. A raw stream's meta (`__`) column is not a data column to rename. Migration: fix the name, use the other method (or `map_columns`), or pass `strict=False` to keep the old behaviour. `strict` is not part of the operator's identity, so no hash moves; it is saved in `to_config()`. +- `map_tags` / `MapTags` and `map_data` / `MapData` refuse, by default, a mapping key that is not a column of the kind they rename (ITL-1160). Before, such a key was accepted and renamed nothing, which hid mistakes such as `map_tags` of a column that had become data after `group_by`. The call now raises `InputValidationError` naming the missing names and the columns of that kind; a name that is a column of the other kind says so (`'age' is a data column: use map_data`). The check runs when the operator is called on a stream, before any row is processed, and on the async path against the first row. A raw stream's meta (`__`) column is not a data column to rename. Migration: fix the name, use the other method (or `map_columns`), or pass `strict=False` to keep the old behaviour. +- `strict` is part of the identity of `MapTags` and `MapData`, as it is of the column selection operators: an operator's identity is its full configuration (developer decision, ITL-1160). Their identity is now `(class name, name_map, drop_unmapped, strict)`, and it is saved in `to_config()`. This moves the content and pipeline hash of every `MapTags` and `MapData` step and of everything downstream of one, and with them the paths of the tables that are scoped by those hashes. - The `strict` and `label` arguments of `map_tags` and `map_data` are keyword-only: `stream.map_tags(name_map, False, "label")` raises `TypeError`; write `label="label"`. ### Added -- `stream.map_columns(name_map)` and the operator `MapColumns` rename tag and data columns in one step (ITL-1160). A renamed tag stays a tag and a renamed data column stays data, with its source-info column; system tags pass through. `drop_unmapped=True` drops every tag and data column not in the mapping. As with `map_tags` and `map_data`, a name that is neither a tag nor a data column is refused unless `strict=False`, a new name taken by a column that is not renamed, or given to two columns, is refused, a `__` new name is refused when the operator is built, and a result with no data column is refused. Like `SelectColumns`, it carries no data meta (`__`) column of the input. Its identity is `("MapColumns", name_map, drop_unmapped)`. +- `stream.map_columns(name_map)` and the operator `MapColumns` rename tag and data columns in one step (ITL-1160). A renamed tag stays a tag and a renamed data column stays data, with its source-info column; system tags pass through. `drop_unmapped=True` drops every tag and data column not in the mapping. As with `map_tags` and `map_data`, a name that is neither a tag nor a data column is refused unless `strict=False`, a new name taken by a column that is not renamed, or given to two columns, is refused, a `__` new name is refused when the operator is built, and a result with no data column is refused. Like `SelectColumns`, it carries no data meta (`__`) column of the input. Its identity is `("MapColumns", name_map, drop_unmapped, strict)`. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index eca8a4752..883811834 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -333,7 +333,7 @@ The table below shows what each class returns from `identity_structure()`. | Operators (unary) | `(operator_class_name, upstream_stream)` | Stream reference resolved via `content_resolver` | | Operators (binary/N-ary) | `(operator_class_name, argument_symmetry(streams))` | `frozenset` for commutative (Join, MergeJoin); `tuple` for ordered (SemiJoin) | | `SelectTagColumns`, `SelectDataColumns`, `DropTagColumns`, `DropDataColumns`, `SelectColumns`, `DropColumns` | `(operator_class_name, sorted unique column names as a list, strict)` | The operators act on a set of names: a list, a tuple, a set, a single string, a repeated or a reordered name list give one hash (ITL-942) | -| `MapTags`, `MapData`, `MapColumns` | `(operator_class_name, name_map, drop_unmapped)` | `strict` is not part of it: it decides only whether an input is refused, never what an accepted input gives (ITL-1160) | +| `MapTags`, `MapData`, `MapColumns` | `(operator_class_name, name_map, drop_unmapped, strict)` | An operator's identity is its full configuration, `strict` included, as for the column selection operators (ITL-1160) | **Inputs of a commutative operator are a set (kept by decision, ITL-942).** A pipeline hash covers the structure of the pipeline, not only its schemas. `MergeJoin()(m1, m1)`, a stream diff --git a/orcapod-design.md b/orcapod-design.md index 080d02065..e889386f8 100644 --- a/orcapod-design.md +++ b/orcapod-design.md @@ -87,7 +87,7 @@ Keep only specified tag or data columns. Optional `strict` mode raises on missin Remove specified tag or data columns. `DropDataColumns` also removes associated source-info columns. ### MapTags / MapData / MapColumns -Rename tag or data columns via a name mapping; `MapColumns` renames both in one step, and a tag stays a tag. A renamed data column's source-info column is renamed with it. Optional `drop_unmapped` mode removes columns not in the mapping. A name that is not a column of the mapped kind is refused by default (`strict=True`, not part of the identity; ITL-1160). +Rename tag or data columns via a name mapping; `MapColumns` renames both in one step, and a tag stays a tag. A renamed data column's source-info column is renamed with it. Optional `drop_unmapped` mode removes columns not in the mapping. A name that is not a column of the mapped kind is refused by default (`strict=True`, part of the identity; ITL-1160). ### PolarsFilter Applies Polars filtering predicates to rows. Output schema is unchanged from input. diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 7afbb822e..0258f482c 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -53,9 +53,8 @@ class MapData(UnaryOperator): dropped. strict: When ``True`` (the default), a key of ``name_map`` that is not a data column of the input raises ``InputValidationError``; when - ``False`` it is ignored. Not part of the operator's identity: it - decides only whether an input is refused, never what an accepted - input gives. + ``False`` it is ignored. Part of the operator's identity, as for + ``SelectTagColumns`` and the other column operators. """ def __init__( @@ -252,6 +251,7 @@ def identity_structure(self) -> Any: self.__class__.__name__, self.name_map, self.drop_unmapped, + self.strict, ) @@ -264,9 +264,8 @@ class MapTags(UnaryOperator): dropped. strict: When ``True`` (the default), a key of ``name_map`` that is not a tag column of the input raises ``InputValidationError``; when - ``False`` it is ignored. Not part of the operator's identity: it - decides only whether an input is refused, never what an accepted - input gives. + ``False`` it is ignored. Part of the operator's identity, as for + ``SelectTagColumns`` and the other column operators. """ def __init__( @@ -452,6 +451,7 @@ def identity_structure(self) -> Any: self.__class__.__name__, self.name_map, self.drop_unmapped, + self.strict, ) @@ -472,9 +472,8 @@ class MapColumns(UnaryOperator): data column); system tags are kept. strict: When ``True`` (the default), a key of ``name_map`` that is neither a tag nor a data column of the input raises - ``InputValidationError``; when ``False`` it is ignored. Not part of - the operator's identity: it decides only whether an input is - refused, never what an accepted input gives. + ``InputValidationError``; when ``False`` it is ignored. Part of the + operator's identity, as for ``SelectColumns``. Raises: InputValidationError: When called on a stream where a new name is taken @@ -518,6 +517,7 @@ def identity_structure(self) -> Any: self.__class__.__name__, self.name_map, self.drop_unmapped, + self.strict, ) @staticmethod diff --git a/tests/test_core/operators/test_mappers_strict.py b/tests/test_core/operators/test_mappers_strict.py index a8f889b84..494a1a727 100644 --- a/tests/test_core/operators/test_mappers_strict.py +++ b/tests/test_core/operators/test_mappers_strict.py @@ -5,8 +5,8 @@ columns of that kind; a name that is a column of the other kind says which method renames it. The check runs when the operator is called on a stream, before any row is processed, and on the async path against the first row. ``strict=False`` keeps -the earlier behaviour: such a name is ignored. ``strict`` is not part of the -operator's identity. ``MapColumns`` renames tag and data columns in one step; a tag +the earlier behaviour: such a name is ignored. ``strict`` is part of the +operator's identity, as for the column selection operators. ``MapColumns`` renames tag and data columns in one step; a tag stays a tag and data stays data. Tests drive the operators through ``op(stream)`` and the stream methods. """ @@ -244,7 +244,7 @@ def test_non_strict_renames_the_names_that_exist( assert getattr(_source(), method)(name_map, strict=False).keys() == keys -# ── strict is not part of the identity ───────────────────────────────────── +# ── strict is part of the identity ───────────────────────────────────────── @pytest.mark.parametrize( @@ -256,13 +256,13 @@ def test_non_strict_renames_the_names_that_exist( ], ids=["map_tags", "map_data", "map_columns"], ) -def test_strict_is_not_part_of_the_identity( - op_cls: type, name_map: dict[str, str] -) -> None: +def test_strict_enters_the_identity(op_cls: type, name_map: dict[str, str]) -> None: strict = op_cls(name_map)(_source()) lenient = op_cls(name_map, strict=False)(_source()) - assert strict.content_hash() == lenient.content_hash() - assert strict.pipeline_hash() == lenient.pipeline_hash() + assert strict.content_hash() != lenient.content_hash() + assert strict.pipeline_hash() != lenient.pipeline_hash() + # The rows are the same: an accepted input gives the same result. + assert _rows(strict) == _rows(lenient) @pytest.mark.parametrize("op_cls", [MapTags, MapData, MapColumns]) From ba4ca3db2ba576370e6d01f9665c7b9a4580bb17 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 11:29:11 -0800 Subject: [PATCH 3/7] refactor(operators): no message-only parameter in the mapper name checks (ITL-1160) MapTags._require_tag_columns and MapData._require_data_columns take only the group they check; the caller annotates the refusal with the names that are columns of the other group (CodeRabbit CLI review of 21b9b8a5, per the "no message-only parameters" rule). Co-Authored-By: Claude Opus 5.5 --- docs/concepts/operators.md | 2 +- src/orcapod/core/operators/mappers.py | 91 +++++++++++++++++++-------- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index d6111ead0..16275feaa 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -418,7 +418,7 @@ try: sessions.map_tags({"age": "age_weeks"}) except InputValidationError as error: print(error) -# MapTags: ['age'] not among the tag columns ['subject_id', 'session']; 'age' is a data column: use map_data; use strict=False to ignore names that are not tag columns +# MapTags: ['age'] not among the tag columns ['subject_id', 'session'] (use strict=False to ignore names that are not tag columns); 'age' is a data column: use map_data ``` A new name already taken by a column that is not renamed is refused too. diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 0258f482c..2d6d9493d 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -22,26 +22,35 @@ pa = LazyModule("pyarrow") -def _other_group_note( - names: Collection[str], other_columns: Collection[str], group: str, method: str -) -> str: - """Name the missing names that are columns of the other group, for an error. +def _with_other_group_note( + error: InputValidationError, + names: Collection[str], + other_columns: Collection[str], + group: str, + method: str, +) -> InputValidationError: + """Add to a refusal which of the names are columns of the other group. + + A reporting helper: the caller, which knows the other group's columns, + annotates the error the mapper raised from its own group alone. Args: - names: The names that are not columns of the renamed group. + error: The refusal of names that are not columns of the renamed group. + names: The names the mapper was given. other_columns: The input's columns of the other group. group: The other group, ``"tag"`` or ``"data"``. method: The stream method that renames a column of the other group. Returns: - One ``"; 'x' is a data column: use map_data"`` clause per such name, or an - empty string. + The error with one ``"; 'x' is a data column: use map_data"`` clause per + name that is a column of the other group. """ - return "".join( + note = "".join( f"; {n!r} is a {group} column: use {method}" for n in names if n in other_columns ) + return InputValidationError(f"{error}{note}") class MapData(UnaryOperator): @@ -72,13 +81,10 @@ def __init__( self.strict = strict super().__init__(**kwargs) - def _require_data_columns( - self, tag_columns: Collection[str], data_columns: Collection[str] - ) -> None: + def _require_data_columns(self, data_columns: Collection[str]) -> None: """Refuse, when ``strict``, a name that is not a data column (ITL-1160). Args: - tag_columns: The input's tag columns. data_columns: The input's data columns, without meta columns. Raises: @@ -89,11 +95,29 @@ def _require_data_columns( if missing and self.strict: raise InputValidationError( f"MapData: {missing} not among the data columns " - f"{list(data_columns)}" - f"{_other_group_note(missing, tag_columns, 'tag', 'map_tags')}; " - f"use strict=False to ignore names that are not data columns" + f"{list(data_columns)} (use strict=False to ignore names that are " + f"not data columns)" ) + def _check_names( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> None: + """Run ``_require_data_columns`` and name the tag columns among the names. + + Args: + tag_columns: The input's tag columns. + data_columns: The input's data columns, without meta columns. + + Raises: + InputValidationError: As ``_require_data_columns``, annotated. + """ + try: + self._require_data_columns(data_columns) + except InputValidationError as error: + raise _with_other_group_note( + error, self.name_map, tag_columns, "tag", "map_tags" + ) from error + def to_config(self) -> dict[str, Any]: """Serialize this MapData operator to a config dict. @@ -155,7 +179,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: def validate_unary_input(self, stream: StreamProtocol) -> None: tag_columns, data_columns = stream.keys() - self._require_data_columns( + self._check_names( tag_columns, [c for c in data_columns if not is_meta_column(c)] ) # verify that renamed value does NOT collide with other columns @@ -227,7 +251,7 @@ async def async_execute( pkt_keys = data.keys() # The names are checked against the first row, as # ``validate_unary_input`` checks them against the stream. - self._require_data_columns( + self._check_names( tag.keys(), [k for k in pkt_keys if not is_meta_column(k)] ) rename_map = { @@ -283,14 +307,11 @@ def __init__( self.strict = strict super().__init__(**kwargs) - def _require_tag_columns( - self, tag_columns: Collection[str], data_columns: Collection[str] - ) -> None: + def _require_tag_columns(self, tag_columns: Collection[str]) -> None: """Refuse, when ``strict``, a name that is not a tag column (ITL-1160). Args: tag_columns: The input's tag columns. - data_columns: The input's data columns, without meta columns. Raises: InputValidationError: When ``strict`` and a key of ``name_map`` is @@ -299,11 +320,29 @@ def _require_tag_columns( missing = [n for n in self.name_map if n not in tag_columns] if missing and self.strict: raise InputValidationError( - f"MapTags: {missing} not among the tag columns {list(tag_columns)}" - f"{_other_group_note(missing, data_columns, 'data', 'map_data')}; " - f"use strict=False to ignore names that are not tag columns" + f"MapTags: {missing} not among the tag columns {list(tag_columns)} " + f"(use strict=False to ignore names that are not tag columns)" ) + def _check_names( + self, tag_columns: Collection[str], data_columns: Collection[str] + ) -> None: + """Run ``_require_tag_columns`` and name the data columns among the names. + + Args: + tag_columns: The input's tag columns. + data_columns: The input's data columns, without meta columns. + + Raises: + InputValidationError: As ``_require_tag_columns``, annotated. + """ + try: + self._require_tag_columns(tag_columns) + except InputValidationError as error: + raise _with_other_group_note( + error, self.name_map, data_columns, "data", "map_data" + ) from error + def to_config(self) -> dict[str, Any]: """Serialize this MapTags operator to a config dict. @@ -360,7 +399,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: It takes two streams as input and raises an error if the inputs are not valid. """ tag_columns, data_columns = stream.keys() - self._require_tag_columns( + self._check_names( tag_columns, [c for c in data_columns if not is_meta_column(c)] ) # verify that renamed value does NOT collide with other columns @@ -426,7 +465,7 @@ async def async_execute( tag_keys = tag.keys() # The names are checked against the first row, as # ``validate_unary_input`` checks them against the stream. - self._require_tag_columns( + self._check_names( tag_keys, [k for k in data.keys() if not is_meta_column(k)] ) rename_map = { From 0394da89cd053e0f3f0e82e0ac743ecb78ad25ad Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 12:37:02 -0800 Subject: [PATCH 4/7] fix(operators): mappers refuse a new name a stream reads as a system column (ITL-1160) MapTags, MapData and MapColumns refuse, when built, a new name that starts with the system-tag or the source-info prefix, or is the context key column (schema_utils.require_names_not_system, beside require_names_not_meta). Before, such a rename was accepted and the column vanished or the stream failed on read. Found by the CodeRabbit CLI review of ba4ca3db. Sources, Pick and Index are left to ITL-1165 (design_issues/ITL-1165-RN1.md). No pinned hash moves. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +- CLAUDE.md | 4 +- changelog.d/ITL-1160.md | 4 ++ design_issues/ITL-1165-RN1.md | 27 +++++++++ src/orcapod/core/operators/mappers.py | 3 + src/orcapod/utils/schema_utils.py | 38 ++++++++++++ .../operators/test_mappers_strict.py | 59 +++++++++++++++++++ 7 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 design_issues/ITL-1165-RN1.md diff --git a/.zed/rules b/.zed/rules index 77d5f8dfc..2c57c020d 100644 --- a/.zed/rules +++ b/.zed/rules @@ -630,7 +630,9 @@ raw ArrowTableStream's `__` columns and for the meta a cached pod's or function carry. One rule, core/operators/meta_columns.py. A carry-forward mode is ITL-867. The meta prefix (`__`) is reserved: ArrowTableStream and the source builder refuse a tag named with it, and MapTags / MapData / MapColumns (targets) and Pick / Index (out) refuse it as a new name -(schema_utils.require_names_not_meta, at construction). +(schema_utils.require_names_not_meta, at construction). The three mappers also refuse a new name +that a stream reads as a system column (system-tag or source-info prefix, the context key column; +schema_utils.require_names_not_system, ITL-1160). Operator or derived source? (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/CLAUDE.md b/CLAUDE.md index 8e5d3dc5d..00c308882 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -650,7 +650,9 @@ carry. The rule is in one place, `core/operators/meta_columns.py` (sync results operator base classes, async rows through `WithoutMetaWriter`). A carry-forward mode is ITL-867. The meta prefix (`__`) is reserved: `ArrowTableStream` and the source builder refuse a tag named with it, and `MapTags` / `MapData` / `MapColumns` (targets) and `Pick` / `Index` (`out`) refuse it as a new name -(`schema_utils.require_names_not_meta`, at construction). +(`schema_utils.require_names_not_meta`, at construction). The three mappers also refuse a new name +that a stream reads as a system column (system-tag or source-info prefix, the context key column; +`schema_utils.require_names_not_system`, ITL-1160). **Operator or derived source?** (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/changelog.d/ITL-1160.md b/changelog.d/ITL-1160.md index 040f10452..4a62a8c37 100644 --- a/changelog.d/ITL-1160.md +++ b/changelog.d/ITL-1160.md @@ -7,3 +7,7 @@ ### Added - `stream.map_columns(name_map)` and the operator `MapColumns` rename tag and data columns in one step (ITL-1160). A renamed tag stays a tag and a renamed data column stays data, with its source-info column; system tags pass through. `drop_unmapped=True` drops every tag and data column not in the mapping. As with `map_tags` and `map_data`, a name that is neither a tag nor a data column is refused unless `strict=False`, a new name taken by a column that is not renamed, or given to two columns, is refused, a `__` new name is refused when the operator is built, and a result with no data column is refused. Like `SelectColumns`, it carries no data meta (`__`) column of the input. Its identity is `("MapColumns", name_map, drop_unmapped, strict)`. + +### Fixed + +- `MapTags`, `MapData` and `MapColumns` refuse, when built, a new name that a stream reads as a system column: one that starts with the system-tag prefix (`_tag_`) or the source-info prefix (`_source_`), or the context key column (`_context_key`) (ITL-1160). Before, such a rename was accepted and the column then vanished from the stream or the stream failed on read with `No data columns found`. diff --git a/design_issues/ITL-1165-RN1.md b/design_issues/ITL-1165-RN1.md new file mode 100644 index 000000000..a71fd0d3a --- /dev/null +++ b/design_issues/ITL-1165-RN1.md @@ -0,0 +1,27 @@ +## Cross-cutting + +### RN1 — Reserved `_` column names: sources silently drop them, `Pick` / `Index` accept them as new names +**Status:** open +**Severity:** medium +**Issue:** ITL-1165 + +Columns whose names a stream reads as system columns are handled inconsistently (found on +November 16, 2026 while fixing ITL-1160, probed on that branch): + +* `ArrowTableSource` silently drops a data column named `_foo`, `_tag_y`, `_source_y`, + `_content_hash` or `_context_key`: the source builds without error and `keys()` lists only + the other data columns. +* `Index(..., out=...)` accepts `_tag_y`, `_source_y` and `_context_key`; `keys()` then lists + the column but `as_table()` does not have it. `Pick` has the same out-name check (only the + meta prefix `__` is refused, `schema_utils.require_names_not_meta`). +* ITL-1160 fixed the mappers only: `MapTags`, `MapData` and `MapColumns` refuse, when built, a + new name that starts with the system-tag prefix (`_tag_`) or the source-info prefix + (`_source_`), or is the context key column (`_context_key`) + (`schema_utils.require_names_not_system`). +* Open question: is a bare `_` name reserved at all? A source drops `_foo` and `_content_hash`, + while `MapData({"x": "_foo"})` and `Index(out="_foo")` give a working data column, and so + does renaming to `_content_hash`. + +To decide: which names are reserved; then a source refuses such a column with an error naming +it instead of dropping it, and `Pick` / `Index` refuse a reserved `out` name through the same +helper as the mappers. diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 2d6d9493d..73fd3d8e3 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -76,6 +76,7 @@ def __init__( # A data column never gets a meta column's name (ITL-1080): refused here, # so no path (sync, async or the prediction) can produce one. schema_utils.require_names_not_meta(name_map.values()) + schema_utils.require_names_not_system(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped self.strict = strict @@ -302,6 +303,7 @@ def __init__( # A tag never gets a meta column's name (ITL-1080): refused here, so no # path (sync, async or the prediction) can produce one. schema_utils.require_names_not_meta(name_map.values()) + schema_utils.require_names_not_system(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped self.strict = strict @@ -530,6 +532,7 @@ def __init__( # No column gets a meta column's name (ITL-1080): refused here, so no # path (sync, async or the prediction) can produce one. schema_utils.require_names_not_meta(name_map.values()) + schema_utils.require_names_not_system(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped self.strict = strict diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index 890522ef0..e2ac12849 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -603,3 +603,41 @@ def require_names_not_meta(names: Collection[str]) -> None: f"column name {name!r} starts with {constants.META_PREFIX!r}, which is " "reserved for meta columns; a tag or data column needs another name." ) + + +def require_names_not_system(names: Collection[str]) -> None: + """Raise if a tag or data column name is one a stream reads as a system column. + + A stream reads a column whose name starts with the system-tag prefix + (``_tag_``) as a system tag, one that starts with the source-info prefix + (``_source_``) as source info, and the context key column (``_context_key``) + as the data context, so a tag or data column renamed to such a name stops + being one (ITL-1160). The meta prefix has its own check, + ``require_names_not_meta``. + + Args: + names: Tag or data column names. + + Raises: + InputValidationError: If a name starts with the system-tag or the + source-info prefix, or is the context key column. The message names + the column. + """ + from orcapod.system_constants import constants + + reserved_prefixes = { + constants.SYSTEM_TAG_PREFIX: "system tags", + constants.SOURCE_PREFIX: "source info", + } + for name in sorted(names): + for prefix, kind in reserved_prefixes.items(): + if name.startswith(prefix): + raise InputValidationError( + f"column name {name!r} starts with {prefix!r}, which is reserved " + f"for {kind}; a tag or data column needs another name." + ) + if name == constants.CONTEXT_KEY: + raise InputValidationError( + f"column name {name!r} is reserved for the data context; a tag or " + "data column needs another name." + ) diff --git a/tests/test_core/operators/test_mappers_strict.py b/tests/test_core/operators/test_mappers_strict.py index 494a1a727..2b047872f 100644 --- a/tests/test_core/operators/test_mappers_strict.py +++ b/tests/test_core/operators/test_mappers_strict.py @@ -444,3 +444,62 @@ def test_map_columns_keeps_arrow_types() -> None: assert schema.field("animal").type == source_schema.field("subject").type assert schema.field("years").type == source_schema.field("age").type assert schema.field("years").type == pa.int64() + + +# ── a new name a stream would read as a system column ────────────────────── + +RESERVED_NEW_NAMES = [ + f"{constants.SYSTEM_TAG_PREFIX}y", + f"{constants.SOURCE_PREFIX}y", + constants.CONTEXT_KEY, +] + + +@pytest.mark.parametrize( + ("op_cls", "old_name"), + [ + (MapTags, "subject"), + (MapData, "age"), + (MapColumns, "subject"), + (MapColumns, "age"), + ], + ids=["map_tags", "map_data", "map_columns_tag", "map_columns_data"], +) +@pytest.mark.parametrize( + "new_name", RESERVED_NEW_NAMES, ids=["system_tag", "source", "context_key"] +) +@pytest.mark.parametrize("strict", [True, False]) +def test_a_system_column_name_is_refused_as_a_new_name( + op_cls: type, old_name: str, new_name: str, strict: bool +) -> None: + # Refused when built, so neither the sync nor the async path can produce one. + with pytest.raises(InputValidationError, match=f"{new_name!r}.*reserved"): + op_cls({old_name: new_name}, strict=strict) + + +@pytest.mark.parametrize( + ("method", "old_name"), + [("map_tags", "subject"), ("map_data", "age"), ("map_columns", "age")], +) +@pytest.mark.parametrize( + "new_name", RESERVED_NEW_NAMES, ids=["system_tag", "source", "context_key"] +) +def test_the_stream_methods_refuse_a_system_column_name( + method: str, old_name: str, new_name: str +) -> None: + with pytest.raises(InputValidationError, match="reserved"): + getattr(_source(), method)({old_name: new_name}) + + +@pytest.mark.parametrize("new_name", ["_age", "_content_hash"]) +def test_other_underscore_names_stay_usable_as_new_names(new_name: str) -> None: + """Only names a stream reads as system columns are refused (see ITL-1160).""" + source = _source() + for stream, async_op in ( + (source.map_data({"age": new_name}), MapData({"age": new_name})), + (source.map_columns({"age": new_name}), MapColumns({"age": new_name})), + ): + assert stream.keys() == (("cohort", "subject"), (new_name, "weight")) + assert stream.as_table().column_names[2] == new_name + rows = async_rows(async_op, source) + assert all(data.keys() == (new_name, "weight") for _, data in rows) From 75c03828a94f5e856845ba48c8a1bbd1f54adc46 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:42:17 -0800 Subject: [PATCH 5/7] docs(tutorial): joined columns follow the new map_data pipeline hash (ITL-1160) Join orders its inputs by pipeline hash; strict in the MapData identity moved those hashes, so the side-by-side sample shows n_imec0 before n_imec1. Co-Authored-By: Claude Opus 5.5 --- docs/tutorial/03-operators.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial/03-operators.md b/docs/tutorial/03-operators.md index aad5bb24d..1b9ecdd04 100644 --- a/docs/tutorial/03-operators.md +++ b/docs/tutorial/03-operators.md @@ -263,10 +263,10 @@ side_by_side = imec0.map_data({"n_spikes": "n_imec0"}).join( ) pair_tags = ["subject_id", "session", "unit"] print(side_by_side.as_pandas_df().sort_values(pair_tags, ignore_index=True).head(3)) -# subject_id session unit n_imec1 n_imec0 -# 0 mouse_01 s1 0 215 149 -# 1 mouse_01 s1 1 299 326 -# 2 mouse_01 s1 2 135 662 +# subject_id session unit n_imec0 n_imec1 +# 0 mouse_01 s1 0 149 215 +# 1 mouse_01 s1 1 326 299 +# 2 mouse_01 s1 2 662 135 ``` **`MergeJoin`**, when they are repeated measurements of the same quantity and you want them From d7b7b6faace5cde039101e81fc6ac56d6e3ef282 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:47:20 -0800 Subject: [PATCH 6/7] test(golden): pin MapColumns and the mapper strict-identity moves (ITL-1160) Approved regeneration of the open v1 revision (developer, 2026-11-16): 7 pipeline entries moved because strict entered the MapData identity (map_data and the drop_nulls_job nodes downstream of its map_data step), one drop_nulls_job table entry re-keyed under its moved path with an identical value, and 24 map_columns entries added. Nothing else changed: generate_golden_hashes.py --check is clean. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/contexts/data/v1.json | 4 +- .../golden/data/v1/pipeline-01.json | 46 +++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 87303ee5e..ff9ce885d 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -235,13 +235,13 @@ "added": { "arrow-01.json": "c66fa55619178f2631d04232555cd58059813e97e9ecd2b9de9e63699e60c964", "datagram-01.json": "598d4a2851819d76ba674e189222e61733ca1861ca82a59e990a8ee5e1b0a7b3", - "pipeline-01.json": "f6fd38d591b20876725d4050ce4f11ad541460e9913262b05d61c1b398770711", + "pipeline-01.json": "ec0049a39e85645d59d9a276a17528bfa64a91f349aa0094052a6d55bc930be5", "schema-01.json": "79aa2a068e0724891cf43ee528ce8f569e65f08864faa5e34773bfc69ac9576e", "semantic-01.json": "8e5d9dfff461f9f1d6bdd49a3019e7bf6122e75d799d725b17531d31b080003c", "torch-01.json": "af2e8b68d66dffaa37e1d0ad75acd42f595d65b9fae00ddcde81bf9e13a265c8" }, "previous": null, - "digest": "2a2dcc4dd5e52a906f79f0ca3a26989ef0869b5531882831da004570f70c4a4d" + "digest": "0a789cf9e8d1df9fb762d6dc2fabffcd3383cd6a21ef147e8c6583f963280c2e" } ] }, diff --git a/tests/test_hashing/golden/data/v1/pipeline-01.json b/tests/test_hashing/golden/data/v1/pipeline-01.json index cbbe3e75a..4bf77eab0 100644 --- a/tests/test_hashing/golden/data/v1/pipeline-01.json +++ b/tests/test_hashing/golden/data/v1/pipeline-01.json @@ -191,8 +191,8 @@ "_tag_source_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f", "_tag_record_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f" ], - "drop_nulls_job.node[ratio].content_hash": "semantic_v1:d52ae96e0035a6a05d2e48b12c8e4850c2b2ad3def9733c69ad2ff2c4d354f7d", - "drop_nulls_job.node[ratio].pipeline_hash": "semantic_v1:c4956a859077ac1fa38cab2fa7bb4f129c567667ec68d248d273e47356f1c48c", + "drop_nulls_job.node[ratio].content_hash": "semantic_v1:91d4c2de93394d5bc615786dbe01bbb8581ef6a3628b6b76bb271ab10721eaab", + "drop_nulls_job.node[ratio].pipeline_hash": "semantic_v1:180bd73b49abfa39858cc882112527ce1d053dc50b348425e9e09b7f492df58e", "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f": "row_0", "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f": "golden_nullable", "drop_nulls_job.node[ratio].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v1:00000213e161255a575a960327448c972d8a05c86fd407479b52b964dd417d8e09eae8", @@ -203,8 +203,8 @@ "_tag_source_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f", "_tag_record_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f" ], - "drop_nulls_job.node[renamed].content_hash": "semantic_v1:5d2af7b40bcb93c1d12ac684cef0be1a7073d1447bbaef940f0ffac340dc68f9", - "drop_nulls_job.node[renamed].pipeline_hash": "semantic_v1:0d2ac225b76434bbb043a19840b957a2583b33ca7345075b3bad25c58ea32cb7", + "drop_nulls_job.node[renamed].content_hash": "semantic_v1:2f461039fecf4d121059cc5133bd80eb9bcdf440d7ec782fe0e3f3da730e21fa", + "drop_nulls_job.node[renamed].pipeline_hash": "semantic_v1:1d6f3da02138530cc8d6c29a51419be6dad8cf45da673844ef78996a7f96c816", "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_record_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f": "row_0", "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}]._tag_source_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f": "golden_nullable", "drop_nulls_job.node[renamed].row[{\"k\":\"r1\",\"site\":\"a\"}].data_hash": "arrow_v1:000002c619a23da2daa78b89cb32837af1561a1e6613d3ef0af83cf95506abeeedab1f", @@ -219,13 +219,13 @@ "arrow_v1:0000026c8d10d614b78d874ef224a78d613befa1a0c773932aaa368237211673d18380", "arrow_v1:000002c619a23da2daa78b89cb32837af1561a1e6613d3ef0af83cf95506abeeedab1f" ], - "drop_nulls_job.table[golden_drop_null_job/golden_ratio/21e6485805726eb7ceed63ef6b677f0b6b11d2eaf48bd9fc44c1dccdf718aa1d/v0/python.function.v0/schema:c4956a859077ac1fa38cab2fa7bb4f129c567667ec68d248d273e47356f1c48c/pdb_v2].record_ids": [ + "drop_nulls_job.table[golden_drop_null_job/golden_ratio/21e6485805726eb7ceed63ef6b677f0b6b11d2eaf48bd9fc44c1dccdf718aa1d/v0/python.function.v0/schema:180bd73b49abfa39858cc882112527ce1d053dc50b348425e9e09b7f492df58e/pdb_v2].record_ids": [ "arrow_v1:0000025e66eb58363aca967d1e9fa33dc979d6b78d90c1420ef3031defc7b74df5eafc", "arrow_v1:000002fc063fe04b01f009f373104bcc0156b4763c503a27468253e3278f81914b1272" ], "drop_nulls_job.table_paths": [ "golden_drop_null_job/_result/golden_ratio/21e6485805726eb7ceed63ef6b677f0b6b11d2eaf48bd9fc44c1dccdf718aa1d/v0/python.function.v0/rdb_v2", - "golden_drop_null_job/golden_ratio/21e6485805726eb7ceed63ef6b677f0b6b11d2eaf48bd9fc44c1dccdf718aa1d/v0/python.function.v0/schema:c4956a859077ac1fa38cab2fa7bb4f129c567667ec68d248d273e47356f1c48c/pdb_v2" + "golden_drop_null_job/golden_ratio/21e6485805726eb7ceed63ef6b677f0b6b11d2eaf48bd9fc44c1dccdf718aa1d/v0/python.function.v0/schema:180bd73b49abfa39858cc882112527ce1d053dc50b348425e9e09b7f492df58e/pdb_v2" ], "group_by.content_hash": "semantic_v1:e39fd80bf63f3549d59902281171f1d94fba5202fad99b7b29eb1a48d4e31364", "group_by.pipeline_hash": "semantic_v1:a54462063efcb67b3d3df58840d7b63d5d5b6d2d772efc154655859fe7588400", @@ -436,8 +436,38 @@ ], "list_tag_producerless_stream.as_given": "semantic_v1:4b99af058c91851f2998544963e3b2d6080b7078e5a29b3aaca0ec4fe172e5e2", "list_tag_producerless_stream.reversed": "semantic_v1:ad76e4cf890d6cf18777dfc3e1998d322a2e7079a9f5824abe1f34f42f3265da", - "map_data.content_hash": "semantic_v1:0b6a94acb6eb95ec5a5d46d4ee9212ddcacffefc20823c164fc8eabaadaef107", - "map_data.pipeline_hash": "semantic_v1:3f213330745687310dfb37749ef2f5103b375016fcda3887ed5a30b2b163a6c8", + "map_columns.drop_unmapped.content_hash": "semantic_v1:dbee292987b03cb1bdb3ef0fd6be097240b308d08f9eeeb91d180b2740be69cb", + "map_columns.drop_unmapped.pipeline_hash": "semantic_v1:27ad3d292121b825fe85b28e54cfb432cecaaa53978431c9039f47df038af54f", + "map_columns.drop_unmapped.row[{\"animal\":\"s1\"}]._tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "row_0", + "map_columns.drop_unmapped.row[{\"animal\":\"s1\"}]._tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "cohort_subjects", + "map_columns.drop_unmapped.row[{\"animal\":\"s1\"}].data_hash": "arrow_v1:0000025eca48b03061872990e7d71bc1d2ff2780795d168481bba072960cf91368c3bf", + "map_columns.drop_unmapped.row[{\"animal\":\"s2\"}]._tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "row_1", + "map_columns.drop_unmapped.row[{\"animal\":\"s2\"}]._tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "cohort_subjects", + "map_columns.drop_unmapped.row[{\"animal\":\"s2\"}].data_hash": "arrow_v1:000002ef0edcfe57d8308fb8f035d58bdd0d4423161cbd05cf90985b32c98aa66e8ccc", + "map_columns.drop_unmapped.row[{\"animal\":\"s3\"}]._tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "row_2", + "map_columns.drop_unmapped.row[{\"animal\":\"s3\"}]._tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "cohort_subjects", + "map_columns.drop_unmapped.row[{\"animal\":\"s3\"}].data_hash": "arrow_v1:000002371aa9a6b18f88c2e770fdae5f1c47c467fa25742439d7dc773bd9aa217d0ee9", + "map_columns.drop_unmapped.system_tag_columns": [ + "_tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec", + "_tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec" + ], + "map_columns.rename.content_hash": "semantic_v1:288ad1168e3be6273284ceed01a56b9e45c8213182f50b19bbe1274e2e2e5432", + "map_columns.rename.pipeline_hash": "semantic_v1:a6d3125ac50d611791a691fb8f05be49b3e0350a1b071fc274839ff71d0f79ed", + "map_columns.rename.row[{\"animal\":\"s1\",\"cohort\":\"a\"}]._tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "row_0", + "map_columns.rename.row[{\"animal\":\"s1\",\"cohort\":\"a\"}]._tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "cohort_subjects", + "map_columns.rename.row[{\"animal\":\"s1\",\"cohort\":\"a\"}].data_hash": "arrow_v1:0000022ad1363967c26697bb148943f0b193f748f21da43e4ba5f3763197a04bf53289", + "map_columns.rename.row[{\"animal\":\"s2\",\"cohort\":\"a\"}]._tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "row_1", + "map_columns.rename.row[{\"animal\":\"s2\",\"cohort\":\"a\"}]._tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "cohort_subjects", + "map_columns.rename.row[{\"animal\":\"s2\",\"cohort\":\"a\"}].data_hash": "arrow_v1:000002a61fcaac02c3fd261c22c4fab3c89182aef1912fb4a19ae1c01af9cf69b4b6b6", + "map_columns.rename.row[{\"animal\":\"s3\",\"cohort\":\"b\"}]._tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "row_2", + "map_columns.rename.row[{\"animal\":\"s3\",\"cohort\":\"b\"}]._tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec": "cohort_subjects", + "map_columns.rename.row[{\"animal\":\"s3\",\"cohort\":\"b\"}].data_hash": "arrow_v1:000002ef21c37efbb54c93fff65d16c5222fed1364ebf8ab9a49758b064f3edd83f3a1", + "map_columns.rename.system_tag_columns": [ + "_tag_source_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec", + "_tag_record_id::220185c669ba2e7bb644abc11150f3b11d9c41c70b04a6397c68bda95f8d24ec" + ], + "map_data.content_hash": "semantic_v1:5dc048a18ed101f311e58853fe170a5d6951af7aca7b67c79bdb5252f3f4fd31", + "map_data.pipeline_hash": "semantic_v1:e56188c74c72cb812622267bee5fef68c5b9237c8f5b38571c5f5d2a3a4176de", "map_data.row[{\"subject\":\"s1\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_0", "map_data.row[{\"subject\":\"s1\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", "map_data.row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:0000022ad1363967c26697bb148943f0b193f748f21da43e4ba5f3763197a04bf53289", From 6589afc98453bd2370c12bcf1a1ecbea49057d36 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 14:14:21 -0800 Subject: [PATCH 7/7] feat(streams)!: map_tags / map_data / map_columns take renames as keyword arguments (ITL-1171) stream.map_columns(old="new") builds the same operator, with the same name_map and hashes, as stream.map_columns({"old": "new"}). The dict is the only positional argument; drop_unmapped, strict and label are keyword-only options, so a column with one of those names, or a name that is not a Python identifier, is renamed through the dict. A dict and keyword renames together, or a name that is not a string, raise TypeError. The operator classes keep their single mapping argument. No golden value moves. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1160.md | 3 +- docs/concepts/operators.md | 12 +++ docs/tutorial/03-operators.md | 9 ++ src/orcapod/core/streams/base.py | 98 +++++++++++++++++-- .../protocols/core_protocols/streams.py | 25 +++-- .../operators/test_mappers_strict.py | 76 ++++++++++++++ 6 files changed, 205 insertions(+), 18 deletions(-) diff --git a/changelog.d/ITL-1160.md b/changelog.d/ITL-1160.md index 4a62a8c37..699d0cf2d 100644 --- a/changelog.d/ITL-1160.md +++ b/changelog.d/ITL-1160.md @@ -2,11 +2,12 @@ - `map_tags` / `MapTags` and `map_data` / `MapData` refuse, by default, a mapping key that is not a column of the kind they rename (ITL-1160). Before, such a key was accepted and renamed nothing, which hid mistakes such as `map_tags` of a column that had become data after `group_by`. The call now raises `InputValidationError` naming the missing names and the columns of that kind; a name that is a column of the other kind says so (`'age' is a data column: use map_data`). The check runs when the operator is called on a stream, before any row is processed, and on the async path against the first row. A raw stream's meta (`__`) column is not a data column to rename. Migration: fix the name, use the other method (or `map_columns`), or pass `strict=False` to keep the old behaviour. - `strict` is part of the identity of `MapTags` and `MapData`, as it is of the column selection operators: an operator's identity is its full configuration (developer decision, ITL-1160). Their identity is now `(class name, name_map, drop_unmapped, strict)`, and it is saved in `to_config()`. This moves the content and pipeline hash of every `MapTags` and `MapData` step and of everything downstream of one, and with them the paths of the tables that are scoped by those hashes. -- The `strict` and `label` arguments of `map_tags` and `map_data` are keyword-only: `stream.map_tags(name_map, False, "label")` raises `TypeError`; write `label="label"`. +- The stream methods `map_tags`, `map_data` and `map_columns` take the renames dict as their only positional argument, and `drop_unmapped`, `strict` and `label` are keyword-only (ITL-1160, ITL-1171). `stream.map_tags(name_map, True)` and `stream.map_tags(name_map, False, "label")` raise `TypeError`: write `drop_unmapped=True`, `label="label"`. `stream.map_data(name_map={...})` no longer names the dict (it reads as a rename of a column called `name_map` and raises `TypeError`): pass the dict by position. The operator classes `MapTags`, `MapData` and `MapColumns` keep their arguments. ### Added - `stream.map_columns(name_map)` and the operator `MapColumns` rename tag and data columns in one step (ITL-1160). A renamed tag stays a tag and a renamed data column stays data, with its source-info column; system tags pass through. `drop_unmapped=True` drops every tag and data column not in the mapping. As with `map_tags` and `map_data`, a name that is neither a tag nor a data column is refused unless `strict=False`, a new name taken by a column that is not renamed, or given to two columns, is refused, a `__` new name is refused when the operator is built, and a result with no data column is refused. Like `SelectColumns`, it carries no data meta (`__`) column of the input. Its identity is `("MapColumns", name_map, drop_unmapped, strict)`. +- `map_tags`, `map_data` and `map_columns` take the renames as keyword arguments too: `stream.map_data(age="age_weeks")` builds the same step, with the same hashes, as `stream.map_data({"age": "age_weeks"})` (ITL-1171). Give one dict or keyword arguments, not both (`TypeError`). `drop_unmapped`, `strict` and `label` are options, never renames: a column with one of those names, or a name that is not a Python identifier, is renamed through the dict. An old or new name that is not a string raises `TypeError` in both forms. ### Fixed diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index c10ef1e13..2ec639c2b 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -406,6 +406,18 @@ renamed = sessions.map_columns({"session": "session_no", "age": "age_weeks"}) print(renamed.keys()) # (('subject_id', 'session_no'), ('age_weeks', 'weight')) ``` +The three methods also take the renames as keyword arguments, `old="new"`. Both forms build +the same operator, with the same hashes: + +```python +by_keyword = sessions.map_columns(session="session_no", age="age_weeks") +print(by_keyword.pipeline_hash() == renamed.pipeline_hash()) # True +``` + +Give the renames as one dict (the only positional argument) or as keyword arguments, not both. +`drop_unmapped`, `strict` and `label` are options, never renames: a column with one of those +names, or a name that is not a Python identifier (`"spike count"`), is renamed through the dict. + A name that is not a column the operator renames raises `InputValidationError` when the operator is called: a tag name for `map_data`, a data name for `map_tags`, and for all three a name that is no column at all. The message names the columns that exist and, for a column of diff --git a/docs/tutorial/03-operators.md b/docs/tutorial/03-operators.md index 1b9ecdd04..1f0498da6 100644 --- a/docs/tutorial/03-operators.md +++ b/docs/tutorial/03-operators.md @@ -210,6 +210,15 @@ print(renamed.keys()) # (('subject_id',), ('sex', 'age', 'genotype')) ``` +or the same renames as keyword arguments, which build the same step: + +```python +print(subjects.map_data(age_weeks="age").keys()) +# (('subject_id',), ('sex', 'age', 'genotype')) +``` + +A name that is not a Python identifier, such as `"spike count"`, needs the dictionary. + Columns you do not rename are kept. A name that is not a column of that kind is refused: a `map_data` of a tag column raises `InputValidationError` and says to use `map_tags`. `map_columns` renames tag and data columns in one call. Pass `drop_unmapped=True` to keep only diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 603dc7f40..56c1fbe1a 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -230,6 +230,45 @@ def _column_names(columns: tuple[str | Collection[str], ...]) -> tuple[str, ...] return tuple(str(name) for name in names) +def _rename_map( + name_map: Mapping[str, str] | None, renames: Mapping[str, object] +) -> dict[str, str]: + """Normalize the renames of ``map_tags``, ``map_data`` and ``map_columns``. + + The renames are given either as one mapping, the only positional argument, or + as keyword arguments, ``old="new"`` (ITL-1171). A name that is not a Python + identifier, or that is one of the methods' options (``drop_unmapped``, + ``strict``, ``label``), can be renamed only through the mapping. + + Args: + name_map: The positional mapping, or ``None`` when none was given. + renames: The keyword renames. + + Returns: + The renames as one dict, in the order given; empty when there are none. + + Raises: + TypeError: When both a mapping and keyword renames are given, or an old or + new name is not a string. + """ + if name_map is not None and renames: + raise TypeError( + "give the renames either as one dict (map_columns({'old': 'new'})) or as " + "keyword arguments (map_columns(old='new')), not both" + ) + pairs = dict(name_map if name_map is not None else renames) + not_strings = { + old: new + for old, new in pairs.items() + if not isinstance(old, str) or not isinstance(new, str) + } + if not_strings: + raise TypeError( + f"old and new column names must be strings, got {not_strings!r}" + ) + return {str(old): str(new) for old, new in pairs.items()} + + class StreamBase(TraceableBase): @property @abstractmethod @@ -304,14 +343,21 @@ def semi_join( def map_tags( self, - name_map: Mapping[str, str], - drop_unmapped: bool = False, + name_map: Mapping[str, str] | None = None, + /, *, + drop_unmapped: bool = False, strict: bool = True, label: str | None = None, + **renames: str, ) -> StreamBase: """Rename tag columns. See ``orcapod.core.operators.MapTags``. + The renames are one dict, the only positional argument (``map_tags({"subject": + "animal"})``), or keyword arguments (``map_tags(subject="animal")``), not both. + A name that is not a Python identifier, or a column named ``drop_unmapped``, + ``strict`` or ``label`` (the options), is renamed through the dict. + Args: name_map: Each tag column to rename, mapped to its new name. drop_unmapped: When ``True``, the tags not in ``name_map`` are dropped. @@ -319,28 +365,42 @@ def map_tags( a tag column raises ``InputValidationError``; when ``False`` it is ignored. label: Optional node label for the pipeline graph. + **renames: Each column to rename, mapped to its new name, as + keyword arguments. Returns: The stream with the tag columns renamed. Raises: + TypeError: When both a dict and keyword renames are given, or a + name is not a string. InputValidationError: When a ``strict`` name is not a tag column, or a new name collides with an existing column. """ from orcapod.core.operators import MapTags - return MapTags(name_map, drop_unmapped, strict=strict)(self, label=label) + return MapTags(_rename_map(name_map, renames), drop_unmapped, strict=strict)( + self, label=label + ) def map_data( self, - name_map: Mapping[str, str], - drop_unmapped: bool = False, + name_map: Mapping[str, str] | None = None, + /, *, + drop_unmapped: bool = False, strict: bool = True, label: str | None = None, + **renames: str, ) -> StreamBase: """Rename data columns. See ``orcapod.core.operators.MapData``. + The renames are one dict, the only positional argument (``map_data({"age": + "age_weeks"})``), or keyword arguments (``map_data(age="age_weeks")``), not + both. A name that is not a Python identifier, or a column named + ``drop_unmapped``, ``strict`` or ``label`` (the options), is renamed through the + dict. + Args: name_map: Each data column to rename, mapped to its new name. drop_unmapped: When ``True``, the data columns not in ``name_map`` @@ -349,32 +409,46 @@ def map_data( a data column raises ``InputValidationError``; when ``False`` it is ignored. label: Optional node label for the pipeline graph. + **renames: Each column to rename, mapped to its new name, as + keyword arguments. Returns: The stream with the data columns renamed. Raises: + TypeError: When both a dict and keyword renames are given, or a + name is not a string. InputValidationError: When a ``strict`` name is not a data column, a new name collides with an existing column, or no data column would remain. """ from orcapod.core.operators import MapData - return MapData(name_map, drop_unmapped, strict=strict)(self, label=label) + return MapData(_rename_map(name_map, renames), drop_unmapped, strict=strict)( + self, label=label + ) def map_columns( self, - name_map: Mapping[str, str], - drop_unmapped: bool = False, + name_map: Mapping[str, str] | None = None, + /, *, + drop_unmapped: bool = False, strict: bool = True, label: str | None = None, + **renames: str, ) -> StreamBase: """Rename columns, tag or data, in one step. A renamed tag stays a tag and a renamed data column stays data. See ``orcapod.core.operators.MapColumns``. + The renames are one dict, the only positional argument (``map_columns({"age": + "age_weeks"})``), or keyword arguments (``map_columns(age="age_weeks")``), not + both. A name that is not a Python identifier, or a column named + ``drop_unmapped``, ``strict`` or ``label`` (the options), is renamed through the + dict. + Args: name_map: Each column to rename, tag or data, mapped to its new name. drop_unmapped: When ``True``, every tag and data column not in @@ -383,17 +457,23 @@ def map_columns( neither a tag nor a data column raises ``InputValidationError``; when ``False`` it is ignored. label: Optional node label for the pipeline graph. + **renames: Each column to rename, mapped to its new name, as + keyword arguments. Returns: The stream with the columns renamed. Raises: + TypeError: When both a dict and keyword renames are given, or a + name is not a string. InputValidationError: When a ``strict`` name is missing, a new name collides with another column, or no data column would remain. """ from orcapod.core.operators import MapColumns - return MapColumns(name_map, drop_unmapped, strict=strict)(self, label=label) + return MapColumns(_rename_map(name_map, renames), drop_unmapped, strict=strict)( + self, label=label + ) def batch( self, diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index e806025fd..a329b3e16 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -348,47 +348,56 @@ def semi_join( def map_tags( self, - name_map: Mapping[str, str], - drop_unmapped: bool = False, + name_map: Mapping[str, str] | None = None, + /, *, + drop_unmapped: bool = False, strict: bool = True, label: str | None = None, + **renames: str, ) -> "StreamProtocol": """ Map tag names in this stream to new names based on the provided mapping. Tags not in the mapping are kept unless ``drop_unmapped`` is True. With - ``strict`` (the default), a name that is not a tag column is refused. + ``strict`` (the default), a name that is not a tag column is refused. The + renames are one positional dict or keyword arguments (``old="new"``). """ ... def map_data( self, - name_map: Mapping[str, str], - drop_unmapped: bool = False, + name_map: Mapping[str, str] | None = None, + /, *, + drop_unmapped: bool = False, strict: bool = True, label: str | None = None, + **renames: str, ) -> "StreamProtocol": """ Map data names in this stream to new names based on the provided mapping. Data columns not in the mapping are kept unless ``drop_unmapped`` is True. With ``strict`` (the default), a name that is not a data column is refused. + The renames are one positional dict or keyword arguments (``old="new"``). """ ... def map_columns( self, - name_map: Mapping[str, str], - drop_unmapped: bool = False, + name_map: Mapping[str, str] | None = None, + /, *, + drop_unmapped: bool = False, strict: bool = True, label: str | None = None, + **renames: str, ) -> "StreamProtocol": """Rename columns, tag or data, in one step; a tag stays a tag. Columns not in the mapping are kept unless ``drop_unmapped`` is True. With ``strict`` (the default), a name that is neither a tag nor a data column - is refused. + is refused. The renames are one positional dict or keyword arguments + (``old="new"``). """ ... diff --git a/tests/test_core/operators/test_mappers_strict.py b/tests/test_core/operators/test_mappers_strict.py index 04ab579f4..d2cab25e3 100644 --- a/tests/test_core/operators/test_mappers_strict.py +++ b/tests/test_core/operators/test_mappers_strict.py @@ -522,3 +522,79 @@ def test_a_new_name_with_the_token_separator_is_refused( """A source-info token splits on ``::`` (ITL-1154): no column name may blur it.""" with pytest.raises(InputValidationError, match="separator"): op_cls({old_name: new_name}) + + +# ── renames as keyword arguments (ITL-1171) ──────────────────────────────── + +KEYWORD_FORMS = [ + ("map_tags", {"subject": "animal"}), + ("map_data", {"age": "years", "weight": "kg"}), + ("map_columns", {"subject": "animal", "age": "years"}), +] + + +@pytest.mark.parametrize(("method", "renames"), KEYWORD_FORMS) +@pytest.mark.parametrize("drop_unmapped", [False, True]) +def test_keyword_renames_build_the_dict_form( + method: str, renames: dict[str, str], drop_unmapped: bool +) -> None: + source = _source() + by_dict = getattr(source, method)(renames, drop_unmapped=drop_unmapped) + by_keyword = getattr(source, method)(drop_unmapped=drop_unmapped, **renames) + assert by_keyword.content_hash() == by_dict.content_hash() + assert by_keyword.pipeline_hash() == by_dict.pipeline_hash() + assert by_keyword.keys() == by_dict.keys() + assert _rows(by_keyword) == _rows(by_dict) + + +@pytest.mark.parametrize("method", ["map_tags", "map_data", "map_columns"]) +def test_a_dict_and_keyword_renames_together_are_refused(method: str) -> None: + with pytest.raises(TypeError, match="either as one dict .* or as keyword"): + getattr(_source(), method)({"age": "years"}, weight="kg") + + +@pytest.mark.parametrize("method", ["map_tags", "map_data", "map_columns"]) +def test_the_dict_is_positional_and_drop_unmapped_keyword_only(method: str) -> None: + source = _source() + with pytest.raises(TypeError): + getattr(source, method)({"age": "years"}, True) + # ``name_map=`` is no longer the dict: it reads as a rename of a column + # called ``name_map``, whose new name is not a string. + with pytest.raises(TypeError, match="must be strings"): + getattr(source, method)(name_map={"age": "years"}) + + +@pytest.mark.parametrize("method", ["map_tags", "map_data", "map_columns"]) +@pytest.mark.parametrize( + "call", + [lambda m, s: getattr(s, m)(age=3), lambda m, s: getattr(s, m)({"age": 3})], + ids=["keyword", "dict"], +) +def test_a_name_that_is_not_a_string_is_refused( + method: str, call: Callable[[str, StreamProtocol], Any] +) -> None: + with pytest.raises(TypeError, match="must be strings"): + call(method, _source()) + + +@pytest.mark.parametrize("method", ["map_tags", "map_data", "map_columns"]) +def test_no_rename_is_the_empty_mapping(method: str) -> None: + source = _source() + none = getattr(source, method)() + empty = getattr(source, method)({}) + assert none.content_hash() == empty.content_hash() + assert none.keys() == source.keys() + + +def test_an_option_named_column_is_renamed_through_the_dict() -> None: + source = DictSource( + data=[{"k": "a", "label": 1, "strict": 2}], + tag_columns=["k"], + tag_only_columns=True, + source_id="options", + ) + renamed = source.map_data({"label": "lbl", "strict": "st"}, label="renamed") + assert renamed.keys() == (("k",), ("lbl", "st")) + assert renamed.label == "renamed" + # As keywords, ``label`` and ``strict`` are the options, not renames. + assert source.map_data(label="node").keys() == source.keys()