From 71eecef7495783a780ad0aaf7d9553b11adfe492 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 01:29:57 -0800 Subject: [PATCH 1/6] feat(referential_tags): SelectColumns/DropColumns act on both roles; errors name the way out (ITL-1022) SelectColumns and DropColumns treat a referential tag as the one physical column it is: naming it keeps (select) or removes (drop) it in both roles, never splitting them, so nothing is materialized. Predicted schema, sync, async and zero-row results agree. Error messages, no behaviour change: a bind refused for a role difference caused by a referential tag says which side holds it and how to make them agree (ITL-1142, still strict); pod.process / strict=True refusing an unread referenced key names the referential tags and the two ways out; "not supported with referential tags yet" names tag_only_columns=True. Docs: SelectColumns/DropColumns rows, renaming a grouped non-key with map_data, GroupBy dropping non-key tags itself. Refs ITL-1022, ITL-1142 Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1022.md | 21 +++ docs/concepts/referential-tags.md | 46 ++++- src/orcapod/core/cached_function_pod.py | 4 +- src/orcapod/core/function_pod.py | 52 ++++- src/orcapod/core/nodes/source_node.py | 92 +++++++++ .../core/operators/column_selection.py | 13 ++ src/orcapod/core/referential_tags.py | 3 +- .../operators/test_async_sync_equivalence.py | 11 ++ .../referential_tags/test_error_messages.py | 177 ++++++++++++++++++ .../referential_tags/test_operators.py | 100 ++++++++++ .../test_zero_row_column_order.py | 9 +- 11 files changed, 505 insertions(+), 23 deletions(-) create mode 100644 changelog.d/ITL-1022.md create mode 100644 tests/test_core/referential_tags/test_error_messages.py diff --git a/changelog.d/ITL-1022.md b/changelog.d/ITL-1022.md new file mode 100644 index 000000000..02078b969 --- /dev/null +++ b/changelog.d/ITL-1022.md @@ -0,0 +1,21 @@ +### Added + +- `SelectColumns` and `DropColumns` (`select_columns`, `drop_columns`) handle referential + tags (ITL-1022). A referential tag is one column in both roles, so naming it acts on both: + selecting it keeps it referential, dropping it (or leaving it out of a select) removes the + column from both roles. The roles are never split and nothing is materialized. Before, + both operators raised "not supported with referential tags yet". + +### Changed + +- Errors that a referential tag causes name it and the way out; no behaviour changes. + - A bind refused because a column is a referential tag on one side and a plain tag or a + data column on the other (listed as a missing or unexpected column) says which side + holds the referential tag, and how to make the two agree: `tag_only_columns` on the + source, or a slot declared with the column in both roles (ITL-1142). The bind stays + strict. + - `pod.process(stream)` and `pod(stream, strict=True)` refusing a referential tag the + function does not read name the referential tags and the two ways out: `pod(stream)`, + or `tag_only_columns`. + - "Not supported with referential tags yet" names `tag_only_columns=True`, which keeps + every tag plain as in v0.3, beside its issue. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 6fb99d58e..35712caef 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -82,10 +82,11 @@ does not make it compute again. | `GroupBy` | A group key stays scalar in both roles, and its source info is one merged token (below). A non-key loses its tag role, as every non-key tag does, but keeps its data column: a list like every other data column, one source-info token per member. The members are ordered by the non-key tags, so that list is in value order and the other data lists follow it. | | `SelectDataColumns`, `DropDataColumns` | Removing the data column leaves `x` a plain tag with the same values, with a `ReferenceMaterializationWarning`. | | `SelectTagColumns`, `DropTagColumns` | Removing the tag leaves `x` a data column. | +| `SelectColumns`, `DropColumns` | `x` is one column, so naming it acts on both roles: selecting it keeps `x` referential, dropping it (or not selecting it) removes the column from both roles. The roles are never split, so nothing is materialized. | | `MapTags` / `map_tags` | Renames the column in both roles; `tag_only=True` renames the tag only and leaves a plain tag. | | `MapData` / `map_data` | Always a data-only move: the data column is renamed and `x` stays a plain tag. | | Filters, `DropNulls`, `SemiJoin`, `Pick`, `Index` | Keep the reference. | -| `Batch`, `MergeJoin`, `SelectColumns`, `DropColumns` | Not supported with referential tags yet: they raise. | +| `Batch`, `MergeJoin` | Not supported with referential tags yet: they raise. | ```python units = op.sources.DictSource( @@ -98,6 +99,31 @@ print(joined.keys()) # (('unit', 'time'), ('unit', 'area', 'time', 'amp')) ``` +```python +print(spikes.select_columns(["time", "amp"]).keys()) +# (('time',), ('time', 'amp')) +print(spikes.drop_columns(["time"]).keys()) +# (('unit',), ('unit', 'amp')) +``` + +### After `GroupBy`: rename a non-key column with `map_data` + +`GroupBy` keeps only its key tags, and drops every other tag itself: there is nothing left +to drop with `drop_tag_columns` afterwards (that raises, since the tag is gone). A +referential non-key column loses its tag role there, but keeps its data column, a list +like every other data column. So after the grouping it is data only, and is renamed with +`map_data`; `map_tags` would find no tag of that name and rename nothing. + +```python +by_unit = spikes.group_by(["unit"]) +print(by_unit.keys()) +# (('unit',), ('unit', 'time', 'amp')) + +renamed = by_unit.map_data({"time": "times"}) +print(renamed.keys()) +# (('unit',), ('unit', 'times', 'amp')) +``` + ### Source info of a merged value A source-info column belongs to its data column and has its shape: a scalar value has one @@ -136,6 +162,20 @@ referential_tags.reference_materialization = "warn" # the default ## Not supported yet These raise "not supported with referential tags yet" when a referential tag reaches them: -`Batch`, `MergeJoin`, `SelectColumns`, `DropColumns`, storing an operator node's results (`LOG` and `REPLAY` cache modes), +`Batch`, `MergeJoin`, storing an operator node's results (`LOG` and `REPLAY` cache modes), `DerivedSource` over a stream with referential tags, and saving a pipeline or job. -`PollingSource` keeps its tags plain. +`PollingSource` keeps its tags plain. Each error names its issue and the way out: build the +source with `tag_only_columns=True` to keep every tag plain, as in v0.3. + +## Errors that name a referential tag + +- `pod.process(stream)` and `pod(stream, strict=True)` are exact: they hand every data + column to the function, so a referential tag the function does not read is refused like + any extra column. The error names the referential tags and the two ways out: call the + pod, `pod(stream)`, which lets them pass through, or keep them plain with + `tag_only_columns`. +- Binding a source to a job's input slot is strict: a column that is a referential tag on + one side and a plain tag or a data column on the other is refused (it is listed as a + missing or unexpected column). The error says which side holds it as a referential tag, + and how to make the two agree: `tag_only_columns` on the source, or a slot declared with + the column in both roles. diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index ea1fea3bf..9dae9cfab 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -431,9 +431,7 @@ def process( active trackers (so a ``PipelineJob`` records this pod). """ input_stream = self.handle_input_streams(*streams) - tag_schema, data_schema = input_stream.output_schema() - self._validate_input_schema(data_schema) - self._validate_output_keys(tag_schema.keys()) + self._validate_input(*input_stream.output_schema()) # The pod has no ``from_config`` (it cannot be rebuilt without its cache # database), the one member of FunctionPodProtocol it lacks; the tracker uses # none of it. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index a2e8e3498..7cbebf82b 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -18,6 +18,7 @@ from orcapod.core.referential_tags import ( materialization_diagnostic_off, not_supported_yet, + reference_names, stream_references, ) from orcapod.core.streams.base import ( @@ -252,9 +253,46 @@ def validate_inputs(self, *streams: StreamProtocol) -> None: ValueError: If inputs are incompatible with the data function schema. """ input_stream = self.handle_input_streams(*streams) - incoming_tag_schema, incoming_data_schema = input_stream.output_schema() - self._validate_input_schema(incoming_data_schema) - self._validate_output_keys(incoming_tag_schema.keys()) + self._validate_input(*input_stream.output_schema()) + + def _validate_input(self, tag_schema: Schema, data_schema: Schema) -> None: + """Validate a (joined) input's schemas against the function. + + Its data must fit the function's parameters (``_validate_input_schema``) and no + output key may be one of its tags (``_validate_output_keys``). When the data + does not fit and the input has referential tags the function does not read, + the error says so and names the ways out: this is the exact form, which hands + every data column to the function (ITL-1022). + + Args: + tag_schema: The input's tag schema. + data_schema: The input's data schema. + + Raises: + ValueError: If the data does not fit the function's parameters. + InputValidationError: If an output key of the function is a tag name. + """ + try: + self._validate_input_schema(data_schema) + except ValueError as error: + parameters = self.input_data_schema + unread = [ + name + for name in reference_names(tag_schema, data_schema) + if name not in parameters + ] + if not unread: + raise + raise ValueError( + f"{error}. {unread} are referential tags of the input (each a tag " + "and a data column of one name) that the function does not read, and " + "the exact form, pod.process(stream) or pod(stream, strict=True), " + "hands every data column to the function. Call the pod as " + "pod(stream) to let them pass through as referential tags, or build " + f"the source with tag_only_columns={unread} (or True) to keep them " + "plain tags" + ) from error + self._validate_output_keys(tag_schema.keys()) def _validate_output_keys(self, tag_names: Collection[str]) -> None: """Reject an output key that is also a tag column of the input. @@ -808,9 +846,7 @@ def output_schema( tag_schema, incoming_data_schema = self.multi_stream_handler().output_schema( *streams, columns=columns, all_info=all_info ) - # validate that incoming_data_schema is valid - self._validate_input_schema(incoming_data_schema) - self._validate_output_keys(tag_schema.keys()) + self._validate_input(tag_schema, incoming_data_schema) # The output schema of the FunctionPodProtocol is determined by the data function # TODO: handle and extend to include additional columns # Namely, the source columns @@ -907,9 +943,7 @@ def process( input_stream = self.handle_input_streams(*streams) # perform input stream schema validation - tag_schema, data_schema = input_stream.output_schema() - self._validate_input_schema(data_schema) - self._validate_output_keys(tag_schema.keys()) + self._validate_input(*input_stream.output_schema()) self.tracker_manager.record_function_pod_invocation( self, input_stream, label=label ) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index b694c5dd4..35af0ea4a 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -248,6 +248,84 @@ def _one_system_tag_pair_issue( return f"{issue}: {remedy} (binding such a stream is planned, ITL-1127)" +_TAG_ROLE = frozenset({"tag"}) +_BOTH_ROLES = frozenset({"tag", "data"}) + + +def _roles( + name: str, tag_schema: Mapping[str, Any], data_schema: Mapping[str, Any] +) -> frozenset[str]: + """Return the roles ``name`` has in a pair of schemas: ``"tag"``, ``"data"``, both, none. + + A name in both is a referential tag (ITL-1023): one column in both roles. + """ + return frozenset( + role + for role, schema in (("tag", tag_schema), ("data", data_schema)) + if name in schema + ) + + +def _referential_role_hint( + name: str, *, source_roles: frozenset[str], slot_roles: frozenset[str] +) -> str | None: + """Explain a role difference that a referential tag on one side causes, with the remedy. + + The bind refuses every role difference (ITL-1142: strict for now). The missing and + unexpected columns it lists do not say that one side holds the column as a + referential tag, a tag and a data column of one name; this does, and names how to + make the two agree (ITL-1022). + + Args: + name: The column name. + source_roles: The roles the column has in the source (see ``_roles``). + slot_roles: The roles the column has in the slot. + + Returns: + The explanation, or ``None`` when the roles agree, when neither side holds the + column as a referential tag, or when one side lacks the column altogether. + """ + if source_roles == slot_roles or not source_roles or not slot_roles: + return None + if source_roles == _BOTH_ROLES: + if slot_roles == _TAG_ROLE: + remedy = ( + f"build the source with tag_only_columns=[{name!r}] (or True) to keep " + f"it a plain tag, or declare the slot with {name!r} as both a tag and " + "a data column" + ) + slot_role = "a plain tag" + else: + remedy = ( + f"build the source without {name!r} in tag_columns, or declare the " + f"slot with {name!r} as both a tag and a data column" + ) + slot_role = "a data column only" + return ( + f"{name!r} is a referential tag of the source (a tag and a data column of " + f"one name) but {slot_role} of the slot: {remedy}" + ) + if slot_roles == _BOTH_ROLES: + if source_roles == _TAG_ROLE: + source_role = "a plain tag" + remedy = ( + f"build the source without {name!r} in tag_only_columns (a declared " + f"tag is referential by default), or declare the slot with {name!r} " + "as a plain tag" + ) + else: + source_role = "a data column only" + remedy = ( + f"build the source with {name!r} in tag_columns, or declare the slot " + f"with {name!r} as a data column only" + ) + return ( + f"{name!r} is {source_role} of the source but a referential tag of the " + f"slot (a tag and a data column of one name): {remedy}" + ) + return None + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -657,6 +735,20 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] f"{_type_text(given[name])}" ) + # A name whose roles differ only because one side holds it as a referential tag + # (a tag and a data column of one name) is refused like any role difference + # (ITL-1142: strict for now); say so, and how to make the two agree (ITL-1022). + for name in sorted( + {*self._tag_schema, *self._data_schema, *source_tag, *source_data} + ): + hint = _referential_role_hint( + name, + source_roles=_roles(name, source_tag, source_data), + slot_roles=_roles(name, self._tag_schema, self._data_schema), + ) + if hint is not None: + issues.append(hint) + # The slot's rows carry one system-tag pair under the slot's names, which only # a source with exactly one pair can take. A joined or merged stream (one pair # per input) or a stream with none is refused (developer, November 14, 2026, diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index 0cb3e699b..5a4998d1c 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -644,8 +644,16 @@ class _TagAndDataColumnsOperator(UnaryOperator, ABC): Unlike those operators, no data meta (``__``) column of the input is carried, on either path, as ``Join`` does; and a result without a data column is refused when the operator is called, on both paths. + + A referential tag (ITL-1022) is one physical column whose name is in both + groups, so naming it acts on that one column in both roles: it is kept as a + referential tag or removed from both, never split into a plain tag and a data + column. Nothing is materialized, so these operators never report it. """ + # Referential tags: handled (ITL-1022). A name acts on both roles alike. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + _verb: ClassVar[str] = "" def __init__(self, columns: str | Collection[str], strict: bool = True, **kwargs): @@ -757,6 +765,11 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: ] ), tag_columns=[c for c in tag_columns if c not in tags_removed], + # A referential tag is kept or removed in both roles at once, so every + # kept one is still referential. + reference_tags=[ + c for c in stream_references(stream) if c not in tags_removed + ], data_context=stream.data_context_key, ) diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index 61142366f..eeb3535dd 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -176,7 +176,8 @@ def not_supported_yet(operation: str, issue: str) -> NotImplementedError: """ return NotImplementedError( f"{operation}: not supported with referential tags yet ({issue}). Build the " - "source with tag_only_columns=[...] to keep those tags plain." + "source with tag_only_columns=True to keep every tag plain, as in v0.3, or " + "tag_only_columns=[...] to keep the named tags plain." ) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index f6f765913..489be16da 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -518,6 +518,17 @@ class Case: lambda: DropDataColumns(["k"]), lambda: (source_a(referential=True),), ), + # A referential tag named or not named is kept or removed in both roles (ITL-1022). + Case( + "ref_select_columns", + lambda: SelectColumns(["k", "i8", "st"]), + lambda: (source_a(referential=True),), + ), + Case( + "ref_drop_columns", + lambda: DropColumns(["ta", "s"]), + lambda: (source_a(referential=True),), + ), Case( "ref_index", lambda: Index("l8", 0, out="first"), diff --git a/tests/test_core/referential_tags/test_error_messages.py b/tests/test_core/referential_tags/test_error_messages.py new file mode 100644 index 000000000..c56e49315 --- /dev/null +++ b/tests/test_core/referential_tags/test_error_messages.py @@ -0,0 +1,177 @@ +"""Errors that a referential tag causes say so and name the way out (ITL-1022, ITL-1142). + +No behaviour changes here: the bind stays strict (a role difference is refused, +ITL-1142 audit Q3), ``pod.process`` stays exact, and the operations not supported yet +still raise. Only the messages explain the referential tag and the remedy. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest + +import orcapod as op +from orcapod import PipelineJob +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import Batch, DropNulls, MergeJoin +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError +from orcapod.types import Schema + +ROWS = [{"k": 1, "x": 10, "y": 2}, {"k": 2, "x": 20, "y": 3}] + + +def _source(tags: list[str], **kwargs: Any) -> op.sources.DictSource: + return op.sources.DictSource(ROWS, tag_columns=tags, source_id="s", **kwargs) + + +def _job(tag_schema: dict[str, type], data_schema: dict[str, type]) -> PipelineJob: + """A job with slot ``src`` of the given schemas, followed by one operator.""" + slot = SourceNode( + name="src", tag_schema=Schema(tag_schema), data_schema=Schema(data_schema) + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = DropNulls()(slot, label="op") + return job + + +# Slots: ``x`` a plain tag, a referential tag (both roles), or data only. +PLAIN_SLOT = ({"k": int, "x": int}, {"y": int}) +REFERENTIAL_SLOT = ({"k": int, "x": int}, {"x": int, "y": int}) +DATA_SLOT = ({"k": int}, {"x": int, "y": int}) + + +def test_a_referential_source_fits_a_referential_slot() -> None: + job = _job(*REFERENTIAL_SLOT) + job.bind(sources={"src": _source(["k", "x"], tag_only_columns=["k"])}) + + +@pytest.mark.parametrize( + ("slot", "source", "listed", "explained"), + [ + ( + PLAIN_SLOT, + lambda: _source(["k", "x"], tag_only_columns=["k"]), + "unexpected data columns: ['x']", + [ + "'x' is a referential tag of the source", + "a plain tag of the slot", + "tag_only_columns=['x']", + "declare the slot with 'x' as both a tag and a data column", + ], + ), + ( + DATA_SLOT, + lambda: _source(["k", "x"], tag_only_columns=["k"]), + "unexpected tag columns: ['x']", + [ + "'x' is a referential tag of the source", + "a data column only of the slot", + "without 'x' in tag_columns", + ], + ), + ( + REFERENTIAL_SLOT, + lambda: _source(["k", "x"], tag_only_columns=True), + "missing data columns: ['x']", + [ + "'x' is a plain tag of the source but a referential tag of the slot", + "without 'x' in tag_only_columns", + "declare the slot with 'x' as a plain tag", + ], + ), + ( + REFERENTIAL_SLOT, + lambda: _source(["k"], tag_only_columns=True), + "missing tag columns: ['x']", + [ + "'x' is a data column only of the source but a referential tag", + "with 'x' in tag_columns", + ], + ), + ], + ids=["ref_into_plain", "ref_into_data", "plain_into_ref", "data_into_ref"], +) +def test_a_role_difference_is_refused_and_explained( + slot: tuple[dict[str, type], dict[str, type]], + source: Callable[[], op.sources.DictSource], + listed: str, + explained: list[str], +) -> None: + job = _job(*slot) + with pytest.raises(SourceSpecMismatchError) as excinfo: + job.bind(sources={"src": source()}) + message = str(excinfo.value) + assert listed in message + for text in explained: + assert text in message + + +def test_a_plain_mismatch_says_nothing_about_referential_tags() -> None: + job = _job({"k": int}, {"y": int}) + with pytest.raises(SourceSpecMismatchError) as excinfo: + job.bind(sources={"src": _source(["k", "x"], tag_only_columns=True)}) + assert "referential" not in str(excinfo.value) + + +# ── pod.process and strict=True: the exact form (ITL-1022) ────────────────── + + +@op.function_pod(output_keys=["half"]) +def half(y: int) -> float: + return y / 2 + + +@pytest.mark.parametrize( + "call", + [lambda s: half.pod.process(s), lambda s: half.pod(s, strict=True)], + ids=["process", "strict"], +) +def test_the_exact_form_names_unread_references_and_the_ways_out( + call: Callable[[op.sources.DictSource], object], +) -> None: + source = _source(["k", "x"], tag_only_columns=["k"]) + with pytest.raises(ValueError, match="column 'x' is not a parameter") as excinfo: + _ = call(source) + message = str(excinfo.value) + assert "['x'] are referential tags of the input" in message + assert "pod(stream)" in message + assert "tag_only_columns=['x']" in message + + +def test_the_exact_form_on_plain_tags_says_nothing_about_referential_tags() -> None: + plain = _source(["k", "x"], tag_only_columns=True) + assert half.pod.process(plain).keys() == (("k", "x"), ("half",)) + extra = op.sources.DictSource( + [{"k": 1, "y": 2, "z": 3}], + tag_columns=["k"], + tag_only_columns=True, + source_id="e", + ) + with pytest.raises(ValueError, match="column 'z' is not a parameter") as excinfo: + _ = half.pod.process(extra) + assert "referential" not in str(excinfo.value) + + +# ── not supported yet: the plain-tag way out and the issue ────────────────── + + +@pytest.mark.parametrize( + ("call", "issue"), + [ + (lambda s: Batch(batch_size=2)(s), "ITL-1017"), + (lambda s: MergeJoin()(s, s), "ITL-1018"), + ], + ids=["batch", "merge_join"], +) +def test_not_supported_yet_names_tag_only_columns_true_and_the_issue( + call: Callable[[op.sources.DictSource], object], issue: str +) -> None: + with pytest.raises(NotImplementedError) as excinfo: + _ = call(_source(["k", "x"])) + message = str(excinfo.value) + assert issue in message + assert "tag_only_columns=True" in message diff --git a/tests/test_core/referential_tags/test_operators.py b/tests/test_core/referential_tags/test_operators.py index e081d84fe..89599cc30 100644 --- a/tests/test_core/referential_tags/test_operators.py +++ b/tests/test_core/referential_tags/test_operators.py @@ -22,6 +22,7 @@ from orcapod.core import referential_tags from orcapod.core.operators import ( Batch, + DropColumns, DropDataColumns, DropNulls, DropTagColumns, @@ -33,6 +34,7 @@ MergeJoin, Pick, PolarsFilter, + SelectColumns, SelectDataColumns, SelectTagColumns, SemiJoin, @@ -317,6 +319,104 @@ def test_removing_a_referential_tag_keeps_its_data(drop) -> None: assert data.source_info()["x"] == "S::row_0::x" +# ── SelectColumns / DropColumns: one physical column (ITL-1022) ──────────── + + +def _xky(**kw: Any): + """Referential tags ``k`` and ``x``; data ``y``.""" + return _ref( + [{"k": 1, "x": 10, "y": 2}, {"k": 2, "x": 20, "y": 3}], ["k", "x"], "C", **kw + ) + + +@pytest.mark.parametrize( + ("make_op", "keys", "references"), + [ + # Naming a referential tag keeps its one column in both roles. + (lambda: SelectColumns(["x", "y"]), (("x",), ("x", "y")), ("x",)), + (lambda: SelectColumns(["x"]), (("x",), ("x",)), ("x",)), + # Not naming it removes the column, in both roles. + (lambda: SelectColumns(["k", "y"]), (("k",), ("k", "y")), ("k",)), + # Dropping it removes the column, in both roles. + (lambda: DropColumns(["x"]), (("k",), ("k", "y")), ("k",)), + (lambda: DropColumns(["k", "x"]), ((), ("y",)), ()), + # Not naming it keeps it referential. + (lambda: DropColumns(["y"]), (("k", "x"), ("k", "x")), ("k", "x")), + ], + ids=["select_x_y", "select_x", "select_k_y", "drop_x", "drop_k_x", "drop_y"], +) +def test_select_and_drop_columns_act_on_the_one_column_in_both_roles( + make_op, keys, references +) -> None: + with warnings.catch_warnings(): + # Neither splits the roles, so nothing is materialized. + warnings.simplefilter("error", ReferenceMaterializationWarning) + result = make_op()(_xky()) + assert result.keys() == keys + assert stream_references(result) == references + table = result.as_table(columns={"source": True}) + tag_keys, data_keys = keys + for name in ("k", "x", "y"): + present = name in tag_keys or name in data_keys + assert table.column_names.count(name) == int(present) + assert (f"_source_{name}" in table.column_names) == (name in data_keys) + for tag, data in result.iter_data(): + for name in references: + assert tag[name] == data[name] + + +def test_selected_reference_keeps_its_values_and_source_info() -> None: + result = SelectColumns(["x", "y"])(_xky()) + assert _rows(result) == [{"x": 10, "y": 2}, {"x": 20, "y": 3}] + assert result.as_table(columns={"source": True}).column( + "_source_x" + ).to_pylist() == ["C::row_0::x", "C::row_1::x"] + + +@pytest.mark.parametrize( + "make_op", + [ + lambda: SelectColumns(["x", "y"]), + lambda: SelectColumns(["k", "y"]), + lambda: SelectColumns(["x"]), + lambda: DropColumns(["x"]), + lambda: DropColumns(["k", "x"]), + lambda: DropColumns(["y"]), + ], +) +def test_select_and_drop_columns_predict_their_result(make_op) -> None: + source = _xky() + predicted = make_op().output_schema(source) + # Checked against static_process (CLAUDE.md standing rule); the binding is the + # name intersection of the two halves, so the schemas also carry it. + result = make_op().static_process(source) + assert predicted == result.output_schema() + assert referential_tags.reference_names(*predicted) == stream_references(result) + + +@pytest.mark.parametrize( + "make_op", + [lambda: SelectColumns(["x", "y"]), lambda: DropColumns(["k"])], + ids=["select", "drop"], +) +def test_select_and_drop_columns_with_no_rows_keep_the_populated_shape( + make_op, +) -> None: + source = _xky() + populated = make_op()(source).as_table(columns={"source": True}) + empty = make_op()(PolarsFilter(constraints={"k": 0})(source)) + assert stream_references(empty) == stream_references(make_op()(source)) + table = empty.as_table(columns={"source": True}) + assert table.num_rows == 0 + assert table.schema == populated.schema + + +def test_select_and_drop_columns_on_plain_tags_are_unchanged() -> None: + plain = _xky(tag_only_columns=True) + assert SelectColumns(["x", "y"])(plain).keys() == (("x",), ("y",)) + assert DropColumns(["x"])(plain).keys() == (("k",), ("y",)) + + # ── row-preserving operators (section 10) ─────────────────────────────────── diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index d2f982ba2..89453c78b 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -531,13 +531,8 @@ def test_job_node(self, step: str, pairs: int, config: str) -> None: # ── referential tags (ITL-1023) ───────────────────────────────────────────── -# Not supported with referential tags yet: Batch (ITL-1017), MergeJoin (ITL-1018), -# SelectColumns and DropColumns (ITL-1022). -REFERENTIAL_STEPS = [ - step - for step in STEPS - if step not in ("merge_join", "batch", "select_columns", "drop_columns") -] +# Not supported with referential tags yet: Batch (ITL-1017), MergeJoin (ITL-1018). +REFERENTIAL_STEPS = [step for step in STEPS if step not in ("merge_join", "batch")] @pytest.fixture From b0c443683e1db456dac568be648e27ac544fb5ea Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 01:40:38 -0800 Subject: [PATCH 2/6] fix(referential_tags): remedies keep plain tags and promise fall-through only where it exists (ITL-1022) The bind remedy lists the source's existing plain tags with the column, since tag_only_columns names every plain tag; following it now binds. The exact-form error tells a wrapper pod (CachedFunctionPod) that pod(stream) keeps the references as plain tags, since only a plain FunctionPod's call lets them fall through, and says to add the names to the declaring source's tag_only_columns. CodeRabbit CLI round 1. Refs ITL-1022, ITL-1142 Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 14 ++++++--- src/orcapod/core/nodes/source_node.py | 16 ++++++++-- .../referential_tags/test_error_messages.py | 30 +++++++++++++++++-- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 7cbebf82b..a98cf85df 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -283,14 +283,20 @@ def _validate_input(self, tag_schema: Schema, data_schema: Schema) -> None: ] if not unread: raise + # Only a plain ``FunctionPod``'s call lets references fall through + # (``__call__``); a wrapper pod's call drops their data columns. + call = ( + "pod(stream) to let them pass through as referential tags" + if isinstance(self, FunctionPod) + else "pod(stream), which keeps them as plain tags" + ) raise ValueError( f"{error}. {unread} are referential tags of the input (each a tag " "and a data column of one name) that the function does not read, and " "the exact form, pod.process(stream) or pod(stream, strict=True), " - "hands every data column to the function. Call the pod as " - "pod(stream) to let them pass through as referential tags, or build " - f"the source with tag_only_columns={unread} (or True) to keep them " - "plain tags" + f"hands every data column to the function. Call the pod as {call}, " + f"or add {unread} to the tag_only_columns of the source that declares " + "them (tag_only_columns=True keeps every tag plain)" ) from error self._validate_output_keys(tag_schema.keys()) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 35af0ea4a..6d38ae528 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -267,7 +267,11 @@ def _roles( def _referential_role_hint( - name: str, *, source_roles: frozenset[str], slot_roles: frozenset[str] + name: str, + *, + source_roles: frozenset[str], + slot_roles: frozenset[str], + tag_only_with_name: list[str], ) -> str | None: """Explain a role difference that a referential tag on one side causes, with the remedy. @@ -280,6 +284,9 @@ def _referential_role_hint( name: The column name. source_roles: The roles the column has in the source (see ``_roles``). slot_roles: The roles the column has in the slot. + tag_only_with_name: The source's plain tags with ``name`` among them, in tag + order: the ``tag_only_columns`` that keep ``name`` plain and change no other + tag. Returns: The explanation, or ``None`` when the roles agree, when neither side holds the @@ -290,7 +297,7 @@ def _referential_role_hint( if source_roles == _BOTH_ROLES: if slot_roles == _TAG_ROLE: remedy = ( - f"build the source with tag_only_columns=[{name!r}] (or True) to keep " + f"build the source with tag_only_columns={tag_only_with_name} to keep " f"it a plain tag, or declare the slot with {name!r} as both a tag and " "a data column" ) @@ -745,6 +752,11 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] name, source_roles=_roles(name, source_tag, source_data), slot_roles=_roles(name, self._tag_schema, self._data_schema), + # ``tag_only_columns`` lists every plain tag, so the remedy keeps the + # source's plain tags plain. + tag_only_with_name=[ + tag for tag in source_tag if tag not in source_data or tag == name + ], ) if hint is not None: issues.append(hint) diff --git a/tests/test_core/referential_tags/test_error_messages.py b/tests/test_core/referential_tags/test_error_messages.py index c56e49315..686d18e05 100644 --- a/tests/test_core/referential_tags/test_error_messages.py +++ b/tests/test_core/referential_tags/test_error_messages.py @@ -14,6 +14,8 @@ import orcapod as op from orcapod import PipelineJob +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Batch, DropNulls, MergeJoin from orcapod.databases import InMemoryArrowDatabase @@ -59,7 +61,8 @@ def test_a_referential_source_fits_a_referential_slot() -> None: [ "'x' is a referential tag of the source", "a plain tag of the slot", - "tag_only_columns=['x']", + # ``tag_only_columns`` lists every plain tag: ``k`` stays plain. + "tag_only_columns=['k', 'x']", "declare the slot with 'x' as both a tag and a data column", ], ), @@ -110,6 +113,12 @@ def test_a_role_difference_is_refused_and_explained( assert text in message +def test_the_named_remedy_binds() -> None: + """Following the first remedy of the plain-slot refusal binds the source.""" + job = _job(*PLAIN_SLOT) + job.bind(sources={"src": _source(["k", "x"], tag_only_columns=["k", "x"])}) + + def test_a_plain_mismatch_says_nothing_about_referential_tags() -> None: job = _job({"k": int}, {"y": int}) with pytest.raises(SourceSpecMismatchError) as excinfo: @@ -138,8 +147,23 @@ def test_the_exact_form_names_unread_references_and_the_ways_out( _ = call(source) message = str(excinfo.value) assert "['x'] are referential tags of the input" in message - assert "pod(stream)" in message - assert "tag_only_columns=['x']" in message + assert "pod(stream) to let them pass through as referential tags" in message + assert "add ['x'] to the tag_only_columns of the source" in message + + +def test_a_pod_without_fall_through_does_not_promise_it() -> None: + """A ``CachedFunctionPod`` called directly keeps its plain call (no fall-through).""" + pod = half.pod + assert isinstance(pod, FunctionPod) + cached = CachedFunctionPod(pod, result_database=InMemoryArrowDatabase()) + source = _source(["k", "x"], tag_only_columns=["k"]) + with pytest.raises(ValueError, match="column 'x' is not a parameter") as excinfo: + _ = cached.process(source) + message = str(excinfo.value) + assert "pass through as referential tags" not in message + assert "pod(stream), which keeps them as plain tags" in message + # And that is what the call does. + assert cached(source).keys() == (("k", "x"), ("half",)) def test_the_exact_form_on_plain_tags_says_nothing_about_referential_tags() -> None: From 5f9ff7916119e951512710fd496398a86becdf64 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:14:37 -0800 Subject: [PATCH 3/6] fix(referential_tags): exact tag_only_columns in bind remedies; wrappers word refusals for their own call (ITL-1022) - A plain source tag or a data column for a referential slot tag: the remedy names the exact tag_only_columns (the source's other plain tags), since adding the column to tag_columns alone leaves it plain under tag_only_columns=True. Following it binds (regression test). - WrappedFunctionPod.validate_inputs and output_schema delegate to the inner pod, which words a referential-tag refusal for its own call; on a refusal they validate again as the wrapper, so a CachedFunctionPod never promises a fall-through its call does not do. - Docs: the error section covers both bind directions and qualifies the fall-through to a plain FunctionPod. GitHub CodeRabbit review of 0bf7a921. Refs ITL-1022, ITL-1142 Co-Authored-By: Claude Opus 5.5 --- docs/concepts/referential-tags.md | 13 +++++--- src/orcapod/core/function_pod.py | 29 ++++++++++++++--- src/orcapod/core/nodes/source_node.py | 31 ++++++++++--------- .../referential_tags/test_error_messages.py | 30 ++++++++++++++---- 4 files changed, 74 insertions(+), 29 deletions(-) diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 35712caef..1af137789 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -172,10 +172,15 @@ source with `tag_only_columns=True` to keep every tag plain, as in v0.3. - `pod.process(stream)` and `pod(stream, strict=True)` are exact: they hand every data column to the function, so a referential tag the function does not read is refused like any extra column. The error names the referential tags and the two ways out: call the - pod, `pod(stream)`, which lets them pass through, or keep them plain with - `tag_only_columns`. + pod, `pod(stream)`, which for a plain `FunctionPod` lets them pass through as + referential tags (a wrapper pod such as `CachedFunctionPod` has no fall-through: its + call keeps them as plain tags), or keep them plain with `tag_only_columns`. - Binding a source to a job's input slot is strict: a column that is a referential tag on one side and a plain tag or a data column on the other is refused (it is listed as a missing or unexpected column). The error says which side holds it as a referential tag, - and how to make the two agree: `tag_only_columns` on the source, or a slot declared with - the column in both roles. + and how to make the two agree, in either direction. A referential source tag for a + plain slot tag: keep it plain with `tag_only_columns`, or declare the slot with the + column in both roles. A plain source tag (or a data column) for a referential slot + tag: make it a declared tag that `tag_only_columns` does not name, or declare the slot + with it plain. `tag_only_columns` lists every plain tag of the source, so the error + names the exact list. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index a98cf85df..aa5cfd9d4 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1901,7 +1901,24 @@ def uri(self) -> tuple[str, ...]: return self._function_pod.uri def validate_inputs(self, *streams: StreamProtocol) -> None: - self._function_pod.validate_inputs(*streams) + try: + self._function_pod.validate_inputs(*streams) + except ValueError: + self._refuse_as_this_pod(*streams) + raise + + def _refuse_as_this_pod(self, *streams: StreamProtocol) -> None: + """Validate ``streams`` as this pod, so a refusal names this pod's call. + + The inner pod words a referential-tag refusal for its own call, which may let + references fall through where this pod's does not (ITL-1022). Called only when + the inner pod refused; when this check passes, the caller re-raises the inner + pod's error. + + Raises: + ValueError: If the input does not fit this pod, with this pod's advice. + """ + self._validate_input(*self.handle_input_streams(*streams).output_schema()) def argument_symmetry(self, streams: Collection[StreamProtocol]) -> ArgumentGroup: return self._function_pod.argument_symmetry(streams) @@ -1912,9 +1929,13 @@ def output_schema( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> tuple[Schema, Schema]: - return self._function_pod.output_schema( - *streams, columns=columns, all_info=all_info - ) + try: + return self._function_pod.output_schema( + *streams, columns=columns, all_info=all_info + ) + except ValueError: + self._refuse_as_this_pod(*streams) + raise # TODO: reconsider whether to return FunctionPodStream here in the signature def process( diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 6d38ae528..8acfe6f5e 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -271,7 +271,7 @@ def _referential_role_hint( *, source_roles: frozenset[str], slot_roles: frozenset[str], - tag_only_with_name: list[str], + source_plain_tags: list[str], ) -> str | None: """Explain a role difference that a referential tag on one side causes, with the remedy. @@ -284,9 +284,9 @@ def _referential_role_hint( name: The column name. source_roles: The roles the column has in the source (see ``_roles``). slot_roles: The roles the column has in the slot. - tag_only_with_name: The source's plain tags with ``name`` among them, in tag - order: the ``tag_only_columns`` that keep ``name`` plain and change no other - tag. + source_plain_tags: The source's plain tags, in tag order. ``tag_only_columns`` + lists every plain tag, so the remedy names the exact list that changes + ``name`` only. Returns: The explanation, or ``None`` when the roles agree, when neither side holds the @@ -297,7 +297,8 @@ def _referential_role_hint( if source_roles == _BOTH_ROLES: if slot_roles == _TAG_ROLE: remedy = ( - f"build the source with tag_only_columns={tag_only_with_name} to keep " + f"build the source with tag_only_columns={[*source_plain_tags, name]} " + "to keep " f"it a plain tag, or declare the slot with {name!r} as both a tag and " "a data column" ) @@ -313,18 +314,20 @@ def _referential_role_hint( f"one name) but {slot_role} of the slot: {remedy}" ) if slot_roles == _BOTH_ROLES: + # A declared tag is referential unless ``tag_only_columns`` names it. + others = [tag for tag in source_plain_tags if tag != name] + tag_only = f"tag_only_columns={others}" if others else "no tag_only_columns" if source_roles == _TAG_ROLE: source_role = "a plain tag" remedy = ( - f"build the source without {name!r} in tag_only_columns (a declared " - f"tag is referential by default), or declare the slot with {name!r} " - "as a plain tag" + f"build the source with {tag_only} (a declared tag is referential " + f"unless named there), or declare the slot with {name!r} as a plain tag" ) else: source_role = "a data column only" remedy = ( - f"build the source with {name!r} in tag_columns, or declare the slot " - f"with {name!r} as a data column only" + f"build the source with {name!r} in tag_columns and {tag_only}, or " + f"declare the slot with {name!r} as a data column only" ) return ( f"{name!r} is {source_role} of the source but a referential tag of the " @@ -752,11 +755,9 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] name, source_roles=_roles(name, source_tag, source_data), slot_roles=_roles(name, self._tag_schema, self._data_schema), - # ``tag_only_columns`` lists every plain tag, so the remedy keeps the - # source's plain tags plain. - tag_only_with_name=[ - tag for tag in source_tag if tag not in source_data or tag == name - ], + # ``tag_only_columns`` lists every plain tag, so a remedy keeps the + # source's other plain tags plain. + source_plain_tags=[tag for tag in source_tag if tag not in source_data], ) if hint is not None: issues.append(hint) diff --git a/tests/test_core/referential_tags/test_error_messages.py b/tests/test_core/referential_tags/test_error_messages.py index 686d18e05..664ab689c 100644 --- a/tests/test_core/referential_tags/test_error_messages.py +++ b/tests/test_core/referential_tags/test_error_messages.py @@ -82,7 +82,8 @@ def test_a_referential_source_fits_a_referential_slot() -> None: "missing data columns: ['x']", [ "'x' is a plain tag of the source but a referential tag of the slot", - "without 'x' in tag_only_columns", + # ``k`` stays plain: the exact ``tag_only_columns`` is named. + "build the source with tag_only_columns=['k']", "declare the slot with 'x' as a plain tag", ], ), @@ -92,7 +93,7 @@ def test_a_referential_source_fits_a_referential_slot() -> None: "missing tag columns: ['x']", [ "'x' is a data column only of the source but a referential tag", - "with 'x' in tag_columns", + "with 'x' in tag_columns and tag_only_columns=['k']", ], ), ], @@ -113,10 +114,21 @@ def test_a_role_difference_is_refused_and_explained( assert text in message -def test_the_named_remedy_binds() -> None: - """Following the first remedy of the plain-slot refusal binds the source.""" - job = _job(*PLAIN_SLOT) - job.bind(sources={"src": _source(["k", "x"], tag_only_columns=["k", "x"])}) +@pytest.mark.parametrize( + ("slot", "remedy"), + [ + (PLAIN_SLOT, lambda: _source(["k", "x"], tag_only_columns=["k", "x"])), + (REFERENTIAL_SLOT, lambda: _source(["k", "x"], tag_only_columns=["k"])), + ], + ids=["ref_into_plain", "plain_or_data_into_ref"], +) +def test_the_named_remedy_binds( + slot: tuple[dict[str, type], dict[str, type]], + remedy: Callable[[], op.sources.DictSource], +) -> None: + """Following the first remedy a refusal names binds the source.""" + job = _job(*slot) + job.bind(sources={"src": remedy()}) def test_a_plain_mismatch_says_nothing_about_referential_tags() -> None: @@ -162,6 +174,12 @@ def test_a_pod_without_fall_through_does_not_promise_it() -> None: message = str(excinfo.value) assert "pass through as referential tags" not in message assert "pod(stream), which keeps them as plain tags" in message + # The wrapper's other entry points, which delegate to the inner pod, say the same. + for check in (cached.validate_inputs, cached.output_schema): + with pytest.raises(ValueError) as delegated: + _ = check(source) + assert "pod(stream), which keeps them as plain tags" in str(delegated.value) + assert "pass through as referential tags" not in str(delegated.value) # And that is what the call does. assert cached(source).keys() == (("k", "x"), ("half",)) From c98e7af018b5527f7852a00c2b1ffa346171e426 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:19:26 -0800 Subject: [PATCH 4/6] docs(referential_tags): the slot fix for a data-only source column is data only (ITL-1022) Co-Authored-By: Claude Opus 5.5 --- docs/concepts/referential-tags.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 1af137789..7aa7ff74d 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -180,7 +180,7 @@ source with `tag_only_columns=True` to keep every tag plain, as in v0.3. missing or unexpected column). The error says which side holds it as a referential tag, and how to make the two agree, in either direction. A referential source tag for a plain slot tag: keep it plain with `tag_only_columns`, or declare the slot with the - column in both roles. A plain source tag (or a data column) for a referential slot - tag: make it a declared tag that `tag_only_columns` does not name, or declare the slot - with it plain. `tag_only_columns` lists every plain tag of the source, so the error + column in both roles. A plain source tag (or a data-only source column) for a + referential slot tag: make it a declared tag that `tag_only_columns` does not name, or + declare the slot with it as the source has it (a plain tag, or a data column only). `tag_only_columns` lists every plain tag of the source, so the error names the exact list. From 56a0bd10ce587dadf8e07934af12d459d02f1cb3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:23:06 -0800 Subject: [PATCH 5/6] docs(referential_tags): the plain-tag way out needs the value as data for a function that reads it (ITL-1022) Co-Authored-By: Claude Opus 5.5 --- docs/concepts/referential-tags.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 7aa7ff74d..c9b9eb771 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -165,7 +165,11 @@ These raise "not supported with referential tags yet" when a referential tag rea `Batch`, `MergeJoin`, storing an operator node's results (`LOG` and `REPLAY` cache modes), `DerivedSource` over a stream with referential tags, and saving a pipeline or job. `PollingSource` keeps its tags plain. Each error names its issue and the way out: build the -source with `tag_only_columns=True` to keep every tag plain, as in v0.3. +source with `tag_only_columns=True` to keep every tag plain, as in v0.3. A plain tag is not +data, so a function that reads a tag's value (as `scale` above reads `time`) can no longer +take it from that column: as in v0.3, the value must also be a data column of its own, for +example a copy in the source's data under another name, or +`DerivedSource(origin, tags_as_data={...})`. ## Errors that name a referential tag From d55119ab7e7b5906d23cfa62c21af2363311f7fb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 12:28:50 -0800 Subject: [PATCH 6/6] fix(function_pod): offer only the referential-tag remedies that can work (ITL-1022) When no data column of the input is a parameter of the function, a wrapper pod's call and tag_only_columns both leave the function nothing to read, so the refusal no longer recommends them; it says why instead. A plain FunctionPod's call still lets the references fall through. CodeRabbit review of 227d370c. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1022.md | 3 +- docs/concepts/referential-tags.md | 5 +- src/orcapod/core/function_pod.py | 36 +++++++++---- .../referential_tags/test_error_messages.py | 51 +++++++++++++++++++ .../referential_tags/test_operators.py | 6 ++- 5 files changed, 88 insertions(+), 13 deletions(-) diff --git a/changelog.d/ITL-1022.md b/changelog.d/ITL-1022.md index 02078b969..df5ee3ab3 100644 --- a/changelog.d/ITL-1022.md +++ b/changelog.d/ITL-1022.md @@ -16,6 +16,7 @@ strict. - `pod.process(stream)` and `pod(stream, strict=True)` refusing a referential tag the function does not read name the referential tags and the two ways out: `pod(stream)`, - or `tag_only_columns`. + or `tag_only_columns`, each only where it can work (not when no data column of the + input is a parameter of the function, except a plain `FunctionPod`'s call). - "Not supported with referential tags yet" names `tag_only_columns=True`, which keeps every tag plain as in v0.3, beside its issue. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 04ffb1b40..ba61b9983 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -191,7 +191,10 @@ example a copy in the source's data under another name, or any extra column. The error names the referential tags and the two ways out: call the pod, `pod(stream)`, which for a plain `FunctionPod` lets them pass through as referential tags (a wrapper pod such as `CachedFunctionPod` has no fall-through: its - call keeps them as plain tags), or keep them plain with `tag_only_columns`. + call keeps them as plain tags), or keep them plain with `tag_only_columns`. It offers + only a way out that can work: when no data column of the input is a parameter, a + wrapper pod's call and `tag_only_columns` would leave the function nothing to read, + and the error says so instead. - Binding a source to a job's input slot is strict: a column that is a referential tag on one side and a plain tag or a data column on the other is refused (it is listed as a missing or unexpected column). The error says which side holds it as a referential tag, diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index aa5cfd9d4..c78c43e87 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -283,20 +283,38 @@ def _validate_input(self, tag_schema: Schema, data_schema: Schema) -> None: ] if not unread: raise - # Only a plain ``FunctionPod``'s call lets references fall through - # (``__call__``); a wrapper pod's call drops their data columns. - call = ( - "pod(stream) to let them pass through as referential tags" - if isinstance(self, FunctionPod) - else "pod(stream), which keeps them as plain tags" + # Name only the ways out that can work for this input. Only a plain + # ``FunctionPod``'s call lets references fall through (``__call__``), + # whatever the other columns. A wrapper pod's call drops their data + # columns, and keeping them plain removes them: either leaves the + # function a column to read only when a parameter column remains. + has_parameter_column = any(name in parameters for name in data_schema) + remedies: list[str] = [] + if isinstance(self, FunctionPod): + remedies.append( + "call the pod as pod(stream) to let them pass through as " + "referential tags" + ) + elif has_parameter_column: + remedies.append( + "call the pod as pod(stream), which keeps them as plain tags" + ) + if has_parameter_column: + remedies.append( + f"add {unread} to the tag_only_columns of the source that " + "declares them (tag_only_columns=True keeps every tag plain)" + ) + way_out = ( + ", or ".join(remedies) + if remedies + else "no data column of the input is a parameter of the function, " + "so neither calling the pod nor keeping them plain leaves it one" ) raise ValueError( f"{error}. {unread} are referential tags of the input (each a tag " "and a data column of one name) that the function does not read, and " "the exact form, pod.process(stream) or pod(stream, strict=True), " - f"hands every data column to the function. Call the pod as {call}, " - f"or add {unread} to the tag_only_columns of the source that declares " - "them (tag_only_columns=True keeps every tag plain)" + f"hands every data column to the function: {way_out}" ) from error self._validate_output_keys(tag_schema.keys()) diff --git a/tests/test_core/referential_tags/test_error_messages.py b/tests/test_core/referential_tags/test_error_messages.py index 032c0f2e6..60bc81566 100644 --- a/tests/test_core/referential_tags/test_error_messages.py +++ b/tests/test_core/referential_tags/test_error_messages.py @@ -184,6 +184,57 @@ def test_a_pod_without_fall_through_does_not_promise_it() -> None: assert cached(source).keys() == (("k", "x"), ("half",)) +@op.function_pod(output_keys=["out"]) +def defaulted(y: int = 1) -> int: + return y + + +def _only_an_unread_reference() -> op.sources.DictSource: + """Data ``x`` only: a referential tag that ``defaulted`` does not read.""" + return op.sources.DictSource( + [{"k": 1, "x": 10}], + tag_columns=["k", "x"], + tag_only_columns=["k"], + source_id="u", + ) + + +def test_no_remedy_is_offered_that_cannot_work() -> None: + """Every data column is an unread reference: name only the ways out that work.""" + source = _only_an_unread_reference() + # Keeping x plain leaves the source no data column, so tag_only_columns cannot help. + with pytest.raises(ValueError, match="At least one data column is required"): + _ = op.sources.DictSource( + [{"k": 1, "x": 10}], + tag_columns=["k", "x"], + tag_only_columns=True, + source_id="u", + ) + # A plain FunctionPod's call lets x fall through: that is the one way out. + pod = defaulted.pod + assert isinstance(pod, FunctionPod) + with pytest.raises(ValueError, match="column 'x' is not a parameter") as excinfo: + _ = pod.process(source) + message = str(excinfo.value) + assert "['x'] are referential tags of the input" in message + assert "pod(stream) to let them pass through as referential tags" in message + assert "tag_only_columns" not in message + assert pod(source).keys() == (("k", "x"), ("x", "out")) + # A wrapper's call keeps no data column for x, so it fails the same way: neither + # remedy is offered, and the error says why. + cached = CachedFunctionPod(pod, result_database=InMemoryArrowDatabase()) + for call in (cached.process, cached): + with pytest.raises( + ValueError, match="column 'x' is not a parameter" + ) as excinfo: + _ = call(source) + message = str(excinfo.value) + assert "['x'] are referential tags of the input" in message + assert "pod(stream)," not in message + assert "tag_only_columns" not in message + assert "no data column of the input is a parameter" in message + + def test_the_exact_form_on_plain_tags_says_nothing_about_referential_tags() -> None: plain = _source(["k", "x"], tag_only_columns=True) assert half.pod.process(plain).keys() == (("k", "x"), ("half",)) diff --git a/tests/test_core/referential_tags/test_operators.py b/tests/test_core/referential_tags/test_operators.py index 524b811f0..986e09b7f 100644 --- a/tests/test_core/referential_tags/test_operators.py +++ b/tests/test_core/referential_tags/test_operators.py @@ -12,7 +12,7 @@ from __future__ import annotations import warnings -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Any import pyarrow as pa @@ -416,7 +416,9 @@ def test_select_and_drop_columns_with_no_rows_keep_the_populated_shape( [lambda: SelectColumns(["x", "y"]), lambda: DropColumns(["k"])], ids=["select", "drop"], ) -def test_select_and_drop_columns_commute_with_batch(make_op) -> None: +def test_select_and_drop_columns_commute_with_batch( + make_op: Callable[[], SelectColumns | DropColumns], +) -> None: # Batch keeps a referential tag one list column in both roles (ITL-1017), so # selecting or dropping before or after it gives the same rows and reference. after = make_op()(Batch(batch_size=2)(_xky()))