From b8192e79aaa22a0b6f3c961e397b086df96ce6cf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:17:53 -0800 Subject: [PATCH 01/11] test(semijoin): failing tests for a tag key nullable on one side only (ITL-829) Co-Authored-By: Claude Opus 5.5 --- ...026-11-13-itl-829-semijoin-nullable-key.md | 86 ++++++++ .../operators/test_semijoin_nullable_keys.py | 187 ++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md create mode 100644 tests/test_core/operators/test_semijoin_nullable_keys.py diff --git a/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md b/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md new file mode 100644 index 000000000..0347bba86 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md @@ -0,0 +1,86 @@ +# ITL-829 (second case) SemiJoin over a key nullable on one side only — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `SemiJoin(left, right)` accepts a shared tag key that is `T` on one input and +`T | None` on the other, by applying `DropNulls()` to the nullable side first. + +**Architecture:** Rule approved by the coordinator (developer decision, November 13, 2026): +for each name that is a TAG column on both inputs and whose Python type admits `None` on +exactly one side, SemiJoin narrows that side as `DropNulls` would, then runs today's semi +join. Keys nullable on both sides or on neither are untouched (byte-identical path). The +result equals `DropNulls(nullable-side keys)` on each input followed by `SemiJoin`, schema +included. Reuse `DropNulls._narrow_schema` / `DropNulls._narrow_table`; never copy them. + +**Tech Stack:** pyarrow, polars, asyncio, pytest. + +Branch: `eywalker/itl-829-semi_join-rejects-compatible-keys-string-vs` (the issue's +gitBranchName; it did not exist remotely: PR #291 used `eywalker/itl-847-...`), from +`origin/main` `512cece9`. + +## Diagnosis (main, 512cece9) + +- (left T, right T|None), (left T|None, right T), composite mixed keys: `InputValidationError` + "Type conflict for key 'id': ..." from `validate_binary_inputs` + (`schema_utils.intersection_schemas`), so a pipeline never reaches async. `async_execute` + called directly gives the null-never-matches rows. +- Both T|None: works, schema `int | None`, null matches nothing (Join section of + `docs/concepts/operators.md`; `core/operators/match_keys.py`). +- Join and MergeJoin raise the same mismatch at graph build (report only, PR body). +- A shared DATA column is a match key too today (report only, PR body). + +## Files + +- Modify: `src/orcapod/core/operators/semijoin.py` +- Create: `tests/test_core/operators/test_semijoin_nullable_keys.py` +- Modify: `tests/test_channels/test_node_async_execute.py` (remove strict xfail) +- Modify: `tests/test_core/operators/test_async_sync_equivalence.py` (cases) +- Docs: `docs/concepts/operators.md` (SemiJoin section, two sentences), `CLAUDE.md` + + `.zed/rules` (one bullet), `changelog.d/ITL-829.md`, `design_issues/ITL-829-SJ2.md` + +## Tasks + +### Task 1: failing tests + +`tests/test_core/operators/test_semijoin_nullable_keys.py`, public interface: +- for each case (left T/right T|None; left T|None/right T; composite a:T,b:T|None vs + a:T|None,b:T): `SemiJoin()(left, right)` rows and Arrow schema equal + `SemiJoin()(DropNulls(lnames)(left), DropNulls(rnames)(right))` (or the un-narrowed input + when a side has no names); `output_schema()` equals + `static_process(...).output_schema()` (comment: schema-prediction exception); left key + type `int` when left was nullable. +- async rows (`async_execute` over channels, after `op(left, right)` validation) equal sync + rows (`assert_equivalent`, `assert_same_provenance` from the equivalence module). +- both nullable: schema stays `int | None`, rows as today, Arrow field nullable. +- a shared DATA column with mismatched nullability still raises `InputValidationError`. +- `int` vs `str | None` tag key still raises. +- identity: `SemiJoin()(l, r).pipeline_hash()` on an all-non-nullable pair unchanged vs + main (covered by golden tests; no new golden entries). +- [ ] Run, confirm they fail on main with the type conflict. WIP commit. + +### Task 2: implement in `semijoin.py` + +- `_nullable_key_names(left_tag_schema, right_tag_schema) -> (left_names, right_names)`: + names in both tag schemas whose type differs in admitting `None` (`_without_none(t) != t`). +- `validate_binary_inputs`: narrow both tag schemas with `DropNulls._narrow_schema` before + `intersection_schemas`. +- `binary_output_schema`: narrow the left output schemas by `left_names`. +- `binary_static_process`: `left_table = DropNulls._narrow_table(left_table, left_names)` + and same for right, when the name sets are non-empty; `logger.info` naming them. +- `async_execute`: in the first-left-row block decide the names from the first left and + first right row's Arrow tag field nullability; emitted left rows go through + `DropNulls._narrow_table` (tag and data tables, as `DropNulls.async_execute`). Right rows + need nothing (a null key never matches). +- [ ] Tests pass; remove the strict xfail in `test_node_async_execute.py`; add cases to + `test_async_sync_equivalence.py`. Commit. + +### Task 3: docs, changelog, design issue; ruff, basedpyright, targeted pytest; PR. + +## Conflicts + +PR #367 rewrites the async input reading (`_collect_right_reading_left`, `_LeftRows`); PR #369 +adds `output = WithoutMetaWriter(output)`. Coordinator: merge main after both land. + +## Progress notes + +- [x] Diagnosis and proposal approved (coordinator, Nov 13). diff --git a/tests/test_core/operators/test_semijoin_nullable_keys.py b/tests/test_core/operators/test_semijoin_nullable_keys.py new file mode 100644 index 000000000..c7654315e --- /dev/null +++ b/tests/test_core/operators/test_semijoin_nullable_keys.py @@ -0,0 +1,187 @@ +"""SemiJoin over a shared tag key that is nullable on one input only (ITL-829, second case). + +Developer decision, November 13, 2026: when a shared tag key is ``T`` on one input and +``T | None`` on the other, SemiJoin drops the rows with a null in that key on the +nullable side and semi-joins. The result is exactly ``DropNulls()`` applied +to the nullable side followed by the semi join, schema included, so a left key that was +nullable comes out ``T``. A key nullable on both sides is not narrowed, and a shared data +column is never narrowed. +""" + +from __future__ import annotations + +import pyarrow as pa +import pytest + +from orcapod.core.operators import DropNulls, SemiJoin +from orcapod.core.sources import ArrowTableSource +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import StreamProtocol +from tests.test_core.operators.test_async_sync_equivalence import ( + _source, # pyright: ignore[reportPrivateUsage] # shared test helper + assert_equivalent, + assert_same_provenance, + async_rows, + sync_rows, +) + +FULL = {"source": True, "system_tags": True} + + +def left_source(nullable: tuple[str, ...]) -> ArrowTableSource: + """Tags ``a`` and ``b`` (int64), data ``x``; a null in each nullable tag.""" + return _source( + { + "a": pa.array([1, 1, 2, 2, 3], pa.int64()), + "b": pa.array( + [1, None if "b" in nullable else 5, 1, 2, 2], pa.int64() + ), + "x": pa.array([10, 20, 30, 40, 50], pa.int64()), + }, + tags=("a", "b"), + source_id="left", + nullable=nullable, + ) + + +def right_source(nullable: tuple[str, ...]) -> ArrowTableSource: + """Tags ``a`` and ``b`` (int64), data ``z``; a null in each nullable tag.""" + return _source( + { + "a": pa.array([1, 2, None if "a" in nullable else 9, 3], pa.int64()), + "b": pa.array([1, 2, 1, None if "b" in nullable else 7], pa.int64()), + "z": pa.array([100, 200, 300, 400], pa.int64()), + }, + tags=("a", "b"), + source_id="right", + nullable=nullable, + ) + + +def narrowed(stream: StreamProtocol, names: tuple[str, ...]) -> StreamProtocol: + """``DropNulls(names)(stream)``, or the stream itself when there are no names.""" + return DropNulls(names)(stream) if names else stream + + +# (left nullable tags, right nullable tags, names DropNulls takes on left, on right) +MIXED = [ + pytest.param(("b",), (), ("b",), (), id="left_nullable"), + pytest.param((), ("b",), (), ("b",), id="right_nullable"), + pytest.param(("b",), ("a",), ("b",), ("a",), id="composite_crossed"), + pytest.param(("a", "b"), ("b",), ("a",), (), id="composite_one_shared_nullable"), +] + + +@pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) +def test_result_equals_drop_nulls_then_semijoin( + left_null: tuple[str, ...], + right_null: tuple[str, ...], + left_drop: tuple[str, ...], + right_drop: tuple[str, ...], +) -> None: + """Rows, Arrow schema, system tags and source info equal the explicit composition.""" + left, right = left_source(left_null), right_source(right_null) + result = SemiJoin()(left, right) + expected = SemiJoin()(narrowed(left, left_drop), narrowed(right, right_drop)) + + assert result.output_schema() == expected.output_schema() + assert result.as_table(columns=FULL).equals( + expected.as_table(columns=FULL), check_metadata=True + ) + assert result.as_table(columns=FULL).num_rows > 0 + + +@pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) +def test_prediction_matches_computed_schema( + left_null: tuple[str, ...], + right_null: tuple[str, ...], + left_drop: tuple[str, ...], + right_drop: tuple[str, ...], +) -> None: + """``output_schema()`` without computing equals the computed stream's schema.""" + left, right = left_source(left_null), right_source(right_null) + op = SemiJoin() + predicted = op(left, right).output_schema(columns=FULL) + # Schema-prediction check: compared against the computed stream (CLAUDE.md). + computed = op.static_process(left, right).output_schema(columns=FULL) + assert predicted == computed + + +def test_left_nullable_key_narrows_to_t() -> None: + """A left key nullable against a non-nullable right key comes out ``T``.""" + result = SemiJoin()(left_source(("b",)), right_source(())) + tag_schema, _ = result.output_schema() + assert tag_schema["b"] is int + assert not result.as_table().schema.field("b").nullable + assert sorted(result.as_table().column("x").to_pylist()) == [10, 40] + + +def test_right_nullable_key_keeps_left_schema() -> None: + """A right key nullable against a non-nullable left key leaves the left schema.""" + left = left_source(()) + result = SemiJoin()(left, right_source(("b",))) + assert result.output_schema() == left.output_schema() + assert result.as_table().schema == left.as_table().schema + assert sorted(result.as_table().column("x").to_pylist()) == [10, 40] + + +def test_key_nullable_on_both_sides_is_not_narrowed() -> None: + """Both nullable: as before, the key stays ``T | None`` and a null matches nothing.""" + left = left_source(("b",)) + result = SemiJoin()(left, right_source(("b",))) + tag_schema, _ = result.output_schema() + assert tag_schema["b"] == int | None + assert result.as_table().schema.field("b").nullable + assert result.output_schema() == left.output_schema() + assert sorted(result.as_table().column("x").to_pylist()) == [10, 40] + + +@pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) +def test_async_rows_equal_sync_rows( + left_null: tuple[str, ...], + right_null: tuple[str, ...], + left_drop: tuple[str, ...], + right_drop: tuple[str, ...], +) -> None: + """The async path yields the sync rows: types, nullability, tags, hashes, sources.""" + left, right = left_source(left_null), right_source(right_null) + expected = sync_rows(SemiJoin(), left, right) + actual = async_rows(SemiJoin(), left, right) + assert expected + assert_equivalent(expected, actual) + assert_same_provenance(expected, actual) + + +def test_shared_data_column_nullability_mismatch_still_raises() -> None: + """Only tag keys are narrowed; a shared data column keeps today's error.""" + left = _source( + {"a": pa.array([1, 2], pa.int64()), "y": pa.array([1, 2], pa.int64())}, + tags=("a",), + source_id="l", + ) + right = _source( + {"a": pa.array([1, 2], pa.int64()), "y": pa.array([1, None], pa.int64())}, + tags=("a",), + source_id="r", + nullable=("y",), + ) + with pytest.raises(InputValidationError, match="'y'"): + _ = SemiJoin()(left, right) + + +def test_other_type_mismatch_still_raises() -> None: + """Narrowing removes only ``None``: ``int`` against ``str | None`` still raises.""" + left = _source( + {"a": pa.array([1, 2], pa.int64()), "x": pa.array([1, 2], pa.int64())}, + tags=("a",), + source_id="l", + ) + right = _source( + {"a": pa.array(["1", None], pa.large_string()), "z": pa.array([1, 2])}, + tags=("a",), + source_id="r", + nullable=("a",), + ) + with pytest.raises(InputValidationError, match="'a'"): + _ = SemiJoin()(left, right) + From 9a45e2165ce6d77564cad89db19644863a940581 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:21:42 -0800 Subject: [PATCH 02/11] fix(semijoin): drop nulls in a tag key nullable on one side only, then semi join (ITL-829) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/semijoin.py | 150 +++++++++++++++++- .../test_channels/test_node_async_execute.py | 10 -- .../operators/test_async_sync_equivalence.py | 25 +++ .../operators/test_semijoin_nullable_keys.py | 11 +- 4 files changed, 172 insertions(+), 24 deletions(-) diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index fe1929cec..20a2c81a9 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,8 +1,11 @@ +import logging from collections.abc import Sequence from typing import TYPE_CHECKING, Any from orcapod.channels import ReadableChannel, WritableChannel +from orcapod.core.datagrams import Data, Tag from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type +from orcapod.core.operators.drop_nulls import DropNulls from orcapod.core.operators.match_keys import frozen_scalar from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError @@ -20,6 +23,13 @@ pc = LazyModule("pyarrow.compute") pl = LazyModule("polars") +logger = logging.getLogger(__name__) + +# DropNulls' own rule, reused so that SemiJoin narrows a key exactly as DropNulls +# does (ITL-829); never a copy of it. +_narrow_schema = DropNulls._narrow_schema # pyright: ignore[reportPrivateUsage] # shared on purpose with DropNulls +_narrow_table = DropNulls._narrow_table # pyright: ignore[reportPrivateUsage] # shared on purpose with DropNulls + class SemiJoin(BinaryOperator): """ @@ -32,9 +42,56 @@ class SemiJoin(BinaryOperator): If there are no overlapping columns between the streams, the entire left stream is returned unchanged. - The output stream preserves the schema of the left stream exactly. + The output stream preserves the schema of the left stream exactly, with + one exception. A key that is a tag column on both inputs and is nullable + (``T | None``) on one of them only is first narrowed on that side as + ``DropNulls`` would: rows with a null in it are dropped and it becomes + ``T``. So a left key nullable against a non-nullable right key comes out + ``T`` (ITL-829). A null key never matches anyway, so this changes types, + never which rows match. A key nullable on both sides is not narrowed. """ + @staticmethod + def _nullable_key_names( + left_tag_schema: Schema, right_tag_schema: Schema + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Return the shared tag keys to narrow on the left and on the right. + + A key is narrowed on a side when it is a tag column of both inputs and + its type admits ``None`` on that side only. Whether a type admits + ``None`` is decided by ``DropNulls``' own schema rule. + + Args: + left_tag_schema: The left input's tag schema. + right_tag_schema: The right input's tag schema. + + Returns: + The names to narrow on the left and on the right, each in the left + input's tag order. + """ + shared = [name for name in left_tag_schema if name in right_tag_schema] + left_narrow = _narrow_schema(left_tag_schema, shared) + right_narrow = _narrow_schema(right_tag_schema, shared) + left_nullable = [n for n in shared if left_narrow[n] != left_tag_schema[n]] + right_nullable = [n for n in shared if right_narrow[n] != right_tag_schema[n]] + return ( + tuple(n for n in left_nullable if n not in right_nullable), + tuple(n for n in right_nullable if n not in left_nullable), + ) + + @classmethod + def _narrowed_tag_schemas( + cls, left_tag_schema: Schema, right_tag_schema: Schema + ) -> tuple[Schema, Schema]: + """Return both tag schemas with the keys nullable on one side only narrowed.""" + left_names, right_names = cls._nullable_key_names( + left_tag_schema, right_tag_schema + ) + return ( + _narrow_schema(left_tag_schema, left_names), + _narrow_schema(right_tag_schema, right_names), + ) + def binary_static_process( self, left_stream: StreamProtocol, right_stream: StreamProtocol ) -> StreamProtocol: @@ -44,11 +101,18 @@ def binary_static_process( """ left_tag_schema, left_data_schema = left_stream.output_schema() right_tag_schema, right_data_schema = right_stream.output_schema() + left_names, right_names = self._nullable_key_names( + left_tag_schema, right_tag_schema + ) + narrowed_left_tags = _narrow_schema(left_tag_schema, left_names) + narrowed_right_tags = _narrow_schema(right_tag_schema, right_names) # Find overlapping columns across all columns (tags + data) - left_all_schema = schema_utils.union_schemas(left_tag_schema, left_data_schema) + left_all_schema = schema_utils.union_schemas( + narrowed_left_tags, left_data_schema + ) right_all_schema = schema_utils.union_schemas( - right_tag_schema, right_data_schema + narrowed_right_tags, right_data_schema ) common_keys = tuple( @@ -63,6 +127,20 @@ def binary_static_process( left_table = left_stream.as_table(columns={"source": True, "system_tags": True}) right_table = right_stream.as_table() + # A shared tag key nullable on one side only: that side goes through + # DropNulls' rule first, so the result is DropNulls then SemiJoin (ITL-829). + if left_names or right_names: + logger.info( + "SemiJoin: dropping rows with a null key and narrowing the key to" + + " non-nullable, on the left for %s and on the right for %s", + list(left_names), + list(right_names), + ) + if left_names: + left_table = _narrow_table(left_table, left_names) + if right_names: + right_table = _narrow_table(right_table, right_names) + # A key must have one Arrow type on both sides (up to large layouts); # otherwise Polars would silently unify, e.g., int8 with int16. for key in common_keys: @@ -115,10 +193,19 @@ def binary_output_schema( ) -> tuple[Schema, Schema]: """ Returns the output types for the semi-join operation. - The output preserves the exact schema of the left stream. + + The output has the left stream's schema, except that a shared tag key + nullable on the left only is narrowed to ``T``. """ - # Semi-join preserves the left stream's schema exactly - return left_stream.output_schema(columns=columns, all_info=all_info) + tag_schema, data_schema = left_stream.output_schema( + columns=columns, all_info=all_info + ) + left_names, _ = self._nullable_key_names( + left_stream.output_schema()[0], right_stream.output_schema()[0] + ) + if not left_names: + return tag_schema, data_schema + return _narrow_schema(tag_schema, left_names), data_schema def validate_binary_inputs( self, left_stream: StreamProtocol, right_stream: StreamProtocol @@ -134,6 +221,11 @@ def validate_binary_inputs( left_tag_schema, left_data_schema = left_stream.output_schema() right_tag_schema, right_data_schema = right_stream.output_schema() + # A shared tag key nullable on one side only is narrowed there first. + left_tag_schema, right_tag_schema = self._narrowed_tag_schemas( + left_tag_schema, right_tag_schema + ) + # Check that overlapping columns have compatible types across all columns left_all_schema = schema_utils.union_schemas( left_tag_schema, left_data_schema @@ -221,6 +313,7 @@ async def async_execute( # Phase 2: Probe — stream left rows common_keys: tuple[str, ...] | None = None right_lookup: set[tuple[object, ...]] = set() + left_names: tuple[str, ...] = () async for tag, data in left_ch: if common_keys is None: @@ -246,6 +339,21 @@ async def async_execute( self._key_arrow_type(right_tag, right_data, key), ) + # A shared tag key nullable on the left only is narrowed on + # every emitted left row, as on the sync path (ITL-829). The + # right side needs nothing: a null key never matches. + left_names, right_names = self._nullable_key_names( + tag.schema(), right_tag.schema() + ) + if left_names or right_names: + logger.info( + "SemiJoin: dropping rows with a null key and narrowing" + + " the key to non-nullable, on the left for %s and on" + + " the right for %s", + list(left_names), + list(right_names), + ) + # Build right-side lookup; a row with a null key never matches. for rt, rp in right_rows: right_key = self._match_key(rt, rp, common_keys) @@ -255,10 +363,38 @@ async def async_execute( # Probe left_key = self._match_key(tag, data, common_keys) if left_key is not None and left_key in right_lookup: - await output.send((tag, data)) + await output.send(self._narrowed_row(tag, data, left_names)) finally: await output.close() + @staticmethod + def _narrowed_row( + tag: TagProtocol, data: DataProtocol, names: Sequence[str] + ) -> tuple[TagProtocol, DataProtocol]: + """Return a left row with ``names`` narrowed by ``DropNulls``' rule. + + The row's tag and data tables go through ``DropNulls._narrow_table``, as + the sync path's left table does, so the row has the sync row's types. + + Args: + tag: A left row's tag, non-null in every name. + data: The row's data. + names: The shared tag keys nullable on the left only. + + Returns: + The row itself when ``names`` is empty, otherwise the rebuilt row. + """ + if not names: + return tag, data + tag_table = _narrow_table(tag.as_table(columns={"system_tags": True}), names) + data_table = _narrow_table( + data.as_table(columns={"source": True, "meta": True}), names + ) + return ( + Tag(tag_table, data_context=tag.data_context_key), + Data(data_table, data_context=data.data_context_key), + ) + @staticmethod def _key_table(tag: TagProtocol, data: DataProtocol, key: str) -> "pa.Table": """Return the one-row Arrow table, the tag's or the data's, holding ``key``.""" diff --git a/tests/test_channels/test_node_async_execute.py b/tests/test_channels/test_node_async_execute.py index e2046ce49..bdec92872 100644 --- a/tests/test_channels/test_node_async_execute.py +++ b/tests/test_channels/test_node_async_execute.py @@ -28,7 +28,6 @@ from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.streams import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase -from orcapod.errors import InputValidationError from orcapod.types import CacheMode, PodConfig @@ -475,15 +474,6 @@ async def test_unary_op_delegation(self): assert "x" in pkt_dict assert "y" not in pkt_dict - @pytest.mark.xfail( - strict=True, - raises=InputValidationError, - reason=( - 'SemiJoin rejects a non-nullable vs nullable key: "Type conflict for ' - "key 'id': vs int | None\" (ITL-829). This test was " - "shadowed by a duplicate class name and never ran until ITL-826." - ), - ) @pytest.mark.asyncio async def test_binary_op_delegation(self): left = make_stream(5) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index da3790e47..133640f53 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -156,6 +156,20 @@ def source_c() -> ArrowTableSource: ) +def source_nullable_k() -> ArrowTableSource: + """Tag ``k: int16``, nullable with a null; for ``SemiJoin`` against a non-null ``k``.""" + return _source( + { + "k": pa.array([1, None, 3, 4], pa.int16()), + "kn": pa.array([1, 2, 3, 4], pa.int8()), + "ks": pa.array(["a", "b", "c", "d"], pa.string()), + }, + tags=("k",), + source_id="kn", + nullable=("k",), + ) + + def merge_source(source_id: str, offset: int) -> ArrowTableSource: """A ``MergeJoin`` input; two of them collide on every data column but one.""" only = f"only_{source_id}" @@ -285,6 +299,17 @@ class Case: ), Case("semijoin", SemiJoin, lambda: (source_c(), source_b(nested=False))), Case("semijoin_nested", SemiJoin, lambda: (source_a(), source_b())), + # A shared tag key nullable on one side only is narrowed there (ITL-829). + Case( + "semijoin_left_nullable_key", + SemiJoin, + lambda: (source_nullable_k(), source_b(nested=False)), + ), + Case( + "semijoin_right_nullable_key", + SemiJoin, + lambda: (source_c(), source_nullable_k()), + ), Case("batch_2", lambda: Batch(batch_size=2), lambda: (source_a(),)), Case("batch_all", Batch, lambda: (source_a(),)), Case("group_by", lambda: GroupBy(["ta"]), lambda: (source_a(),)), diff --git a/tests/test_core/operators/test_semijoin_nullable_keys.py b/tests/test_core/operators/test_semijoin_nullable_keys.py index c7654315e..66fdebb1f 100644 --- a/tests/test_core/operators/test_semijoin_nullable_keys.py +++ b/tests/test_core/operators/test_semijoin_nullable_keys.py @@ -33,9 +33,7 @@ def left_source(nullable: tuple[str, ...]) -> ArrowTableSource: return _source( { "a": pa.array([1, 1, 2, 2, 3], pa.int64()), - "b": pa.array( - [1, None if "b" in nullable else 5, 1, 2, 2], pa.int64() - ), + "b": pa.array([1, None if "b" in nullable else 5, 1, 2, 2], pa.int64()), "x": pa.array([10, 20, 30, 40, 50], pa.int64()), }, tags=("a", "b"), @@ -113,7 +111,7 @@ def test_left_nullable_key_narrows_to_t() -> None: tag_schema, _ = result.output_schema() assert tag_schema["b"] is int assert not result.as_table().schema.field("b").nullable - assert sorted(result.as_table().column("x").to_pylist()) == [10, 40] + assert result.as_table().column("x").to_pylist() == [10, 40] def test_right_nullable_key_keeps_left_schema() -> None: @@ -122,7 +120,7 @@ def test_right_nullable_key_keeps_left_schema() -> None: result = SemiJoin()(left, right_source(("b",))) assert result.output_schema() == left.output_schema() assert result.as_table().schema == left.as_table().schema - assert sorted(result.as_table().column("x").to_pylist()) == [10, 40] + assert result.as_table().column("x").to_pylist() == [10, 40] def test_key_nullable_on_both_sides_is_not_narrowed() -> None: @@ -133,7 +131,7 @@ def test_key_nullable_on_both_sides_is_not_narrowed() -> None: assert tag_schema["b"] == int | None assert result.as_table().schema.field("b").nullable assert result.output_schema() == left.output_schema() - assert sorted(result.as_table().column("x").to_pylist()) == [10, 40] + assert result.as_table().column("x").to_pylist() == [10, 40] @pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) @@ -184,4 +182,3 @@ def test_other_type_mismatch_still_raises() -> None: ) with pytest.raises(InputValidationError, match="'a'"): _ = SemiJoin()(left, right) - From c5571236624c718eca4310cf63c1a1cf5da9b8db Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:31:56 -0800 Subject: [PATCH 03/11] fix(join, merge_join, semijoin): accept a shared tag key that differs only in nullability (ITL-829) Replaces the earlier SemiJoin narrowing: the developer's revised rule keeps SemiJoin's output the left schema, and gives Join and MergeJoin a key that is non-nullable when any input has it non-nullable. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/join.py | 67 ++++- src/orcapod/core/operators/key_nullability.py | 56 ++++ src/orcapod/core/operators/merge_join.py | 17 +- src/orcapod/core/operators/semijoin.py | 153 +--------- .../operators/test_async_sync_equivalence.py | 17 +- tests/test_core/operators/test_drop_nulls.py | 30 +- .../operators/test_join_key_nullability.py | 265 ++++++++++++++++++ .../operators/test_semijoin_nullable_keys.py | 184 ------------ 8 files changed, 445 insertions(+), 344 deletions(-) create mode 100644 src/orcapod/core/operators/key_nullability.py create mode 100644 tests/test_core/operators/test_join_key_nullability.py delete mode 100644 tests/test_core/operators/test_semijoin_nullable_keys.py diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index c53f07b9a..ed7960b5d 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -10,6 +10,10 @@ require_shared_arrow_type, require_tag_data_roles_agree, ) +from orcapod.core.operators.key_nullability import ( + key_nullable, + reconcile_key_nullability, +) from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import ( @@ -111,7 +115,11 @@ def _joined_user_schemas(self, *streams: StreamProtocol) -> tuple[Schema, Schema for other_stream in ordered_streams[1:]: other_tag_schema, other_data_schema = other_stream.output_schema() other_data_schema = self._without_meta_fields(other_data_schema) - tag_schema = schema_utils.union_schemas(tag_schema, other_tag_schema) + # A shared tag key that differs in nullability only is non-nullable + # (ITL-829): a null key matches nothing. + tag_schema = schema_utils.union_schemas( + *reconcile_key_nullability(tag_schema, other_tag_schema) + ) intersection_data_schema = schema_utils.intersection_schemas( data_schema, other_data_schema ) @@ -250,6 +258,9 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: table = arrow_utils.restore_schema_nullability( table, table_ref_schema, next_ref_schema ) + table = self._shared_key_nullability( + table, table_ref_schema, next_ref_schema, common_tag_keys + ) tag_keys.update(next_tag_keys) @@ -278,6 +289,49 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: tag_columns=tuple(tag_order), ) + @staticmethod + def _shared_key_nullability( + table: "pa.Table", + left_schema: "pa.Schema", + right_schema: "pa.Schema", + keys: Collection[str], + ) -> "pa.Table": + """Flag each shared tag key nullable only when both inputs have it nullable. + + A null key matches nothing, so a key non-nullable in either input holds + no null after the join (ITL-829). The same arrays are re-wrapped under the + new field flags: no value or type changes. Equal flags keep the flag. + + Args: + table: The joined table. + left_schema: The left input table's schema. + right_schema: The right input table's schema. + keys: The join keys; names absent from either schema are skipped. + + Returns: + The table with each shared key's nullable flag decided by ``key_nullable``. + """ + schema = table.schema + changed = False + for name in keys: + left_index = left_schema.get_field_index(name) + right_index = right_schema.get_field_index(name) + index = schema.get_field_index(name) + if left_index < 0 or right_index < 0 or index < 0: + continue + nullable = key_nullable( + left_schema.field(left_index).nullable, + right_schema.field(right_index).nullable, + ) + if schema.field(index).nullable != nullable: + changed = True + schema = schema.set( # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + index, schema.field(index).with_nullable(nullable) + ) + if not changed: + return table + return pa.Table.from_arrays(table.columns, schema=schema) + @staticmethod def _without_meta(table: "pa.Table") -> "pa.Table": """Drop the meta (``__``-prefixed) columns from an input table. @@ -757,7 +811,8 @@ def _merge_tag_tables( left lacks. The async path joins in canonical order, so this is the order ``_joined_user_schemas`` defines. A shared (join-key) column takes the right tag's field, as ``static_process`` does - (``restore_schema_nullability`` lets the last reference win), and the + (``restore_schema_nullability`` lets the last reference win), nullable + only when both tags have it nullable (``key_nullable``), and the left tag's values, as its Polars join does (they are equal because the rows matched on it, except ``0.0`` against ``-0.0``). @@ -787,10 +842,16 @@ def _merge_tag_tables( # A tag without user columns is a zero-column, zero-row table. for index, name in enumerate(table.column_names): arrow_type = table.schema.types[index] + field = table.schema.field(index) if name in types: _ = require_shared_arrow_type("Join", name, types[name], arrow_type) + # A shared key is nullable only when both tags have it + # nullable: a null key matches nothing (ITL-829). + field = field.with_nullable( + key_nullable(fields[name].nullable, field.nullable) + ) types[name] = arrow_type - fields[name] = table.schema.field(index) + fields[name] = field # A shared column keeps the left tag's values (the sync join's # Polars join does): rows that match on ``0.0`` and ``-0.0`` # keep the left spelling. diff --git a/src/orcapod/core/operators/key_nullability.py b/src/orcapod/core/operators/key_nullability.py new file mode 100644 index 000000000..1958c9d96 --- /dev/null +++ b/src/orcapod/core/operators/key_nullability.py @@ -0,0 +1,56 @@ +"""A shared tag key that differs only in nullability between join inputs (ITL-829). + +Developer decision, November 13, 2026: ``Join``, ``MergeJoin`` and ``SemiJoin`` accept +a shared tag key that is ``T`` in one input and ``T | None`` in another. A null key +matches nothing, so no joined row can hold a null there: the key is nullable in a join's +output only when it is nullable in every input. ``SemiJoin``'s output is the left input's +schema as it is. Only tag schemas go through this module; data columns never do. +""" + +from __future__ import annotations + +from orcapod.core.operators.drop_nulls import DropNulls +from orcapod.types import Schema + +# DropNulls' own rule for removing ``None`` from a type, reused, never copied. +_narrow_schema = DropNulls._narrow_schema # pyright: ignore[reportPrivateUsage] # shared on purpose with DropNulls + + +def key_nullable(*nullable: bool) -> bool: + """Return whether a shared tag key is nullable in a join's output. + + Args: + *nullable: The key's nullable flag in each input that has it. + + Returns: + True only when the key is nullable in every input. + """ + return all(nullable) + + +def reconcile_key_nullability(first: Schema, second: Schema) -> tuple[Schema, Schema]: + """Give a shared tag key that differs only in nullability its non-nullable type. + + For each name in both tag schemas whose types differ but are equal once + ``None`` is removed, both schemas get that type without ``None``. Every other + pair is left as it is, so a real type difference still raises where the + schemas are combined, with the message it has always had. + + Args: + first: One input's tag schema. + second: The other input's tag schema. + + Returns: + The two tag schemas, with those keys non-nullable in both. + """ + shared = [name for name in first if name in second] + first_narrow = _narrow_schema(first, shared) + second_narrow = _narrow_schema(second, shared) + names = [ + name + for name in shared + if first[name] != second[name] and first_narrow[name] == second_narrow[name] + ] + if not names: + return first, second + return _narrow_schema(first, names), _narrow_schema(second, names) diff --git a/src/orcapod/core/operators/merge_join.py b/src/orcapod/core/operators/merge_join.py index 6044088aa..fe98429fe 100644 --- a/src/orcapod/core/operators/merge_join.py +++ b/src/orcapod/core/operators/merge_join.py @@ -6,6 +6,10 @@ require_shared_arrow_type, require_tag_data_roles_agree, ) +from orcapod.core.operators.key_nullability import ( + key_nullable, + reconcile_key_nullability, +) from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import StreamProtocol @@ -216,8 +220,11 @@ def binary_output_schema( left_tag_schema, left_data_schema = first_stream.output_schema() right_tag_schema, right_data_schema = second_stream.output_schema() - # Tag schema: union of both tag schemas - tag_schema = schema_utils.union_schemas(left_tag_schema, right_tag_schema) + # Tag schema: union of both tag schemas. A shared tag key that differs in + # nullability only is non-nullable (ITL-829): a null key matches nothing. + tag_schema = schema_utils.union_schemas( + *reconcile_key_nullability(left_tag_schema, right_tag_schema) + ) # Data schema: colliding columns become list[T], non-colliding stay scalar colliding_schema = schema_utils.intersection_schemas( @@ -382,6 +389,12 @@ def binary_static_process( for name, nullable in right_nullable.items(): if name not in output_nullable: output_nullable[name] = nullable + # A shared tag key is nullable only when nullable in both inputs: a null + # key matches nothing (ITL-829). Equal flags give the flag as before. + for name in shared_tag_keys: + output_nullable[name] = key_nullable( + left_nullable[name], right_nullable[name] + ) # Merged (colliding) data columns become 2-element lists — always non-null for col in colliding_keys: output_nullable[col] = False diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index 20a2c81a9..f5246aa23 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,11 +1,9 @@ -import logging from collections.abc import Sequence from typing import TYPE_CHECKING, Any from orcapod.channels import ReadableChannel, WritableChannel -from orcapod.core.datagrams import Data, Tag from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type -from orcapod.core.operators.drop_nulls import DropNulls +from orcapod.core.operators.key_nullability import reconcile_key_nullability from orcapod.core.operators.match_keys import frozen_scalar from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError @@ -23,13 +21,6 @@ pc = LazyModule("pyarrow.compute") pl = LazyModule("polars") -logger = logging.getLogger(__name__) - -# DropNulls' own rule, reused so that SemiJoin narrows a key exactly as DropNulls -# does (ITL-829); never a copy of it. -_narrow_schema = DropNulls._narrow_schema # pyright: ignore[reportPrivateUsage] # shared on purpose with DropNulls -_narrow_table = DropNulls._narrow_table # pyright: ignore[reportPrivateUsage] # shared on purpose with DropNulls - class SemiJoin(BinaryOperator): """ @@ -42,56 +33,11 @@ class SemiJoin(BinaryOperator): If there are no overlapping columns between the streams, the entire left stream is returned unchanged. - The output stream preserves the schema of the left stream exactly, with - one exception. A key that is a tag column on both inputs and is nullable - (``T | None``) on one of them only is first narrowed on that side as - ``DropNulls`` would: rows with a null in it are dropped and it becomes - ``T``. So a left key nullable against a non-nullable right key comes out - ``T`` (ITL-829). A null key never matches anyway, so this changes types, - never which rows match. A key nullable on both sides is not narrowed. + The output stream preserves the schema of the left stream exactly. A shared + tag key that is ``T`` in one input and ``T | None`` in the other is accepted + (a null key matches nothing) and keeps the left input's type (ITL-829). """ - @staticmethod - def _nullable_key_names( - left_tag_schema: Schema, right_tag_schema: Schema - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Return the shared tag keys to narrow on the left and on the right. - - A key is narrowed on a side when it is a tag column of both inputs and - its type admits ``None`` on that side only. Whether a type admits - ``None`` is decided by ``DropNulls``' own schema rule. - - Args: - left_tag_schema: The left input's tag schema. - right_tag_schema: The right input's tag schema. - - Returns: - The names to narrow on the left and on the right, each in the left - input's tag order. - """ - shared = [name for name in left_tag_schema if name in right_tag_schema] - left_narrow = _narrow_schema(left_tag_schema, shared) - right_narrow = _narrow_schema(right_tag_schema, shared) - left_nullable = [n for n in shared if left_narrow[n] != left_tag_schema[n]] - right_nullable = [n for n in shared if right_narrow[n] != right_tag_schema[n]] - return ( - tuple(n for n in left_nullable if n not in right_nullable), - tuple(n for n in right_nullable if n not in left_nullable), - ) - - @classmethod - def _narrowed_tag_schemas( - cls, left_tag_schema: Schema, right_tag_schema: Schema - ) -> tuple[Schema, Schema]: - """Return both tag schemas with the keys nullable on one side only narrowed.""" - left_names, right_names = cls._nullable_key_names( - left_tag_schema, right_tag_schema - ) - return ( - _narrow_schema(left_tag_schema, left_names), - _narrow_schema(right_tag_schema, right_names), - ) - def binary_static_process( self, left_stream: StreamProtocol, right_stream: StreamProtocol ) -> StreamProtocol: @@ -101,18 +47,15 @@ def binary_static_process( """ left_tag_schema, left_data_schema = left_stream.output_schema() right_tag_schema, right_data_schema = right_stream.output_schema() - left_names, right_names = self._nullable_key_names( + # A shared tag key may differ in nullability only (ITL-829). + left_tag_schema, right_tag_schema = reconcile_key_nullability( left_tag_schema, right_tag_schema ) - narrowed_left_tags = _narrow_schema(left_tag_schema, left_names) - narrowed_right_tags = _narrow_schema(right_tag_schema, right_names) # Find overlapping columns across all columns (tags + data) - left_all_schema = schema_utils.union_schemas( - narrowed_left_tags, left_data_schema - ) + left_all_schema = schema_utils.union_schemas(left_tag_schema, left_data_schema) right_all_schema = schema_utils.union_schemas( - narrowed_right_tags, right_data_schema + right_tag_schema, right_data_schema ) common_keys = tuple( @@ -127,20 +70,6 @@ def binary_static_process( left_table = left_stream.as_table(columns={"source": True, "system_tags": True}) right_table = right_stream.as_table() - # A shared tag key nullable on one side only: that side goes through - # DropNulls' rule first, so the result is DropNulls then SemiJoin (ITL-829). - if left_names or right_names: - logger.info( - "SemiJoin: dropping rows with a null key and narrowing the key to" - + " non-nullable, on the left for %s and on the right for %s", - list(left_names), - list(right_names), - ) - if left_names: - left_table = _narrow_table(left_table, left_names) - if right_names: - right_table = _narrow_table(right_table, right_names) - # A key must have one Arrow type on both sides (up to large layouts); # otherwise Polars would silently unify, e.g., int8 with int16. for key in common_keys: @@ -193,19 +122,10 @@ def binary_output_schema( ) -> tuple[Schema, Schema]: """ Returns the output types for the semi-join operation. - - The output has the left stream's schema, except that a shared tag key - nullable on the left only is narrowed to ``T``. + The output preserves the exact schema of the left stream. """ - tag_schema, data_schema = left_stream.output_schema( - columns=columns, all_info=all_info - ) - left_names, _ = self._nullable_key_names( - left_stream.output_schema()[0], right_stream.output_schema()[0] - ) - if not left_names: - return tag_schema, data_schema - return _narrow_schema(tag_schema, left_names), data_schema + # Semi-join preserves the left stream's schema exactly + return left_stream.output_schema(columns=columns, all_info=all_info) def validate_binary_inputs( self, left_stream: StreamProtocol, right_stream: StreamProtocol @@ -220,9 +140,8 @@ def validate_binary_inputs( try: left_tag_schema, left_data_schema = left_stream.output_schema() right_tag_schema, right_data_schema = right_stream.output_schema() - - # A shared tag key nullable on one side only is narrowed there first. - left_tag_schema, right_tag_schema = self._narrowed_tag_schemas( + # A shared tag key may differ in nullability only (ITL-829). + left_tag_schema, right_tag_schema = reconcile_key_nullability( left_tag_schema, right_tag_schema ) @@ -313,7 +232,6 @@ async def async_execute( # Phase 2: Probe — stream left rows common_keys: tuple[str, ...] | None = None right_lookup: set[tuple[object, ...]] = set() - left_names: tuple[str, ...] = () async for tag, data in left_ch: if common_keys is None: @@ -339,21 +257,6 @@ async def async_execute( self._key_arrow_type(right_tag, right_data, key), ) - # A shared tag key nullable on the left only is narrowed on - # every emitted left row, as on the sync path (ITL-829). The - # right side needs nothing: a null key never matches. - left_names, right_names = self._nullable_key_names( - tag.schema(), right_tag.schema() - ) - if left_names or right_names: - logger.info( - "SemiJoin: dropping rows with a null key and narrowing" - + " the key to non-nullable, on the left for %s and on" - + " the right for %s", - list(left_names), - list(right_names), - ) - # Build right-side lookup; a row with a null key never matches. for rt, rp in right_rows: right_key = self._match_key(rt, rp, common_keys) @@ -363,38 +266,10 @@ async def async_execute( # Probe left_key = self._match_key(tag, data, common_keys) if left_key is not None and left_key in right_lookup: - await output.send(self._narrowed_row(tag, data, left_names)) + await output.send((tag, data)) finally: await output.close() - @staticmethod - def _narrowed_row( - tag: TagProtocol, data: DataProtocol, names: Sequence[str] - ) -> tuple[TagProtocol, DataProtocol]: - """Return a left row with ``names`` narrowed by ``DropNulls``' rule. - - The row's tag and data tables go through ``DropNulls._narrow_table``, as - the sync path's left table does, so the row has the sync row's types. - - Args: - tag: A left row's tag, non-null in every name. - data: The row's data. - names: The shared tag keys nullable on the left only. - - Returns: - The row itself when ``names`` is empty, otherwise the rebuilt row. - """ - if not names: - return tag, data - tag_table = _narrow_table(tag.as_table(columns={"system_tags": True}), names) - data_table = _narrow_table( - data.as_table(columns={"source": True, "meta": True}), names - ) - return ( - Tag(tag_table, data_context=tag.data_context_key), - Data(data_table, data_context=data.data_context_key), - ) - @staticmethod def _key_table(tag: TagProtocol, data: DataProtocol, key: str) -> "pa.Table": """Return the one-row Arrow table, the tag's or the data's, holding ``key``.""" diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 133640f53..389fb44d0 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -157,7 +157,7 @@ def source_c() -> ArrowTableSource: def source_nullable_k() -> ArrowTableSource: - """Tag ``k: int16``, nullable with a null; for ``SemiJoin`` against a non-null ``k``.""" + """Tag ``k: int16``, nullable with a null; for a join against a non-nullable ``k``.""" return _source( { "k": pa.array([1, None, 3, 4], pa.int16()), @@ -292,6 +292,18 @@ class Case: Case("join_2_reversed", Join, lambda: (source_b(), source_a())), Case("join_3", Join, lambda: (source_a(), source_b(), source_c())), Case("join_3_rotated", Join, lambda: (source_c(), source_a(), source_b())), + # A shared tag key nullable in one input only (ITL-829). + Case("join_nullable_key", Join, lambda: (source_nullable_k(), source_b())), + Case( + "join_3_nullable_key", + Join, + lambda: (source_c(), source_nullable_k(), source_b()), + ), + Case( + "merge_join_nullable_key", + MergeJoin, + lambda: (source_b(nested=False), source_nullable_k()), + ), Case( "merge_join", MergeJoin, @@ -299,7 +311,7 @@ class Case: ), Case("semijoin", SemiJoin, lambda: (source_c(), source_b(nested=False))), Case("semijoin_nested", SemiJoin, lambda: (source_a(), source_b())), - # A shared tag key nullable on one side only is narrowed there (ITL-829). + # A shared tag key nullable in one input only (ITL-829). Case( "semijoin_left_nullable_key", SemiJoin, @@ -801,6 +813,7 @@ def make_sources() -> tuple[StreamProtocol, ...]: ) META_XFAIL: dict[str, str] = { "merge_join": _MATERIALIZE_DROPS_META, + "merge_join_nullable_key": _MATERIALIZE_DROPS_META, "batch_2": _MATERIALIZE_DROPS_META, "batch_all": _MATERIALIZE_DROPS_META, "group_by": _MATERIALIZE_DROPS_META, diff --git a/tests/test_core/operators/test_drop_nulls.py b/tests/test_core/operators/test_drop_nulls.py index 654f4be06..f41cec8a9 100644 --- a/tests/test_core/operators/test_drop_nulls.py +++ b/tests/test_core/operators/test_drop_nulls.py @@ -244,7 +244,8 @@ def test_tag_and_data_columns_in_one_call() -> None: def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: """A source whose key columns are nullable by specification (as a Spiral table's - are) cannot join a source whose keys are ``T``; after ``DropNulls`` it can.""" + are) joins a source whose keys are ``T`` as it does after ``DropNulls``: a null + key matches nothing, so the join gives the key ``T`` either way (ITL-829).""" events = _source( { "session": pa.array(["s1", None, "s3"], pa.large_string()), @@ -262,19 +263,20 @@ def test_narrowed_nullable_key_joins_with_a_non_nullable_key() -> None: tags=("session",), source_id="sessions", ) - with pytest.raises(InputValidationError, match="not compatible"): - Join()(events, sessions) - - joined = Join()(DropNulls("session")(events), sessions) - assert joined.output_schema() == ( - {"session": str}, - {"start": int | None, "animal": str}, - ) - rows = sorted(joined.as_table().to_pylist(), key=lambda r: r["session"]) - assert rows == [ - {"session": "s1", "start": 100, "animal": "m1"}, - {"session": "s3", "start": 300, "animal": "m3"}, - ] + for joined in ( + Join()(events, sessions), + Join()(DropNulls("session")(events), sessions), + ): + assert joined.output_schema() == ( + {"session": str}, + {"start": int | None, "animal": str}, + ) + assert not joined.as_table().schema.field("session").nullable + rows = sorted(joined.as_table().to_pylist(), key=lambda r: r["session"]) + assert rows == [ + {"session": "s1", "start": 100, "animal": "m1"}, + {"session": "s3", "start": 300, "animal": "m3"}, + ] def test_stream_without_tag_columns() -> None: diff --git a/tests/test_core/operators/test_join_key_nullability.py b/tests/test_core/operators/test_join_key_nullability.py new file mode 100644 index 000000000..773cd765a --- /dev/null +++ b/tests/test_core/operators/test_join_key_nullability.py @@ -0,0 +1,265 @@ +"""Joins over a shared tag key that differs only in nullability (ITL-829, second case). + +Developer decision, November 13, 2026: ``Join``, ``MergeJoin`` and ``SemiJoin`` accept a +shared tag key that is ``T`` in one input and ``T | None`` in another, because a null key +matches nothing. ``Join`` and ``MergeJoin`` give the key ``T`` when any input has it +non-nullable, and ``T | None`` only when every input has it nullable. ``SemiJoin``'s +output is the left input's schema as it is. Data columns are untouched: a data column +that differs in nullability behaves as before. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pyarrow as pa +import pytest + +from orcapod.core.operators import Join, MergeJoin, SemiJoin +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.core.sources import ArrowTableSource +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import StreamProtocol +from tests.test_core.operators.test_async_sync_equivalence import ( + _source, # pyright: ignore[reportPrivateUsage] # shared test helper + assert_equivalent, + assert_same_provenance, + async_rows, + sync_rows, +) + +SYSTEM_TAGS = {"system_tags": True} + + +def keyed( + source_id: str, + data: str, + nullable: tuple[str, ...] = (), + b: tuple[int | None, ...] = (1, 5, 1, 2, 2), +) -> ArrowTableSource: + """Tags ``a`` and ``b`` (int64) and one int64 data column named ``data``. + + A tag listed in ``nullable`` is declared nullable; ``b`` may hold a null. + """ + return _source( + { + "a": pa.array([1, 1, 2, 2, 3], pa.int64()), + "b": pa.array(list(b), pa.int64()), + data: pa.array([10, 20, 30, 40, 50], pa.int64()), + }, + tags=("a", "b"), + source_id=source_id, + nullable=nullable, + ) + + +def with_null(source_id: str, data: str, nullable: tuple[str, ...]) -> ArrowTableSource: + """Like ``keyed``, with a null in ``b`` when ``b`` is nullable.""" + b = (1, None, 1, 2, 2) if "b" in nullable else (1, 5, 1, 2, 2) + return keyed(source_id, data, nullable, b) + + +def tag_types(stream: StreamProtocol) -> dict[str, object]: + """The stream's predicted tag types.""" + return dict(stream.output_schema()[0]) + + +def arrow_nullable(stream: StreamProtocol, name: str) -> bool: + """Whether the computed table's field ``name`` is nullable.""" + return stream.as_table().schema.field(name).nullable + + +# ── Join and MergeJoin: T if any input has T ─────────────────────────────── + +JOINS: list[Callable[[], StaticOutputOperatorPod]] = [Join, MergeJoin] +JOIN_IDS = ["join", "merge_join"] + + +@pytest.mark.parametrize("make_op", JOINS, ids=JOIN_IDS) +@pytest.mark.parametrize( + ("left_null", "right_null", "expected_b"), + [ + pytest.param(("b",), (), int, id="left_nullable"), + pytest.param((), ("b",), int, id="right_nullable"), + pytest.param(("b",), ("b",), int | None, id="both_nullable"), + pytest.param((), (), int, id="neither_nullable"), + ], +) +def test_join_key_is_nullable_only_when_every_input_has_it_nullable( + make_op: Callable[[], StaticOutputOperatorPod], + left_null: tuple[str, ...], + right_null: tuple[str, ...], + expected_b: object, +) -> None: + """Prediction and computed Arrow field agree, in both operand orders.""" + left, right = with_null("l", "x", left_null), with_null("r", "z", right_null) + for result in (make_op()(left, right), make_op()(right, left)): + assert tag_types(result) == {"a": int, "b": expected_b} + assert arrow_nullable(result, "b") == (expected_b is not int) + assert not arrow_nullable(result, "a") + # A null key matches nothing: the row with b null finds no partner. + matched = [10, 30, 40, 50] if left_null or right_null else [10, 20, 30, 40, 50] + assert result.as_table().sort_by("x").column("x").to_pylist() == matched + + +@pytest.mark.parametrize("make_op", JOINS, ids=JOIN_IDS) +@pytest.mark.parametrize( + ("left_null", "right_null"), + [ + pytest.param(("b",), (), id="left_nullable"), + pytest.param((), ("b",), id="right_nullable"), + pytest.param(("a", "b"), ("b",), id="composite"), + ], +) +def test_join_prediction_matches_computed_schema( + make_op: Callable[[], StaticOutputOperatorPod], + left_null: tuple[str, ...], + right_null: tuple[str, ...], +) -> None: + """``output_schema()`` without computing equals the computed stream's schema.""" + left, right = with_null("l", "x", left_null), with_null("r", "z", right_null) + op = make_op() + predicted = op(left, right).output_schema(columns=SYSTEM_TAGS) + # Schema-prediction check: compared against the computed stream (CLAUDE.md). + computed = op.static_process(left, right).output_schema(columns=SYSTEM_TAGS) + assert predicted == computed + + +@pytest.mark.parametrize( + ("nullables", "expected_b"), + [ + pytest.param((("b",), ("b",), ()), int, id="one_non_nullable"), + pytest.param(((), ("b",), ("b",)), int, id="first_non_nullable"), + pytest.param((("b",), ("b",), ("b",)), int | None, id="all_nullable"), + ], +) +def test_nary_join_key_is_nullable_only_when_every_input_has_it_nullable( + nullables: tuple[tuple[str, ...], ...], expected_b: object +) -> None: + """Across three inputs the key is ``T`` as soon as one input has it ``T``.""" + sources = [ + with_null(f"s{i}", f"d{i}", nullable) for i, nullable in enumerate(nullables) + ] + result = Join()(*sources) + assert tag_types(result)["b"] == expected_b + assert arrow_nullable(result, "b") == (expected_b is not int) + # Schema-prediction check: compared against the computed stream (CLAUDE.md). + computed = Join().static_process(*sources).output_schema(columns=SYSTEM_TAGS) + assert result.output_schema(columns=SYSTEM_TAGS) == computed + expected = sync_rows(Join(), *sources) + assert expected + assert_equivalent(expected, async_rows(Join(), *sources)) + + +# ── SemiJoin: the left input's schema as it is ───────────────────────────── + + +@pytest.mark.parametrize( + ("left_null", "right_null"), + [ + pytest.param(("b",), (), id="left_nullable"), + pytest.param((), ("b",), id="right_nullable"), + pytest.param(("b",), ("b",), id="both_nullable"), + pytest.param(("b",), ("a",), id="composite_crossed"), + ], +) +def test_semijoin_keeps_the_left_schema( + left_null: tuple[str, ...], right_null: tuple[str, ...] +) -> None: + """The result has the left input's schema exactly; a null key matches nothing.""" + left, right = with_null("l", "x", left_null), with_null("r", "z", right_null) + result = SemiJoin()(left, right) + assert result.output_schema() == left.output_schema() + assert result.as_table().schema == left.as_table().schema + assert result.as_table().column("x").to_pylist() == [10, 30, 40, 50] + # Schema-prediction check: compared against the computed stream (CLAUDE.md). + computed = SemiJoin().static_process(left, right).output_schema(columns=SYSTEM_TAGS) + assert result.output_schema(columns=SYSTEM_TAGS) == computed + + +# ── every operator: async equals sync ────────────────────────────────────── + +OPERATORS: list[Callable[[], StaticOutputOperatorPod]] = [Join, MergeJoin, SemiJoin] +OPERATOR_IDS = ["join", "merge_join", "semijoin"] + + +@pytest.mark.parametrize("make_op", OPERATORS, ids=OPERATOR_IDS) +@pytest.mark.parametrize( + ("left_null", "right_null"), + [ + pytest.param(("b",), (), id="left_nullable"), + pytest.param((), ("b",), id="right_nullable"), + pytest.param(("b",), ("a",), id="composite_crossed"), + ], +) +def test_async_rows_equal_sync_rows( + make_op: Callable[[], StaticOutputOperatorPod], + left_null: tuple[str, ...], + right_null: tuple[str, ...], +) -> None: + """The async path yields the sync rows: types, nullability, tags, hashes, sources.""" + left, right = with_null("l", "x", left_null), with_null("r", "z", right_null) + expected = sync_rows(make_op(), left, right) + actual = async_rows(make_op(), left, right) + assert expected + assert_equivalent(expected, actual) + assert_same_provenance(expected, actual) + + +# ── what does not change ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize("make_op", OPERATORS, ids=OPERATOR_IDS) +def test_other_key_type_difference_still_raises( + make_op: Callable[[], StaticOutputOperatorPod], +) -> None: + """Only ``None`` is reconciled: ``int`` against ``str | None`` still raises.""" + left = _source( + {"a": pa.array([1, 2], pa.int64()), "x": pa.array([1, 2], pa.int64())}, + tags=("a",), + source_id="l", + ) + right = _source( + { + "a": pa.array(["1", None], pa.large_string()), + "z": pa.array([1, 2], pa.int64()), + }, + tags=("a",), + source_id="r", + nullable=("a",), + ) + with pytest.raises(InputValidationError, match="not compatible|Type conflict"): + _ = make_op()(left, right) + + +def _data_pair() -> tuple[ArrowTableSource, ArrowTableSource]: + """Two inputs sharing tag ``a`` and data column ``y``: ``int`` left, ``int | None`` right.""" + left = _source( + {"a": pa.array([1, 2], pa.int64()), "y": pa.array([1, 2], pa.int64())}, + tags=("a",), + source_id="l", + ) + right = _source( + {"a": pa.array([1, 2], pa.int64()), "y": pa.array([1, None], pa.int64())}, + tags=("a",), + source_id="r", + nullable=("y",), + ) + return left, right + + +@pytest.mark.parametrize( + ("make_op", "message"), + [ + pytest.param(Join, "Type conflict for key .y.", id="join"), + pytest.param(MergeJoin, "incompatible types", id="merge_join"), + pytest.param(SemiJoin, "Type conflict for key 'y'", id="semijoin"), + ], +) +def test_data_column_nullability_difference_behaves_as_before( + make_op: Callable[[], StaticOutputOperatorPod], message: str +) -> None: + """A data column is never reconciled: each operator raises as it did.""" + left, right = _data_pair() + with pytest.raises(InputValidationError, match=message): + _ = make_op()(left, right) diff --git a/tests/test_core/operators/test_semijoin_nullable_keys.py b/tests/test_core/operators/test_semijoin_nullable_keys.py deleted file mode 100644 index 66fdebb1f..000000000 --- a/tests/test_core/operators/test_semijoin_nullable_keys.py +++ /dev/null @@ -1,184 +0,0 @@ -"""SemiJoin over a shared tag key that is nullable on one input only (ITL-829, second case). - -Developer decision, November 13, 2026: when a shared tag key is ``T`` on one input and -``T | None`` on the other, SemiJoin drops the rows with a null in that key on the -nullable side and semi-joins. The result is exactly ``DropNulls()`` applied -to the nullable side followed by the semi join, schema included, so a left key that was -nullable comes out ``T``. A key nullable on both sides is not narrowed, and a shared data -column is never narrowed. -""" - -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.operators import DropNulls, SemiJoin -from orcapod.core.sources import ArrowTableSource -from orcapod.errors import InputValidationError -from orcapod.protocols.core_protocols import StreamProtocol -from tests.test_core.operators.test_async_sync_equivalence import ( - _source, # pyright: ignore[reportPrivateUsage] # shared test helper - assert_equivalent, - assert_same_provenance, - async_rows, - sync_rows, -) - -FULL = {"source": True, "system_tags": True} - - -def left_source(nullable: tuple[str, ...]) -> ArrowTableSource: - """Tags ``a`` and ``b`` (int64), data ``x``; a null in each nullable tag.""" - return _source( - { - "a": pa.array([1, 1, 2, 2, 3], pa.int64()), - "b": pa.array([1, None if "b" in nullable else 5, 1, 2, 2], pa.int64()), - "x": pa.array([10, 20, 30, 40, 50], pa.int64()), - }, - tags=("a", "b"), - source_id="left", - nullable=nullable, - ) - - -def right_source(nullable: tuple[str, ...]) -> ArrowTableSource: - """Tags ``a`` and ``b`` (int64), data ``z``; a null in each nullable tag.""" - return _source( - { - "a": pa.array([1, 2, None if "a" in nullable else 9, 3], pa.int64()), - "b": pa.array([1, 2, 1, None if "b" in nullable else 7], pa.int64()), - "z": pa.array([100, 200, 300, 400], pa.int64()), - }, - tags=("a", "b"), - source_id="right", - nullable=nullable, - ) - - -def narrowed(stream: StreamProtocol, names: tuple[str, ...]) -> StreamProtocol: - """``DropNulls(names)(stream)``, or the stream itself when there are no names.""" - return DropNulls(names)(stream) if names else stream - - -# (left nullable tags, right nullable tags, names DropNulls takes on left, on right) -MIXED = [ - pytest.param(("b",), (), ("b",), (), id="left_nullable"), - pytest.param((), ("b",), (), ("b",), id="right_nullable"), - pytest.param(("b",), ("a",), ("b",), ("a",), id="composite_crossed"), - pytest.param(("a", "b"), ("b",), ("a",), (), id="composite_one_shared_nullable"), -] - - -@pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) -def test_result_equals_drop_nulls_then_semijoin( - left_null: tuple[str, ...], - right_null: tuple[str, ...], - left_drop: tuple[str, ...], - right_drop: tuple[str, ...], -) -> None: - """Rows, Arrow schema, system tags and source info equal the explicit composition.""" - left, right = left_source(left_null), right_source(right_null) - result = SemiJoin()(left, right) - expected = SemiJoin()(narrowed(left, left_drop), narrowed(right, right_drop)) - - assert result.output_schema() == expected.output_schema() - assert result.as_table(columns=FULL).equals( - expected.as_table(columns=FULL), check_metadata=True - ) - assert result.as_table(columns=FULL).num_rows > 0 - - -@pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) -def test_prediction_matches_computed_schema( - left_null: tuple[str, ...], - right_null: tuple[str, ...], - left_drop: tuple[str, ...], - right_drop: tuple[str, ...], -) -> None: - """``output_schema()`` without computing equals the computed stream's schema.""" - left, right = left_source(left_null), right_source(right_null) - op = SemiJoin() - predicted = op(left, right).output_schema(columns=FULL) - # Schema-prediction check: compared against the computed stream (CLAUDE.md). - computed = op.static_process(left, right).output_schema(columns=FULL) - assert predicted == computed - - -def test_left_nullable_key_narrows_to_t() -> None: - """A left key nullable against a non-nullable right key comes out ``T``.""" - result = SemiJoin()(left_source(("b",)), right_source(())) - tag_schema, _ = result.output_schema() - assert tag_schema["b"] is int - assert not result.as_table().schema.field("b").nullable - assert result.as_table().column("x").to_pylist() == [10, 40] - - -def test_right_nullable_key_keeps_left_schema() -> None: - """A right key nullable against a non-nullable left key leaves the left schema.""" - left = left_source(()) - result = SemiJoin()(left, right_source(("b",))) - assert result.output_schema() == left.output_schema() - assert result.as_table().schema == left.as_table().schema - assert result.as_table().column("x").to_pylist() == [10, 40] - - -def test_key_nullable_on_both_sides_is_not_narrowed() -> None: - """Both nullable: as before, the key stays ``T | None`` and a null matches nothing.""" - left = left_source(("b",)) - result = SemiJoin()(left, right_source(("b",))) - tag_schema, _ = result.output_schema() - assert tag_schema["b"] == int | None - assert result.as_table().schema.field("b").nullable - assert result.output_schema() == left.output_schema() - assert result.as_table().column("x").to_pylist() == [10, 40] - - -@pytest.mark.parametrize(("left_null", "right_null", "left_drop", "right_drop"), MIXED) -def test_async_rows_equal_sync_rows( - left_null: tuple[str, ...], - right_null: tuple[str, ...], - left_drop: tuple[str, ...], - right_drop: tuple[str, ...], -) -> None: - """The async path yields the sync rows: types, nullability, tags, hashes, sources.""" - left, right = left_source(left_null), right_source(right_null) - expected = sync_rows(SemiJoin(), left, right) - actual = async_rows(SemiJoin(), left, right) - assert expected - assert_equivalent(expected, actual) - assert_same_provenance(expected, actual) - - -def test_shared_data_column_nullability_mismatch_still_raises() -> None: - """Only tag keys are narrowed; a shared data column keeps today's error.""" - left = _source( - {"a": pa.array([1, 2], pa.int64()), "y": pa.array([1, 2], pa.int64())}, - tags=("a",), - source_id="l", - ) - right = _source( - {"a": pa.array([1, 2], pa.int64()), "y": pa.array([1, None], pa.int64())}, - tags=("a",), - source_id="r", - nullable=("y",), - ) - with pytest.raises(InputValidationError, match="'y'"): - _ = SemiJoin()(left, right) - - -def test_other_type_mismatch_still_raises() -> None: - """Narrowing removes only ``None``: ``int`` against ``str | None`` still raises.""" - left = _source( - {"a": pa.array([1, 2], pa.int64()), "x": pa.array([1, 2], pa.int64())}, - tags=("a",), - source_id="l", - ) - right = _source( - {"a": pa.array(["1", None], pa.large_string()), "z": pa.array([1, 2])}, - tags=("a",), - source_id="r", - nullable=("a",), - ) - with pytest.raises(InputValidationError, match="'a'"): - _ = SemiJoin()(left, right) From 9cd2a5b37fd1ea274a686834a80c5a2efead4888 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:34:03 -0800 Subject: [PATCH 04/11] docs(operators): joins accept a tag key that differs only in nullability (ITL-829); golden entries Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 8 +++ CLAUDE.md | 8 +++ changelog.d/ITL-829.md | 3 + design_issues/ITL-829-SJ2.md | 26 ++++++++ docs/concepts/operators.md | 7 +++ ...026-11-13-itl-829-semijoin-nullable-key.md | 10 ++++ tests/test_hashing/golden/cases.py | 52 ++++++++++++++++ tests/test_hashing/golden/data/pipeline.json | 60 +++++++++++++++++++ 8 files changed, 174 insertions(+) create mode 100644 changelog.d/ITL-829.md create mode 100644 design_issues/ITL-829-SJ2.md diff --git a/.zed/rules b/.zed/rules index f14340ed6..ef1dcc390 100644 --- a/.zed/rules +++ b/.zed/rules @@ -796,6 +796,14 @@ data. The rules: async path) and never stored, so its identity is the fixed ("DropNulls", None), different from any list of names. System-tag, source-info, context and meta columns are never included. An explicitly empty collection is not "no argument": it raises ValueError. +- `Join`, `MergeJoin` and `SemiJoin` accept a shared TAG key whose types differ only in + nullability (`T` against `T | None`), because a null key matches nothing (developer, + November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it + `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output is the left + input's schema as it is. The rule is `core/operators/key_nullability.py` + (`reconcile_key_nullability` on tag schemas before they are combined, `key_nullable` for the + Arrow field flag, set by re-wrapping arrays, never by a cast). Equal nullability takes the + unchanged path; any other type difference, and every data column, behaves as before. - GroupBy requires every column in `by` to be a scalar tag column; raises InputValidationError for unknown, data, or list-valued columns (list-valued usually means the stream came from Batch — group before batching, not after). Members sort by non-key tag values with system diff --git a/CLAUDE.md b/CLAUDE.md index 95deee60a..c863c0f35 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -840,6 +840,14 @@ data. The rules: async path) and never stored, so its identity is the fixed `("DropNulls", None)`, different from any list of names. System-tag, source-info, context and meta columns are never included. An explicitly empty collection is not "no argument": it raises `ValueError`. +- `Join`, `MergeJoin` and `SemiJoin` accept a shared TAG key whose types differ only in + nullability (`T` against `T | None`), because a null key matches nothing (developer, + November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it + `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output is the left + input's schema as it is. The rule is `core/operators/key_nullability.py` + (`reconcile_key_nullability` on tag schemas before they are combined, `key_nullable` for the + Arrow field flag, set by re-wrapping arrays, never by a cast). Equal nullability takes the + unchanged path; any other type difference, and every data column, behaves as before. - `GroupBy` requires every column in `by` to be a scalar tag column; it raises `InputValidationError` for unknown, data, or list-valued columns (the last usually means the stream came from `Batch` — group before batching, not after). Members are sorted by non-key diff --git a/changelog.d/ITL-829.md b/changelog.d/ITL-829.md new file mode 100644 index 000000000..5e8969279 --- /dev/null +++ b/changelog.d/ITL-829.md @@ -0,0 +1,3 @@ +### Fixed + +- `Join`, `MergeJoin` and `SemiJoin` accept a shared tag key that is non-nullable in one input and nullable in another (ITL-829). They raised `InputValidationError` ("Type conflict for key 'id': vs int | None", or "Types int | None and are not compatible") when the graph was built. A null key matches nothing, so `Join` and `MergeJoin` now give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output keeps the left input's schema as it is. Output prediction and computed Arrow fields agree on the sync and async paths. Any other type difference still raises, a data column that differs in nullability behaves as before, and every pipeline that worked before gives the same rows, schema and hashes. diff --git a/design_issues/ITL-829-SJ2.md b/design_issues/ITL-829-SJ2.md new file mode 100644 index 000000000..dd39ef49c --- /dev/null +++ b/design_issues/ITL-829-SJ2.md @@ -0,0 +1,26 @@ +## `src/orcapod/core/operators/semijoin.py` — nested columns and string/large_string keys + +### SJ2 — Joins rejected a tag key that differs only in nullability +**Status:** resolved +**Severity:** medium +**Issue:** ITL-829 (second case) + +`SemiJoin`, `Join` and `MergeJoin` raised `InputValidationError` when a shared tag key was `T` +in one input and `T | None` in another, e.g. a stream with a declared non-nullable key against +a table built without a schema ("Type conflict for key 'id': vs int | None" from +`schema_utils.intersection_schemas`, or "Types int | None and are not +compatible" from `union_schemas`). The graph could not be built, although a null key matches +nothing and the result is clear. The strict xfail `test_binary_op_delegation` in +`tests/test_channels/test_node_async_execute.py` recorded it for `SemiJoin`. + +**Fix:** developer decision, November 13, 2026: the three joins accept such a key. +`core/operators/key_nullability.py` holds the rule: `reconcile_key_nullability` gives both tag +schemas the non-nullable type for a key whose types are equal once `None` is removed (only tag +schemas; any other difference is left to raise as before), and `key_nullable` makes the output +key nullable only when every input has it nullable. `Join` (prediction, sync field flags after +`restore_schema_nullability`, async `_merge_tag_tables`) and `MergeJoin` (prediction, sync +`output_nullable`) give the key `T` when any input has it `T`; `SemiJoin` validates with the +reconciled schemas and keeps the left input's schema. Flags are set by re-wrapping arrays, +never by a cast. Inputs with equal nullability take the unchanged path, so no stored value or +golden hash moves. Tests in `tests/test_core/operators/test_join_key_nullability.py`, cases in +`test_async_sync_equivalence.py`, golden entries `pipeline/join_key_nullability`. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 60d6656e0..ad90005e0 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -67,6 +67,13 @@ not match each other (the SQL convention for nulls; `MergeJoin` and `SemiJoin` f another null, an empty list is not a null, and NaN equals NaN. The async path gives the same rows. +Because a null key matches nothing, a shared tag column may differ between the inputs in +nullability alone: `T` in one input and `T | None` in another is accepted by `Join`, `MergeJoin` +and `SemiJoin`. In the output of `Join` and `MergeJoin` the column is `T` when any input has it +`T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output keeps the left +input's type. Any other type difference is still an error, and a data column is never +reconciled this way. + A name is a tag or a data column, never both: if one input has `x` as a tag column and another has it as a data column, `Join` (and `MergeJoin`) raises `InputValidationError` when the graph is built; rename one with `map_tags` or `map_data`. Column order follows each operator's argument symmetry: commutative operators (`Join`, diff --git a/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md b/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md index 0347bba86..e1349aff0 100644 --- a/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md +++ b/superpowers/plans/2026-11-13-itl-829-semijoin-nullable-key.md @@ -84,3 +84,13 @@ adds `output = WithoutMetaWriter(output)`. Coordinator: merge main after both la ## Progress notes - [x] Diagnosis and proposal approved (coordinator, Nov 13). +- [x] SUPERSEDED (developer, Nov 13): no narrowing. New rule, approved by the coordinator: + `Join`, `MergeJoin`, `SemiJoin` accept a shared TAG key differing only in nullability; + Join/MergeJoin key is `T` if any input has `T`, `T | None` only if all nullable; SemiJoin + output = left schema as is. Helper `core/operators/key_nullability.py` + (`reconcile_key_nullability`, `key_nullable`). Data columns unchanged. +- [x] Implemented (commit c5571236); tests `test_join_key_nullability.py`, equivalence cases, + `test_drop_nulls.py::test_narrowed_nullable_key_joins_with_a_non_nullable_key` updated + (asserted the old Join error), golden entries `pipeline/join_key_nullability` (adds only). +- [x] Docs (operators.md null-key paragraph), CLAUDE.md/.zed bullet, changelog, design issue SJ2. +- [ ] Merge main after #367 and #369 land (coordinator tells); PR; CodeRabbit; CI. diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 2345129e9..c7f9d942a 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -1189,6 +1189,58 @@ def pipeline_semi_join() -> dict[str, Any]: return stream_summary(SemiJoin()(_subjects(), _scans())) +def _keyed(source_id: str, value: str, nullable_key: bool) -> Any: + """A source on tag ``subject`` (``large_string``), nullable with a null if asked.""" + from orcapod.sources import ArrowTableSource + + subjects = ["s1", None, "s3"] if nullable_key else ["s1", "s2", "s3"] + table = pa.table( + { + "subject": pa.array(subjects, pa.large_string()), + value: pa.array([1, 2, 3], pa.int64()), + }, + schema=pa.schema( + [ + pa.field("subject", pa.large_string(), nullable=nullable_key), + pa.field(value, pa.int64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["subject"], source_id=source_id) + + +@case("pipeline", "join_key_nullability") +def pipeline_join_key_nullability() -> dict[str, Any]: + """Pin the joins over a tag key that is ``T`` in one input and ``T | None`` in the other. + + Accepted since ITL-829 (it raised before, so nothing was stored): ``Join`` and + ``MergeJoin`` give the key ``T``; ``SemiJoin`` keeps the left input's schema. + """ + from orcapod.operators import Join, MergeJoin, SemiJoin + + def nullable() -> Any: + return _keyed("golden_nullable_key", "x", nullable_key=True) + + def plain() -> Any: + return _keyed("golden_plain_key", "y", nullable_key=False) + + out = _prefixed("join", stream_summary(Join()(nullable(), plain()))) + out.update( + _prefixed("merge_join", stream_summary(MergeJoin()(nullable(), plain()))) + ) + out.update( + _prefixed( + "semi_join_left_nullable", stream_summary(SemiJoin()(nullable(), plain())) + ) + ) + out.update( + _prefixed( + "semi_join_right_nullable", stream_summary(SemiJoin()(plain(), nullable())) + ) + ) + return out + + @case("pipeline", "batch") def pipeline_batch() -> dict[str, Any]: from orcapod.operators import Batch diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index 6e3ccda85..70d90a41f 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -331,6 +331,66 @@ "golden_job/_result/golden_adjust/40131bb9828634977dd6065e421c248d7d8b230de8be25eab43601ba72c3568b/v0/python.function.v0/rdb_v2", "golden_job/golden_adjust/40131bb9828634977dd6065e421c248d7d8b230de8be25eab43601ba72c3568b/v0/python.function.v0/schema:b97f130704dffaf3ce77afb43573cb83fe856fa4870b2d190a1a96a297e6ab1b/pdb_v2" ], + "join_key_nullability.join.content_hash": "semantic_v1:654e75e232bab094c059ce1018b290b1f4698ff3b4fbc7a1f4afc5f5d6335545", + "join_key_nullability.join.pipeline_hash": "semantic_v1:120fd1f9567fd2cded849a05508204630170d59f61ac983f522cc08854ebef4f", + "join_key_nullability.join.row[{\"subject\":\"s1\"}]._tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "8736cbb9637655afaafac9fd41a24028", + "join_key_nullability.join.row[{\"subject\":\"s1\"}]._tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "163cfc5912085f089ad82d9d5b4f4e1a", + "join_key_nullability.join.row[{\"subject\":\"s1\"}]._tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "golden_nullable_key", + "join_key_nullability.join.row[{\"subject\":\"s1\"}]._tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "golden_plain_key", + "join_key_nullability.join.row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:0000025019693a2d1d578c8af9aaae0e3dab43933bbf1196b4e42a4c3a83c80a1f61ef", + "join_key_nullability.join.row[{\"subject\":\"s3\"}]._tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "d216b6331f0f5ab59432f4b74ae1f62c", + "join_key_nullability.join.row[{\"subject\":\"s3\"}]._tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "481c8d1815885b53a45cce95c4867f11", + "join_key_nullability.join.row[{\"subject\":\"s3\"}]._tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "golden_nullable_key", + "join_key_nullability.join.row[{\"subject\":\"s3\"}]._tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "golden_plain_key", + "join_key_nullability.join.row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:0000022166408838e16c8aac02628c1ab78d2e6ab30d5751d26b5a603cfaeab2d65df6", + "join_key_nullability.join.system_tag_columns": [ + "_tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0", + "_tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0", + "_tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1", + "_tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1" + ], + "join_key_nullability.merge_join.content_hash": "semantic_v1:7a383aa5faa2ac2ae6fcc9802e133d83d62d46ba9b0eed611e4a3106aa1cddf9", + "join_key_nullability.merge_join.pipeline_hash": "semantic_v1:c08d29cf61eab7d4f064c5d442dca83620d00ef79ae8cd880855f520556a65dd", + "join_key_nullability.merge_join.row[{\"subject\":\"s1\"}]._tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "8736cbb9637655afaafac9fd41a24028", + "join_key_nullability.merge_join.row[{\"subject\":\"s1\"}]._tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "163cfc5912085f089ad82d9d5b4f4e1a", + "join_key_nullability.merge_join.row[{\"subject\":\"s1\"}]._tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "golden_nullable_key", + "join_key_nullability.merge_join.row[{\"subject\":\"s1\"}]._tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "golden_plain_key", + "join_key_nullability.merge_join.row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:0000025019693a2d1d578c8af9aaae0e3dab43933bbf1196b4e42a4c3a83c80a1f61ef", + "join_key_nullability.merge_join.row[{\"subject\":\"s3\"}]._tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "d216b6331f0f5ab59432f4b74ae1f62c", + "join_key_nullability.merge_join.row[{\"subject\":\"s3\"}]._tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "481c8d1815885b53a45cce95c4867f11", + "join_key_nullability.merge_join.row[{\"subject\":\"s3\"}]._tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0": "golden_nullable_key", + "join_key_nullability.merge_join.row[{\"subject\":\"s3\"}]._tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1": "golden_plain_key", + "join_key_nullability.merge_join.row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:0000022166408838e16c8aac02628c1ab78d2e6ab30d5751d26b5a603cfaeab2d65df6", + "join_key_nullability.merge_join.system_tag_columns": [ + "_tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0", + "_tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998:0", + "_tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1", + "_tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90:1" + ], + "join_key_nullability.semi_join_left_nullable.content_hash": "semantic_v1:29ce6ffb5ea7953e60773540f41ae30e28b31ecb983c506876f7a9b09b5dfc6c", + "join_key_nullability.semi_join_left_nullable.pipeline_hash": "semantic_v1:0093f639b07ebb1f68fd0bf11dfbc0ac8f49f4349f17090c68cb775a87e359ba", + "join_key_nullability.semi_join_left_nullable.row[{\"subject\":\"s1\"}]._tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998": "8736cbb9637655afaafac9fd41a24028", + "join_key_nullability.semi_join_left_nullable.row[{\"subject\":\"s1\"}]._tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998": "golden_nullable_key", + "join_key_nullability.semi_join_left_nullable.row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:000002fcbe7eec1821a57679d34084a583d8282798563aa706ecb9c0c24382580de4df", + "join_key_nullability.semi_join_left_nullable.row[{\"subject\":\"s3\"}]._tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998": "d216b6331f0f5ab59432f4b74ae1f62c", + "join_key_nullability.semi_join_left_nullable.row[{\"subject\":\"s3\"}]._tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998": "golden_nullable_key", + "join_key_nullability.semi_join_left_nullable.row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:000002691516abea55461f4ddcafeae1b13911e77feb4a15432a87ab1d331021c016e2", + "join_key_nullability.semi_join_left_nullable.system_tag_columns": [ + "_tag_source_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998", + "_tag_record_id::1b192c3c6d1f48e76d116b9cc8b0b71bb37c57c6042fa59bc250166f9eab8998" + ], + "join_key_nullability.semi_join_right_nullable.content_hash": "semantic_v1:1eb3632295a21af07695b77c985b3dac474e7ed6cae88cdb43ba5fb8baa037e7", + "join_key_nullability.semi_join_right_nullable.pipeline_hash": "semantic_v1:67fa5f4136c0201c9936841bfd1ca7bc0313c14946dd0549204ad1de6267f6ca", + "join_key_nullability.semi_join_right_nullable.row[{\"subject\":\"s1\"}]._tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90": "163cfc5912085f089ad82d9d5b4f4e1a", + "join_key_nullability.semi_join_right_nullable.row[{\"subject\":\"s1\"}]._tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90": "golden_plain_key", + "join_key_nullability.semi_join_right_nullable.row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:0000024b077e94d169475bca0f9ac2be54d8b331e00e5672ccf58d3522d3b87160b156", + "join_key_nullability.semi_join_right_nullable.row[{\"subject\":\"s3\"}]._tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90": "481c8d1815885b53a45cce95c4867f11", + "join_key_nullability.semi_join_right_nullable.row[{\"subject\":\"s3\"}]._tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90": "golden_plain_key", + "join_key_nullability.semi_join_right_nullable.row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:00000265bfaf9aa011289d5a9cbeb6a65530b172df28fcf4dbf77e9dda9c3b61dcd7a6", + "join_key_nullability.semi_join_right_nullable.system_tag_columns": [ + "_tag_source_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90", + "_tag_record_id::4366ff465533eda06d75a1d3c6d7b1c6ce3d532a40c8b6226bbf24513ab24e90" + ], "join_pod.join.content_hash": "semantic_v1:f84f685e1dd94ce2c10410bed706b8f63561604cc7601428d9775e5d9a744b76", "join_pod.join.pipeline_hash": "semantic_v1:c6dfb17a1a41372d5946a9d7462cf98466c2357cce73eaff940967065bec57f7", "join_pod.join.row[{\"subject\":\"s1\"}]._tag_record_id::d22029fcc2b0458566a83389c23c1a850dc69499f2456ae83bc6345284f861a5::d22029fcc2b0458566a83389c23c1a850dc69499f2456ae83bc6345284f861a5:0": "e8e02084627f5e1aad83839e421538e8", From 65020860a95655eafb97e420565a9bfeff03fed8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:40:47 -0800 Subject: [PATCH 05/11] fix(semijoin): key on the shared tag columns only (ITL-1088) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/semijoin.py | 106 +++++++------- .../operators/test_async_sync_equivalence.py | 20 +++ .../operators/test_join_key_nullability.py | 12 +- .../operators/test_semijoin_nested.py | 14 +- .../operators/test_semijoin_tag_keys.py | 134 ++++++++++++++++++ 5 files changed, 229 insertions(+), 57 deletions(-) create mode 100644 tests/test_core/operators/test_semijoin_tag_keys.py diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index f5246aa23..744bd5d03 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,4 +1,4 @@ -from collections.abc import Sequence +from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any from orcapod.channels import ReadableChannel, WritableChannel @@ -27,17 +27,52 @@ class SemiJoin(BinaryOperator): Binary operator that performs a semi-join between two streams. A semi-join returns only the entries from the left stream that have - matching entries in the right stream, based on equality of values - in overlapping columns (columns with the same name and compatible types). + matching entries in the right stream. The match keys are exactly the + columns that are tag columns of both streams (ITL-1088): a row matches + when its values in all of them equal a right row's. A name shared as a + data column, or a tag on one side and a data column on the other, is not a + key and is ignored; the right stream's data columns are never read. - If there are no overlapping columns between the streams, the entire - left stream is returned unchanged. + If the streams share no tag column, the entire left stream is returned + unchanged. The output stream preserves the schema of the left stream exactly. A shared tag key that is ``T`` in one input and ``T | None`` in the other is accepted (a null key matches nothing) and keeps the left input's type (ITL-829). """ + @staticmethod + def _match_keys( + left_tag_keys: Collection[str], right_tag_keys: Collection[str] + ) -> tuple[str, ...]: + """Return the semi join's match keys: the names that are tags on both sides. + + Args: + left_tag_keys: The left input's tag column names. + right_tag_keys: The right input's tag column names. + + Returns: + The names in both, in the left input's order. + """ + return tuple(name for name in left_tag_keys if name in right_tag_keys) + + def _checked_match_keys( + self, left_stream: StreamProtocol, right_stream: StreamProtocol + ) -> tuple[str, ...]: + """Return the match keys after checking that their types agree. + + Raises: + TypeError: If a key's types differ beyond nullability (ITL-829). + """ + left_tag_schema, _ = left_stream.output_schema() + right_tag_schema, _ = right_stream.output_schema() + # A shared tag key may differ in nullability only (ITL-829); any other + # difference raises here. + _ = schema_utils.intersection_schemas( + *reconcile_key_nullability(left_tag_schema, right_tag_schema) + ) + return self._match_keys(tuple(left_tag_schema), tuple(right_tag_schema)) + def binary_static_process( self, left_stream: StreamProtocol, right_stream: StreamProtocol ) -> StreamProtocol: @@ -45,30 +80,17 @@ def binary_static_process( Performs a semi-join between left and right streams. Returns entries from left stream that have matching entries in right stream. """ - left_tag_schema, left_data_schema = left_stream.output_schema() - right_tag_schema, right_data_schema = right_stream.output_schema() - # A shared tag key may differ in nullability only (ITL-829). - left_tag_schema, right_tag_schema = reconcile_key_nullability( - left_tag_schema, right_tag_schema - ) - - # Find overlapping columns across all columns (tags + data) - left_all_schema = schema_utils.union_schemas(left_tag_schema, left_data_schema) - right_all_schema = schema_utils.union_schemas( - right_tag_schema, right_data_schema - ) - - common_keys = tuple( - schema_utils.intersection_schemas(left_all_schema, right_all_schema).keys() - ) + left_tag_schema, _ = left_stream.output_schema() + common_keys = self._checked_match_keys(left_stream, right_stream) - # If no overlapping columns, return the left stream unmodified + # If no shared tag column, return the left stream unmodified if not common_keys: return left_stream # include source info and system tags for left stream left_table = left_stream.as_table(columns={"source": True, "system_tags": True}) - right_table = right_stream.as_table() + # Only the right side's keys are read, never its data columns. + right_table = right_stream.as_table().select(list(common_keys)) # A key must have one Arrow type on both sides (up to large layouts); # otherwise Polars would silently unify, e.g., int8 with int16. @@ -132,32 +154,16 @@ def validate_binary_inputs( ) -> None: """ Validates that the input streams are compatible for semi-join. - Checks that overlapping columns have compatible types. + Checks that the match keys (the shared tag columns) have compatible + types; data columns are not keys and are not checked. Stores the common keys so that ``async_execute`` can use them to determine the correct empty-right behavior without data. """ try: - left_tag_schema, left_data_schema = left_stream.output_schema() - right_tag_schema, right_data_schema = right_stream.output_schema() - # A shared tag key may differ in nullability only (ITL-829). - left_tag_schema, right_tag_schema = reconcile_key_nullability( - left_tag_schema, right_tag_schema - ) - - # Check that overlapping columns have compatible types across all columns - left_all_schema = schema_utils.union_schemas( - left_tag_schema, left_data_schema - ) - right_all_schema = schema_utils.union_schemas( - right_tag_schema, right_data_schema - ) - - # intersection_schemas will raise an error if types are incompatible - common = schema_utils.intersection_schemas( - left_all_schema, right_all_schema + self._validated_common_keys: tuple[str, ...] = self._checked_match_keys( + left_stream, right_stream ) - self._validated_common_keys: tuple[str, ...] = tuple(common.keys()) except Exception as e: raise InputValidationError( @@ -224,10 +230,8 @@ async def async_execute( await output.send((tag, data)) return - # Determine right-side keys from first row - right_tag_keys = set(right_rows[0][0].keys()) - right_pkt_keys = set(right_rows[0][1].keys()) - right_all_keys = right_tag_keys | right_pkt_keys + # The right side's tag columns, from its first row + right_tag_keys = tuple(right_rows[0][0].keys()) # Phase 2: Probe — stream left rows common_keys: tuple[str, ...] | None = None @@ -235,11 +239,9 @@ async def async_execute( async for tag, data in left_ch: if common_keys is None: - # First left row — determine common keys and build index - left_tag_keys = set(tag.keys()) - left_pkt_keys = set(data.keys()) - left_all_keys = left_tag_keys | left_pkt_keys - common_keys = tuple(sorted(left_all_keys & right_all_keys)) + # First left row — determine common keys (the shared tag + # columns, ITL-1088) and build index + common_keys = self._match_keys(tuple(tag.keys()), right_tag_keys) if not common_keys: # No common keys — pass all left rows through diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 389fb44d0..63b490c9b 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -170,6 +170,20 @@ def source_nullable_k() -> ArrowTableSource: ) +def source_shared_data() -> ArrowTableSource: + """Tag ``k: int16``; data ``i32`` and ``ls`` named as ``source_b``'s, other values and types.""" + return _source( + { + "k": pa.array([3, 1], pa.int16()), + "i32": pa.array([-1, -2], pa.int64()), + "ls": pa.array([None, "x"], pa.string()), + }, + tags=("k",), + source_id="shared_data", + nullable=("ls",), + ) + + def merge_source(source_id: str, offset: int) -> ArrowTableSource: """A ``MergeJoin`` input; two of them collide on every data column but one.""" only = f"only_{source_id}" @@ -322,6 +336,12 @@ class Case: SemiJoin, lambda: (source_c(), source_nullable_k()), ), + # A shared data column name is not a key (ITL-1088): only ``k`` decides. + Case( + "semijoin_shared_data_column", + SemiJoin, + lambda: (source_b(nested=False), source_shared_data()), + ), Case("batch_2", lambda: Batch(batch_size=2), lambda: (source_a(),)), Case("batch_all", Batch, lambda: (source_a(),)), Case("group_by", lambda: GroupBy(["ta"]), lambda: (source_a(),)), diff --git a/tests/test_core/operators/test_join_key_nullability.py b/tests/test_core/operators/test_join_key_nullability.py index 773cd765a..91dac7d53 100644 --- a/tests/test_core/operators/test_join_key_nullability.py +++ b/tests/test_core/operators/test_join_key_nullability.py @@ -253,13 +253,21 @@ def _data_pair() -> tuple[ArrowTableSource, ArrowTableSource]: [ pytest.param(Join, "Type conflict for key .y.", id="join"), pytest.param(MergeJoin, "incompatible types", id="merge_join"), - pytest.param(SemiJoin, "Type conflict for key 'y'", id="semijoin"), ], ) def test_data_column_nullability_difference_behaves_as_before( make_op: Callable[[], StaticOutputOperatorPod], message: str ) -> None: - """A data column is never reconciled: each operator raises as it did.""" + """A data column is never reconciled: ``Join`` and ``MergeJoin`` raise as they did.""" left, right = _data_pair() with pytest.raises(InputValidationError, match=message): _ = make_op()(left, right) + + +def test_semijoin_ignores_a_data_column_that_differs_in_nullability() -> None: + """``SemiJoin`` keys on shared tags only (ITL-1088), so a shared data column is + neither a key nor checked: the result is every left row whose tag matches.""" + left, right = _data_pair() + result = SemiJoin()(left, right) + assert result.output_schema() == left.output_schema() + assert result.as_table().column("y").to_pylist() == [1, 2] diff --git a/tests/test_core/operators/test_semijoin_nested.py b/tests/test_core/operators/test_semijoin_nested.py index 309c7fb8e..7d8d3c6d8 100644 --- a/tests/test_core/operators/test_semijoin_nested.py +++ b/tests/test_core/operators/test_semijoin_nested.py @@ -268,10 +268,18 @@ def test_nested_non_nullable_fields_keep_their_flag() -> None: def keyed(name: str, key_column: Column, tag: Sequence[int]) -> ArrowTableSource: - """A source with tag ``t_`` and the shared nullable data column ``key``.""" + """A source with tags ``t_`` and the shared nullable ``key``, data ``d_``. + + ``key`` is the only tag both inputs share, so it is the semi join's match key + (ITL-1088: the keys are the shared tag columns). + """ return _source( - {f"t_{name}": pa.array(tag, pa.int16()), "key": key_column}, - tags=(f"t_{name}",), + { + f"t_{name}": pa.array(tag, pa.int16()), + "key": key_column, + f"d_{name}": pa.array(tag, pa.int16()), + }, + tags=(f"t_{name}", "key"), source_id=name, nullable=("key",), ) diff --git a/tests/test_core/operators/test_semijoin_tag_keys.py b/tests/test_core/operators/test_semijoin_tag_keys.py new file mode 100644 index 000000000..f2336570b --- /dev/null +++ b/tests/test_core/operators/test_semijoin_tag_keys.py @@ -0,0 +1,134 @@ +"""SemiJoin keys on the shared tag columns only (ITL-1088). + +Developer decision, November 13, 2026: "Semijoin should key on shared tags only". The +match keys are exactly the names that are tag columns of both inputs. A name shared as a +data column, or a tag on one side and a data column on the other, is ignored, and the +right input's data columns are never read. With no shared tag, every left row is +returned. Before, every shared name, tag or data, was a match key. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pyarrow as pa +import pytest + +from orcapod.core.operators import PolarsFilter, SemiJoin +from orcapod.core.sources import ArrowTableSource +from orcapod.protocols.core_protocols import StreamProtocol +from tests.test_core.operators.test_async_sync_equivalence import ( + _source, # pyright: ignore[reportPrivateUsage] # shared test helper + assert_equivalent, + assert_same_provenance, + async_rows, + sync_rows, +) + + +def left_source() -> ArrowTableSource: + """Tag ``id`` 1..3, data ``y`` 1..3 and ``x``.""" + return _source( + { + "id": pa.array([1, 2, 3], pa.int64()), + "y": pa.array([1, 2, 3], pa.int64()), + "x": pa.array([10, 20, 30], pa.int64()), + }, + tags=("id",), + source_id="left", + ) + + +def right_same_tag_other_y() -> ArrowTableSource: + """Tag ``id`` 1..3 and data ``y`` that equals the left's only for ``id`` 1 and 3.""" + return _source( + {"id": pa.array([1, 2, 3], pa.int64()), "y": pa.array([1, 9, 3], pa.int64())}, + tags=("id",), + source_id="right_y", + ) + + +def right_tag_y() -> ArrowTableSource: + """Tag ``y`` (the left's data column name) and data ``z``.""" + return _source( + {"y": pa.array([1, 3], pa.int64()), "z": pa.array([0, 0], pa.int64())}, + tags=("y",), + source_id="right_tag_y", + ) + + +def right_nothing_shared() -> ArrowTableSource: + """Tag ``other`` and data ``w``: no name in common with the left.""" + return _source( + {"other": pa.array([7], pa.int64()), "w": pa.array([0], pa.int64())}, + tags=("other",), + source_id="right_nothing", + ) + + +def right_unequal_data_of_another_type() -> ArrowTableSource: + """Tag ``id`` 1 and 3; data ``y`` and ``x`` of other types and values than the left's.""" + return _source( + { + "id": pa.array([3, 1], pa.int64()), + "y": pa.array(["no", "match"], pa.large_string()), + "x": pa.array([None, 0.5], pa.float64()), + }, + tags=("id",), + source_id="right_other_data", + nullable=("x",), + ) + + +def left_ids(stream: StreamProtocol) -> list[int | None]: + """The ``id`` of each result row, in order.""" + return stream.as_table().column("id").to_pylist() + + +CASES = [ + # (right input, ids kept) + pytest.param( + right_same_tag_other_y, [1, 2, 3], id="shared_data_column_is_not_a_key" + ), + pytest.param(right_tag_y, [1, 2, 3], id="tag_on_right_data_on_left_is_not_a_key"), + pytest.param(right_nothing_shared, [1, 2, 3], id="nothing_shared"), + pytest.param( + right_unequal_data_of_another_type, [1, 3], id="right_data_never_read" + ), +] + + +@pytest.mark.parametrize(("make_right", "kept"), CASES) +def test_semijoin_keys_on_shared_tags_only( + make_right: Callable[[], ArrowTableSource], kept: list[int] +) -> None: + """The result is the left rows whose shared tag values match; data plays no part.""" + left = left_source() + result = SemiJoin()(left, make_right()) + assert left_ids(result) == kept + assert result.output_schema() == left.output_schema() + assert result.as_table().schema == left.as_table().schema + + +@pytest.mark.parametrize(("make_right", "kept"), CASES) +def test_async_rows_equal_sync_rows( + make_right: Callable[[], ArrowTableSource], kept: list[int] +) -> None: + """The async path keys on the same columns and yields the sync rows.""" + left, right = left_source(), make_right() + expected = sync_rows(SemiJoin(), left, right) + actual = async_rows(SemiJoin(), left, right) + assert len(expected) == len(kept) + assert_equivalent(expected, actual) + assert_same_provenance(expected, actual) + + +def test_filter_a_stream_by_a_subset_of_itself() -> None: + """The commonest use: every data column name is shared, and only the tags decide.""" + source = left_source() + subset = PolarsFilter(constraints={"id": 2})(left_source()) + result = SemiJoin()(source, subset) + assert left_ids(result) == [2] + assert_equivalent( + sync_rows(SemiJoin(), source, subset), async_rows(SemiJoin(), source, subset) + ) From 6a0700148e7606a8eebcee052120a22c4e1b40dc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:41:43 -0800 Subject: [PATCH 06/11] docs(semijoin): key rule, breaking-change note and design issue (ITL-1088) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 +++++ CLAUDE.md | 5 +++++ changelog.d/ITL-829.md | 35 ++++++++++++++++++++++++++++++++++- design_issues/ITL-1088-SJ3.md | 21 +++++++++++++++++++++ docs/concepts/operators.md | 5 +++++ 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 design_issues/ITL-1088-SJ3.md diff --git a/.zed/rules b/.zed/rules index ef1dcc390..d5732e136 100644 --- a/.zed/rules +++ b/.zed/rules @@ -796,6 +796,11 @@ data. The rules: async path) and never stored, so its identity is the fixed ("DropNulls", None), different from any list of names. System-tag, source-info, context and meta columns are never included. An explicitly empty collection is not "no argument": it raises ValueError. +- `SemiJoin`'s match keys are exactly the names that are tag columns of both inputs + (developer, November 13, 2026, ITL-1088), from one helper, `SemiJoin._match_keys`, used by + validation, the sync path and the async path. A shared data column name, or a name that is a + tag on one side and data on the other, is ignored (never an error); the right input's data + columns are never read. No shared tag column: every left row is returned. - `Join`, `MergeJoin` and `SemiJoin` accept a shared TAG key whose types differ only in nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it diff --git a/CLAUDE.md b/CLAUDE.md index c863c0f35..8cda1d5de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -840,6 +840,11 @@ data. The rules: async path) and never stored, so its identity is the fixed `("DropNulls", None)`, different from any list of names. System-tag, source-info, context and meta columns are never included. An explicitly empty collection is not "no argument": it raises `ValueError`. +- `SemiJoin`'s match keys are exactly the names that are tag columns of both inputs + (developer, November 13, 2026, ITL-1088), from one helper, `SemiJoin._match_keys`, used by + validation, the sync path and the async path. A shared data column name, or a name that is a + tag on one side and data on the other, is ignored (never an error); the right input's data + columns are never read. No shared tag column: every left row is returned. - `Join`, `MergeJoin` and `SemiJoin` accept a shared TAG key whose types differ only in nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it diff --git a/changelog.d/ITL-829.md b/changelog.d/ITL-829.md index 5e8969279..686d8280f 100644 --- a/changelog.d/ITL-829.md +++ b/changelog.d/ITL-829.md @@ -1,3 +1,36 @@ +### Breaking changes and migration + +#### `SemiJoin` keys on the shared tag columns only (ITL-1088) + +`SemiJoin` used every name its two inputs share, tag or data, as a match key. A left row was +kept only when its tags **and** its values in every shared data column equalled those of some +right row, and a name that was a tag on one side and a data column on the other was a key too. +So the operator read data values, which only filters may do. The match keys are now exactly the +columns that are tag columns of both inputs. A shared data column name, or a name that is a tag +on one side and data on the other, is ignored, and the right input's data columns are never +read. + +```python +left = DictSource(data=[{"id": 1, "y": 1}, {"id": 2, "y": 2}], tag_columns=["id"]) +right = DictSource(data=[{"id": 1, "y": 1}, {"id": 2, "y": 9}], tag_columns=["id"]) +SemiJoin()(left, right) +# v0.2.0: only id 1 (id and y must both match) +# now: id 1 and id 2 (only the shared tag id decides) +``` + +- **What changes.** A pipeline whose two `SemiJoin` inputs share a data column name now keeps + the left rows whose shared tags match, whatever the data values. Before, rows whose shared data + values differed were dropped. The new result always contains the old one. Filtering a stream + by a subset of itself (every data name shared, equal values) gives the same rows as before. A + shared data column whose type or nullability differs no longer raises, because it is not a + key. +- **Unchanged.** `SemiJoin`'s identity and pipeline hash do not change. The output schema is + still the left input's. With no shared tag column, every left row is still returned. +- **Stored operator caches.** A cached `SemiJoin` node keeps its table path. With + `CacheMode.LOG` the node recomputes on its next run, emits the new rows and stores the rows + that were not stored yet. With `CacheMode.REPLAY`, and for a loaded pipeline, the node replays + the stored rows, i.e. the old, smaller result, until a `LOG` run stores the new one. + ### Fixed -- `Join`, `MergeJoin` and `SemiJoin` accept a shared tag key that is non-nullable in one input and nullable in another (ITL-829). They raised `InputValidationError` ("Type conflict for key 'id': vs int | None", or "Types int | None and are not compatible") when the graph was built. A null key matches nothing, so `Join` and `MergeJoin` now give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output keeps the left input's schema as it is. Output prediction and computed Arrow fields agree on the sync and async paths. Any other type difference still raises, a data column that differs in nullability behaves as before, and every pipeline that worked before gives the same rows, schema and hashes. +- `Join`, `MergeJoin` and `SemiJoin` accept a shared tag key that is non-nullable in one input and nullable in another (ITL-829). They raised `InputValidationError` ("Type conflict for key 'id': vs int | None", or "Types int | None and are not compatible") when the graph was built. A null key matches nothing, so `Join` and `MergeJoin` now give the key `T` when any input has it `T`, and `T | None` only when every input has it nullable; `SemiJoin`'s output keeps the left input's schema as it is. Output prediction and computed Arrow fields agree on the sync and async paths. Any other type difference still raises, a data column that differs in nullability behaves as before in `Join` and `MergeJoin`, and every pipeline that worked before gives the same rows, schema and hashes. diff --git a/design_issues/ITL-1088-SJ3.md b/design_issues/ITL-1088-SJ3.md new file mode 100644 index 000000000..c857180da --- /dev/null +++ b/design_issues/ITL-1088-SJ3.md @@ -0,0 +1,21 @@ +## `src/orcapod/core/operators/semijoin.py` — nested columns and string/large_string keys + +### SJ3 — `SemiJoin` matched on shared data columns +**Status:** resolved +**Severity:** high +**Issue:** ITL-1088 + +`SemiJoin` took every name its inputs share, tag or data, as a match key. A shared data column +`y` filtered the left rows by data value, and a name that was a tag on one side and a data column +on the other was matched across the two roles. The operator therefore read data content, +which only filters may do, and a shared data column of another type or nullability made the +graph fail to build. + +**Fix:** developer decision, November 13, 2026: "Semijoin should key on shared tags only". The +keys come from one helper, `SemiJoin._match_keys(left_tag_keys, right_tag_keys)`, used by +validation, `binary_static_process` and `async_execute`: the names that are tag columns of both +inputs. Other shared names are ignored, and the sync path reads only the right input's key +columns. With no shared tag every left row is returned, as before. The identity is +unchanged; a cached node in `REPLAY` serves its stored (old) rows until a `LOG` run stores the +new result. Tests in `tests/test_core/operators/test_semijoin_tag_keys.py`; the nested-key tests +in `test_semijoin_nested.py` now use a shared tag key. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index ad90005e0..3fb7c3e4e 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -152,6 +152,11 @@ stream. The right stream's data columns are discarded. SemiJoin is **not commuta the order of inputs matters. The first stream is the one being filtered; the second stream provides the set of matching tags. +The match keys are exactly the columns that are tag columns of both streams. A name the two +streams share as a data column, or a tag in one and a data column in the other, is not a key +and is ignored: the right stream's data columns are never read. If the streams share no tag +column, every left row is returned. + ```python from orcapod.sources import DictSource from orcapod.operators import SemiJoin From 6de240694cd37c837d37eb0735a2d13dac0f223d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:50:16 -0800 Subject: [PATCH 07/11] feat(semijoin): warn about shared names that are not match keys (ITL-1088) SemiJoinIgnoredColumnWarning names each shared column that is not a tag on both inputs, once per operator and set of names; the first-frame-outside- Orcapod stacklevel helper moves to utils/warning_utils for reuse. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/semijoin.py | 115 +++++++++++- src/orcapod/errors.py | 18 ++ src/orcapod/logical_types/provisional.py | 22 +-- src/orcapod/utils/warning_utils.py | 27 +++ .../operators/test_async_sync_equivalence.py | 6 + .../operators/test_join_key_nullability.py | 5 +- .../operators/test_semijoin_tag_keys.py | 174 +++++++++++++++--- .../operators/test_tag_data_collision.py | 11 +- 8 files changed, 326 insertions(+), 52 deletions(-) create mode 100644 src/orcapod/utils/warning_utils.py diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index 744bd5d03..136e4efe2 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,3 +1,4 @@ +import warnings from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any @@ -6,11 +7,13 @@ from orcapod.core.operators.key_nullability import reconcile_key_nullability from orcapod.core.operators.match_keys import frozen_scalar from orcapod.core.streams import ArrowTableStream -from orcapod.errors import InputValidationError +from orcapod.errors import InputValidationError, SemiJoinIgnoredColumnWarning from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol +from orcapod.system_constants import constants from orcapod.types import ColumnConfig, ContentHash, Schema from orcapod.utils import arrow_utils, schema_utils from orcapod.utils.lazy_module import LazyModule +from orcapod.utils.warning_utils import outside_orcapod_stacklevel if TYPE_CHECKING: import polars as pl @@ -31,7 +34,8 @@ class SemiJoin(BinaryOperator): columns that are tag columns of both streams (ITL-1088): a row matches when its values in all of them equal a right row's. A name shared as a data column, or a tag on one side and a data column on the other, is not a - key and is ignored; the right stream's data columns are never read. + key and is ignored, with a ``SemiJoinIgnoredColumnWarning`` (provisional in + v0.3); the right stream's data columns are never read. If the streams share no tag column, the entire left stream is returned unchanged. @@ -56,6 +60,97 @@ def _match_keys( """ return tuple(name for name in left_tag_keys if name in right_tag_keys) + @staticmethod + def _ignored_shared_names( + left_tag_keys: Collection[str], + left_data_keys: Collection[str], + right_tag_keys: Collection[str], + right_data_keys: Collection[str], + ) -> tuple[tuple[str, str, str], ...]: + """Return the shared names that are not match keys, with each side's role. + + The decision is by name only: no value or type is read. Meta (``__``) + columns are not user columns and never count. + + Args: + left_tag_keys: The left input's tag column names. + left_data_keys: The left input's data column names. + right_tag_keys: The right input's tag column names. + right_data_keys: The right input's data column names. + + Returns: + ``(name, left role, right role)`` for each name shared as data on both + sides or as a tag on one side and data on the other, in the left + input's order; roles are ``"tag"`` or ``"data"``. + """ + right_roles = {name: "tag" for name in right_tag_keys} | { + name: "data" for name in right_data_keys + } + left_roles = [(name, "tag") for name in left_tag_keys] + [ + (name, "data") for name in left_data_keys + ] + return tuple( + (name, left_role, right_roles[name]) + for name, left_role in left_roles + if name in right_roles + and not name.startswith(constants.META_PREFIX) + and (left_role, right_roles[name]) != ("tag", "tag") + ) + + def _warn_once_per_application( + self, left_stream: StreamProtocol, right_stream: StreamProtocol + ) -> None: + """Warn about shared non-key names once per operator and set of names. + + One application is validated more than once: when the operator is + called, when a job builds its node and when the sync orchestrator runs + it over the rows it rebuilt. The ignored names are remembered on the + operator, so each set is warned about once. + + Args: + left_stream: The left input. + right_stream: The right input. + """ + left_tags, left_data = left_stream.keys() + right_tags, right_data = right_stream.keys() + ignored = self._ignored_shared_names( + left_tags, left_data, right_tags, right_data + ) + if not ignored: + return + warned: set[tuple[tuple[str, str, str], ...]] | None = getattr( + self, "_warned_ignored", None + ) + if warned is None: + warned = set() + self._warned_ignored: set[tuple[tuple[str, str, str], ...]] = warned + if ignored in warned: + return + warned.add(ignored) + self._warn_ignored(ignored) + + @staticmethod + def _warn_ignored(ignored: Sequence[tuple[str, str, str]]) -> None: + """Issue ``SemiJoinIgnoredColumnWarning`` for the shared non-key names. + + Args: + ignored: ``(name, left role, right role)`` per ignored name. + """ + described = ", ".join( + f"'{name}' ({left_role} on the left, {right_role} on the right)" + for name, left_role, right_role in ignored + ) + warnings.warn( + f"SemiJoin ignores the shared column(s) {described}: they are not used" + + " for matching. SemiJoin matches only on tag columns shared by both" + + " inputs. To silence this, rename or drop the column(s) on the right" + + " input before the semi join (map_data, drop_data_columns," + + " select_data_columns, or map_tags for a tag), or filter" + + " orcapod.errors.SemiJoinIgnoredColumnWarning.", + SemiJoinIgnoredColumnWarning, + stacklevel=outside_orcapod_stacklevel(), + ) + def _checked_match_keys( self, left_stream: StreamProtocol, right_stream: StreamProtocol ) -> tuple[str, ...]: @@ -159,6 +254,10 @@ def validate_binary_inputs( Stores the common keys so that ``async_execute`` can use them to determine the correct empty-right behavior without data. + + Warns: + SemiJoinIgnoredColumnWarning: Once per operator and set of names, when + the inputs share a name that is not a tag on both sides (ITL-1088). """ try: self._validated_common_keys: tuple[str, ...] = self._checked_match_keys( @@ -169,6 +268,7 @@ def validate_binary_inputs( raise InputValidationError( f"Input streams are not compatible for semi-join: {e}" ) from e + self._warn_once_per_application(left_stream, right_stream) def is_commutative(self) -> bool: return False @@ -242,6 +342,17 @@ async def async_execute( # First left row — determine common keys (the shared tag # columns, ITL-1088) and build index common_keys = self._match_keys(tuple(tag.keys()), right_tag_keys) + # Validation warns about shared non-key names; when it did not + # run (a direct call), warn here, once, from the first rows. + if not hasattr(self, "_validated_common_keys"): + ignored = self._ignored_shared_names( + tag.keys(), + data.keys(), + right_tag_keys, + right_rows[0][1].keys(), + ) + if ignored: + self._warn_ignored(ignored) if not common_keys: # No common keys — pass all left rows through diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index dc557f3c0..3ed27a1f5 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -398,6 +398,24 @@ class ProvisionalHashWarning(UserWarning): """ +class SemiJoinIgnoredColumnWarning(UserWarning): + """``SemiJoin`` inputs share a column name that is not a match key. + + ``SemiJoin`` matches on the columns that are tag columns of both inputs and on + nothing else (ITL-1088). A name the inputs share as data columns, or as a tag on + one side and a data column on the other, is ignored: it does not take part in + matching, and the right input's data is never read. This warning names such + columns, once per operator and set of columns, because before v0.3 they were + match keys. The behaviour for them is provisional in v0.3 and may become an + error. + + To silence it, rename or drop the column on the right input before the semi join + (``map_data``, ``drop_data_columns``, ``select_data_columns``, or ``map_tags`` + for a tag), or call ``warnings.filterwarnings("ignore", + category=SemiJoinIgnoredColumnWarning)``. + """ + + class EmptyDataAccessError(Exception): """Raised when a payload-access method is called on an ``EmptyData`` instance. diff --git a/src/orcapod/logical_types/provisional.py b/src/orcapod/logical_types/provisional.py index 553f0c76c..cb39c1e29 100644 --- a/src/orcapod/logical_types/provisional.py +++ b/src/orcapod/logical_types/provisional.py @@ -9,34 +9,16 @@ from __future__ import annotations -import sys import threading import warnings -from types import FrameType from orcapod.errors import ProvisionalHashWarning +from orcapod.utils.warning_utils import outside_orcapod_stacklevel _lock = threading.RLock() _issued: set[str] = set() -def _outside_orcapod_stacklevel() -> int: - """Return the stacklevel of the first frame outside Orcapod, as seen from the caller. - - Level 1 is the caller of this function (``warn_provisional``). When every frame - is Orcapod's (a worker thread), the caller of ``warn_provisional`` is used. - """ - level = 1 - frame: FrameType | None = sys._getframe(1) # pyright: ignore[reportPrivateUsage] # as the warnings module does - while frame is not None: - module = str(frame.f_globals.get("__name__", "")) - if module != "orcapod" and not module.startswith("orcapod."): - return level - frame = frame.f_back - level += 1 - return 2 - - def warn_provisional(extension_name: str) -> None: """Issue ``ProvisionalHashWarning`` for a type, once per process. @@ -60,7 +42,7 @@ def warn_provisional(extension_name: str) -> None: "silence: warnings.filterwarnings('ignore', " "category=orcapod.errors.ProvisionalHashWarning).", ProvisionalHashWarning, - stacklevel=_outside_orcapod_stacklevel(), + stacklevel=outside_orcapod_stacklevel(), ) _issued.add(extension_name) diff --git a/src/orcapod/utils/warning_utils.py b/src/orcapod/utils/warning_utils.py new file mode 100644 index 000000000..53c9b633c --- /dev/null +++ b/src/orcapod/utils/warning_utils.py @@ -0,0 +1,27 @@ +"""Attribute an Orcapod warning to the user's code.""" + +from __future__ import annotations + +import sys +from types import FrameType + + +def outside_orcapod_stacklevel() -> int: + """Return the stacklevel of the first frame outside Orcapod, as seen from the caller. + + Level 1 is the caller of this function, the function that calls + ``warnings.warn``. When every frame is Orcapod's (a worker thread), the caller + of that function is used. + + Returns: + The ``stacklevel`` to pass to ``warnings.warn``. + """ + level = 1 + frame: FrameType | None = sys._getframe(1) # pyright: ignore[reportPrivateUsage] # as the warnings module does + while frame is not None: + module = str(frame.f_globals.get("__name__", "")) + if module != "orcapod" and not module.startswith("orcapod."): + return level + frame = frame.f_back + level += 1 + return 2 diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 63b490c9b..c5bfc451a 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -57,6 +57,12 @@ ) from orcapod.utils import arrow_utils +# The ``semijoin_shared_data_column`` case shares a data column name on purpose; SemiJoin +# warns that it is not a match key (ITL-1088), which is tested in test_semijoin_tag_keys. +pytestmark = pytest.mark.filterwarnings( + "ignore::orcapod.errors.SemiJoinIgnoredColumnWarning" +) + Row: TypeAlias = tuple[TagProtocol, DataProtocol] Column: TypeAlias = "pa.Array[Any] | pa.ChunkedArray[Any]" diff --git a/tests/test_core/operators/test_join_key_nullability.py b/tests/test_core/operators/test_join_key_nullability.py index 91dac7d53..15a7798a8 100644 --- a/tests/test_core/operators/test_join_key_nullability.py +++ b/tests/test_core/operators/test_join_key_nullability.py @@ -18,7 +18,7 @@ from orcapod.core.operators import Join, MergeJoin, SemiJoin from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod from orcapod.core.sources import ArrowTableSource -from orcapod.errors import InputValidationError +from orcapod.errors import InputValidationError, SemiJoinIgnoredColumnWarning from orcapod.protocols.core_protocols import StreamProtocol from tests.test_core.operators.test_async_sync_equivalence import ( _source, # pyright: ignore[reportPrivateUsage] # shared test helper @@ -268,6 +268,7 @@ def test_semijoin_ignores_a_data_column_that_differs_in_nullability() -> None: """``SemiJoin`` keys on shared tags only (ITL-1088), so a shared data column is neither a key nor checked: the result is every left row whose tag matches.""" left, right = _data_pair() - result = SemiJoin()(left, right) + with pytest.warns(SemiJoinIgnoredColumnWarning, match="'y'"): + result = SemiJoin()(left, right) assert result.output_schema() == left.output_schema() assert result.as_table().column("y").to_pylist() == [1, 2] diff --git a/tests/test_core/operators/test_semijoin_tag_keys.py b/tests/test_core/operators/test_semijoin_tag_keys.py index f2336570b..4323aab63 100644 --- a/tests/test_core/operators/test_semijoin_tag_keys.py +++ b/tests/test_core/operators/test_semijoin_tag_keys.py @@ -1,27 +1,35 @@ """SemiJoin keys on the shared tag columns only (ITL-1088). Developer decision, November 13, 2026: "Semijoin should key on shared tags only". The -match keys are exactly the names that are tag columns of both inputs. A name shared as a -data column, or a tag on one side and a data column on the other, is ignored, and the -right input's data columns are never read. With no shared tag, every left row is -returned. Before, every shared name, tag or data, was a match key. +match keys are exactly the names that are tag columns of both inputs. Every other shared +name (data on both sides, or a tag on one side and data on the other) is ignored, and the +right input's data columns are never read. For v0.3 such a name raises +``SemiJoinIgnoredColumnWarning`` once per application (provisional: it may become an +error). With no shared tag, every left row is returned. Before, every shared name, tag or +data, was a match key. """ from __future__ import annotations +import asyncio +import warnings from collections.abc import Callable import pyarrow as pa import pytest +from orcapod import PipelineJob +from orcapod.channels import Channel from orcapod.core.operators import PolarsFilter, SemiJoin from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import SemiJoinIgnoredColumnWarning from orcapod.protocols.core_protocols import StreamProtocol from tests.test_core.operators.test_async_sync_equivalence import ( + Row, _source, # pyright: ignore[reportPrivateUsage] # shared test helper assert_equivalent, assert_same_provenance, - async_rows, sync_rows, ) @@ -80,55 +88,171 @@ def right_unequal_data_of_another_type() -> ArrowTableSource: ) +def right_tags_only() -> ArrowTableSource: + """Tag ``id`` 1 and 3 and data ``z``: only the tag is shared.""" + return _source( + {"id": pa.array([1, 3], pa.int64()), "z": pa.array([0, 0], pa.int64())}, + tags=("id",), + source_id="right_tags_only", + ) + + +def right_data_id() -> ArrowTableSource: + """Tag ``other`` and data ``id``: the left's tag name is data here.""" + return _source( + {"other": pa.array([7], pa.int64()), "id": pa.array([1], pa.int64())}, + tags=("other",), + source_id="right_data_id", + ) + + def left_ids(stream: StreamProtocol) -> list[int | None]: """The ``id`` of each result row, in order.""" return stream.as_table().column("id").to_pylist() +# (right input, ids kept, ignored (name, left role, right role)) CASES = [ - # (right input, ids kept) pytest.param( - right_same_tag_other_y, [1, 2, 3], id="shared_data_column_is_not_a_key" + right_same_tag_other_y, + [1, 2, 3], + [("y", "data", "data")], + id="data_on_both_sides", + ), + pytest.param( + right_tag_y, [1, 2, 3], [("y", "data", "tag")], id="data_left_tag_right" ), - pytest.param(right_tag_y, [1, 2, 3], id="tag_on_right_data_on_left_is_not_a_key"), - pytest.param(right_nothing_shared, [1, 2, 3], id="nothing_shared"), pytest.param( - right_unequal_data_of_another_type, [1, 3], id="right_data_never_read" + right_data_id, [1, 2, 3], [("id", "tag", "data")], id="tag_left_data_right" ), + pytest.param( + right_unequal_data_of_another_type, + [1, 3], + [("y", "data", "data"), ("x", "data", "data")], + id="right_data_never_read", + ), + pytest.param(right_tags_only, [1, 3], [], id="tags_only"), + pytest.param(right_nothing_shared, [1, 2, 3], [], id="nothing_shared"), ] +Ignored = list[tuple[str, str, str]] + + +def assert_warned_once(record: list[warnings.WarningMessage], ignored: Ignored) -> None: + """Exactly one ``SemiJoinIgnoredColumnWarning`` naming each column and its roles, + or none when nothing is ignored.""" + found = [w for w in record if issubclass(w.category, SemiJoinIgnoredColumnWarning)] + assert len(found) == (1 if ignored else 0) + if ignored: + message = str(found[0].message) + for name, left_role, right_role in ignored: + assert ( + f"'{name}' ({left_role} on the left, {right_role} on the right)" + in message + ) + assert "only on tag columns shared by both inputs" in message + # Attributed to this test file, the user's call. + assert found[0].filename == __file__ -@pytest.mark.parametrize(("make_right", "kept"), CASES) + +@pytest.mark.parametrize(("make_right", "kept", "ignored"), CASES) def test_semijoin_keys_on_shared_tags_only( - make_right: Callable[[], ArrowTableSource], kept: list[int] + make_right: Callable[[], ArrowTableSource], kept: list[int], ignored: Ignored ) -> None: - """The result is the left rows whose shared tag values match; data plays no part.""" + """Only shared tags decide; other shared names warn once and play no part.""" left = left_source() - result = SemiJoin()(left, make_right()) - assert left_ids(result) == kept + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + result = SemiJoin()(left, make_right()) + assert left_ids(result) == kept + assert_warned_once(record, ignored) assert result.output_schema() == left.output_schema() assert result.as_table().schema == left.as_table().schema -@pytest.mark.parametrize(("make_right", "kept"), CASES) +async def _async_rows( + op: SemiJoin, left: StreamProtocol, right: StreamProtocol +) -> list[Row]: + channels: list[Channel[Row]] = [Channel(buffer_size=64) for _ in range(2)] + output: Channel[Row] = Channel(buffer_size=64) + for stream, channel in zip((left, right), channels): + for row in stream.iter_data(): + await channel.writer.send(row) + await channel.writer.close() + await op.async_execute( + [channel.reader for channel in channels], + output.writer, + input_pipeline_hashes=[left.pipeline_hash(), right.pipeline_hash()], + ) + return await output.reader.collect() + + +@pytest.mark.parametrize(("make_right", "kept", "ignored"), CASES) def test_async_rows_equal_sync_rows( - make_right: Callable[[], ArrowTableSource], kept: list[int] + make_right: Callable[[], ArrowTableSource], kept: list[int], ignored: Ignored ) -> None: - """The async path keys on the same columns and yields the sync rows.""" + """The async path keys on the same columns; called without validation it warns once.""" left, right = left_source(), make_right() - expected = sync_rows(SemiJoin(), left, right) - actual = async_rows(SemiJoin(), left, right) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SemiJoinIgnoredColumnWarning) + expected = sync_rows(SemiJoin(), left, right) + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + actual = asyncio.run(_async_rows(SemiJoin(), left, right)) + found = [w for w in record if issubclass(w.category, SemiJoinIgnoredColumnWarning)] + assert len(found) == (1 if ignored else 0) assert len(expected) == len(kept) assert_equivalent(expected, actual) assert_same_provenance(expected, actual) +def test_validated_async_run_does_not_warn_again() -> None: + """An operator applied (and warned) once does not warn again on its async run.""" + left, right = left_source(), right_same_tag_other_y() + op = SemiJoin() + with pytest.warns(SemiJoinIgnoredColumnWarning): + _ = op(left, right) + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + rows = asyncio.run(_async_rows(op, left, right)) + assert len(rows) == 3 + assert not [ + w for w in record if issubclass(w.category, SemiJoinIgnoredColumnWarning) + ] + + +@pytest.mark.parametrize("run_async", [False, True], ids=["sync", "async"]) +def test_job_warns_once(run_async: bool) -> None: + """A semi join in a job warns once, at the call, not again when the job runs.""" + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + job = PipelineJob(name="semijoin_warns_once", store=InMemoryArrowDatabase()) + with job: + _ = SemiJoin(label="sj")(left_source(), right_same_tag_other_y()) + if run_async: + from orcapod.pipeline import AsyncPipelineOrchestrator + + job.run(orchestrator=AsyncPipelineOrchestrator()) + else: + job.run() + assert_warned_once(record, [("y", "data", "data")]) + + def test_filter_a_stream_by_a_subset_of_itself() -> None: - """The commonest use: every data column name is shared, and only the tags decide.""" + """The commonest use: every data name is shared (and warned about); tags decide.""" source = left_source() subset = PolarsFilter(constraints={"id": 2})(left_source()) - result = SemiJoin()(source, subset) + with pytest.warns(SemiJoinIgnoredColumnWarning, match="'y'.*'x'"): + result = SemiJoin()(source, subset) + assert left_ids(result) == [2] + + +def test_filter_without_the_warning() -> None: + """Reduce the right input to its tags plus one renamed data column: no warning.""" + source = left_source() + subset = PolarsFilter(constraints={"id": 2})(left_source()) + keys_only = subset.select_data_columns(["x"]).map_data({"x": "_x"}) + with warnings.catch_warnings(): + warnings.simplefilter("error", SemiJoinIgnoredColumnWarning) + result = SemiJoin()(source, keys_only) assert left_ids(result) == [2] - assert_equivalent( - sync_rows(SemiJoin(), source, subset), async_rows(SemiJoin(), source, subset) - ) diff --git a/tests/test_core/operators/test_tag_data_collision.py b/tests/test_core/operators/test_tag_data_collision.py index 41437c68e..ac6119072 100644 --- a/tests/test_core/operators/test_tag_data_collision.py +++ b/tests/test_core/operators/test_tag_data_collision.py @@ -19,7 +19,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.operators import Index, Join, MergeJoin, Pick, SemiJoin from orcapod.core.sources import ArrowTableSource -from orcapod.errors import InputValidationError +from orcapod.errors import InputValidationError, SemiJoinIgnoredColumnWarning from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from .test_async_sync_equivalence import async_rows @@ -124,8 +124,13 @@ def test_valid_join_still_works() -> None: def test_semijoin_is_not_affected() -> None: - """SemiJoin keys on every common column, tag or data, and returns the left stream's columns.""" - result = SemiJoin()(tag_x(), data_x()) + """SemiJoin keys on tags shared by both inputs only (ITL-1088): ``x``, a tag on the + left and data on the right, is not a key and is named in a warning; with no shared + tag every left row is returned, with the left stream's columns.""" + with pytest.warns( + SemiJoinIgnoredColumnWarning, match="'x' \\(tag on the left, data" + ): + result = SemiJoin()(tag_x(), data_x()) table = result.as_table() assert table.column_names == ["x", "a"] assert table.column("x").to_pylist() == [1, 2] From 974eb0a475edf0c9f40ff0ebfad5a429eba7910c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:51:25 -0800 Subject: [PATCH 08/11] docs(semijoin): ignored shared names warn (provisional in v0.3); cached-node note (ITL-1088) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 6 ++++-- CLAUDE.md | 6 ++++-- changelog.d/ITL-829.md | 39 +++++++++++++++++++++-------------- design_issues/ITL-1088-SJ3.md | 16 +++++++++----- docs/concepts/operators.md | 8 ++++++- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/.zed/rules b/.zed/rules index d5732e136..0fa36929f 100644 --- a/.zed/rules +++ b/.zed/rules @@ -799,8 +799,10 @@ data. The rules: - `SemiJoin`'s match keys are exactly the names that are tag columns of both inputs (developer, November 13, 2026, ITL-1088), from one helper, `SemiJoin._match_keys`, used by validation, the sync path and the async path. A shared data column name, or a name that is a - tag on one side and data on the other, is ignored (never an error); the right input's data - columns are never read. No shared tag column: every left row is returned. + tag on one side and data on the other, is ignored (by name only); the right input's data + columns are never read. No shared tag column: every left row is returned. Ignored names + raise `SemiJoinIgnoredColumnWarning` once per operator and set of names (provisional in + v0.3: may become an error); the async path warns only when no validation ran. - `Join`, `MergeJoin` and `SemiJoin` accept a shared TAG key whose types differ only in nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it diff --git a/CLAUDE.md b/CLAUDE.md index 8cda1d5de..65a4cb98d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -843,8 +843,10 @@ data. The rules: - `SemiJoin`'s match keys are exactly the names that are tag columns of both inputs (developer, November 13, 2026, ITL-1088), from one helper, `SemiJoin._match_keys`, used by validation, the sync path and the async path. A shared data column name, or a name that is a - tag on one side and data on the other, is ignored (never an error); the right input's data - columns are never read. No shared tag column: every left row is returned. + tag on one side and data on the other, is ignored (by name only); the right input's data + columns are never read. No shared tag column: every left row is returned. Ignored names + raise `SemiJoinIgnoredColumnWarning` once per operator and set of names (provisional in + v0.3: may become an error); the async path warns only when no validation ran. - `Join`, `MergeJoin` and `SemiJoin` accept a shared TAG key whose types differ only in nullability (`T` against `T | None`), because a null key matches nothing (developer, November 13, 2026, ITL-829). `Join` and `MergeJoin` give the key `T` when any input has it diff --git a/changelog.d/ITL-829.md b/changelog.d/ITL-829.md index 686d8280f..cb06f5c17 100644 --- a/changelog.d/ITL-829.md +++ b/changelog.d/ITL-829.md @@ -6,30 +6,39 @@ kept only when its tags **and** its values in every shared data column equalled those of some right row, and a name that was a tag on one side and a data column on the other was a key too. So the operator read data values, which only filters may do. The match keys are now exactly the -columns that are tag columns of both inputs. A shared data column name, or a name that is a tag -on one side and data on the other, is ignored, and the right input's data columns are never -read. +columns that are tag columns of both inputs. Any other shared name (data on both sides, or a +tag on one side and data on the other) is ignored, and the right input's data columns are +never read. ```python left = DictSource(data=[{"id": 1, "y": 1}, {"id": 2, "y": 2}], tag_columns=["id"]) right = DictSource(data=[{"id": 1, "y": 1}, {"id": 2, "y": 9}], tag_columns=["id"]) SemiJoin()(left, right) # v0.2.0: only id 1 (id and y must both match) -# now: id 1 and id 2 (only the shared tag id decides) +# now: id 1 and id 2 (only the shared tag id decides), and a SemiJoinIgnoredColumnWarning +# naming 'y' ``` -- **What changes.** A pipeline whose two `SemiJoin` inputs share a data column name now keeps - the left rows whose shared tags match, whatever the data values. Before, rows whose shared data - values differed were dropped. The new result always contains the old one. Filtering a stream - by a subset of itself (every data name shared, equal values) gives the same rows as before. A - shared data column whose type or nullability differs no longer raises, because it is not a - key. +- **What changes.** If the two `SemiJoin` inputs share a data column name, the result now keeps + every left row whose shared tags match, whatever the data values. Before, rows whose shared + data values differed were dropped, so the new result always contains the old one. Filtering a + stream by a subset of itself (every data name shared, equal values) gives the same rows as + before. A shared data column whose type or nullability differs no longer raises, because it + is not a key. +- **The warning.** Each ignored shared name is listed in a `SemiJoinIgnoredColumnWarning` + (new, in `orcapod.errors`), once per operator and set of names. Silence it by renaming or + dropping the column on the right input before the semi join, or with + `warnings.filterwarnings("ignore", category=SemiJoinIgnoredColumnWarning)`. This behaviour is + provisional in v0.3 and may become an error. - **Unchanged.** `SemiJoin`'s identity and pipeline hash do not change. The output schema is - still the left input's. With no shared tag column, every left row is still returned. -- **Stored operator caches.** A cached `SemiJoin` node keeps its table path. With - `CacheMode.LOG` the node recomputes on its next run, emits the new rows and stores the rows - that were not stored yet. With `CacheMode.REPLAY`, and for a loaded pipeline, the node replays - the stored rows, i.e. the old, smaller result, until a `LOG` run stores the new one. + still the left input's. With no shared column at all, every left row is still returned. +- **Stored operator caches.** A cached `SemiJoin` node keeps its table path, so what it stored + before the upgrade is still there. With caching off it computes the new result. With + `CacheMode.LOG` it computes the new result on its next run, passes it downstream, and adds + the rows it had not stored yet. With `CacheMode.REPLAY`, and for a pipeline loaded from a + file, it serves the stored rows, which are the old, smaller result. To get the new result + there, run the pipeline once with the operator cache in `LOG` mode, or clear the operator's + cache. ### Fixed diff --git a/design_issues/ITL-1088-SJ3.md b/design_issues/ITL-1088-SJ3.md index c857180da..1fb1a7405 100644 --- a/design_issues/ITL-1088-SJ3.md +++ b/design_issues/ITL-1088-SJ3.md @@ -14,8 +14,14 @@ graph fail to build. **Fix:** developer decision, November 13, 2026: "Semijoin should key on shared tags only". The keys come from one helper, `SemiJoin._match_keys(left_tag_keys, right_tag_keys)`, used by validation, `binary_static_process` and `async_execute`: the names that are tag columns of both -inputs. Other shared names are ignored, and the sync path reads only the right input's key -columns. With no shared tag every left row is returned, as before. The identity is -unchanged; a cached node in `REPLAY` serves its stored (old) rows until a `LOG` run stores the -new result. Tests in `tests/test_core/operators/test_semijoin_tag_keys.py`; the nested-key tests -in `test_semijoin_nested.py` now use a shared tag key. +inputs. Every other shared name is ignored (by name only), and the sync path reads only the +right input's key columns. With no shared tag every left row is returned, as before. Ignored +names raise `SemiJoinIgnoredColumnWarning` (from `_ignored_shared_names`) once per operator and +set of names. The developer considered making them an error and settled, for v0.3, on the +warning, because a stream semi-joined with a filtered copy of itself shares every data name; +the final behaviour is to be decided after use. Developer's note for that decision: once +referential tags are available, the suggestion should be to promote the data column to a tag +on both sides so that it is matched. The identity is unchanged; a cached node in `REPLAY` +serves its stored (old) rows until a `LOG` run stores the new result. Tests in +`tests/test_core/operators/test_semijoin_tag_keys.py`; the nested-key tests in +`test_semijoin_nested.py` now use a shared tag key. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 3fb7c3e4e..d6301c387 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -155,7 +155,13 @@ provides the set of matching tags. The match keys are exactly the columns that are tag columns of both streams. A name the two streams share as a data column, or a tag in one and a data column in the other, is not a key and is ignored: the right stream's data columns are never read. If the streams share no tag -column, every left row is returned. +column, every left row is returned. Because such names were match keys before v0.3, SemiJoin +names them in a `SemiJoinIgnoredColumnWarning` (from `orcapod.errors`). This behaviour is +provisional in v0.3 and may become an error. To silence the warning, rename or drop the +column on the right stream before the semi join (`map_data`, `select_data_columns`, +`drop_data_columns`, or `map_tags` for a tag). A right stream must keep at least one data +column, so a filtered copy of the left stream (which shares every data name) can be reduced +to one renamed column, e.g. `subset.select_data_columns(["age"]).map_data({"age": "_age"})`. ```python from orcapod.sources import DictSource From 270059e8e3df465762ccf1b3d30c2915c22ed167 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 19:09:39 -0800 Subject: [PATCH 09/11] test(spiraldb): a T key joined with a T | None key comes out T (ITL-829) Co-Authored-By: Claude Opus 5.5 --- .../test_spiraldb_table_source_fake.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_core/sources/test_spiraldb_table_source_fake.py b/tests/test_core/sources/test_spiraldb_table_source_fake.py index f4364331e..23b214d81 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_fake.py +++ b/tests/test_core/sources/test_spiraldb_table_source_fake.py @@ -19,7 +19,7 @@ from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase -from orcapod.errors import InputValidationError, NullabilityRecordError +from orcapod.errors import NullabilityRecordError from orcapod.operators import Join, SelectDataColumns from orcapod.databases import SpiralDBConnector from orcapod.sources import SpiralDBTableSource @@ -484,14 +484,20 @@ def halve(weight: float) -> float: with pytest.raises(ValueError, match="float \\| None"): halve.pod(SelectDataColumns(["weight"])(source)) - def test_join_of_recorded_and_unrecorded_keys_is_refused(self, strict_root): - # Type strictness: a key that is T on one side and T | None on the other. - """A T key joined with a T | None key is refused.""" - with pytest.raises(InputValidationError, match="str \\| None"): - Join()( - SpiralDBTableSource(_PROJECT, "unit"), - SpiralDBTableSource(_PROJECT, "session_unrecorded"), - ) + def test_join_of_recorded_and_unrecorded_keys_gives_the_recorded_type( + self, strict_root + ): + """A T key joined with a T | None key is accepted and comes out T (ITL-829). + + A null key matches nothing, so no joined row can hold a null there. + """ + unit = SpiralDBTableSource(_PROJECT, "unit") + unrecorded = SpiralDBTableSource(_PROJECT, "session_unrecorded") + assert unrecorded.output_schema()[0]["subject_id"] == str | None + joined = Join()(unit, unrecorded) + tag_schema, _ = joined.output_schema() + assert tag_schema["subject_id"] is str + assert not joined.as_table().schema.field("subject_id").nullable def test_plain_typed_pod_runs_through_join_and_save_load( self, strict_root, tmp_path From 16b93c263be111a985b9a921b9dec97e959cab27 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 19:10:44 -0800 Subject: [PATCH 10/11] fix(semijoin): record an ignored-column warning only once it is issued; docs: Join accepts T | None against T (review) Co-Authored-By: Claude Opus 5.5 --- docs/concepts/operators.md | 7 ++++--- src/orcapod/core/operators/semijoin.py | 4 +++- tests/test_core/operators/test_semijoin_tag_keys.py | 10 ++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index d6301c387..becba55a9 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -494,9 +494,10 @@ The rules: was). The new hash is the one the row has in a stream that declared the column non-nullable from the start. -Tag columns work the same way. A stream whose key column is `str | None` cannot be joined -with one whose key is `str` (`Join` never reconciles types), so narrow the key first. The -stream method is `drop_nulls`: +Tag columns work the same way. A stream whose key column is `str | None` can be joined with +one whose key is `str`: a null key matches nothing, so the joined key comes out `str` (see +`Join`). Narrowing still matters wherever a column must be `T` before it is consumed, such +as a pod parameter typed `str`. The stream method is `drop_nulls`: ```python by_method = recordings.drop_nulls(["session", "start_ms"]) diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index 136e4efe2..573efd29a 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -126,8 +126,10 @@ def _warn_once_per_application( self._warned_ignored: set[tuple[tuple[str, str, str], ...]] = warned if ignored in warned: return - warned.add(ignored) + # Recorded only once the warning is issued: under an error filter it + # raises, and a retry must raise again. self._warn_ignored(ignored) + warned.add(ignored) @staticmethod def _warn_ignored(ignored: Sequence[tuple[str, str, str]]) -> None: diff --git a/tests/test_core/operators/test_semijoin_tag_keys.py b/tests/test_core/operators/test_semijoin_tag_keys.py index 4323aab63..120ef58f5 100644 --- a/tests/test_core/operators/test_semijoin_tag_keys.py +++ b/tests/test_core/operators/test_semijoin_tag_keys.py @@ -238,6 +238,16 @@ def test_job_warns_once(run_async: bool) -> None: assert_warned_once(record, [("y", "data", "data")]) +def test_warning_promoted_to_an_error_raises_again_on_retry() -> None: + """Under an error filter the warning raises, and a retry raises again.""" + op = SemiJoin() + with warnings.catch_warnings(): + warnings.simplefilter("error", SemiJoinIgnoredColumnWarning) + for _ in range(2): + with pytest.raises(SemiJoinIgnoredColumnWarning): + _ = op(left_source(), right_same_tag_other_y()) + + def test_filter_a_stream_by_a_subset_of_itself() -> None: """The commonest use: every data name is shared (and warned about); tags decide.""" source = left_source() From 73ba3badb995bb4406a3859be66fef7890bb0409 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 19:26:03 -0800 Subject: [PATCH 11/11] test(semijoin): PipelineJob takes database= after the main merge Co-Authored-By: Claude Opus 5.5 --- tests/test_core/operators/test_semijoin_tag_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core/operators/test_semijoin_tag_keys.py b/tests/test_core/operators/test_semijoin_tag_keys.py index 120ef58f5..2070ccde5 100644 --- a/tests/test_core/operators/test_semijoin_tag_keys.py +++ b/tests/test_core/operators/test_semijoin_tag_keys.py @@ -226,7 +226,7 @@ def test_job_warns_once(run_async: bool) -> None: """A semi join in a job warns once, at the call, not again when the job runs.""" with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always") - job = PipelineJob(name="semijoin_warns_once", store=InMemoryArrowDatabase()) + job = PipelineJob(name="semijoin_warns_once", database=InMemoryArrowDatabase()) with job: _ = SemiJoin(label="sj")(left_source(), right_same_tag_other_y()) if run_async: