From 227f1b6c333818865700ba7f84152b883efcfa6e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:55:46 -0800 Subject: [PATCH 01/26] test(pipeline): bind refuses a source whose column types differ (failing, ITL-1112) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-1112-bind-refuses-type.md | 72 +++++++ tests/test_pipeline/test_bind_type_check.py | 181 ++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md create mode 100644 tests/test_pipeline/test_bind_type_check.py diff --git a/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md new file mode 100644 index 000000000..223ca6415 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md @@ -0,0 +1,72 @@ +# ITL-1112 bind refuses a source whose column types differ — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `PipelineJob.bind(sources=...)` and the `SourceJobNode.bound_source` setter refuse a +source whose column types differ from the slot's declared schema, so an accepted bind never +moves a pipeline hash and never sends rows of another type to a stored path. + +**Architecture:** `SourceNodeBase.validate` (the node owns the declared schema and its name) +compares, column by column, the source's `output_schema()` with the declared tag and data +schemas: names, role (tag/data), Python type. `bind` validates every slot before it changes +anything (sources and database); the setter calls the same `validate` before assigning. +`SourceSpecMismatchError` becomes a subclass of `InputValidationError` and `ValueError` (E1). + +**Tech Stack:** Python, pytest, `uv run`. + +--- + +## Decisions (coordinator, 2026-11-14) + +- Developer: "Definitely refuse". No recompute of downstream identities. +- A1: compare names, roles, Python types (the pipeline-hash inputs); `optional_fields` and + column order are not compared. +- E1: `SourceSpecMismatchError(InputValidationError, ValueError)` for name and type mismatches. +- Move `self._database = database` in `bind` after source validation. +- Setter calls `validate` (setting `None` unbinds, unchecked). +- **Nullability (T vs T | None): PENDING with the developer.** No tests or code may fix either + answer until the coordinator replies. Tests use int vs str only. +- ITL-949 (int32 vs int64) out of scope. + +## Reproduction (main a9b50350) + +Scratchpad scripts `repro.py`, `repro3.py`: slot `src` (id:int, x:int) -> MapData(x->y) -> +`const(y: int) -> int`. Binding a str source is accepted by bind and the setter, fresh and +loaded job; pipeline hashes unchanged (src 70bf0c37, renamed e777d53b); a run sends str values +into the int-declared paths (`schema:f289a35f.../pdb_v2`, `_result/.../rdb_v2`). + +## Files + +- Modify: `src/orcapod/errors.py` (`SourceSpecMismatchError` bases) +- Modify: `src/orcapod/core/nodes/source_node.py` (`validate`, `bound_source` setter) +- Modify: `src/orcapod/pipeline/job.py` (`bind`: database assignment after validation; docstring) +- Test: `tests/test_pipeline/test_bind_type_check.py` (new) +- Add: `changelog.d/ITL-1112.md`, `design_issues/ITL-1112-BT1.md` +- Docs: `docs/concepts/function-pods.md` near line 783 (one sentence on bind) + +## Tasks + +- [ ] 1. Failing tests (`tests/test_pipeline/test_bind_type_check.py`), public interface: + - bind with x: str in place of x: int raises `SourceSpecMismatchError`, which is an + `InputValidationError`; message names slot `src`, column `x`, `int` and `str`. + - same through `job.nodes["src"].bound_source = ...`. + - both on a fresh job and on a job from `PipelineJob.load`. + - after the refusal the previous binding is kept (`job.nodes["src"].bound_source is old`), + and `bind(database=new, sources={bad})` leaves `job.database` unchanged. + - a tag column given as str for a tag declared int is refused too. + - after an accepted rebind (same types, other values/source id) every node's + `pipeline_hash()` equals its value before the bind (fresh and loaded). + - an accepted bind still works and the run stores rows. + Run: `uv run pytest tests/test_pipeline/test_bind_type_check.py -v` -> FAIL. +- [ ] 2. Implement: errors.py bases; `validate` compares types after names; setter calls + `validate`; `bind` moves database assignment. Run tests -> PASS. Commit. +- [ ] 3. Nullability: after the coordinator's reply, add tests and the rule. +- [ ] 4. Changelog fragment (Breaking changes and migration), design_issues entry, docs + sentence; `uv run python scripts/assemble_changelog.py --check`, + `uv run python scripts/design_issues.py --check`. +- [ ] 5. ruff, basedpyright, full `uv run pytest tests/`; merge origin/main; push; PR. + +## Notes for a successor + +- Conflicts: PR #384 (ITL-1102) edits the `bound_source` setter (adds `_update_modified_time()`); + ITL-1111 adds a memo clear inside `bind`. Whoever merges second resolves. diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py new file mode 100644 index 000000000..77fc8ad37 --- /dev/null +++ b/tests/test_pipeline/test_bind_type_check.py @@ -0,0 +1,181 @@ +"""Binding a source whose column types differ from the slot's is refused (ITL-1112). + +A source slot's declared tag and data schemas are what every downstream pipeline hash is +built from, and the pipeline hash scopes the stored table paths. A source whose columns have +other types would send rows of those types to the paths of the declared ones, so both ways +of binding refuse it: ``PipelineJob.bind(sources=...)`` and the source node's +``bound_source`` setter, on a fresh job and on a loaded one. An accepted bind never moves a +pipeline hash. +""" + +from __future__ import annotations + +import pathlib +from collections.abc import Callable + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import MapData +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError, SourceSpecMismatchError +from orcapod.sources import DictSource +from orcapod.types import Schema + + +@function_pod(output_keys="z") +def const(y: int) -> int: + """Return 0 for any *y*.""" + return 0 + + +def _ints(source_id: str = "ints", start: int = 1) -> DictSource: + """Return a source with tag ``id: int`` and data ``x: int``.""" + return DictSource( + [{"id": 0, "x": start}, {"id": 1, "x": start + 1}], + tag_columns=["id"], + source_id=source_id, + ) + + +def _strs() -> DictSource: + """Return a source with tag ``id: int`` and data ``x: str``.""" + return DictSource( + [{"id": 0, "x": "a"}, {"id": 1, "x": "b"}], tag_columns=["id"], source_id="strs" + ) + + +def _str_tags() -> DictSource: + """Return a source with tag ``id: str`` and data ``x: int``.""" + return DictSource( + [{"id": "a", "x": 1}, {"id": "b", "x": 2}], tag_columns=["id"], source_id="stags" + ) + + +def _fresh_job(tmp_path: pathlib.Path) -> PipelineJob: + """Return a job with slot ``src`` (id: int, x: int) -> MapData -> ``const``, bound to ints.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = const.pod(MapData({"x": "y"})(slot, label="renamed"), label="const") + job.bind(sources={"src": _ints()}) + return job + + +def _loaded_job(tmp_path: pathlib.Path) -> PipelineJob: + """Return the fresh job, run and saved, loaded again from its file.""" + job = _fresh_job(tmp_path) + _ = job.run() + path = tmp_path / "job.json" + job.save(str(path)) + return PipelineJob.load(str(path)) + + +JOBS: dict[str, Callable[[pathlib.Path], PipelineJob]] = { + "fresh": _fresh_job, + "loaded": _loaded_job, +} + + +def _bind(job: PipelineJob, source: DictSource) -> None: + """Bind *source* to slot ``src`` with ``PipelineJob.bind``.""" + job.bind(sources={"src": source}) + + +def _assign(job: PipelineJob, source: DictSource) -> None: + """Bind *source* to slot ``src`` by assigning the node's ``bound_source``.""" + job.nodes["src"].bound_source = source # pyright: ignore[reportAttributeAccessIssue] # nodes is typed by the base node; "src" is a SourceJobNode + + +BINDS: dict[str, Callable[[PipelineJob, DictSource], None]] = { + "bind": _bind, + "bound_source": _assign, +} + + +def _pipeline_hashes(job: PipelineJob) -> dict[str, str]: + """Return every node's pipeline hash, by node label.""" + return {name: node.pipeline_hash().to_string() for name, node in job.nodes.items()} + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_data_column_of_another_type_is_refused( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, DictSource], None], +) -> None: + """A source with ``x: str`` for a slot declaring ``x: int`` is refused, naming both types.""" + job = make_job(tmp_path) + with pytest.raises(SourceSpecMismatchError) as excinfo: + bind(job, _strs()) + assert isinstance(excinfo.value, InputValidationError) + message = str(excinfo.value) + assert "'src'" in message + assert "'x'" in message + assert "int" in message and "str" in message + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_tag_column_of_another_type_is_refused( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, DictSource], None], +) -> None: + """A source with tag ``id: str`` for a slot declaring ``id: int`` is refused.""" + job = make_job(tmp_path) + with pytest.raises(SourceSpecMismatchError, match="'id'"): + bind(job, _str_tags()) + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_refused_source_leaves_the_binding_unchanged( + tmp_path: pathlib.Path, bind: Callable[[PipelineJob, DictSource], None] +) -> None: + """After a refusal the slot keeps its source, its schema and every pipeline hash.""" + job = _fresh_job(tmp_path) + before_source = job.nodes["src"].bound_source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + before_hashes = _pipeline_hashes(job) + with pytest.raises(SourceSpecMismatchError): + bind(job, _strs()) + assert job.nodes["src"].bound_source is before_source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + assert dict(job.nodes["renamed"].output_schema()[1]) == {"y": int} + assert _pipeline_hashes(job) == before_hashes + + +def test_a_refused_bind_leaves_the_database_unchanged(tmp_path: pathlib.Path) -> None: + """``bind(database=..., sources=...)`` with a refused source changes nothing.""" + job = _fresh_job(tmp_path) + database = job.database + with pytest.raises(SourceSpecMismatchError): + job.bind(sources={"src": _strs()}, database=InMemoryArrowDatabase()) + assert job.database is database + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_an_accepted_bind_moves_no_pipeline_hash( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, DictSource], None], +) -> None: + """Binding another source of the declared types leaves every pipeline hash as it was.""" + job = make_job(tmp_path) + before = _pipeline_hashes(job) + bind(job, _ints(source_id="other", start=10)) + assert _pipeline_hashes(job) == before + + +def test_an_accepted_rebind_runs_and_stores_the_new_rows(tmp_path: pathlib.Path) -> None: + """A rebind to a source of the declared types runs and stores its rows.""" + job = _fresh_job(tmp_path) + _ = job.run() + job.bind(sources={"src": _ints(source_id="other", start=10)}) + _ = job.run() + assert job.nodes["const"].as_table().column("z").to_pylist() == [0, 0] From 715b36295dce2a110f78ef80609def3d3c60913e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:59:06 -0800 Subject: [PATCH 02/26] fix(pipeline)!: bind and bound_source refuse a source whose column types differ from the slot's (ITL-1112) SourceNodeBase.validate compares Python types per column after names and roles; the bound_source setter validates; bind validates every source before it changes the database. SourceSpecMismatchError is now an InputValidationError as well as a ValueError. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 66 +++++++++++++++++---------- src/orcapod/errors.py | 16 +++---- src/orcapod/pipeline/job.py | 18 ++++++-- 3 files changed, 62 insertions(+), 38 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8d7cf8923..e98c015de 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -43,6 +43,11 @@ logger = logging.getLogger(__name__) +def _type_text(data_type: Any) -> str: + """Return *data_type* as it is written in an annotation (``int``, ``int | None``).""" + return data_type.__name__ if isinstance(data_type, type) else str(data_type) + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -237,43 +242,45 @@ def keys( # ------------------------------------------------------------------ def validate(self, source: "StreamProtocol") -> None: - """Check that *source* is schema-compatible with this node's declared schema. + """Check that *source* has this slot's declared schema. + + The source's tag and data columns must have the declared names, roles and + Python types (ITL-1112): those are what every downstream pipeline hash, and so + every stored path, is built from, so a source that passes never moves one. + Column order and ``optional_fields`` are not compared; neither enters the + pipeline hash. Args: source: A concrete stream to validate. Raises: - SourceSpecMismatchError: If schema columns don't match. + SourceSpecMismatchError: If a column is missing, unexpected, or of another + type than declared. """ source_tag, source_data = source.output_schema() - tag_issues: list[str] = [] - data_issues: list[str] = [] - - spec_tag_cols = set(self._tag_schema.keys()) - src_tag_cols = set(source_tag.keys()) - if spec_tag_cols != src_tag_cols: - missing = spec_tag_cols - src_tag_cols - extra = src_tag_cols - spec_tag_cols - if missing: - tag_issues.append(f"missing tag columns: {sorted(missing)}") - if extra: - tag_issues.append(f"unexpected tag columns: {sorted(extra)}") - - spec_data_cols = set(self._data_schema.keys()) - src_data_cols = set(source_data.keys()) - if spec_data_cols != src_data_cols: - missing = spec_data_cols - src_data_cols - extra = src_data_cols - spec_data_cols + issues: list[str] = [] + for role, declared, given in ( + ("tag", self._tag_schema, source_tag), + ("data", self._data_schema, source_data), + ): + missing = set(declared.keys()) - set(given.keys()) + extra = set(given.keys()) - set(declared.keys()) if missing: - data_issues.append(f"missing data columns: {sorted(missing)}") + issues.append(f"missing {role} columns: {sorted(missing)}") if extra: - data_issues.append(f"unexpected data columns: {sorted(extra)}") + issues.append(f"unexpected {role} columns: {sorted(extra)}") + issues.extend( + f"{role} column {name!r} is declared {_type_text(declared[name])} " + f"but the source has {_type_text(given[name])}" + for name in declared.keys() + if name in given and given[name] != declared[name] + ) - if tag_issues or data_issues: + if issues: raise SourceSpecMismatchError( f"SourceNode '{self._name}' is not compatible with the provided source. " - + "; ".join(tag_issues + data_issues) + + "; ".join(issues) ) def as_table( @@ -597,7 +604,16 @@ def bound_source(self) -> StreamProtocol | None: @bound_source.setter def bound_source(self, value: StreamProtocol | None) -> None: - """Bind *value* as the concrete source and invalidate both hash caches.""" + """Bind *value* as the concrete source and invalidate both hash caches. + + A source is checked against the slot's declared schema first (``validate``), so + one of other column types is refused before anything changes. ``None`` unbinds. + + Raises: + SourceSpecMismatchError: If *value*'s schema differs from the declared one. + """ + if value is not None: + self.validate(value) self._bound_source = value self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index d776adff9..fb73eb656 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -213,14 +213,14 @@ class UnboundSourceError(RuntimeError): """ -class SourceSpecMismatchError(ValueError): - """Raised when a concrete source's schema is incompatible with a SourceNode slot. - - The class name ``SourceSpecMismatchError`` is preserved for compatibility - with any code that catches it by name. - - Contains the slot name and a description of the incompatible field(s). - Raised at ``bind()`` time — schema mismatches are rejected before execution. +class SourceSpecMismatchError(InputValidationError, ValueError): + """Raised when a concrete source's schema differs from a source slot's declared schema. + + A source fits a slot when its tag and data columns have the slot's names, roles and + Python types (ITL-1112): those are what every downstream pipeline hash, and so every + stored path, is built from. The message names the slot and the column, and for a type + difference both types. Raised by ``PipelineJob.bind(sources=...)`` and by assigning a + source node's ``bound_source``, before anything is bound. """ diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c38146edc..f015f1c51 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -417,7 +417,11 @@ def bind( When *sources* is provided, each concrete source is validated against its matching ``SourceJobNode`` slot schema, then the corresponding - ``SourceJobNode.bound_source`` is updated in-place. + ``SourceJobNode.bound_source`` is updated in-place. A source fits its slot + when its tag and data columns have the declared names, roles and Python + types, so a bind never changes a pipeline hash (ITL-1112). Every source is + validated before anything is changed: a refused source leaves the job's + bindings and database as they were. When *database* is provided and differs from the current database, ``_distribute_databases()`` is called so that all job nodes receive @@ -429,16 +433,14 @@ def bind( execution_context: Replaces the current execution context. Raises: - SourceSpecMismatchError: If any source's schema is incompatible. + SourceSpecMismatchError: If any source's columns differ from its slot's + declared names, roles or types. ValueError: If a source key has no matching ``SourceJobNode`` slot. """ from orcapod.core.nodes.source_node import SourceNodeBase database_changed = database is not None and database is not self._database - if database is not None: - self._database = database - if sources is not None: source_names = { node.name @@ -456,6 +458,12 @@ def bind( if isinstance(node, SourceNodeBase) and node.name in sources: node.validate(sources[node.name]) + # Every source is validated before anything changes: a refused source leaves + # the job as it was, its database included (ITL-1112). + if database is not None: + self._database = database + + if sources is not None: # Update bound_source on SourceJobNodes and maintain both dicts. for h, job_node in (self._persistent_node_map or {}).items(): if isinstance(job_node, SourceJobNode) and job_node.name in sources: From e024d40bab614775bc8722ec13ff9f58508eac8d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:11:06 -0800 Subject: [PATCH 03/26] docs: changelog fragment, design issue BT1 and bind docs sentence (ITL-1112) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1112.md | 25 +++++++++++++++++++++++++ design_issues/ITL-1112-BT1.md | 24 ++++++++++++++++++++++++ docs/concepts/function-pods.md | 5 ++++- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ITL-1112.md create mode 100644 design_issues/ITL-1112-BT1.md diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md new file mode 100644 index 000000000..f96eefd64 --- /dev/null +++ b/changelog.d/ITL-1112.md @@ -0,0 +1,25 @@ +### Breaking changes and migration + +#### Binding refuses a source whose column types differ from the slot's (ITL-1112) + +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and +assigning `job.nodes[name].bound_source` checked nothing. A source with other column types was +accepted: every pipeline hash, built from the slot's declared schema, stayed the same, so a run +wrote rows of the new types to the stored paths of the declared ones. Both ways of binding now +compare the source's tag and data columns with the slot's: names, role (tag or data) and Python +type, the inputs of the pipeline hash. Any difference raises `SourceSpecMismatchError` naming +the slot, the column and both types, before anything is bound; a refused `bind` leaves the job +as it was, its database included. A bind that is accepted never changes a pipeline hash. + +```python +slot = SourceNode(name="src", tag_schema={"id": int}, data_schema={"x": int}) +job.bind(sources={"src": DictSource([{"id": 0, "x": "a"}], tag_columns=["id"])}) +# v0.2.0: accepted; rows with x: str were stored under the path of x: int +# now: SourceSpecMismatchError: ... data column 'x' is declared int but the source has str +``` + +`SourceSpecMismatchError` is now an `InputValidationError` as well as a `ValueError`. +Migration: bind a source whose columns have the slot's types, converting them before the bind, +or record the pipeline with a slot of the source's types (which gives it its own stored paths). +Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart +(ITL-949). diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md new file mode 100644 index 000000000..76757682d --- /dev/null +++ b/design_issues/ITL-1112-BT1.md @@ -0,0 +1,24 @@ +## `src/orcapod/core/nodes/source_node.py` + +### BT1 — Binding a source of other column types left every pipeline hash unchanged +**Status:** resolved +**Severity:** high +**Issue:** ITL-1112 + +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and +assigning a source node's `bound_source` checked nothing. A source whose `x` is `str`, bound to +a slot declaring `x: int`, was accepted on a fresh and on a loaded job. Every pipeline hash is +built from the slot's declared schema, so none moved, while `output_schema()` passed the bound +source through: a downstream `MapData` reported `y: str` under the pipeline hash of `y: int`. A +run then sent `str` values to a function typed `int` and wrote its rows to the stored paths of +the declared types (`schema:/pdb_v2` and the pod's result table). + +**Fix:** `SourceNodeBase.validate` compares, column by column, the source's tag and data schemas +with the declared ones: names, role (tag or data) and Python type, the inputs of the pipeline +hash, so an accepted bind never moves one. Column order and `optional_fields` are not compared; +neither enters the pipeline hash. The `bound_source` setter calls the same check before it +assigns, and `bind` checks every source before it changes anything, its database included. +`SourceSpecMismatchError` names the slot, the column and both types and is now an +`InputValidationError` as well as a `ValueError`. Two Arrow types that are one Python type +(`int32` and `int64`) are still not told apart (ITL-949). No hash, golden value or stored path +changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index c04e4e8cc..6cbaaf03c 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -780,7 +780,10 @@ rows under that key is an empty result, not an error. paths. In-memory sources (`DictSource`, `ListSource`, `ArrowTableSource`, `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs this at INFO level. This does not affect reading the other nodes. To read such a source - node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. + node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. The source must + have the slot's tag and data columns with the same Python types; `bind` refuses any other + (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source never changes a + stored path. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each From 4a92072285d841d37fa0dfaa50e2c4e60affd627 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:14:59 -0800 Subject: [PATCH 04/26] style(tests): ruff format the bind type check tests Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_bind_type_check.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py index 77fc8ad37..044f8314f 100644 --- a/tests/test_pipeline/test_bind_type_check.py +++ b/tests/test_pipeline/test_bind_type_check.py @@ -49,7 +49,9 @@ def _strs() -> DictSource: def _str_tags() -> DictSource: """Return a source with tag ``id: str`` and data ``x: int``.""" return DictSource( - [{"id": "a", "x": 1}, {"id": "b", "x": 2}], tag_columns=["id"], source_id="stags" + [{"id": "a", "x": 1}, {"id": "b", "x": 2}], + tag_columns=["id"], + source_id="stags", ) @@ -172,7 +174,9 @@ def test_an_accepted_bind_moves_no_pipeline_hash( assert _pipeline_hashes(job) == before -def test_an_accepted_rebind_runs_and_stores_the_new_rows(tmp_path: pathlib.Path) -> None: +def test_an_accepted_rebind_runs_and_stores_the_new_rows( + tmp_path: pathlib.Path, +) -> None: """A rebind to a source of the declared types runs and stores its rows.""" job = _fresh_job(tmp_path) _ = job.run() From 582049aa2a6250f852ddeeed2969d0956fa9734c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:17:23 -0800 Subject: [PATCH 05/26] docs(plan): ITL-1112 round-2 decisions (narrower bind, cast, load) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-1112-bind-refuses-type.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md index 223ca6415..e488167f0 100644 --- a/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md +++ b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md @@ -70,3 +70,32 @@ into the int-declared paths (`schema:f289a35f.../pdb_v2`, `_result/.../rdb_v2`). - Conflicts: PR #384 (ITL-1102) edits the `bound_source` setter (adds `_update_modified_time()`); ITL-1111 adds a memo clear inside `bind`. Whoever merges second resolves. + +## Decisions, round 2 (coordinator/developer, 2026-11-14) + +- Nullability: a slot accepts a STRICTER source. Declared `T | None` accepts a source with `T`; + declared `T` refuses `T | None`. Nothing else is compatible (no other union, subclass, width). +- The node presents the DECLARED schema: `output_schema()`/`keys()` and its rows. Rows are cast + to the declared form at one helper: same arrays re-wrapped under nullable fields (no cast) and + the two system-tag columns renamed from the source's schema-hash suffix to the declared one + (N1; names from `system_tag_column_names` on both hashes). Row hashes, record ids, stored + schema are those of a `T | None` source (the flag reaches the row hash today: measured). +- `content_hash` and `source_id` stay the source's own. +- A narrower source whose system-tag columns are not exactly the two of its schema hash is + refused at bind (never mid-run). +- Entry points: `iter_data`, `async_iter_data` (per row: Tag/Data rebuilt with the row's data + context), `as_table` and `head` (an `ArrowTableStream` over the cast table, so content-hash + column, sort and column selection are computed on the cast rows), `output_schema`. +- L1: `SourceJobNode.__init__` binds through the setter; `PipelineJob.load` handles a rebuilt + source the slot refuses like one that could not be rebuilt (warning, slot unbound). +- PR as draft; CodeRabbit reviews by itself (do not request). + +## Tasks, round 2 + +- [ ] 6. Tests `tests/test_pipeline/test_bind_narrower_source.py`: accepted narrower bind (bind, + setter; fresh, loaded), reverse refused, downstream schema and pipeline hashes unchanged, + rows table-equal to a nullable-flag source (names, flags, row hashes, record ids), sync and + async, same-path append (narrower run then nullable run with a null), zero rows, head. + Loaded job: CSV changed type (warning, unbound, run fails as unbound, rebind works); CSV now + narrower (bound, cast). +- [ ] 7. Implement `_DeclaredForm` + entry points + `__init__` through setter + load handling. From 0ced55c252305ad46bd7afe43be2387274e19607 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:19:03 -0800 Subject: [PATCH 06/26] test(pipeline): a slot of T | None accepts a source of T, read as declared (failing, ITL-1112) Co-Authored-By: Claude Opus 5.5 --- .../test_bind_narrower_source.py | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 tests/test_pipeline/test_bind_narrower_source.py diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py new file mode 100644 index 000000000..737df30d4 --- /dev/null +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -0,0 +1,353 @@ +"""A slot declared ``T | None`` accepts a source of ``T``, presented as ``T | None`` (ITL-1112). + +Developer decision, November 14, 2026: a source slot accepts a source that is stricter than +it declares (a column ``x: int`` for a slot declaring ``x: int | None``) and refuses the +reverse. The slot gives the rows of its declared schema: the bound source is treated as +``x: int | None``. Its rows are exactly those of a source with a nullable ``x``, the same +values and the same source id: Arrow fields, system-tag column names, row hashes and record +ids alike, so both write to one stored path and append to one table. Nothing downstream +moves: every pipeline hash and predicted schema is the declared one. +""" + +from __future__ import annotations + +import asyncio +import logging +import pathlib +from collections.abc import Callable +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import MapData +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.types import Schema + + +@function_pod(output_keys="z") +def inc(y: int | None) -> int: + """Return ``y + 1``, or 0 for a missing ``y``.""" + return 0 if y is None else y + 1 + + +def _arrow_source( + xs: list[int | None], *, x_nullable: bool, ids: list[int] | None = None +) -> ArrowTableSource: + """Return a source of tag ``id`` (non-null) and data ``x`` with the given flag.""" + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), x_nullable)] + ) + ids = list(range(len(xs))) if ids is None else ids + return ArrowTableSource( + pa.table({"id": ids, "x": xs}, schema=schema), + tag_columns=["id"], + source_id="same", + ) + + +def _slot(x_type: Any) -> SourceNode: + """Return slot ``src`` declaring tag ``id: int`` and data ``x`` of ``x_type``.""" + return SourceNode( + name="src", tag_schema=Schema({"id": int}), data_schema=Schema({"x": x_type}) + ) + + +def _job(tmp_path: pathlib.Path, x_type: Any = int | None) -> PipelineJob: + """Return an unbound job: slot ``src`` -> MapData x->y ``renamed`` -> ``inc``.""" + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = inc.pod(MapData({"x": "y"})(_slot(x_type), label="renamed"), label="inc") + return job + + +def _fresh(tmp_path: pathlib.Path) -> PipelineJob: + """Return the job bound to a nullable source, as recorded.""" + job = _job(tmp_path) + job.bind(sources={"src": _arrow_source([1, None], x_nullable=True)}) + return job + + +def _loaded(tmp_path: pathlib.Path) -> PipelineJob: + """Return the job, run with a nullable source and saved, loaded again.""" + job = _fresh(tmp_path) + _ = job.run() + path = tmp_path / "job.json" + job.save(str(path)) + return PipelineJob.load(str(path)) + + +JOBS: dict[str, Callable[[pathlib.Path], PipelineJob]] = { + "fresh": _fresh, + "loaded": _loaded, +} + + +def _bind(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` with ``PipelineJob.bind``.""" + job.bind(sources={"src": source}) + + +def _assign(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` by assigning the node's ``bound_source``.""" + job.nodes["src"].bound_source = source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +BINDS: dict[str, Callable[[PipelineJob, Any], None]] = { + "bind": _bind, + "bound_source": _assign, +} + + +def _pipeline_hashes(job: PipelineJob) -> dict[str, str]: + """Return every node's pipeline hash, by node label.""" + return {name: node.pipeline_hash().to_string() for name, node in job.nodes.items()} + + +def _row_hashes(pairs: Any) -> list[tuple[str, str]]: + """Return the tag and data content hashes of each ``(tag, data)`` pair.""" + return [ + (t.content_hash().to_string(), d.content_hash().to_string()) for t, d in pairs + ] + + +async def _collect(node: Any) -> list[Any]: + """Return every pair of ``node.async_iter_data()``.""" + return [pair async for pair in node.async_iter_data()] + + +# --------------------------------------------------------------------------- +# Accepted and refused +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_narrower_source_is_accepted_and_moves_nothing( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, Any], None], +) -> None: + """``x: int`` binds to a slot of ``x: int | None``; schemas and pipeline hashes stay.""" + job = make_job(tmp_path) + before = _pipeline_hashes(job) + bind(job, DictSource([{"id": 0, "x": 1}], tag_columns=["id"], source_id="d")) + assert _pipeline_hashes(job) == before + assert dict(job.nodes["src"].output_schema()[1]) == {"x": int | None} + assert dict(job.nodes["renamed"].output_schema()[1]) == {"y": int | None} + assert job.nodes["src"].as_table().schema.field("x").nullable + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_nullable_source_is_refused_by_a_non_null_slot( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A source with ``x: int | None`` is refused by a slot declaring ``x: int``.""" + job = _job(tmp_path, x_type=int) + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + if make_job is _loaded: + _ = job.run() + job.save(str(tmp_path / "job.json")) + job = PipelineJob.load(str(tmp_path / "job.json")) + with pytest.raises( + SourceSpecMismatchError, + match=r"'x' is declared int but the source has int \| None", + ): + bind(job, _arrow_source([1, None], x_nullable=True)) + + +def test_a_narrower_tag_column_is_accepted(tmp_path: pathlib.Path) -> None: + """A tag ``id: int`` binds to a slot declaring ``id: int | None`` and reads as one.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int | None}), + data_schema=Schema({"x": int}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(slot, label="renamed") + job.bind(sources={"src": DictSource([{"id": 0, "x": 1}], tag_columns=["id"])}) + assert dict(job.nodes["src"].output_schema()[0]) == {"id": int | None} + assert job.nodes["src"].as_table().schema.field("id").nullable + + +@pytest.mark.parametrize( + "declared, given", + [(int | None, str), (int | str | None, int), (int | None, float)], + ids=["other-type", "other-union-member", "numeric"], +) +def test_only_the_none_member_may_be_missing( + tmp_path: pathlib.Path, declared: Any, given: type +) -> None: + """``T | None`` accepts ``T`` only: no other type, union member or numeric type.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(declared), label="renamed") + value: Any = {str: "a", int: 1, float: 1.5}[given] + with pytest.raises(SourceSpecMismatchError, match="'x'"): + job.bind( + sources={"src": DictSource([{"id": 0, "x": value}], tag_columns=["id"])} + ) + + +# --------------------------------------------------------------------------- +# The rows are those of the declared schema +# --------------------------------------------------------------------------- + + +def _bound_slot(tmp_path: pathlib.Path, source: ArrowTableSource) -> Any: + """Return slot ``src`` of a fresh ``x: int | None`` job, bound to *source*.""" + job = _job(tmp_path) + job.bind(sources={"src": source}) + return job.nodes["src"] + + +def test_rows_equal_those_of_a_nullable_source(tmp_path: pathlib.Path) -> None: + """Narrower rows equal a nullable source's: fields, system tags, hashes, record ids.""" + narrower = _bound_slot(tmp_path / "a", _arrow_source([1, 2], x_nullable=False)) + nullable = _bound_slot(tmp_path / "b", _arrow_source([1, 2], x_nullable=True)) + assert narrower.as_table(all_info=True).equals( + nullable.as_table(all_info=True), check_metadata=True + ) + assert _row_hashes(narrower.iter_data()) == _row_hashes(nullable.iter_data()) + assert _row_hashes(asyncio.run(_collect(narrower))) == _row_hashes( + nullable.iter_data() + ) + assert narrower.keys(all_info=True) == nullable.keys(all_info=True) + + +def test_rows_of_each_read_carry_the_declared_flags(tmp_path: pathlib.Path) -> None: + """``iter_data``, ``async_iter_data`` and ``head`` give nullable ``x``.""" + slot = _bound_slot(tmp_path, _arrow_source([1, 2], x_nullable=False)) + for _, data in slot.iter_data(): + assert data.as_table().schema.field("x").nullable + for _, data in asyncio.run(_collect(slot)): + assert data.as_table().schema.field("x").nullable + head = slot.head(1, all_info=True).table + assert head.schema.field("x").nullable + assert head.equals(slot.as_table(all_info=True).slice(0, 1)) + + +def test_zero_rows_have_the_types_of_rows(tmp_path: pathlib.Path) -> None: + """A narrower source with no rows reads with the types it has with rows.""" + + @function_pod(output_keys="x") + def same(v: int) -> int: + """Return ``v``.""" + return v + + origin = PipelineJob(name="origin", database=InMemoryArrowDatabase()) + with origin: + _ = same.pod( + DictSource([{"id": 0, "v": 1}], tag_columns=["id"], source_id="o"), + label="same", + ) + empty = origin.nodes["same"].as_source() # pyright: ignore[reportAttributeAccessIssue] # a function job node; as_source before a run has no rows + job = _job(tmp_path) + job.bind(sources={"src": empty}) + with_rows = _bound_slot(tmp_path / "rows", _arrow_source([1], x_nullable=False)) + table = job.nodes["src"].as_table() + assert table.num_rows == 0 + assert table.schema == with_rows.as_table().schema + + +# --------------------------------------------------------------------------- +# One stored path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +def test_narrower_then_nullable_runs_append_to_one_path( + tmp_path: pathlib.Path, orchestrator: type +) -> None: + """A run of a narrower source, then of a nullable one, append under the same path.""" + job = _job(tmp_path) + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + first = job.run(orchestrator=orchestrator()) + assert first.ok + paths = set(job.database.list_table_paths(())) # pyright: ignore[reportOptionalMemberAccess] # the job has a database + job.bind(sources={"src": _arrow_source([None, 3], x_nullable=True, ids=[2, 3])}) + second = job.run(orchestrator=orchestrator()) + assert second.ok + assert set(job.database.list_table_paths(())) == paths # pyright: ignore[reportOptionalMemberAccess] # the job has a database + assert sorted(job.nodes["inc"].as_table().column("z").to_pylist()) == [0, 4] + stored = [ + job.database.get_all_records(path) # pyright: ignore[reportOptionalMemberAccess] # the job has a database + for path in paths + if path[-1] == "pdb_v2" + ] + assert [table.num_rows for table in stored] == [4] + + +# --------------------------------------------------------------------------- +# A loaded job's rebuilt file source +# --------------------------------------------------------------------------- + + +def _csv_job(tmp_path: pathlib.Path, text: str) -> str: + """Save a job over a CSV source of *text*; return the job file's path.""" + csv = tmp_path / "data.csv" + _ = csv.write_text(text) + job = PipelineJob(name="c", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = MapData({"x": "y"})( + CSVSource(str(csv), tag_columns=["id"], source_id="csv"), label="renamed" + ) + _ = job.run() + path = tmp_path / "job.json" + job.save(str(path)) + return str(path) + + +def _csv_slot(job: PipelineJob) -> Any: + """Return the CSV source's slot of *job*.""" + return job.nodes["csv"] + + +def test_loaded_csv_of_another_type_is_left_unbound( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """A CSV whose column changed type loads unbound, with a warning naming the slot.""" + path = _csv_job(tmp_path, "id,x\n0,1\n1,2\n") + _ = (tmp_path / "data.csv").write_text("id,x\n0,a\n1,b\n") + with caplog.at_level(logging.WARNING): + job = PipelineJob.load(path) + assert any( + "'csv'" in r.getMessage() and "declared int" in r.getMessage() + for r in caplog.records + ) + assert job.unbound_sources == ["csv"] + with pytest.raises(UnboundSourceError): + _ = _csv_slot(job).as_table() + _ = (tmp_path / "fixed.csv").write_text("id,x\n0,7\n") + job.bind( + sources={ + "csv": CSVSource( + str(tmp_path / "fixed.csv"), tag_columns=["id"], source_id="csv" + ) + } + ) + assert _csv_slot(job).as_table().column("x").to_pylist() == [7] + + +def test_loaded_csv_now_narrower_is_bound_as_declared(tmp_path: pathlib.Path) -> None: + """A CSV saved with an empty cell (``x: int | None``) and now without loads bound.""" + path = _csv_job(tmp_path, "id,x\n0,1\n1,\n") + _ = (tmp_path / "data.csv").write_text("id,x\n0,1\n1,2\n") + job = PipelineJob.load(path) + slot = _csv_slot(job) + assert dict(slot.output_schema()[1]) == {"x": int | None} + assert slot.as_table().schema.field("x").nullable + assert slot.as_table().column("x").to_pylist() == [1, 2] From 0484c23131e9c1652afd8b282f7a443e1b2a7c76 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:21:55 -0800 Subject: [PATCH 07/26] feat(pipeline): a slot of T | None accepts a source of T and gives its rows as declared (ITL-1112) A narrower bound source is cast to the slot's declared schema at one place (_DeclaredForm): the same arrays under nullable fields and the two system-tag columns renamed after the declared schema hash. Applied on iter_data, async_iter_data, as_table, head and output_schema. The reverse (a nullable source for a non-null slot) is refused. SourceJobNode.__init__ binds through the setter; PipelineJob.load treats a rebuilt source its slot refuses as one that could not be rebuilt. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 262 +++++++++++++++++++++++--- src/orcapod/pipeline/job.py | 10 + 2 files changed, 242 insertions(+), 30 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index e98c015de..7b7fd9550 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -7,15 +7,19 @@ from __future__ import annotations +import dataclasses import functools import logging from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any from orcapod import contexts from orcapod.config import OrcapodConfig from orcapod.core.base import TraceableBase +from orcapod.core.datagrams import Data, Tag +from orcapod.core.operators.drop_nulls import _without_none # pyright: ignore[reportPrivateUsage] # DropNulls' rule for removing None from a type, shared, never copied +from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import schema_requires_io from orcapod.core.streams.view import ( ViewStatus, @@ -31,6 +35,7 @@ from orcapod.types import ColumnConfig, Schema from orcapod.utils import arrow_utils from orcapod.utils.arrow_utils import system_tag_column_names +from orcapod.utils.lazy_module import LazyModule from orcapod.utils.schema_utils import compute_source_schema_hash if TYPE_CHECKING: @@ -39,6 +44,8 @@ from orcapod.channels import WritableChannel from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol +else: + pa = LazyModule("pyarrow") logger = logging.getLogger(__name__) @@ -48,6 +55,63 @@ def _type_text(data_type: Any) -> str: return data_type.__name__ if isinstance(data_type, type) else str(data_type) +def _is_narrower(declared: Any, given: Any) -> bool: + """Return whether *given* is *declared* without its ``None`` member. + + ``int`` is narrower than ``int | None`` and ``int | str`` than ``int | str | None``; + nothing else is (developer, November 14, 2026, ITL-1112). + + Args: + declared: The slot's declared Python type of a column. + given: The bound source's Python type of that column. + """ + return declared != given and _without_none(declared) == given + + +@dataclasses.dataclass(frozen=True) +class _DeclaredForm: + """How the rows of a narrower bound source become rows of the declared schema. + + A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows + as rows of ``x: int | None`` (ITL-1112): the same arrays under nullable fields, and + the two system-tag columns, named after the source's schema hash, renamed after the + declared schema's. Nothing else differs: values, types, source-info columns and + their tokens pass unchanged, so the rows equal those of a source of the declared + schema with the same source id. + + Attributes: + nullable: The columns declared ``T | None`` that the source gives as ``T``. + renames: The source's system-tag column names, mapped to the declared ones. + """ + + nullable: frozenset[str] + renames: Mapping[str, str] + + def apply(self, table: pa.Table) -> pa.Table: + """Return *table*'s arrays under the declared fields and system-tag names. + + Serves a whole stream table and a single row's tag or data table: columns that + are not in the table are skipped. + + Args: + table: Rows read from the bound source. + + Returns: + The same arrays, with the columns in ``nullable`` flagged nullable and the + columns in ``renames`` renamed. + """ + fields = [] + for field in table.schema: + if field.name in self.nullable: + field = field.with_nullable(True) + if field.name in self.renames: + field = field.with_name(self.renames[field.name]) + fields.append(field) + schema = pa.schema(fields, metadata=table.schema.metadata) + # Same arrays under the declared fields: no cast, so no value or type changes. + return pa.Table.from_arrays(table.columns, schema=schema) + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -242,24 +306,40 @@ def keys( # ------------------------------------------------------------------ def validate(self, source: "StreamProtocol") -> None: - """Check that *source* has this slot's declared schema. + """Check that *source* fits this slot's declared schema. - The source's tag and data columns must have the declared names, roles and - Python types (ITL-1112): those are what every downstream pipeline hash, and so - every stored path, is built from, so a source that passes never moves one. - Column order and ``optional_fields`` are not compared; neither enters the - pipeline hash. + The source's tag and data columns must have the declared names and roles, and + each the declared Python type, or, where the slot declares ``T | None``, ``T`` + (ITL-1112). Those are what every downstream pipeline hash, and so every stored + path, is built from, so a source that fits never moves one. Column order and + ``optional_fields`` are not compared; neither enters the pipeline hash. Args: source: A concrete stream to validate. Raises: - SourceSpecMismatchError: If a column is missing, unexpected, or of another - type than declared. + SourceSpecMismatchError: If a column is missing, unexpected, or of a type + the declared one does not accept. + """ + _ = self._fit(source) + + def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str]]: + """Check *source* against the declared schema, as ``validate`` describes. + + Args: + source: A concrete stream to check. + + Returns: + The source's tag and data schemas, and the names of the columns it gives as + ``T`` where the slot declares ``T | None``. + + Raises: + SourceSpecMismatchError: If *source* does not fit. """ source_tag, source_data = source.output_schema() issues: list[str] = [] + narrower: set[str] = set() for role, declared, given in ( ("tag", self._tag_schema, source_tag), ("data", self._data_schema, source_data), @@ -270,18 +350,24 @@ def validate(self, source: "StreamProtocol") -> None: issues.append(f"missing {role} columns: {sorted(missing)}") if extra: issues.append(f"unexpected {role} columns: {sorted(extra)}") - issues.extend( - f"{role} column {name!r} is declared {_type_text(declared[name])} " - f"but the source has {_type_text(given[name])}" - for name in declared.keys() - if name in given and given[name] != declared[name] - ) + for name in declared.keys(): + if name not in given or given[name] == declared[name]: + continue + if _is_narrower(declared[name], given[name]): + narrower.add(name) + else: + issues.append( + f"{role} column {name!r} is declared " + f"{_type_text(declared[name])} but the source has " + f"{_type_text(given[name])}" + ) if issues: raise SourceSpecMismatchError( f"SourceNode '{self._name}' is not compatible with the provided source. " + "; ".join(issues) ) + return source_tag, source_data, frozenset(narrower) def as_table( self, @@ -588,10 +674,13 @@ def __init__( data_schema=data_schema, data_context=data_context, ) - # Direct assignment to the backing attribute — super().__init__() has - # already initialised _content_hash_cache, so the property setter is - # safe to use here; we bypass it only to make the init path explicit. - self._bound_source: StreamProtocol | None = bound_source + self._bound_source: StreamProtocol | None = None + self._declared_form: _DeclaredForm | None = None + if bound_source is not None: + # Through the setter, which checks the source and, for a narrower one, + # records how its rows become rows of the declared schema (ITL-1112). + # super().__init__() has initialised the hash caches it invalidates. + self.bound_source = bound_source # ------------------------------------------------------------------ # bound_source property — explicit binding with cache invalidation @@ -607,17 +696,89 @@ def bound_source(self, value: StreamProtocol | None) -> None: """Bind *value* as the concrete source and invalidate both hash caches. A source is checked against the slot's declared schema first (``validate``), so - one of other column types is refused before anything changes. ``None`` unbinds. + one that does not fit is refused before anything changes. A narrower source + (``x: int`` for a slot declaring ``x: int | None``) is accepted, and the node + gives its rows as rows of the declared schema (ITL-1112). ``None`` unbinds. Raises: - SourceSpecMismatchError: If *value*'s schema differs from the declared one. + SourceSpecMismatchError: If *value* does not fit the declared schema. """ - if value is not None: - self.validate(value) + form = None if value is None else self._declared_form_for(value) self._bound_source = value + self._declared_form = form self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: + """Check *source* and return how its rows become rows of the declared schema. + + Args: + source: The stream being bound. + + Returns: + ``None`` when the source's schema equals the declared one, so its rows pass + unchanged; otherwise the ``_DeclaredForm`` of its narrower columns. + + Raises: + SourceSpecMismatchError: If *source* does not fit the declared schema. + """ + source_tag, source_data, narrower = self._fit(source) + if not narrower: + return None + source_schema_hash = compute_source_schema_hash( + source_tag, source_data, self.data_context, self.orcapod_config + ) + return _DeclaredForm( + nullable=narrower, + renames=dict( + zip( + system_tag_column_names(source_schema_hash), + system_tag_column_names(self._schema_hash_str), + strict=True, + ) + ), + ) + + def _declared_rows( + self, rows: Iterator[tuple[TagProtocol, DataProtocol]], form: _DeclaredForm + ) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield each row of a narrower source as a row of the declared schema.""" + for tag, data in rows: + yield self._declared_row(tag, data, form) + + @staticmethod + def _declared_row( + tag: TagProtocol, data: DataProtocol, form: _DeclaredForm + ) -> tuple[TagProtocol, DataProtocol]: + """Return one row of a narrower source as a row of the declared schema. + + The row's tag (with its system tags) and data (with its source info) Arrow + tables go through ``form``, and the row is rebuilt from them with its own + data context. + """ + tag_table = form.apply(tag.as_table(columns={"system_tags": True})) + data_table = form.apply(data.as_table(columns={"source": True, "meta": True})) + new_tag = ( + Tag(tag_table, data_context=tag.data_context_key) + if tag_table.num_columns + else tag + ) + return new_tag, Data(data_table, data_context=data.data_context_key) + + def _declared_stream( + self, source: StreamProtocol, form: _DeclaredForm, table: pa.Table + ) -> ArrowTableStream: + """Return a stream over *table*, read from *source*, as the declared schema. + + *table* must hold the system-tag, source-info and context columns, so the + stream selects, sorts and hashes the declared rows as any stream does. + """ + return ArrowTableStream( + form.apply(table), + tag_columns=tuple(self._tag_schema.keys()), + data_context=source.data_context, # pyright: ignore[reportAttributeAccessIssue] # StreamProtocol omits data_context, which every concrete stream provides (ITL-844) + ) + # ------------------------------------------------------------------ # Identity — delegate to bound source when set # ------------------------------------------------------------------ @@ -648,7 +809,9 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before running." ) - return self._bound_source.iter_data() + if self._declared_form is None: + return self._bound_source.iter_data() + return self._declared_rows(self._bound_source.iter_data(), self._declared_form) def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll: as for the bound source.""" @@ -671,8 +834,33 @@ def head( ViewStatus.UNAVAILABLE, note="no source bound", ) - return head_of( - self._bound_source.head(n, columns=columns, all_info=all_info), self + if self._declared_form is None: + return head_of( + self._bound_source.head(n, columns=columns, all_info=all_info), self + ) + # The source's own head, read with every column group the declared stream + # needs; the declared stream then selects the asked-for groups and computes + # the content hashes of the declared rows. Nothing is computed. + config = ColumnConfig.handle_config(columns, all_info=all_info) + view = head_of( + self._bound_source.head( + n, + columns=ColumnConfig( + source=True, + system_tags=True, + context=True, + sort_by_tags=config.sort_by_tags, + ), + ), + self, + ) + table = self._declared_stream( + self._bound_source, self._declared_form, view.table + ).as_table(columns=config) + return dataclasses.replace( + view, + table=table, + tag_columns=tuple(c for c in view.tag_columns if c in table.column_names), ) async def async_iter_data( @@ -694,8 +882,9 @@ async def async_iter_data( f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before running." ) - async for pair in self._bound_source.async_iter_data(): - yield pair + form = self._declared_form + async for tag, data in self._bound_source.async_iter_data(): + yield (tag, data) if form is None else self._declared_row(tag, data, form) def output_schema( self, @@ -719,7 +908,8 @@ def output_schema( Returns: Tuple of ``(tag_schema, data_schema)``. """ - if self._bound_source is None: + if self._bound_source is None or self._declared_form is not None: + # Unbound, or bound to a narrower source given as the declared schema. return super().output_schema(columns=columns, all_info=all_info) return self._bound_source.output_schema(columns=columns, all_info=all_info) @@ -743,7 +933,19 @@ def as_table( f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before calling as_table()." ) - return self._bound_source.as_table(columns=columns, all_info=all_info) + if self._declared_form is None: + return self._bound_source.as_table(columns=columns, all_info=all_info) + table = self._bound_source.as_table( + columns={ + "source": True, + "system_tags": True, + "context": True, + "sort_by_tags": False, + } + ) + return self._declared_stream( + self._bound_source, self._declared_form, table + ).as_table(columns=columns, all_info=all_info) @classmethod def from_stream( diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index f015f1c51..f89297e3d 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1437,6 +1437,7 @@ def load( import tempfile from pathlib import Path as _Path + from orcapod.core.nodes.source_node import SourceNodeBase from orcapod.pipeline.graph import Pipeline from orcapod.pipeline.serialization import ( SUPPORTED_JOB_FORMAT_VERSIONS, @@ -1464,6 +1465,11 @@ def load( os.unlink(tmp_path) # Reconstruct sources (best-effort) + slots = { + node.name: node + for node in pipeline._persistent_node_map.values() + if isinstance(node, SourceNodeBase) + } sources: dict[str, cp.StreamProtocol] = {} for spec_name, src_config in ( data.get("bindings", {}).get("sources", {}).items() @@ -1472,6 +1478,10 @@ def load( try: source = resolve_source_from_config(src_config) if source is not None: + # A rebuilt source its slot refuses (a file whose column types + # changed) is one that could not be rebuilt (ITL-1112). + if spec_name in slots: + slots[spec_name].validate(source) sources[spec_name] = source except NotImplementedError: # In-memory sources (DictSource, ListSource, ArrowTableSource, From acc423bb1f8303bf95c5bdbdb3769e4ea35c2f41 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:29:27 -0800 Subject: [PATCH 08/26] docs: changelog, design issue and bind docs for the narrower-source rule (ITL-1112) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1112.md | 48 +++++++++++++------ design_issues/ITL-1112-BT1.md | 39 +++++++++------ docs/concepts/function-pods.md | 7 +-- src/orcapod/core/nodes/source_node.py | 7 ++- .../test_bind_narrower_source.py | 16 +++---- 5 files changed, 74 insertions(+), 43 deletions(-) diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index f96eefd64..ede494318 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -1,15 +1,16 @@ ### Breaking changes and migration -#### Binding refuses a source whose column types differ from the slot's (ITL-1112) +#### Binding refuses a source whose column types the slot does not declare (ITL-1112) -`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and -assigning `job.nodes[name].bound_source` checked nothing. A source with other column types was -accepted: every pipeline hash, built from the slot's declared schema, stayed the same, so a run -wrote rows of the new types to the stored paths of the declared ones. Both ways of binding now -compare the source's tag and data columns with the slot's: names, role (tag or data) and Python -type, the inputs of the pipeline hash. Any difference raises `SourceSpecMismatchError` naming -the slot, the column and both types, before anything is bound; a refused `bind` leaves the job -as it was, its database included. A bind that is accepted never changes a pipeline hash. +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only; +assigning `job.nodes[name].bound_source` and the file sources `PipelineJob.load` rebuilds were +not checked at all. A source with other column types was accepted: every pipeline hash, built +from the slot's declared schema, stayed the same, so a run wrote rows of the new types to the +stored paths of the declared ones. Every way of binding now compares the source's tag and data +columns with the slot's: names, role (tag or data) and Python type, the inputs of the pipeline +hash. A column must have the declared type, or `T` where the slot declares `T | None`. Anything +else raises `SourceSpecMismatchError` naming the slot, the column and both types, before +anything is bound; a refused `bind` leaves the job as it was, its database included. ```python slot = SourceNode(name="src", tag_schema={"id": int}, data_schema={"x": int}) @@ -18,8 +19,27 @@ job.bind(sources={"src": DictSource([{"id": 0, "x": "a"}], tag_columns=["id"])}) # now: SourceSpecMismatchError: ... data column 'x' is declared int but the source has str ``` -`SourceSpecMismatchError` is now an `InputValidationError` as well as a `ValueError`. -Migration: bind a source whose columns have the slot's types, converting them before the bind, -or record the pipeline with a slot of the source's types (which gives it its own stored paths). -Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart -(ITL-949). +Refused now, accepted before: a column of another type, and a nullable column (`int | None`) +for a slot that declares it non-null (`int`). A slot recorded from data without a null (a +`DictSource`, `ListSource`, `CSVSource` or `DataFrameSource` infers `int` then) refuses a +later source whose data has one; a `pa.table` and a Delta table written from one are nullable +(`int | None`) in every column, tags included, unless their schema says otherwise. +`SourceSpecMismatchError` is now an `InputValidationError` as well as a `ValueError`. A file +source that `PipelineJob.load` rebuilds and its slot refuses is handled like one that could not +be rebuilt: a warning names the slot and the slot stays unbound. + +Migration: bind a source whose columns have the slot's types (convert them, or give the +Arrow table a schema with the slot's nullability), or record the pipeline with a slot of the +source's types, which gives it its own stored paths. Two Arrow types that are one Python type +(`int32` and `int64`) are still not told apart (ITL-949). + +### Changed + +- A slot that declares `x: int | None` accepts a source whose `x` is `int` (ITL-1112). The slot + gives that source's rows as rows of its declared schema: `output_schema()`, `as_table()`, + `iter_data()`, `head()` and the async path show `x` nullable, and the system-tag columns + carry the declared schema's names. The rows are exactly those of a source with a nullable + `x`, the same values and the same source id (row hashes and record ids included), so both + write to the same stored tables. Nothing downstream changes: every pipeline hash and + predicted schema is the declared one. The source node's `content_hash` and source id stay + the bound source's own. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 76757682d..c5318d82f 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -5,20 +5,31 @@ **Severity:** high **Issue:** ITL-1112 -`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and -assigning a source node's `bound_source` checked nothing. A source whose `x` is `str`, bound to -a slot declaring `x: int`, was accepted on a fresh and on a loaded job. Every pipeline hash is -built from the slot's declared schema, so none moved, while `output_schema()` passed the bound -source through: a downstream `MapData` reported `y: str` under the pipeline hash of `y: int`. A -run then sent `str` values to a function typed `int` and wrote its rows to the stored paths of -the declared types (`schema:/pdb_v2` and the pod's result table). +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only; +assigning a source node's `bound_source`, and the file sources `PipelineJob.load` rebuilds, were +not checked at all. A source whose `x` is `str`, bound to a slot declaring `x: int`, was +accepted on a fresh and on a loaded job. Every pipeline hash is built from the slot's declared +schema, so none moved, while `output_schema()` passed the bound source through: a downstream +`MapData` reported `y: str` under the pipeline hash of `y: int`. A run then sent `str` values to +a function typed `int` and wrote its rows to the stored paths of the declared types +(`schema:/pdb_v2` and the pod's result table). The same held for a +nullability difference (`int` against `int | None`), which has its own pipeline hash. **Fix:** `SourceNodeBase.validate` compares, column by column, the source's tag and data schemas with the declared ones: names, role (tag or data) and Python type, the inputs of the pipeline -hash, so an accepted bind never moves one. Column order and `optional_fields` are not compared; -neither enters the pipeline hash. The `bound_source` setter calls the same check before it -assigns, and `bind` checks every source before it changes anything, its database included. -`SourceSpecMismatchError` names the slot, the column and both types and is now an -`InputValidationError` as well as a `ValueError`. Two Arrow types that are one Python type -(`int32` and `int64`) are still not told apart (ITL-949). No hash, golden value or stored path -changed. +hash. A column must have the declared type or, where the slot declares `T | None`, `T` +(developer, November 14, 2026); nothing else fits. Column order and `optional_fields` are not +compared; neither enters the pipeline hash. The `bound_source` setter (which `__init__` now goes +through) calls the same check before it assigns, `bind` checks every source before it changes +anything, its database included, and `load` handles a rebuilt source its slot refuses like one +it could not rebuild. `SourceSpecMismatchError` names the slot, the column and both types and +is now an `InputValidationError` as well as a `ValueError`. + +A narrower source is given as the declared schema at one place, `_DeclaredForm`: the same +arrays under nullable fields, and the two system-tag columns renamed from the source's schema +hash to the declared one (their names carry the schema hash). `iter_data`, `async_iter_data`, +`as_table`, `head` and `output_schema` go through it, so the rows equal those of a nullable +source with the same values and source id, row hashes and record ids included, and both append +to one stored table. The node's `content_hash` and source id stay the source's own. Two Arrow +types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949). No +hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2d420a771..050fc0e20 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -833,9 +833,10 @@ rows under that key is an empty result, not an error. `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs this at INFO level. This does not affect reading the other nodes. To read such a source node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. The source must - have the slot's tag and data columns with the same Python types; `bind` refuses any other - (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source never changes a - stored path. + have the slot's tag and data columns with the same Python types, except that a column the + slot declares `T | None` may be `T` (the slot then reads it as `T | None`); `bind` refuses + any other (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source + never changes a stored path. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 7b7fd9550..e22ad7210 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -100,14 +100,13 @@ def apply(self, table: pa.Table) -> pa.Table: The same arrays, with the columns in ``nullable`` flagged nullable and the columns in ``renames`` renamed. """ - fields = [] - for field in table.schema: + schema = table.schema + for position, field in enumerate(table.schema): if field.name in self.nullable: field = field.with_nullable(True) if field.name in self.renames: field = field.with_name(self.renames[field.name]) - fields.append(field) - schema = pa.schema(fields, metadata=table.schema.metadata) + schema = schema.set(position, field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized # Same arrays under the declared fields: no cast, so no value or type changes. return pa.Table.from_arrays(table.columns, schema=schema) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 737df30d4..4d25ff5b5 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -275,20 +275,18 @@ def test_narrower_then_nullable_runs_append_to_one_path( """A run of a narrower source, then of a nullable one, append under the same path.""" job = _job(tmp_path) job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + database = job.database + assert database is not None first = job.run(orchestrator=orchestrator()) assert first.ok - paths = set(job.database.list_table_paths(())) # pyright: ignore[reportOptionalMemberAccess] # the job has a database + paths = set(database.list_table_paths(())) job.bind(sources={"src": _arrow_source([None, 3], x_nullable=True, ids=[2, 3])}) second = job.run(orchestrator=orchestrator()) assert second.ok - assert set(job.database.list_table_paths(())) == paths # pyright: ignore[reportOptionalMemberAccess] # the job has a database + assert set(database.list_table_paths(())) == paths assert sorted(job.nodes["inc"].as_table().column("z").to_pylist()) == [0, 4] - stored = [ - job.database.get_all_records(path) # pyright: ignore[reportOptionalMemberAccess] # the job has a database - for path in paths - if path[-1] == "pdb_v2" - ] - assert [table.num_rows for table in stored] == [4] + stored = [database.get_all_records(path) for path in paths if path[-1] == "pdb_v2"] + assert [None if table is None else table.num_rows for table in stored] == [4] # --------------------------------------------------------------------------- @@ -331,6 +329,8 @@ def test_loaded_csv_of_another_type_is_left_unbound( assert job.unbound_sources == ["csv"] with pytest.raises(UnboundSourceError): _ = _csv_slot(job).as_table() + with pytest.raises(RuntimeError, match="cannot execute a loaded job"): + _ = job.run() _ = (tmp_path / "fixed.csv").write_text("id,x\n0,7\n") job.bind( sources={ From 091bf52d7ca2dd2e847da3a5cb57d605595b718c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:29:52 -0800 Subject: [PATCH 09/26] docs(source_node): docstrings of the read paths name the declared-schema cast (ITL-1112) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index e22ad7210..769c9fd3b 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -825,7 +825,10 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the bound source's head, or no rows when no source is bound.""" + """Return the bound source's head, or no rows when no source is bound. + + A narrower bound source's rows are shown as rows of the declared schema. + """ if self._bound_source is None: check_head_size(n) return self._head_without_rows( @@ -893,9 +896,11 @@ def output_schema( ) -> tuple[Schema, Schema]: """Return ``(tag_schema, data_schema)``, delegating to ``bound_source`` when set. - When ``bound_source`` is present, this is a transparent pass-through to - ``bound_source.output_schema(columns=columns, all_info=all_info)`` so callers - get the same result as querying the source directly. + When ``bound_source`` has the declared schema, this is a transparent + pass-through to ``bound_source.output_schema(columns=columns, all_info=all_info)`` + so callers get the same result as querying the source directly. A narrower + bound source (``x: int`` for a declared ``x: int | None``) is given as the + declared schema (ITL-1112). When unbound, delegates to ``SourceNodeBase.output_schema()`` which includes system-tag schema entries derived from the declared schemas. @@ -920,6 +925,9 @@ def as_table( ) -> pa.Table: """Materialize the concrete source as a PyArrow Table. + A narrower bound source's rows are given as rows of the declared schema + (ITL-1112). + Args: columns: Column selection config. all_info: If True, include all metadata columns. From 5c9cfb96711beffc9b786a8e21de1a8d32642299 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:33:43 -0800 Subject: [PATCH 10/26] test(pipeline): no accepted bind changes a node's path or table schema; system-tag values are the source's own (ITL-1112) Adds the invariant to the docs, CLAUDE.md and .zed/rules, with its limit (Arrow width only, ITL-949). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 11 ++ CLAUDE.md | 11 ++ docs/concepts/function-pods.md | 6 +- .../test_bind_keeps_node_schemas.py | 138 ++++++++++++++++++ .../test_bind_narrower_source.py | 40 +++++ 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 tests/test_pipeline/test_bind_keeps_node_schemas.py diff --git a/.zed/rules b/.zed/rules index af3ce036a..ebdde7893 100644 --- a/.zed/rules +++ b/.zed/rules @@ -771,6 +771,17 @@ data. The rules: ### Important implementation details - ArrowTableSource raises ValueError if any tag_columns are not in the table. +- The pipeline determines every node's schema; a bound source never changes it (developer, + November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the + schema of any node's or stored table (column names with system tags, types, nullable + flags, order); only values differ. SourceNodeBase.validate is the one check, used by + PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and + PipelineJob.load: per column the declared Python type, or T where the slot declares + T | None; anything else raises SourceSpecMismatchError before anything is bound. A + narrower source is given as the declared schema at one place, _DeclaredForm (nullable + fields over the same arrays; the system-tag column NAMES of the declared schema hash, the + VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow + width only (int32 against int64) is not yet refused (ITL-949). - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index d66e4d9c6..387d5ada9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -813,6 +813,17 @@ data. The rules: ### Important implementation details - `ArrowTableSource.__init__` raises `ValueError` if any `tag_columns` are not in the table. +- The pipeline determines every node's schema; a bound source never changes it (developer, + November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the + schema of any node's or stored table (column names with system tags, types, nullable + flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by + `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and + `PipelineJob.load`: per column the declared Python type, or `T` where the slot declares + `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. A + narrower source is given as the declared schema at one place, `_DeclaredForm` (nullable + fields over the same arrays; the system-tag column NAMES of the declared schema hash, the + VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow + width only (`int32` against `int64`) is not yet refused (ITL-949). - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 050fc0e20..eafed3358 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -835,8 +835,10 @@ rows under that key is an empty result, not an error. node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. The source must have the slot's tag and data columns with the same Python types, except that a column the slot declares `T | None` may be `T` (the slot then reads it as `T | None`); `bind` refuses - any other (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source - never changes a stored path. + any other (`SourceSpecMismatchError`, an `InputValidationError`). The pipeline determines + every node's schema: binding a source never changes a node's stored path or the schema of + any node's or stored table, only the values. The check compares Python types, so a + difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py new file mode 100644 index 000000000..c01b037b8 --- /dev/null +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -0,0 +1,138 @@ +"""No accepted bind changes a node's stored path or table schema (ITL-1112). + +Developer, November 14, 2026: "for pipeline, all node's path/table schema should NOT change +regardless of what source you bind to the source nodes in the pipeline". One job is run over +every kind of accepted bind: a source of the declared schema, a narrower source into a +``T | None`` slot, sources of different kinds, through ``bind``, the ``bound_source`` setter +and ``PipelineJob.load``. After each, the stored table paths, the Arrow schema of every stored +table and of every node's ``as_table(all_info=True)`` (column names with system tags, types, +nullable flags, order) are those of the first bind: only values differ. + +The bind check compares Python types, so two Arrow widths of one Python type (``int32`` and +``int64``) are not covered here (ITL-949). +""" + +from __future__ import annotations + +import pathlib +from collections.abc import Callable +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import MapData +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.types import Schema + + +@function_pod(output_keys="z") +def inc(y: int | None) -> int: + """Return ``y + 1``, or 0 for a missing ``y``.""" + return 0 if y is None else y + 1 + + +def _arrow( + ids: list[int], xs: list[int | None], *, x_nullable: bool +) -> ArrowTableSource: + """Return an Arrow source of tag ``id`` (non-null) and data ``x``.""" + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), x_nullable)] + ) + return ArrowTableSource( + pa.table({"id": ids, "x": xs}, schema=schema), tag_columns=["id"] + ) + + +def _csv(tmp_path: pathlib.Path, name: str, text: str) -> CSVSource: + """Return a CSV source over *text*, written to *name* in *tmp_path*.""" + path = tmp_path / name + _ = path.write_text(text) + return CSVSource(str(path), tag_columns=["id"], source_id=name) + + +NODES = ("src", "renamed", "inc") +# A loaded job reads stored results only; its uncached operator has none to read. +LOADED_NODES = ("src", "inc") + + +def _snapshot(job: PipelineJob, nodes: tuple[str, ...] = NODES) -> dict[str, Any]: + """Return the stored paths, stored table schemas and *nodes*' table schemas of *job*.""" + database = job.database + assert database is not None + paths = sorted(database.list_table_paths(())) + stored = {} + for path in paths: + table = database.get_all_records(path) + assert table is not None + stored[path] = table.schema + return { + "paths": paths, + "stored": stored, + "nodes": { + name: job.nodes[name].as_table(all_info=True).schema for name in nodes + }, + } + + +def _assign(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` by assigning the node's ``bound_source``.""" + job.nodes["src"].bound_source = source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +def _bind(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` with ``PipelineJob.bind``.""" + job.bind(sources={"src": source}) + + +def test_no_accepted_bind_changes_a_path_or_a_table_schema( + tmp_path: pathlib.Path, +) -> None: + """Every accepted bind leaves paths and every stored and node table schema as they were.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int | None}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = inc.pod(MapData({"x": "y"})(slot, label="renamed"), label="inc") + + binds: list[tuple[str, Callable[[PipelineJob, Any], None], Any]] = [ + ("declared schema", _bind, _arrow([0, 1], [1, None], x_nullable=True)), + ("narrower arrow", _bind, _arrow([2, 3], [2, 3], x_nullable=False)), + ( + "narrower dict", + _assign, + DictSource([{"id": 4, "x": 4}], tag_columns=["id"], source_id="d"), + ), + ( + "dict with a null", + _assign, + DictSource([{"id": 5, "x": None}, {"id": 6, "x": 6}], tag_columns=["id"]), + ), + ("csv with an empty cell", _bind, _csv(tmp_path, "a.csv", "id,x\n7,\n8,8\n")), + ("narrower csv", _bind, _csv(tmp_path, "b.csv", "id,x\n9,9\n")), + ] + first: dict[str, Any] | None = None + for label, bind, source in binds: + bind(job, source) + assert job.run().ok, label + snapshot = _snapshot(job) + if first is None: + first = snapshot + assert snapshot == first, label + + # The last bound source, the narrower CSV, is rebuilt from its file on load. + job.save(str(tmp_path / "job.json")) + loaded = PipelineJob.load(str(tmp_path / "job.json")) + before = _snapshot(loaded, LOADED_NODES) + assert before["paths"] == first["paths"] + assert before["stored"] == first["stored"] + assert before["nodes"]["src"] == first["nodes"]["src"] + _bind(loaded, _arrow([0, 1], [1, None], x_nullable=True)) + # A loaded function node orders its meta columns unlike a fresh one, bind or not + # (reported separately), so the loaded job is compared with itself. + assert _snapshot(loaded, LOADED_NODES) == before diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 4d25ff5b5..3fc6208b7 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -351,3 +351,43 @@ def test_loaded_csv_now_narrower_is_bound_as_declared(tmp_path: pathlib.Path) -> assert dict(slot.output_schema()[1]) == {"x": int | None} assert slot.as_table().schema.field("x").nullable assert slot.as_table().column("x").to_pylist() == [1, 2] + + +# --------------------------------------------------------------------------- +# System-tag values are the source's own +# --------------------------------------------------------------------------- + + +def test_system_tag_values_and_source_info_are_the_sources_own( + tmp_path: pathlib.Path, +) -> None: + """Names follow the slot; source id, record ids and source-info tokens the source. + + The source is given no source id, so its default hash-based id is computed from its + true table (``x: int``) and is carried unchanged. + """ + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), False)] + ) + source = ArrowTableSource( + pa.table({"id": [0, 1], "x": [1, 2]}, schema=schema), tag_columns=["id"] + ) + direct = source.as_table(all_info=True) + slot = _bound_slot(tmp_path, source) + bound = slot.as_table(all_info=True) + + def system_tag(table: pa.Table, prefix: str) -> list[Any]: + """Return the values of the one column of *table* whose name starts with *prefix*.""" + (name,) = [c for c in table.column_names if c.startswith(prefix)] + return table.column(name).to_pylist() + + for prefix in ("_tag_source_id::", "_tag_record_id::"): + assert system_tag(bound, prefix) == system_tag(direct, prefix) + assert system_tag(bound, "_tag_source_id::") == [source.source_id] * 2 + assert ( + bound.column("_source_x").to_pylist() == direct.column("_source_x").to_pylist() + ) + # The names are the slot's, not the source's. + names = [c for c in bound.column_names if c.startswith("_tag_")] + assert names == list(slot.keys(columns={"system_tags": True})[0][1:]) + assert not set(names) & set(direct.column_names) From 493252c15ca1958c051e2963c11fad89f3f85094 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:57:45 -0800 Subject: [PATCH 11/26] test(pipeline): narrow the first snapshot before the loaded-job checks (basedpyright) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_bind_keeps_node_schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index c01b037b8..75718f7bb 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -125,6 +125,7 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( first = snapshot assert snapshot == first, label + assert first is not None # The last bound source, the narrower CSV, is rebuilt from its file on load. job.save(str(tmp_path / "job.json")) loaded = PipelineJob.load(str(tmp_path / "job.json")) From b5e69147507f740eab2437bfdd17269a3b909863 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 04:34:53 -0800 Subject: [PATCH 12/26] fix(source_node): columns in declared order, a stream's system tags take the slot's names, meta kept; narrower joined stream refused (ITL-1112) CodeRabbit round 1 on #386: _DeclaredForm also orders tag, data and source-info columns as declared and renames a stream's one system-tag pair (found by prefix) to the slot's names, on every bind. A joined or merged stream of the declared schema passes through (known gap); a narrower one is refused at bind. The narrower as_table and head paths read meta columns too, and the head view lists the renamed system tags. The rebind test's pod now returns its input. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 +- CLAUDE.md | 13 +- changelog.d/ITL-1112.md | 17 +- design_issues/ITL-1112-BT1.md | 10 +- docs/concepts/function-pods.md | 3 +- src/orcapod/core/nodes/source_node.py | 134 ++++++++++++--- .../test_bind_keeps_node_schemas.py | 7 + .../test_bind_narrower_source.py | 155 +++++++++++++++++- tests/test_pipeline/test_bind_type_check.py | 7 +- 9 files changed, 314 insertions(+), 45 deletions(-) diff --git a/.zed/rules b/.zed/rules index ebdde7893..b7f4d8285 100644 --- a/.zed/rules +++ b/.zed/rules @@ -777,11 +777,14 @@ data. The rules: flags, order); only values differ. SourceNodeBase.validate is the one check, used by PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and PipelineJob.load: per column the declared Python type, or T where the slot declares - T | None; anything else raises SourceSpecMismatchError before anything is bound. A - narrower source is given as the declared schema at one place, _DeclaredForm (nullable - fields over the same arrays; the system-tag column NAMES of the declared schema hash, the - VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow - width only (int32 against int64) is not yet refused (ITL-949). + T | None; anything else raises SourceSpecMismatchError before anything is bound. The + source's rows are given as the declared schema at one place, _DeclaredForm (nullable + fields over the same arrays; the declared column order; the system-tag column NAMES of the + declared schema hash, taken from the source's own pair by prefix, the VALUES the source's + own). Limits: the check compares Python types, so a difference in Arrow width only + (int32 against int64) is not yet refused (ITL-949); a joined or merged stream (several + system-tag pairs) of the declared schema passes through with its own system tags, and a + narrower one is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 387d5ada9..1c9d867e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -819,11 +819,14 @@ data. The rules: flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and `PipelineJob.load`: per column the declared Python type, or `T` where the slot declares - `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. A - narrower source is given as the declared schema at one place, `_DeclaredForm` (nullable - fields over the same arrays; the system-tag column NAMES of the declared schema hash, the - VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow - width only (`int32` against `int64`) is not yet refused (ITL-949). + `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. The + source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable + fields over the same arrays; the declared column order; the system-tag column NAMES of the + declared schema hash, taken from the source's own pair by prefix, the VALUES the source's + own). Limits: the check compares Python types, so a difference in Arrow width only + (`int32` against `int64`) is not yet refused (ITL-949); a joined or merged stream (several + system-tag pairs) of the declared schema passes through with its own system tags, and a + narrower one is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index ede494318..726d036d1 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -30,8 +30,7 @@ be rebuilt: a warning names the slot and the slot stays unbound. Migration: bind a source whose columns have the slot's types (convert them, or give the Arrow table a schema with the slot's nullability), or record the pipeline with a slot of the -source's types, which gives it its own stored paths. Two Arrow types that are one Python type -(`int32` and `int64`) are still not told apart (ITL-949). +source's types, which gives it its own stored paths. ### Changed @@ -43,3 +42,17 @@ source's types, which gives it its own stored paths. Two Arrow types that are on write to the same stored tables. Nothing downstream changes: every pipeline hash and predicted schema is the declared one. The source node's `content_hash` and source id stay the bound source's own. +- The pipeline determines every node's schema (ITL-1112): no accepted bind changes a node's + stored path or the column names, types, nullable flags or order of any node's or stored + table. A source whose columns come in another order than the slot declares gives them in + the declared order, and a stream bound to a slot (a source passed through a filter or a + rename, with one system-tag pair) gives its system-tag columns the slot's names; their + values stay the stream's own. Both passed through as they were before. + +### Known limitations + +- The bind check compares Python types, so a source that differs from its slot only in + Arrow width (`int32` against `int64`) is still accepted (ITL-949). +- A joined or merged stream (one system-tag pair per input) of the slot's declared schema + is bound as before, with its own system-tag columns, which the slot does not predict; a + narrower one is refused. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index c5318d82f..747d25b9d 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -30,6 +30,10 @@ arrays under nullable fields, and the two system-tag columns renamed from the so hash to the declared one (their names carry the schema hash). `iter_data`, `async_iter_data`, `as_table`, `head` and `output_schema` go through it, so the rows equal those of a nullable source with the same values and source id, row hashes and record ids included, and both append -to one stored table. The node's `content_hash` and source id stay the source's own. Two Arrow -types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949). No -hash, golden value or stored path changed. +to one stored table. The node's `content_hash` and source id stay the source's own. The same +form gives a source's columns in the declared order, and a stream's one system-tag pair the +slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a +node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) +are still not told apart (ITL-949), and a joined or merged stream of the declared schema +(several system-tag pairs) is still bound with its own system tags; a narrower one is +refused. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index eafed3358..960391f85 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -838,7 +838,8 @@ rows under that key is an empty result, not an error. any other (`SourceSpecMismatchError`, an `InputValidationError`). The pipeline determines every node's schema: binding a source never changes a node's stored path or the schema of any node's or stored table, only the values. The check compares Python types, so a - difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). + difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949), and + a joined stream of the slot's schema is bound with its own system-tag columns. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 769c9fd3b..7bd8b43a6 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -11,7 +11,7 @@ import functools import logging from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Collection, Iterator, Mapping from typing import TYPE_CHECKING, Any from orcapod import contexts @@ -31,6 +31,7 @@ check_head_size, ) from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import ColumnConfig, Schema from orcapod.utils import arrow_utils @@ -70,22 +71,27 @@ def _is_narrower(declared: Any, given: Any) -> bool: @dataclasses.dataclass(frozen=True) class _DeclaredForm: - """How the rows of a narrower bound source become rows of the declared schema. + """How the rows of a bound source become rows of the declared schema. - A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows - as rows of ``x: int | None`` (ITL-1112): the same arrays under nullable fields, and - the two system-tag columns, named after the source's schema hash, renamed after the - declared schema's. Nothing else differs: values, types, source-info columns and - their tokens pass unchanged, so the rows equal those of a source of the declared - schema with the same source id. + The pipeline determines every node's schema (ITL-1112). A slot declaring + ``x: int | None`` accepts a source of ``x: int`` and gives its rows as rows of + ``x: int | None``: the same arrays under nullable fields. The source's system-tag + columns take the slot's names (their values stay the source's own), and its tag and + data columns, with their source-info columns, the declared order. Nothing else + differs: values, types, source-info tokens and meta columns pass unchanged, so the + rows equal those of a source of the declared schema with the same source id. Attributes: nullable: The columns declared ``T | None`` that the source gives as ``T``. renames: The source's system-tag column names, mapped to the declared ones. + tag_order: The declared tag columns, in declared order. + data_order: The declared data columns, in declared order. """ nullable: frozenset[str] renames: Mapping[str, str] + tag_order: tuple[str, ...] + data_order: tuple[str, ...] def apply(self, table: pa.Table) -> pa.Table: """Return *table*'s arrays under the declared fields and system-tag names. @@ -97,8 +103,9 @@ def apply(self, table: pa.Table) -> pa.Table: table: Rows read from the bound source. Returns: - The same arrays, with the columns in ``nullable`` flagged nullable and the - columns in ``renames`` renamed. + The same arrays, with the columns in ``nullable`` flagged nullable, the + columns in ``renames`` renamed, and the tag, data and source-info columns in + declared order within the positions each group holds. """ schema = table.schema for position, field in enumerate(table.schema): @@ -108,7 +115,52 @@ def apply(self, table: pa.Table) -> pa.Table: field = field.with_name(self.renames[field.name]) schema = schema.set(position, field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized # Same arrays under the declared fields: no cast, so no value or type changes. - return pa.Table.from_arrays(table.columns, schema=schema) + table = pa.Table.from_arrays(table.columns, schema=schema) + return table.select(self._declared_positions(table.column_names)) + + def _declared_positions(self, names: list[str]) -> list[int]: + """Return the column positions of *names* with each group in declared order. + + The tag columns, the data columns and the data columns' source-info columns each + fill the positions their group holds, in declared order; every other column + keeps its position. + """ + position = {name: index for index, name in enumerate(names)} + order = list(range(len(names))) + source_info = tuple(f"{constants.SOURCE_PREFIX}{n}" for n in self.data_order) + for group in (self.tag_order, self.data_order, source_info): + present = [name for name in group if name in position] + for slot, name in zip( + sorted(position[n] for n in present), present, strict=True + ): + order[slot] = position[name] + return order + + +def _system_tag_columns( + source: StreamProtocol, tag_names: Collection[str] +) -> tuple[list[str], list[str]]: + """Return *source*'s source-id and record-id system-tag columns. + + A root source, or a stream that keeps its system tags (a filter, a rename), has one + of each; a stream that joins or merges inputs has one pair per input; a source with + no rows may have none. + + Args: + source: The stream being bound. + tag_names: Its user tag columns. + """ + names = [ + name + for name in source.keys(columns={"system_tags": True})[0] + if name not in tag_names + ] + source_id = f"{constants.SYSTEM_TAG_SOURCE_ID_PREFIX}{constants.BLOCK_SEPARATOR}" + record_id = f"{constants.SYSTEM_TAG_RECORD_ID_PREFIX}{constants.BLOCK_SEPARATOR}" + return ( + [name for name in names if name.startswith(source_id)], + [name for name in names if name.startswith(record_id)], + ) class SourceNodeBase(TraceableBase, ABC): @@ -361,6 +413,16 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] f"{_type_text(given[name])}" ) + if narrower and not issues: + # A narrower source's rows are given with the slot's one system-tag pair, + # which a joined or merged stream's several pairs cannot become. + source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) > 1 or len(record_ids) > 1: + issues.append( + "a narrower source must carry one source-id and one record-id " + f"system-tag column, not {sorted(source_ids + record_ids)}" + ) + if issues: raise SourceSpecMismatchError( f"SourceNode '{self._name}' is not compatible with the provided source. " @@ -715,27 +777,47 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: source: The stream being bound. Returns: - ``None`` when the source's schema equals the declared one, so its rows pass - unchanged; otherwise the ``_DeclaredForm`` of its narrower columns. + ``None`` when the source's rows already are rows of the declared schema + (same types, column order and system-tag names), or are a joined or merged + stream of the declared schema, so its rows pass unchanged; otherwise the + ``_DeclaredForm`` that makes them so. Raises: SourceSpecMismatchError: If *source* does not fit the declared schema. """ source_tag, source_data, narrower = self._fit(source) - if not narrower: + source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) > 1 or len(record_ids) > 1: + # A joined or merged stream of the declared schema passes through, its + # system tags as they are: a known gap (ITL-1112). A narrower one was + # refused by _fit. return None - source_schema_hash = compute_source_schema_hash( - source_tag, source_data, self.data_context, self.orcapod_config + declared_source_id, declared_record_id = system_tag_column_names( + self._schema_hash_str ) + renames = { + name: declared + for names, declared in ( + (source_ids, declared_source_id), + (record_ids, declared_record_id), + ) + for name in names + if name != declared + } + tag_order = tuple(self._tag_schema.keys()) + data_order = tuple(self._data_schema.keys()) + if ( + not narrower + and not renames + and tuple(source_tag.keys()) == tag_order + and tuple(source_data.keys()) == data_order + ): + return None return _DeclaredForm( nullable=narrower, - renames=dict( - zip( - system_tag_column_names(source_schema_hash), - system_tag_column_names(self._schema_hash_str), - strict=True, - ) - ), + renames=renames, + tag_order=tag_order, + data_order=data_order, ) def _declared_rows( @@ -848,6 +930,7 @@ def head( self._bound_source.head( n, columns=ColumnConfig( + meta=True, source=True, system_tags=True, context=True, @@ -859,10 +942,12 @@ def head( table = self._declared_stream( self._bound_source, self._declared_form, view.table ).as_table(columns=config) + renames = self._declared_form.renames + tag_columns = {renames.get(c, c) for c in view.tag_columns} return dataclasses.replace( view, table=table, - tag_columns=tuple(c for c in view.tag_columns if c in table.column_names), + tag_columns=tuple(c for c in table.column_names if c in tag_columns), ) async def async_iter_data( @@ -944,6 +1029,7 @@ def as_table( return self._bound_source.as_table(columns=columns, all_info=all_info) table = self._bound_source.as_table( columns={ + "meta": True, "source": True, "system_tags": True, "context": True, diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index 75718f7bb..f9cc4fee2 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -114,6 +114,13 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( DictSource([{"id": 5, "x": None}, {"id": 6, "x": 6}], tag_columns=["id"]), ), ("csv with an empty cell", _bind, _csv(tmp_path, "a.csv", "id,x\n7,\n8,8\n")), + ( + "stream over a source", + _assign, + MapData({"q": "x"})( + DictSource([{"id": 10, "q": 10}], tag_columns=["id"], source_id="m") + ), + ), ("narrower csv", _bind, _csv(tmp_path, "b.csv", "id,x\n9,9\n")), ] first: dict[str, Any] | None = None diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 3fc6208b7..31093cb0c 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -22,7 +22,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode -from orcapod.core.operators import MapData +from orcapod.core.operators import Join, MapData from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -231,8 +231,10 @@ def test_rows_of_each_read_carry_the_declared_flags(tmp_path: pathlib.Path) -> N assert data.as_table().schema.field("x").nullable for _, data in asyncio.run(_collect(slot)): assert data.as_table().schema.field("x").nullable - head = slot.head(1, all_info=True).table + view = slot.head(1, all_info=True) + head = view.table assert head.schema.field("x").nullable + assert view.tag_columns == slot.keys(columns={"system_tags": True})[0] assert head.equals(slot.as_table(all_info=True).slice(0, 1)) @@ -391,3 +393,152 @@ def system_tag(table: pa.Table, prefix: str) -> list[Any]: names = [c for c in bound.column_names if c.startswith("_tag_")] assert names == list(slot.keys(columns={"system_tags": True})[0][1:]) assert not set(names) & set(direct.column_names) + + +# --------------------------------------------------------------------------- +# Column order, transformed streams and meta columns (CodeRabbit round 1) +# --------------------------------------------------------------------------- + + +def _abc_job(a_type: Any) -> PipelineJob: + """Return a job whose slot declares tag ``id`` and data ``a``, ``b``, ``c`` in order.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"a": a_type, "b": str, "c": float}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"a": "a2"})(slot, label="renamed") + return job + + +@pytest.mark.parametrize("a_type", [int, int | None], ids=["equal", "narrower"]) +def test_columns_follow_the_declared_order(a_type: Any) -> None: + """A source with its columns in another order gives them in the declared order.""" + job = _abc_job(a_type) + job.bind( + sources={ + "src": DictSource( + [{"id": 0, "c": 1.5, "b": "s", "a": 1}], tag_columns=["id"] + ) + } + ) + node = job.nodes["src"] + assert list(node.output_schema()[1]) == ["a", "b", "c"] + table = node.as_table(columns={"source": True}) + assert table.column_names == [ + "id", + "a", + "b", + "c", + "_source_a", + "_source_b", + "_source_c", + ] + for _, data in node.iter_data(): + assert data.as_table().column_names == ["a", "b", "c"] + for _, data in asyncio.run(_collect(node)): + assert data.as_table().column_names == ["a", "b", "c"] + assert node.head(1).table.column_names == ["id", "a", "b", "c"] + assert job.nodes["renamed"].as_table().column_names == ["id", "a2", "b", "c"] + + +def _mapped_stream() -> Any: + """Return a stream (not a root source): a DictSource with ``q`` renamed to ``x``.""" + source = DictSource([{"id": 0, "q": 1}], tag_columns=["id"], source_id="m") + return MapData({"q": "x"})(source) + + +@pytest.mark.parametrize("x_type", [int, int | None], ids=["equal", "narrower"]) +def test_a_transformed_stream_gets_the_slots_system_tag_names( + tmp_path: pathlib.Path, x_type: Any +) -> None: + """A stream's one system-tag pair takes the slot's names; its values stay its own.""" + stream = _mapped_stream() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(x_type), label="renamed") + job.bind(sources={"src": stream}) + node = job.nodes["src"] + predicted = node.keys(columns={"system_tags": True})[0][1:] + table = node.as_table(columns={"system_tags": True}) + names = [c for c in table.column_names if c.startswith("_tag_")] + assert names == list(predicted) + direct = stream.as_table(columns={"system_tags": True}) + direct_names = [c for c in direct.column_names if c.startswith("_tag_")] + assert not set(direct_names) & set(names) + for mine, theirs in zip(names, direct_names, strict=True): + assert table.column(mine).to_pylist() == direct.column(theirs).to_pylist() + + +def _joined_stream(x_nullable: bool) -> Any: + """Return a Join of two sources: two system-tag pairs, data ``x`` and ``w``.""" + left = _arrow_source([1, 2], x_nullable=x_nullable) + right = DictSource([{"id": 0, "w": "p"}, {"id": 1, "w": "q"}], tag_columns=["id"]) + return Join()(left, right) + + +def _xw_job() -> PipelineJob: + """Return a job whose slot declares tag ``id`` and data ``x: int | None``, ``w``.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int | None, "w": str}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(slot, label="renamed") + return job + + +def test_a_joined_stream_of_the_declared_schema_passes_through() -> None: + """An equal-schema stream with two system-tag pairs is bound as it is (known gap).""" + stream = _joined_stream(x_nullable=True) + job = _xw_job() + job.bind(sources={"src": stream}) + assert job.nodes["src"].keys(all_info=True) == stream.keys(all_info=True) + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_narrower_joined_stream_is_refused( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A narrower stream with two system-tag pairs cannot take the slot's one pair.""" + job = _xw_job() + with pytest.raises(SourceSpecMismatchError, match="system-tag"): + bind(job, _joined_stream(x_nullable=False)) + assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +def test_meta_columns_of_a_narrower_stream_are_kept() -> None: + """The ``__`` columns a job node's rows carry reach the rows of a narrower bind.""" + + @function_pod(output_keys="x") + def same(v: int) -> int: + """Return ``v``.""" + return v + + origin = PipelineJob(name="origin", database=InMemoryArrowDatabase()) + with origin: + _ = same.pod( + DictSource([{"id": 0, "v": 1}], tag_columns=["id"], source_id="o"), + label="same", + ) + _ = origin.run() + stream = origin.nodes["same"] + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + job.bind(sources={"src": stream}) + node = job.nodes["src"] + own = [ + sorted(data.as_table(columns={"meta": True}).column_names) + for _, data in stream.iter_data() + ] + assert any("__computed" in names for names in own) + for rows in (list(node.iter_data()), asyncio.run(_collect(node))): + assert [ + sorted(data.as_table(columns={"meta": True}).column_names) + for _, data in rows + ] == own diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py index 044f8314f..b5ab4d84d 100644 --- a/tests/test_pipeline/test_bind_type_check.py +++ b/tests/test_pipeline/test_bind_type_check.py @@ -26,8 +26,8 @@ @function_pod(output_keys="z") def const(y: int) -> int: - """Return 0 for any *y*.""" - return 0 + """Return *y*, so stored results show which source they came from.""" + return y def _ints(source_id: str = "ints", start: int = 1) -> DictSource: @@ -182,4 +182,5 @@ def test_an_accepted_rebind_runs_and_stores_the_new_rows( _ = job.run() job.bind(sources={"src": _ints(source_id="other", start=10)}) _ = job.run() - assert job.nodes["const"].as_table().column("z").to_pylist() == [0, 0] + # Results of the first source (x 1, 2) would be [1, 2]. + assert sorted(job.nodes["const"].as_table().column("z").to_pylist()) == [10, 11] From 390c5d204b76fdee88c18eb24b25e1024fdbeb59 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:01:29 -0800 Subject: [PATCH 13/26] fix(source_node): a narrower bind needs exactly one system-tag pair; pass-through kept in one function (ITL-1112) CodeRabbit round 2 on #386: a narrower stream without system tags is refused at bind like a narrower joined one. The equal-schema pass-through of streams without exactly one pair is the known gap pending the developer, in _keeps_its_own_system_tags. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 ++- CLAUDE.md | 5 ++- changelog.d/ITL-1112.md | 6 +-- design_issues/ITL-1112-BT1.md | 4 +- docs/concepts/function-pods.md | 3 +- src/orcapod/core/nodes/source_node.py | 41 +++++++++++++------ .../test_bind_narrower_source.py | 20 +++++++++ 7 files changed, 62 insertions(+), 22 deletions(-) diff --git a/.zed/rules b/.zed/rules index b7f4d8285..075fdefff 100644 --- a/.zed/rules +++ b/.zed/rules @@ -783,8 +783,9 @@ data. The rules: declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (int32 against int64) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) of the declared schema passes through with its own system tags, and a - narrower one is refused. + system-tag pairs) or a stream with none, of the declared schema, passes through with its + own system tags (_keeps_its_own_system_tags, pending the developer), and a narrower one + is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 1c9d867e0..9d6b344a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -825,8 +825,9 @@ data. The rules: declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) of the declared schema passes through with its own system tags, and a - narrower one is refused. + system-tag pairs) or a stream with none, of the declared schema, passes through with its + own system tags (`_keeps_its_own_system_tags`, pending the developer), and a narrower one + is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 726d036d1..bfe75979f 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -53,6 +53,6 @@ source's types, which gives it its own stored paths. - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A joined or merged stream (one system-tag pair per input) of the slot's declared schema - is bound as before, with its own system-tag columns, which the slot does not predict; a - narrower one is refused. +- A joined or merged stream (one system-tag pair per input), or a stream with no system-tag + columns, of the slot's declared schema is bound as before, with its own system-tag + columns, which the slot does not predict; a narrower one is refused. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 747d25b9d..1f1d542b6 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -35,5 +35,5 @@ form gives a source's columns in the declared order, and a stream's one system-t slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949), and a joined or merged stream of the declared schema -(several system-tag pairs) is still bound with its own system tags; a narrower one is -refused. No hash, golden value or stored path changed. +(several system-tag pairs), or one with no system tags, is still bound with its own system +tags; a narrower one is refused. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 960391f85..b8638263c 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -839,7 +839,8 @@ rows under that key is an empty result, not an error. every node's schema: binding a source never changes a node's stored path or the schema of any node's or stored table, only the values. The check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949), and - a joined stream of the slot's schema is bound with its own system-tag columns. + a joined stream, or one without system tags, of the slot's schema is bound with its own + system-tag columns. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 7bd8b43a6..1b4764ac1 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -163,6 +163,24 @@ def _system_tag_columns( ) +def _keeps_its_own_system_tags(source_ids: list[str], record_ids: list[str]) -> bool: + """Return whether a bound stream's system tags stay as they are, not the slot's. + + The slot's rows carry one system-tag pair, under the slot's names. A stream with + exactly one pair takes those names; a joined or merged stream (one pair per input) + or a stream with none cannot. A narrower such stream is refused at bind; one of the + declared schema passes through with its own system tags, a known gap against the + rule that the pipeline determines every node's schema. Pending the developer's + decision (ITL-1112, "S1"): refusing these binds too is a change here and in + ``SourceNodeBase._fit``. + + Args: + source_ids: The stream's source-id system-tag columns. + record_ids: The stream's record-id system-tag columns. + """ + return len(source_ids) != 1 or len(record_ids) != 1 + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -415,12 +433,13 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] if narrower and not issues: # A narrower source's rows are given with the slot's one system-tag pair, - # which a joined or merged stream's several pairs cannot become. + # which only a source with exactly one pair can take. source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if len(source_ids) > 1 or len(record_ids) > 1: + if _keeps_its_own_system_tags(source_ids, record_ids): issues.append( - "a narrower source must carry one source-id and one record-id " - f"system-tag column, not {sorted(source_ids + record_ids)}" + "a narrower source must carry exactly one source-id and one " + "record-id system-tag column, not " + f"{sorted(source_ids + record_ids)}" ) if issues: @@ -787,21 +806,19 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: """ source_tag, source_data, narrower = self._fit(source) source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if len(source_ids) > 1 or len(record_ids) > 1: - # A joined or merged stream of the declared schema passes through, its - # system tags as they are: a known gap (ITL-1112). A narrower one was - # refused by _fit. + if _keeps_its_own_system_tags(source_ids, record_ids): + # Of the declared schema (a narrower one was refused by _fit): passes + # through as it is. return None declared_source_id, declared_record_id = system_tag_column_names( self._schema_hash_str ) renames = { name: declared - for names, declared in ( - (source_ids, declared_source_id), - (record_ids, declared_record_id), + for name, declared in ( + (source_ids[0], declared_source_id), + (record_ids[0], declared_record_id), ) - for name in names if name != declared } tag_order = tuple(self._tag_schema.keys()) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 31093cb0c..9647e7ba4 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -23,6 +23,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join, MapData +from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -542,3 +543,22 @@ def same(v: int) -> int: sorted(data.as_table(columns={"meta": True}).column_names) for _, data in rows ] == own + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_narrower_stream_without_system_tags_is_refused( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A narrower stream with no system-tag pair cannot be given the slot's pair.""" + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), False)] + ) + stream = ArrowTableStream( + pa.table({"id": [0], "x": [1]}, schema=schema), tag_columns=["id"] + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + with pytest.raises(SourceSpecMismatchError, match="system-tag"): + bind(job, stream) + assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode From de1ed2587174024a0f5ac08a6e65b6cb1d10c2f5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:31:11 -0800 Subject: [PATCH 14/26] fix(source_node): a slot's zero-row table is its bound source's in the declared form (ITL-1112, ITL-912) The typed zero-row table #375 added for a bound source node returned the source's own (its nullable flags, column order and system-tag names), so a downstream node before a run read with other types than after it. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 12 ++- .../test_bind_narrower_source.py | 84 +++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 0008b5e02..79a2ca8ec 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -912,11 +912,17 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: return self._declared_rows(self._bound_source.iter_data(), self._declared_form) def _typed_zero_row_table(self) -> "pa.Table | None": - """The bound source's zero-row table; ``None`` when unbound or unknown. + """The bound source's zero-row table in the declared form; ``None`` if unknown. - See ``typed_zero_row_table``. + A source given in the declared form (``_DeclaredForm``) has its zero-row table + given in that form too, so it has exactly the schema of the node's populated + table (ITL-1112, ITL-912). ``None`` when unbound or when the source cannot type + its rows without rows. See ``typed_zero_row_table``. """ - return typed_zero_row_table(self._bound_source) + table = typed_zero_row_table(self._bound_source) + if table is None or self._declared_form is None: + return table + return self._declared_form.apply(table) def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll: as for the bound source.""" diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 9647e7ba4..b37a15e5e 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -24,6 +24,7 @@ from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join, MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream +from orcapod.core.streams.base import ZERO_ROW_INPUT_COLUMNS, typed_zero_row_table from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -562,3 +563,86 @@ def test_a_narrower_stream_without_system_tags_is_refused( with pytest.raises(SourceSpecMismatchError, match="system-tag"): bind(job, stream) assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +# --------------------------------------------------------------------------- +# Zero-row tables (ITL-912): the declared form, as the populated table has it +# --------------------------------------------------------------------------- + + +def _declared_bind() -> PipelineJob: + """Return the ``x: int`` job bound to a source of exactly the declared form.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int), label="renamed") + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + return job + + +def _narrower_bind() -> PipelineJob: + """Return the ``x: int | None`` job bound to a source of ``x: int``.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + return job + + +def _reordered_bind() -> PipelineJob: + """Return the ``a, b, c`` job bound to a source whose data come as ``c, b, a``.""" + job = _abc_job(int) + source = DictSource([{"id": 0, "c": 1.5, "b": "s", "a": 1}], tag_columns=["id"]) + job.bind(sources={"src": source}) + return job + + +def _transformed_bind() -> PipelineJob: + """Return the ``x: int`` job bound to a stream whose system tags have other names.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int), label="renamed") + job.bind(sources={"src": _mapped_stream()}) + return job + + +DECLARED_FORM_BINDS: dict[str, Callable[[], PipelineJob]] = { + "declared": _declared_bind, + "narrower": _narrower_bind, + "reordered": _reordered_bind, + "transformed": _transformed_bind, +} + + +@pytest.mark.parametrize( + "make_job", DECLARED_FORM_BINDS.values(), ids=DECLARED_FORM_BINDS.keys() +) +def test_the_zero_row_table_of_a_slot_is_its_populated_declared_form( + make_job: Callable[[], PipelineJob], +) -> None: + """A slot's zero-row table has exactly the schema of its populated table.""" + job = make_job() + node = job.nodes["src"] + populated = node.as_table(columns=ZERO_ROW_INPUT_COLUMNS) + assert populated.num_rows > 0 + table = typed_zero_row_table(node) + assert table is not None + assert table.num_rows == 0 + assert table.schema == populated.schema + + +@pytest.mark.parametrize( + "make_job", DECLARED_FORM_BINDS.values(), ids=DECLARED_FORM_BINDS.keys() +) +def test_a_downstream_node_before_a_run_has_the_types_of_its_rows( + make_job: Callable[[], PipelineJob], +) -> None: + """The node after the slot reads, before a run, with the schema it has after one.""" + job = make_job() + before = job.nodes["renamed"].as_table(all_info=True) + head = job.nodes["renamed"].head(all_info=True).table + assert before.num_rows == 0 + _ = job.run() + after = job.nodes["renamed"].as_table(all_info=True) + assert after.num_rows > 0 + assert before.schema == after.schema + assert head.schema == after.schema From c6dfdc1ba491631b91dfcac82bb8e96bc647288b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:36:34 -0800 Subject: [PATCH 15/26] fix(source_node)!: every bind refuses a source without exactly one system-tag pair (ITL-1126) A joined or merged stream (one system-tag pair per input) or a raw stream with none was bound with its own system-tag columns and column order when its schema equalled the slot's, so the slot's table schema depended on what was bound. Developer, November 14, 2026: refuse it for now; binding one properly is ITL-1127. _fit refuses it on every bind (bind, bound_source setter, construction, load) with SourceSpecMismatchError naming the slot and the columns found; the pass-through function is removed. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 8 +- CLAUDE.md | 8 +- changelog.d/ITL-1112.md | 18 ++- design_issues/ITL-1112-BT1.md | 7 +- docs/concepts/function-pods.md | 7 +- src/orcapod/core/nodes/source_node.py | 61 ++++----- .../test_bind_narrower_source.py | 123 ++++++++++++------ 7 files changed, 137 insertions(+), 95 deletions(-) diff --git a/.zed/rules b/.zed/rules index f6d052c8c..64193bd06 100644 --- a/.zed/rules +++ b/.zed/rules @@ -801,10 +801,10 @@ data. The rules: fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only - (int32 against int64) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) or a stream with none, of the declared schema, passes through with its - own system tags (_keeps_its_own_system_tags, pending the developer), and a narrower one - is refused. + (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns + are not exactly one source-id and one record-id column (a joined or merged stream, a raw + stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); + binding one properly is ITL-1127. Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 4cfc94195..b1575bb8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -843,10 +843,10 @@ data. The rules: fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only - (`int32` against `int64`) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) or a stream with none, of the declared schema, passes through with its - own system tags (`_keeps_its_own_system_tags`, pending the developer), and a narrower one - is refused. + (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns + are not exactly one source-id and one record-id column (a joined or merged stream, a raw + stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); + binding one properly is ITL-1127. Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index bfe75979f..79e783e91 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -32,6 +32,21 @@ Migration: bind a source whose columns have the slot's types (convert them, or g Arrow table a schema with the slot's nullability), or record the pipeline with a slot of the source's types, which gives it its own stored paths. +#### Binding refuses a joined or merged stream, and a stream without system tags (ITL-1126) + +A slot's rows carry one source-id and one record-id system-tag column, under the slot's names. +A stream with one pair per input (the result of a `Join` or `MergeJoin`), or with none (an +`ArrowTableStream` over a table without them), cannot give its rows that way; it was bound +before with its own system-tag columns and column order, so the schema of the slot's table, +and of every table after it, depended on what was bound. Every way of binding (`bind`, the +`bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses +it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever +its column types. + +Migration: make the join part of the pipeline (bind its inputs to slots of their own), or +store the joined stream and bind the stored source (a source has one pair). Binding such a +stream properly is planned (ITL-1127). + ### Changed - A slot that declares `x: int | None` accepts a source whose `x` is `int` (ITL-1112). The slot @@ -53,6 +68,3 @@ source's types, which gives it its own stored paths. - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A joined or merged stream (one system-tag pair per input), or a stream with no system-tag - columns, of the slot's declared schema is bound as before, with its own system-tag - columns, which the slot does not predict; a narrower one is refused. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 1f1d542b6..d9957b835 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -34,6 +34,7 @@ to one stored table. The node's `content_hash` and source id stay the source's o form gives a source's columns in the declared order, and a stream's one system-tag pair the slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) -are still not told apart (ITL-949), and a joined or merged stream of the declared schema -(several system-tag pairs), or one with no system tags, is still bound with its own system -tags; a narrower one is refused. No hash, golden value or stored path changed. +are still not told apart (ITL-949). A source whose system-tag columns are not exactly one +source-id and one record-id column (a joined or merged stream, a raw stream with none) is +refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is +ITL-1127). No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8638263c..d44e5c2be 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -838,9 +838,10 @@ rows under that key is an empty result, not an error. any other (`SourceSpecMismatchError`, an `InputValidationError`). The pipeline determines every node's schema: binding a source never changes a node's stored path or the schema of any node's or stored table, only the values. The check compares Python types, so a - difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949), and - a joined stream, or one without system tags, of the slot's schema is bound with its own - system-tag columns. + difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A + joined or merged stream (one pair of system-tag columns per input), or one with no + system-tag columns, is refused too: make the join part of the pipeline, or store the joined + stream and bind the stored source. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 79a2ca8ec..ca2ac73a1 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -143,8 +143,9 @@ def _system_tag_columns( """Return *source*'s source-id and record-id system-tag columns. A root source, or a stream that keeps its system tags (a filter, a rename), has one - of each; a stream that joins or merges inputs has one pair per input; a source with - no rows may have none. + of each; a stream that joins or merges inputs has one pair per input; a stream over a + table without them (a raw ``ArrowTableStream``) has none. A slot accepts only the + first (ITL-1126). Args: source: The stream being bound. @@ -163,24 +164,6 @@ def _system_tag_columns( ) -def _keeps_its_own_system_tags(source_ids: list[str], record_ids: list[str]) -> bool: - """Return whether a bound stream's system tags stay as they are, not the slot's. - - The slot's rows carry one system-tag pair, under the slot's names. A stream with - exactly one pair takes those names; a joined or merged stream (one pair per input) - or a stream with none cannot. A narrower such stream is refused at bind; one of the - declared schema passes through with its own system tags, a known gap against the - rule that the pipeline determines every node's schema. Pending the developer's - decision (ITL-1112, "S1"): refusing these binds too is a change here and in - ``SourceNodeBase._fit``. - - Args: - source_ids: The stream's source-id system-tag columns. - record_ids: The stream's record-id system-tag columns. - """ - return len(source_ids) != 1 or len(record_ids) != 1 - - class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -381,14 +364,18 @@ def validate(self, source: "StreamProtocol") -> None: each the declared Python type, or, where the slot declares ``T | None``, ``T`` (ITL-1112). Those are what every downstream pipeline hash, and so every stored path, is built from, so a source that fits never moves one. Column order and - ``optional_fields`` are not compared; neither enters the pipeline hash. + ``optional_fields`` are not compared; neither enters the pipeline hash. The + source must also carry exactly one source-id and one record-id system-tag + column, which take the slot's names; a joined or merged stream, or one with no + system tags, is refused (ITL-1126). Args: source: A concrete stream to validate. Raises: SourceSpecMismatchError: If a column is missing, unexpected, or of a type - the declared one does not accept. + the declared one does not accept, or the source's system-tag columns + are not exactly one source-id and one record-id column. """ _ = self._fit(source) @@ -431,16 +418,16 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] f"{_type_text(given[name])}" ) - if narrower and not issues: - # A narrower source's rows are given with the slot's one system-tag pair, - # which only a source with exactly one pair can take. - source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if _keeps_its_own_system_tags(source_ids, record_ids): - issues.append( - "a narrower source must carry exactly one source-id and one " - "record-id system-tag column, not " - f"{sorted(source_ids + record_ids)}" - ) + # The slot's rows carry one system-tag pair under the slot's names, which only + # a source with exactly one pair can take. A joined or merged stream (one pair + # per input) or a stream with none is refused (developer, November 14, 2026, + # ITL-1126); binding one properly is ITL-1127. + source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) != 1 or len(record_ids) != 1: + issues.append( + "a source must carry exactly one source-id and one record-id " + f"system-tag column, not {sorted(source_ids + record_ids)}" + ) if issues: raise SourceSpecMismatchError( @@ -797,19 +784,15 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: Returns: ``None`` when the source's rows already are rows of the declared schema - (same types, column order and system-tag names), or are a joined or merged - stream of the declared schema, so its rows pass unchanged; otherwise the - ``_DeclaredForm`` that makes them so. + (same types, column order and system-tag names), so they pass unchanged; + otherwise the ``_DeclaredForm`` that makes them so. Raises: SourceSpecMismatchError: If *source* does not fit the declared schema. """ source_tag, source_data, narrower = self._fit(source) + # _fit has checked that the source has exactly one pair. source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if _keeps_its_own_system_tags(source_ids, record_ids): - # Of the declared schema (a narrower one was refused by _fit): passes - # through as it is. - return None declared_source_id, declared_record_id = system_tag_column_names( self._schema_hash_str ) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index b37a15e5e..dd2ff2768 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -21,7 +21,7 @@ import pytest from orcapod import PipelineJob, function_pod -from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.operators import Join, MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ZERO_ROW_INPUT_COLUMNS, typed_zero_row_table @@ -494,25 +494,6 @@ def _xw_job() -> PipelineJob: return job -def test_a_joined_stream_of_the_declared_schema_passes_through() -> None: - """An equal-schema stream with two system-tag pairs is bound as it is (known gap).""" - stream = _joined_stream(x_nullable=True) - job = _xw_job() - job.bind(sources={"src": stream}) - assert job.nodes["src"].keys(all_info=True) == stream.keys(all_info=True) - - -@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) -def test_a_narrower_joined_stream_is_refused( - bind: Callable[[PipelineJob, Any], None], -) -> None: - """A narrower stream with two system-tag pairs cannot take the slot's one pair.""" - job = _xw_job() - with pytest.raises(SourceSpecMismatchError, match="system-tag"): - bind(job, _joined_stream(x_nullable=False)) - assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode - - def test_meta_columns_of_a_narrower_stream_are_kept() -> None: """The ``__`` columns a job node's rows carry reach the rows of a narrower bind.""" @@ -546,25 +527,6 @@ def same(v: int) -> int: ] == own -@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) -def test_a_narrower_stream_without_system_tags_is_refused( - bind: Callable[[PipelineJob, Any], None], -) -> None: - """A narrower stream with no system-tag pair cannot be given the slot's pair.""" - schema = pa.schema( - [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), False)] - ) - stream = ArrowTableStream( - pa.table({"id": [0], "x": [1]}, schema=schema), tag_columns=["id"] - ) - job = PipelineJob(name="j", database=InMemoryArrowDatabase()) - with job: - _ = MapData({"x": "y"})(_slot(int | None), label="renamed") - with pytest.raises(SourceSpecMismatchError, match="system-tag"): - bind(job, stream) - assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode - - # --------------------------------------------------------------------------- # Zero-row tables (ITL-912): the declared form, as the populated table has it # --------------------------------------------------------------------------- @@ -646,3 +608,86 @@ def test_a_downstream_node_before_a_run_has_the_types_of_its_rows( assert after.num_rows > 0 assert before.schema == after.schema assert head.schema == after.schema + + +# --------------------------------------------------------------------------- +# Not exactly one system-tag pair: refused (developer, November 14, 2026, ITL-1126) +# --------------------------------------------------------------------------- + + +def _raw_stream(x_nullable: bool) -> Any: + """Return an ``ArrowTableStream`` of tag ``id`` and data ``x``, ``w``: no system tags.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), False), + pa.field("x", pa.int64(), x_nullable), + pa.field("w", pa.large_string(), False), + ] + ) + return ArrowTableStream( + pa.table({"id": [0], "x": [1], "w": ["p"]}, schema=schema), tag_columns=["id"] + ) + + +def _construct(job: PipelineJob, source: Any) -> None: + """Build a node for slot ``src`` with *source* bound at construction.""" + slot = job.nodes["src"] + _ = SourceJobNode( + name="src", + tag_schema=slot.output_schema()[0], + data_schema=slot.output_schema()[1], + bound_source=source, + ) + + +ALL_BINDS: dict[str, Callable[[PipelineJob, Any], None]] = { + **BINDS, + "construction": _construct, +} + +ONE_PAIR_REQUIRED: dict[str, Callable[[bool], Any]] = { + "joined": _joined_stream, + "no_system_tags": _raw_stream, +} + + +@pytest.mark.parametrize("x_nullable", [True, False], ids=["declared", "narrower"]) +@pytest.mark.parametrize( + "make_stream", ONE_PAIR_REQUIRED.values(), ids=ONE_PAIR_REQUIRED.keys() +) +@pytest.mark.parametrize("bind", ALL_BINDS.values(), ids=ALL_BINDS.keys()) +def test_a_stream_without_exactly_one_system_tag_pair_is_refused( + bind: Callable[[PipelineJob, Any], None], + make_stream: Callable[[bool], Any], + x_nullable: bool, +) -> None: + """A joined stream, or one with no system tags, cannot take the slot's one pair. + + The slot's rows carry one source-id and one record-id column, under the slot's + names. Refused whatever the column types; the error names the slot and the + system-tag columns found. + """ + stream = make_stream(x_nullable) + found = [ + name + for name in stream.keys(columns={"system_tags": True})[0] + if name.startswith("_tag") + ] + job = _xw_job() + with pytest.raises(SourceSpecMismatchError) as raised: + bind(job, stream) + message = str(raised.value) + assert "'src'" in message + assert "exactly one source-id and one record-id" in message + assert str(sorted(found)) in message + assert job.nodes["src"].bound_source is None + + +def test_a_refused_joined_stream_keeps_the_previous_binding() -> None: + """After the refusal the slot keeps the source it had.""" + job = _xw_job() + before = DictSource([{"id": 0, "x": 1, "w": "p"}], tag_columns=["id"]) + job.bind(sources={"src": before}) + with pytest.raises(SourceSpecMismatchError): + job.bind(sources={"src": _joined_stream(x_nullable=True)}) + assert job.nodes["src"].bound_source is before From 9b842de2605287cf283fa84015b6bc0d572ce62a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:41:37 -0800 Subject: [PATCH 16/26] fix(source_node): a pipeline's own leaf at compile is taken as it is, pending the developer (ITL-1126) Pipeline.compile binds each leaf through SourceJobNode.from_stream, so the ITL-1126 refusal broke a pipeline built directly over a raw ArrowTableStream. Coordinator decision (option B): compile accepts its leaf as on main, in one place (_accept_pipeline_leaf); every other bind refuses it. Refusing at compile too is the developer's decision. Two MagicMock fixtures carry the slot's system-tag pair. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 +- CLAUDE.md | 5 +- changelog.d/ITL-1112.md | 4 ++ design_issues/ITL-1112-BT1.md | 3 +- docs/concepts/function-pods.md | 4 +- src/orcapod/core/nodes/source_node.py | 47 ++++++++++++++++--- tests/test_core/nodes/test_source_node.py | 46 +++++++++++++----- .../test_bind_narrower_source.py | 22 +++++++++ 8 files changed, 113 insertions(+), 23 deletions(-) diff --git a/.zed/rules b/.zed/rules index 64193bd06..3613fe731 100644 --- a/.zed/rules +++ b/.zed/rules @@ -804,7 +804,10 @@ data. The rules: (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through. + binding one properly is ITL-1127. Never pass a bound stream's own system tags through, + with one exception pending the developer: Pipeline.compile binds each leaf through + SourceJobNode._accept_pipeline_leaf, which takes a leaf without exactly one pair (a raw + ArrowTableStream) as it is, so a pipeline built directly over a raw stream still compiles. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index b1575bb8d..5e6d8ca2f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -846,7 +846,10 @@ data. The rules: (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through. + binding one properly is ITL-1127. Never pass a bound stream's own system tags through, + with one exception pending the developer: `Pipeline.compile` binds each leaf through + `SourceJobNode._accept_pipeline_leaf`, which takes a leaf without exactly one pair (a raw + `ArrowTableStream`) as it is, so a pipeline built directly over a raw stream still compiles. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 79e783e91..ceab9bd41 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -68,3 +68,7 @@ stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). +- A pipeline built directly over a raw `ArrowTableStream` (one without system-tag columns) + still compiles: its source node takes the stream as it is, so that node's rows carry no + system-tag columns although the slot predicts a pair. Binding such a stream to a slot + afterwards is refused (ITL-1126). diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index d9957b835..baf49f2c4 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -37,4 +37,5 @@ node's path or table schema. Two Arrow types that are one Python type (`int32` a are still not told apart (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is -ITL-1127). No hash, golden value or stored path changed. +ITL-1127), except a pipeline's own leaf at compile (`_accept_pipeline_leaf`), which a +pipeline built directly over a raw stream still passes through, pending the developer. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index d44e5c2be..a7bf9e70b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -841,7 +841,9 @@ rows under that key is an empty result, not an error. difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A joined or merged stream (one pair of system-tag columns per input), or one with no system-tag columns, is refused too: make the join part of the pipeline, or store the joined - stream and bind the stored source. + stream and bind the stored source. A pipeline built directly over a raw `ArrowTableStream` + still compiles, for now: its source node's rows carry no system-tag columns, although the + slot predicts a pair. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index ca2ac73a1..06e20fd6e 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -776,6 +776,34 @@ def bound_source(self, value: StreamProtocol | None) -> None: self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + def _accept_pipeline_leaf(self, stream: StreamProtocol) -> None: + """Bind *stream*, a leaf of the pipeline this slot was built from. + + ``Pipeline.compile`` makes each leaf stream a slot through ``from_stream``, with + the slot's schema taken from that same stream. A leaf with exactly one + source-id and one record-id system-tag column is bound as any source is + (``bound_source``). A leaf without exactly one pair (a raw ``ArrowTableStream`` + has none) is bound as it is, as before ITL-1126, so a pipeline built directly + over a raw stream still compiles: its rows carry the stream's own system-tag + columns, not the pair the slot predicts. This is the only place a source + without exactly one pair is bound. PENDING the developer's decision (option + "A"): refuse such a leaf here too, as every other way of binding does. + + Args: + stream: The leaf stream, whose schema the slot was built from. + + Raises: + SourceSpecMismatchError: If a leaf with one pair does not fit the slot. + """ + source_ids, record_ids = _system_tag_columns(stream, self._tag_schema.keys()) + if len(source_ids) == 1 and len(record_ids) == 1: + self.bound_source = stream + return + self._bound_source = stream + self._declared_form = None + self._invalidate_content_hash_cache() + self._invalidate_pipeline_hash_cache() + def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: """Check *source* and return how its rows become rows of the declared schema. @@ -1074,6 +1102,10 @@ def from_stream( **bound** ``SourceJobNode`` whose ``content_hash()`` delegates to the concrete stream. + Used by ``Pipeline.compile`` for each leaf of the pipeline: the stream is + bound through ``_accept_pipeline_leaf``, which also accepts a leaf without + exactly one system-tag pair (a raw ``ArrowTableStream``), as before ITL-1126. + Args: stream: The stream to wrap. name: Optional explicit slot name. In all three cases, when *name* @@ -1089,12 +1121,14 @@ def from_stream( match stream: case SourceJobNode(): # Case 1: copy — preserve bound_source, do NOT wrap the SJN itself. - return cls( + node = cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, - bound_source=stream.bound_source, ) + if stream.bound_source is not None: + node._accept_pipeline_leaf(stream.bound_source) + return node case SourceNode(): # Case 2: unbound — schema placeholder; SJN hash = SourceNode hash. return cls( @@ -1121,12 +1155,11 @@ def from_stream( slot_name = stream.source_id else: slot_name = f"{stream.label}:{stream.content_hash().to_hex()}" - return cls( - name=slot_name, - tag_schema=tag_schema, - data_schema=data_schema, - bound_source=stream, + node = cls( + name=slot_name, tag_schema=tag_schema, data_schema=data_schema ) + node._accept_pipeline_leaf(stream) + return node def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 50532f33b..94470c1a8 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -2,8 +2,11 @@ from __future__ import annotations +from unittest.mock import MagicMock + import pytest +from orcapod.core.nodes.source_node import SourceJobNode from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.types import Schema @@ -283,6 +286,21 @@ def test_source_job_node_as_table_delegates_to_concrete( assert isinstance(table, pa.Table) +def _mock_source_with_the_slots_system_tags( + node: SourceJobNode, tag_schema: Schema, data_schema: Schema +) -> MagicMock: + """Return a mock source of the slot's schema carrying the slot's system-tag pair. + + A bound source must carry exactly one source-id and one record-id system-tag + column (ITL-1126); the slot's own names make the bind pass rows unchanged. + """ + tag_keys = node.keys(columns={"system_tags": True})[0] + mock_source = MagicMock() + mock_source.output_schema.return_value = (tag_schema, data_schema) + mock_source.keys.return_value = (tag_keys, tuple(data_schema.keys())) + return mock_source + + class TestSourceNodeColumnConfig: """SourceNodeBase must honour column_config / all_info in keys() and output_schema().""" @@ -385,15 +403,17 @@ def test_unbound_system_tag_names_match_bound(self, tag_schema, data_schema): def test_bound_keys_delegates_to_source(self, tag_schema, data_schema): """Bound SourceJobNode.keys() delegates to bound_source.output_schema().""" - from unittest.mock import MagicMock from orcapod.core.nodes.source_node import SourceJobNode from orcapod.types import ColumnConfig - mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) - - job_node = SourceJobNode(name="x", bound_source=mock_source) - mock_source.output_schema.reset_mock() # clear init-time schema derivation call + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema + ) + mock_source = _mock_source_with_the_slots_system_tags( + job_node, tag_schema, data_schema + ) + job_node.bound_source = mock_source + mock_source.output_schema.reset_mock() # clear bind-time schema check call cfg = ColumnConfig(system_tags=True) job_node.keys(columns=cfg, all_info=False) @@ -414,15 +434,17 @@ def test_bound_keys_all_info_matches_source_directly(self, tag_schema, data_sche def test_bound_output_schema_delegates_to_source(self, tag_schema, data_schema): """Bound SourceJobNode.output_schema() delegates to bound_source.output_schema().""" - from unittest.mock import MagicMock from orcapod.core.nodes.source_node import SourceJobNode from orcapod.types import ColumnConfig - mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) - - job_node = SourceJobNode(name="x", bound_source=mock_source) - # Reset call history accumulated during __init__ (schema derivation). + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema + ) + mock_source = _mock_source_with_the_slots_system_tags( + job_node, tag_schema, data_schema + ) + job_node.bound_source = mock_source + # Reset call history accumulated during the bind (schema check). mock_source.output_schema.reset_mock() cfg = ColumnConfig(system_tags=True) job_node.output_schema(columns=cfg, all_info=False) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index dd2ff2768..87485b722 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -691,3 +691,25 @@ def test_a_refused_joined_stream_keeps_the_previous_binding() -> None: with pytest.raises(SourceSpecMismatchError): job.bind(sources={"src": _joined_stream(x_nullable=True)}) assert job.nodes["src"].bound_source is before + + +def test_a_pipeline_built_over_a_raw_stream_still_compiles() -> None: + """A raw stream as a pipeline's leaf is accepted as before; binding it is refused. + + The one exception to ITL-1126, pending the developer's decision: the slot built + from the leaf takes the leaf as it is (its rows carry no system tags, although + the slot predicts a pair). Binding the same stream to the slot afterwards goes + through the bind check and is refused. + """ + raw = _raw_stream(x_nullable=True) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(raw, label="renamed") + (slot,) = [name for name in job.nodes if name != "renamed"] + assert job.nodes[slot].bound_source is raw + assert job.nodes["renamed"].as_table().num_rows == 0 + _ = job.run() + assert job.nodes["renamed"].as_table().column("y").to_pylist() == [1] + with pytest.raises(SourceSpecMismatchError, match="exactly one source-id"): + job.bind(sources={slot: raw}) + assert job.nodes[slot].bound_source is raw From 0d9fb60b8c53ba15a7996d17dc1af9bd35a67ff9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:54:21 -0800 Subject: [PATCH 17/26] fix(source_node)!: a pipeline built over a raw stream is refused too (ITL-1126) Developer, November 14, 2026 (option A): until a raw stream is captured into a source wrapper when the pipeline is built (ITL-1127), building a pipeline or job directly over a stream without exactly one system-tag pair is refused like every other bind. _accept_pipeline_leaf is removed. The error names the found columns and the remedy: ArrowTableSource(stream.as_table(), tag_columns=[...]) for a raw stream; the join in the pipeline, or a stored source, for a joined one. Tests that bound a raw stream as a stand-in for a source now use a source; #375's meta-column zero-row test keeps its point outside a job. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 8 +- CLAUDE.md | 8 +- changelog.d/ITL-1112.md | 14 ++-- design_issues/ITL-1112-BT1.md | 4 +- docs/concepts/function-pods.md | 4 +- src/orcapod/core/nodes/source_node.py | 82 +++++++++---------- .../function_pod/test_ephemeral_result.py | 11 ++- tests/test_core/nodes/test_source_node.py | 8 +- tests/test_core/test_tracker.py | 41 ++++++---- .../test_bind_narrower_source.py | 37 +++++---- .../test_operator_node_empty_table_types.py | 24 +++--- 11 files changed, 133 insertions(+), 108 deletions(-) diff --git a/.zed/rules b/.zed/rules index 3613fe731..967eada73 100644 --- a/.zed/rules +++ b/.zed/rules @@ -804,10 +804,10 @@ data. The rules: (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through, - with one exception pending the developer: Pipeline.compile binds each leaf through - SourceJobNode._accept_pipeline_leaf, which takes a leaf without exactly one pair (a raw - ArrowTableStream) as it is, so a pipeline built directly over a raw stream still compiles. + binding one properly is ITL-1127. That includes a pipeline's input: Pipeline.compile + binds each leaf through SourceJobNode.from_stream, so a pipeline built directly over a + raw ArrowTableStream is refused (developer, November 14, 2026; wrap the table as a + source). Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 5e6d8ca2f..1e7dbdbc0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -846,10 +846,10 @@ data. The rules: (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through, - with one exception pending the developer: `Pipeline.compile` binds each leaf through - `SourceJobNode._accept_pipeline_leaf`, which takes a leaf without exactly one pair (a raw - `ArrowTableStream`) as it is, so a pipeline built directly over a raw stream still compiles. + binding one properly is ITL-1127. That includes a pipeline's input: `Pipeline.compile` + binds each leaf through `SourceJobNode.from_stream`, so a pipeline built directly over a + raw `ArrowTableStream` is refused (developer, November 14, 2026; wrap the table as a + source). Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index ceab9bd41..971019738 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -41,11 +41,13 @@ before with its own system-tag columns and column order, so the schema of the sl and of every table after it, depended on what was bound. Every way of binding (`bind`, the `bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever -its column types. +its column types. So does building a pipeline or job directly over such a stream +(`with job: op(ArrowTableStream(...))`), which binds it to the pipeline's input slot. -Migration: make the join part of the pipeline (bind its inputs to slots of their own), or -store the joined stream and bind the stored source (a source has one pair). Binding such a -stream properly is planned (ITL-1127). +Migration: wrap a raw stream's table as a source, +`ArrowTableSource(stream.as_table(), tag_columns=[...])`; make a join part of the pipeline +(bind its inputs to slots of their own), or store the joined stream and bind the stored +source. Binding such a stream properly is planned (ITL-1127). ### Changed @@ -68,7 +70,3 @@ stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A pipeline built directly over a raw `ArrowTableStream` (one without system-tag columns) - still compiles: its source node takes the stream as it is, so that node's rows carry no - system-tag columns although the slot predicts a pair. Binding such a stream to a slot - afterwards is refused (ITL-1126). diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index baf49f2c4..49c2e0a0c 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -37,5 +37,5 @@ node's path or table schema. Two Arrow types that are one Python type (`int32` a are still not told apart (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is -ITL-1127), except a pipeline's own leaf at compile (`_accept_pipeline_leaf`), which a -pipeline built directly over a raw stream still passes through, pending the developer. No hash, golden value or stored path changed. +ITL-1127), also as a pipeline's input: a pipeline built directly over a raw stream is +refused at compile. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index a7bf9e70b..d7ef8e4c6 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -842,8 +842,8 @@ rows under that key is an empty result, not an error. joined or merged stream (one pair of system-tag columns per input), or one with no system-tag columns, is refused too: make the join part of the pipeline, or store the joined stream and bind the stored source. A pipeline built directly over a raw `ArrowTableStream` - still compiles, for now: its source node's rows carry no system-tag columns, although the - slot predicts a pair. + is refused the same way: wrap its table as a source, + `ArrowTableSource(stream.as_table(), tag_columns=[...])`. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 06e20fd6e..5767ec866 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -145,7 +145,7 @@ def _system_tag_columns( A root source, or a stream that keeps its system tags (a filter, a rename), has one of each; a stream that joins or merges inputs has one pair per input; a stream over a table without them (a raw ``ArrowTableStream``) has none. A slot accepts only the - first (ITL-1126). + first, also as a pipeline's input (ITL-1126). Args: source: The stream being bound. @@ -164,6 +164,31 @@ def _system_tag_columns( ) +def _one_system_tag_pair_issue(source_ids: list[str], record_ids: list[str]) -> str: + """Return why a source without exactly one system-tag pair is refused, and the remedy. + + Args: + source_ids: The source's source-id system-tag columns. + record_ids: The source's record-id system-tag columns. + """ + found = sorted(source_ids + record_ids) + if not found: + remedy = ( + "a stream without system-tag columns (a raw ArrowTableStream) cannot be " + "bound or used as a pipeline's input; wrap its table as a source, e.g. " + "ArrowTableSource(stream.as_table(), tag_columns=[...])" + ) + else: + remedy = ( + "a joined or merged stream cannot be bound; make the join part of the " + "pipeline, or store the joined stream and bind the stored source" + ) + return ( + "a source must carry exactly one source-id and one record-id system-tag " + f"column, not {found}: {remedy} (binding such a stream is planned, ITL-1127)" + ) + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -424,10 +449,7 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] # ITL-1126); binding one properly is ITL-1127. source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) if len(source_ids) != 1 or len(record_ids) != 1: - issues.append( - "a source must carry exactly one source-id and one record-id " - f"system-tag column, not {sorted(source_ids + record_ids)}" - ) + issues.append(_one_system_tag_pair_issue(source_ids, record_ids)) if issues: raise SourceSpecMismatchError( @@ -776,34 +798,6 @@ def bound_source(self, value: StreamProtocol | None) -> None: self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() - def _accept_pipeline_leaf(self, stream: StreamProtocol) -> None: - """Bind *stream*, a leaf of the pipeline this slot was built from. - - ``Pipeline.compile`` makes each leaf stream a slot through ``from_stream``, with - the slot's schema taken from that same stream. A leaf with exactly one - source-id and one record-id system-tag column is bound as any source is - (``bound_source``). A leaf without exactly one pair (a raw ``ArrowTableStream`` - has none) is bound as it is, as before ITL-1126, so a pipeline built directly - over a raw stream still compiles: its rows carry the stream's own system-tag - columns, not the pair the slot predicts. This is the only place a source - without exactly one pair is bound. PENDING the developer's decision (option - "A"): refuse such a leaf here too, as every other way of binding does. - - Args: - stream: The leaf stream, whose schema the slot was built from. - - Raises: - SourceSpecMismatchError: If a leaf with one pair does not fit the slot. - """ - source_ids, record_ids = _system_tag_columns(stream, self._tag_schema.keys()) - if len(source_ids) == 1 and len(record_ids) == 1: - self.bound_source = stream - return - self._bound_source = stream - self._declared_form = None - self._invalidate_content_hash_cache() - self._invalidate_pipeline_hash_cache() - def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: """Check *source* and return how its rows become rows of the declared schema. @@ -1102,9 +1096,10 @@ def from_stream( **bound** ``SourceJobNode`` whose ``content_hash()`` delegates to the concrete stream. - Used by ``Pipeline.compile`` for each leaf of the pipeline: the stream is - bound through ``_accept_pipeline_leaf``, which also accepts a leaf without - exactly one system-tag pair (a raw ``ArrowTableStream``), as before ITL-1126. + Used by ``Pipeline.compile`` for each leaf of the pipeline. The stream is + bound as any source is, so a leaf without exactly one source-id and one + record-id system-tag column (a raw ``ArrowTableStream``) is refused + (ITL-1126). Args: stream: The stream to wrap. @@ -1121,14 +1116,12 @@ def from_stream( match stream: case SourceJobNode(): # Case 1: copy — preserve bound_source, do NOT wrap the SJN itself. - node = cls( + return cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, + bound_source=stream.bound_source, ) - if stream.bound_source is not None: - node._accept_pipeline_leaf(stream.bound_source) - return node case SourceNode(): # Case 2: unbound — schema placeholder; SJN hash = SourceNode hash. return cls( @@ -1155,11 +1148,12 @@ def from_stream( slot_name = stream.source_id else: slot_name = f"{stream.label}:{stream.content_hash().to_hex()}" - node = cls( - name=slot_name, tag_schema=tag_schema, data_schema=data_schema + return cls( + name=slot_name, + tag_schema=tag_schema, + data_schema=data_schema, + bound_source=stream, ) - node._accept_pipeline_leaf(stream) - return node def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. diff --git a/tests/test_core/function_pod/test_ephemeral_result.py b/tests/test_core/function_pod/test_ephemeral_result.py index b7fb8dbeb..89244c18d 100644 --- a/tests/test_core/function_pod/test_ephemeral_result.py +++ b/tests/test_core/function_pod/test_ephemeral_result.py @@ -650,7 +650,11 @@ def test_pipeline_job_set_ephemeral_store_propagates(self): """PipelineJob.set_ephemeral_store propagates to all compiled nodes.""" from orcapod.pipeline.job import PipelineJob - stream = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) + # A job's input must be a source (ITL-1126: a raw stream has no system tags). + stream = ArrowTableSource( + _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]).as_table(), + tag_columns=["id"], + ) pod = _make_pod() job = PipelineJob(name="test_pipeline") @@ -670,7 +674,10 @@ def test_pipeline_job_set_ephemeral_store_none_detaches(self): """PipelineJob.set_ephemeral_store(None) detaches the store from all nodes.""" from orcapod.pipeline.job import PipelineJob - stream = _make_stream([{"id": 0, "x": 10}]) + # A job's input must be a source (ITL-1126: a raw stream has no system tags). + stream = ArrowTableSource( + _make_stream([{"id": 0, "x": 10}]).as_table(), tag_columns=["id"] + ) pod = _make_pod() job = PipelineJob(name="test_pipeline") diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 94470c1a8..f1d9bd396 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -501,7 +501,13 @@ def test_source_job_node_from_stream_slot_name_uses_full_hash(self): """SourceJobNode.from_stream(non-RootSource) builds name as label + full content hash.""" from orcapod.core.nodes.source_node import SourceJobNode - stream = self._make_arrow_stream() + # A stream derived from a source: not a RootSource, and it carries the one + # system-tag pair a bound stream must have (ITL-1126; a raw stream has none). + from orcapod.core.operators import MapData + from orcapod.core.sources import DictSource + + source = DictSource([{"id": 1, "value": 1.0}], tag_columns=["id"]) + stream = MapData({"value": "v"})(source) job_node = SourceJobNode.from_stream(stream) expected_name = f"{stream.label}:{stream.content_hash().to_hex()}" diff --git a/tests/test_core/test_tracker.py b/tests/test_core/test_tracker.py index f28e63f1e..a5da5864b 100644 --- a/tests/test_core/test_tracker.py +++ b/tests/test_core/test_tracker.py @@ -105,6 +105,15 @@ def _make_stream(n: int = 3) -> ArrowTableStream: return ArrowTableStream(table, tag_columns=["id"]) +def _make_source(n: int = 3) -> ArrowTableSource: + """The rows of ``_make_stream`` as a source: one system-tag pair, so it can be bound. + + A raw ``ArrowTableStream`` carries no system-tag columns, and a source node refuses + it (ITL-1126). + """ + return ArrowTableSource(_make_stream(n).as_table(), tag_columns=["id"]) + + def _make_two_col_stream(n: int = 3) -> ArrowTableStream: """Stream with tag=id, data={a, b} for binary operator tests. Uses nullable=False schema.""" schema = pa.schema( @@ -154,7 +163,7 @@ class TestSourceNode: def _make_job_node(self, stream=None, name="test_source"): """Create a SourceJobNode wrapping an optional concrete stream.""" if stream is None: - stream = _make_stream() + stream = _make_source() tag_schema = Schema({"id": int}) data_schema = Schema({"x": int}) return SourceJobNode( @@ -165,7 +174,7 @@ def _make_job_node(self, stream=None, name="test_source"): ) def test_construction(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) assert node.bound_source is stream assert node.node_type == "source" @@ -173,32 +182,32 @@ def test_construction(self): assert node.upstreams == () def test_label_from_argument(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="my_source") # label resolves to the name via computed_label() assert node.label == "my_source" def test_label_set_explicitly(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="my_source") node._label = "explicit" assert node.label == "explicit" def test_label_defaults_to_name(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="my_source") # No explicit label → computed_label() returns the slot name assert node.label == "my_source" def test_repr(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="test") r = repr(node) assert "SourceJobNode" in r assert "test" in r def test_content_hash_delegates_to_concrete(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) # SourceJobNode with concrete delegates content_hash to the concrete source assert node.content_hash() == stream.content_hash() @@ -209,28 +218,28 @@ def test_upstreams_setter_rejects_nonempty(self): node.upstreams = (_make_stream(),) def test_output_schema(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) tag_s, data_s = node.output_schema() assert set(tag_s.keys()) == {"id"} assert set(data_s.keys()) == {"x"} def test_keys(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) tag_keys, data_keys = node.keys() assert set(tag_keys) == {"id"} assert set(data_keys) == {"x"} def test_delegates_as_table(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) node_table = node.as_table() stream_table = stream.as_table() assert node_table.equals(stream_table) def test_delegates_iter_data(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) node_data = list(node.iter_data()) stream_data = list(stream.iter_data()) @@ -238,19 +247,19 @@ def test_delegates_iter_data(self): def test_run_equivalent(self): """execute() on SourceJobNode returns all data rows.""" - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) result = node.execute() assert len(result) == stream.as_table().num_rows def test_data_context_key(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) # SourceJobNode uses its own data context (not the concrete stream's) assert node.data_context_key is not None def test_data_context(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) assert node.data_context is not None assert node.data_context.context_key is not None @@ -265,7 +274,7 @@ class TestNodeContextDelegation: """All node types delegate data_context to their wrapped entity.""" def test_source_node_context_matches_stream(self): - stream = _make_stream() + stream = _make_source() node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), @@ -292,7 +301,7 @@ def test_operator_node_context_matches_operator(self): assert node.data_context.context_key == op.data_context_key def test_source_node_hash_consistent_with_stream(self): - stream = _make_stream() + stream = _make_source() node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 87485b722..edabdb5d9 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -693,23 +693,32 @@ def test_a_refused_joined_stream_keeps_the_previous_binding() -> None: assert job.nodes["src"].bound_source is before -def test_a_pipeline_built_over_a_raw_stream_still_compiles() -> None: - """A raw stream as a pipeline's leaf is accepted as before; binding it is refused. +def test_a_pipeline_built_over_a_raw_stream_is_refused() -> None: + """A raw stream as a pipeline's input is refused, telling the user what to do. - The one exception to ITL-1126, pending the developer's decision: the slot built - from the leaf takes the leaf as it is (its rows carry no system tags, although - the slot predicts a pair). Binding the same stream to the slot afterwards goes - through the bind check and is refused. + Developer, November 14, 2026: refuse a pipeline built directly over a stream + without system tags until a raw stream is captured into a source (ITL-1127). """ raw = _raw_stream(x_nullable=True) job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with pytest.raises(SourceSpecMismatchError) as raised: + with job: + _ = MapData({"x": "y"})(raw, label="renamed") + message = str(raised.value) + assert "exactly one source-id and one record-id" in message + assert "ArrowTableSource(stream.as_table(), tag_columns=[...])" in message + assert "ITL-1127" in message + + +def test_a_raw_streams_table_wrapped_as_the_error_advises_runs() -> None: + """``ArrowTableSource(stream.as_table(), tag_columns=[...])`` builds and runs.""" + raw = _raw_stream(x_nullable=True) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: - _ = MapData({"x": "y"})(raw, label="renamed") - (slot,) = [name for name in job.nodes if name != "renamed"] - assert job.nodes[slot].bound_source is raw - assert job.nodes["renamed"].as_table().num_rows == 0 + _ = MapData({"x": "y"})( + ArrowTableSource(raw.as_table(), tag_columns=["id"]), label="renamed" + ) _ = job.run() - assert job.nodes["renamed"].as_table().column("y").to_pylist() == [1] - with pytest.raises(SourceSpecMismatchError, match="exactly one source-id"): - job.bind(sources={slot: raw}) - assert job.nodes[slot].bound_source is raw + table = job.nodes["renamed"].as_table() + assert table.column("y").to_pylist() == [1] + assert table.column("w").to_pylist() == ["p"] diff --git a/tests/test_pipeline/test_operator_node_empty_table_types.py b/tests/test_pipeline/test_operator_node_empty_table_types.py index 2608f8651..e42a3462f 100644 --- a/tests/test_pipeline/test_operator_node_empty_table_types.py +++ b/tests/test_pipeline/test_operator_node_empty_table_types.py @@ -642,8 +642,12 @@ def test_head_of_a_filter_has_the_large_types_its_run_gives() -> None: def test_an_input_with_a_meta_data_column_gives_a_zero_row_table() -> None: - """A raw stream lists a ``__`` column as data; an operator node over it still - reads as zero rows before a run, without that column.""" + """A raw stream lists a ``__`` column as data; an operator over it still reads as + zero rows before it computes, without that column, typed as its computed rows. + + A raw stream cannot be a job's input (ITL-1126: it has no system-tag pair), so + the operator is applied to it directly, outside a job. + """ raw = ArrowTableStream( pa.table( { @@ -654,18 +658,16 @@ def test_an_input_with_a_meta_data_column_gives_a_zero_row_table() -> None: ), tag_columns=["id"], ) - job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) - with job: - _ = SelectDataColumns(["x"])(raw, label="op") - node = _op(job) + stream = SelectDataColumns(["x"])(raw) - head = node.head(all_info=True).table - table = node.as_table(all_info=True) + head = stream.head(all_info=True).table + table = stream.as_table(all_info=True) - assert head.num_rows == table.num_rows == 0 + assert head.num_rows == 0 + assert table.num_rows == 2 assert head.schema == table.schema - assert table.schema.field("x").type == pa.int8() - assert not any("__debug" in name for name in table.column_names) + assert head.schema.field("x").type == pa.int8() + assert not any("__debug" in name for name in head.column_names) # Both branches share the data columns, which SemiJoin ignores: not under test here. From 1ee849088f951fab20f1a0cc6c71738770c86f64 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:01:54 -0800 Subject: [PATCH 18/26] test(polling_source): bind a source, not a raw stream, to a source node (ITL-1126) Co-Authored-By: Claude Opus 5.5 --- tests/test_channels/test_polling_source.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_channels/test_polling_source.py b/tests/test_channels/test_polling_source.py index eccc69523..4cae55f42 100644 --- a/tests/test_channels/test_polling_source.py +++ b/tests/test_channels/test_polling_source.py @@ -12,6 +12,7 @@ from orcapod.channels import Channel from orcapod.core.nodes.source_node import SourceJobNode +from orcapod.core.sources import ArrowTableSource from orcapod.core.sources.polling_source import PollingSource from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.errors import CursorInvalidatedError @@ -244,7 +245,8 @@ async def test_source_node_async_execute_uses_async_iter(self): """SourceJobNode.async_execute routes through async_iter_data — no regression.""" from orcapod.types import Schema - stream = _make_arrow_stream(2) + # A bound source needs one system-tag pair (ITL-1126); a raw stream has none. + stream = ArrowTableSource(_make_arrow_stream(2).as_table(), tag_columns=["id"]) node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), @@ -262,7 +264,8 @@ async def test_source_node_async_execute_uses_async_iter(self): async def test_source_node_closes_channel_after_exhaustion(self): from orcapod.types import Schema - stream = _make_arrow_stream(1) + # A bound source needs one system-tag pair (ITL-1126); a raw stream has none. + stream = ArrowTableSource(_make_arrow_stream(1).as_table(), tag_columns=["id"]) node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), From e83bcb5609e1687e6144cd73d6586cc584201bbf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:16:38 -0800 Subject: [PATCH 19/26] fix(source_node): a slot predicts its record id as bytes; #387's rebind test asserts the refusal (ITL-1112) - test_graph_walk_cost: the bind of another column type is refused and moves no downstream schema (bind and bound_source; the strict xfail for a direct assignment is folded in, since the setter refuses). ITL-1111 fragment and GW1 corrected. - SourceNodeBase.output_schema typed the record-id system tag str; rows have large_binary (bytes). A source bound as declared passed bytes through while an unbound slot and a narrower bind said str, so a bind changed predictions. Now bytes: predictions are equal unbound and under every accepted bind, and equal to the rows' Python schema (test). #387's memo clear is kept with a comment. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1111.md | 2 +- changelog.d/ITL-1112.md | 8 ++ design_issues/ITL-1111-GW1.md | 3 +- design_issues/ITL-1112-BT1.md | 4 +- src/orcapod/core/nodes/source_node.py | 9 +- src/orcapod/pipeline/job.py | 8 +- tests/test_core/nodes/test_source_node.py | 16 +++- .../test_bind_narrower_source.py | 90 +++++++++++++++++++ tests/test_pipeline/test_graph_walk_cost.py | 62 +++++++------ 9 files changed, 161 insertions(+), 41 deletions(-) diff --git a/changelog.d/ITL-1111.md b/changelog.d/ITL-1111.md index 745a398f0..a84d4c6f5 100644 --- a/changelog.d/ITL-1111.md +++ b/changelog.d/ITL-1111.md @@ -1,6 +1,6 @@ ### Fixed -- Defining a pipeline whose stages split a stream into two branches and join them again no longer takes time exponential in the number of stages (about 4x per stage before: 9 stages took 13 seconds to build, with no row computed; 20 stages now build in well under a second, as streams and inside a `PipelineJob`) (ITL-1111). Three graph walks asked a shared upstream once per path: every operator stream, function pod stream and node predicted its output schema again from its upstreams on each call, and a join asks each input several times; `head()` and the notebook preview walked every path to check whether reading the schema would poll a source; and each operator or function pod call formatted its input streams, whose description nests every upstream, into a debug log message even with debug logging off. A stream or node now keeps its predicted output schema per column selection (`output_schema()`, `keys()`), the poll check visits each upstream once, and the log message is formatted only when debug logging is on (see Changed for the description). Job compilation, `save`, `PipelineJob.load` and a run's schema queries are linear in the nodes as well: each node is asked a bounded number of times. A schema that includes the system tags is still as wide as the joins make it, because each join keeps both inputs' system-tag columns: stacked split-and-join stages over one source double their number per stage (ITL-1114). `head()` and the notebook preview of a stream that has not run build their table of no rows from a table that carries the system tags whatever columns are shown, so over such stacked stages they are as wide as the joins make them (ITL-1132). No hash, stored table or golden value changes. `PipelineJob.bind(sources=...)` makes every node of the job predict its schema again, so binding a source whose column has another type still changes the downstream schemas; assigning `bound_source` on a source node directly, or setting a node's `upstreams`, does not reach the nodes downstream of it, so use `bind`. +- Defining a pipeline whose stages split a stream into two branches and join them again no longer takes time exponential in the number of stages (about 4x per stage before: 9 stages took 13 seconds to build, with no row computed; 20 stages now build in well under a second, as streams and inside a `PipelineJob`) (ITL-1111). Three graph walks asked a shared upstream once per path: every operator stream, function pod stream and node predicted its output schema again from its upstreams on each call, and a join asks each input several times; `head()` and the notebook preview walked every path to check whether reading the schema would poll a source; and each operator or function pod call formatted its input streams, whose description nests every upstream, into a debug log message even with debug logging off. A stream or node now keeps its predicted output schema per column selection (`output_schema()`, `keys()`), the poll check visits each upstream once, and the log message is formatted only when debug logging is on (see Changed for the description). Job compilation, `save`, `PipelineJob.load` and a run's schema queries are linear in the nodes as well: each node is asked a bounded number of times. A schema that includes the system tags is still as wide as the joins make it, because each join keeps both inputs' system-tag columns: stacked split-and-join stages over one source double their number per stage (ITL-1114). `head()` and the notebook preview of a stream that has not run build their table of no rows from a table that carries the system tags whatever columns are shown, so over such stacked stages they are as wide as the joins make them (ITL-1132). No hash, stored table or golden value changes. `PipelineJob.bind(sources=...)` makes every node of the job predict its schema again (a source whose column has another type is refused, ITL-1112); assigning `bound_source` on a source node directly, or setting a node's `upstreams`, does not reach the nodes downstream of it, so use `bind`. ### Changed diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 971019738..99d7d78e1 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -66,6 +66,14 @@ source. Binding such a stream properly is planned (ITL-1127). rename, with one system-tag pair) gives its system-tag columns the slot's names; their values stay the stream's own. Both passed through as they were before. +### Fixed + +- A source slot's predicted schema with system tags (`output_schema(columns={"system_tags": True})`, + `all_info=True`) gave the record-id column as `str`; every source's rows have it as `bytes` + (Arrow `large_binary`). It is `bytes` now, for an unbound slot and for every bind, so the + slot's prediction and every prediction after it are what the rows are, whatever is bound + (ITL-1112). + ### Known limitations - The bind check compares Python types, so a source that differs from its slot only in diff --git a/design_issues/ITL-1111-GW1.md b/design_issues/ITL-1111-GW1.md index e96c27484..7746e42a2 100644 --- a/design_issues/ITL-1111-GW1.md +++ b/design_issues/ITL-1111-GW1.md @@ -27,7 +27,8 @@ Call-count tests over 20 stages: `tests/test_pipeline/test_graph_walk_cost.py`. Left as they are: assigning `SourceJobNode.bound_source` directly, or setting a node's `upstreams`, clears no memo of the nodes downstream of it, which keep their schema (their -pipeline hashes are equally stale today; strict xfail tests record both). The risk is bounded: +pipeline hashes are equally stale today; a strict xfail test records the `upstreams` case, and a +`bound_source` of another column type is refused, ITL-1112). The risk is bounded: nothing in `src` calls the `upstreams` setter (the loaders build a node with `__new__` and set `_input_streams` before any schema is asked; only tests call it), and after ITL-1112 a bind cannot change a slot's schema. Cascading the clear (a registry of dependents, or a setter that diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 49c2e0a0c..0a42f71c3 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -38,4 +38,6 @@ are still not told apart (ITL-949). A source whose system-tag columns are not ex source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is ITL-1127), also as a pipeline's input: a pipeline built directly over a raw stream is -refused at compile. No hash, golden value or stored path changed. +refused at compile. The slot's own prediction types the record-id system-tag column `bytes`, +as every source's rows have it (it said `str`), so a slot's prediction is the same unbound +and under every accepted bind. No hash, golden value or stored path changed. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 5767ec866..a5310ef47 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -329,9 +329,10 @@ def output_schema( When ``columns.system_tags`` is ``True`` (or ``all_info=True``), the returned tag schema is extended with the two system-tag entries for this - node's schema (both typed as ``str``). Their names are deterministic from - the declared schemas and match what a concrete source with the same schema - would produce. + node's schema: the source id as ``str`` and the record id as ``bytes``, the + types every source's rows have (Arrow ``large_string`` and ``large_binary``). + Their names are deterministic from the declared schemas and match what a + concrete source with the same schema would produce. Other ``ColumnConfig`` flags (``meta``, ``context``, ``source``, ``content_hash``, ``sort_by_tags``) are no-ops at the node level — @@ -352,7 +353,7 @@ def output_schema( self._schema_hash_str ) tag_schema = Schema( - {**dict(tag_schema), source_id_col: str, record_id_col: str} + {**dict(tag_schema), source_id_col: str, record_id_col: bytes} ) return tag_schema, self._data_schema diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 5dbad8e9d..632d06084 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -474,8 +474,12 @@ def bind( schema_hash = job_node.as_node().content_hash().to_string() self._sources_by_hash[schema_hash] = sources[job_node.name] self._sources.update(sources) - # ``validate`` checks column names only, so a rebound source may give a - # column another type: every node predicts its schema again (ITL-1111). + # Every node predicts its schema again (ITL-1111). Since ITL-1112 a bind + # should never change a prediction: ``validate`` refuses a source of other + # column names or types, and the slot gives the declared schema, so this + # clear is expected to find nothing to change. It is kept so that a bind + # can never leave a stale prediction if a bound source's schema ever + # differs in a way the check does not see. for job_node in (self._persistent_node_map or {}).values(): if isinstance(job_node, StreamBase): job_node._forget_output_schema() # pyright: ignore[reportPrivateUsage] # the job owns its nodes' bindings diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index f1d9bd396..e2c8dae9b 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -340,10 +340,13 @@ def test_unbound_keys_all_info_same_as_system_tags(self, tag_schema, data_schema via_all_info = node.keys(all_info=True) assert via_system_tags == via_all_info - def test_unbound_output_schema_system_tags_adds_str_entries( + def test_unbound_output_schema_system_tags_adds_typed_entries( self, tag_schema, data_schema ): - """output_schema(system_tags=True) tag schema includes two str-typed system-tag entries.""" + """output_schema(system_tags=True) adds the source id (str) and record id (bytes). + + The types a source's rows have (ITL-1112): ``large_string`` and ``large_binary``. + """ from orcapod.core.nodes.source_node import SourceNode from orcapod.system_constants import constants from orcapod.types import ColumnConfig @@ -361,7 +364,14 @@ def test_unbound_output_schema_system_tags_adds_str_entries( if k.startswith(constants.SYSTEM_TAG_PREFIX) } assert len(system_tag_entries) == 2 - assert all(v is str for v in system_tag_entries.values()) + types = { + k.split(constants.BLOCK_SEPARATOR)[0]: v + for k, v in system_tag_entries.items() + } + assert types == { + constants.SYSTEM_TAG_SOURCE_ID_PREFIX: str, + constants.SYSTEM_TAG_RECORD_ID_PREFIX: bytes, + } def test_unbound_output_schema_default_unchanged(self, tag_schema, data_schema): """Regression: output_schema() with no args still returns plain schemas.""" diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index edabdb5d9..883598913 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -722,3 +722,93 @@ def test_a_raw_streams_table_wrapped_as_the_error_advises_runs() -> None: table = job.nodes["renamed"].as_table() assert table.column("y").to_pylist() == [1] assert table.column("w").to_pylist() == ["p"] + + +# --------------------------------------------------------------------------- +# The slot's predicted schema says what its rows are, bound or unbound +# --------------------------------------------------------------------------- + + +def _x_job() -> PipelineJob: + """Return an unbound job: slot ``src`` (``id: int``, ``x: int | None``) -> rename.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + return job + + +def _abc_unbound() -> PipelineJob: + """Return the unbound ``a, b, c`` job.""" + return _abc_job(int) + + +def _abc_bound(order: str) -> Callable[[], PipelineJob]: + """Return a maker of the ``a, b, c`` job bound to a source with data in *order*.""" + + def make() -> PipelineJob: + job = _abc_job(int) + values = {"a": 1, "b": "s", "c": 1.5} + row = {"id": 0, **{name: values[name] for name in order}} + job.bind(sources={"src": DictSource([row], tag_columns=["id"])}) + return job + + return make + + +def _x_bound(source: Callable[[], Any]) -> Callable[[], PipelineJob]: + """Return a maker of the ``x: int | None`` job bound to ``source()``.""" + + def make() -> PipelineJob: + job = _x_job() + job.bind(sources={"src": source()}) + return job + + return make + + +PREDICTION_CASES: dict[str, tuple[Callable[[], PipelineJob], ...]] = { + "x": ( + _x_job, + _x_bound(lambda: _arrow_source([1, None], x_nullable=True)), + _x_bound(lambda: _arrow_source([1, 2], x_nullable=False)), + _x_bound(_mapped_stream), + ), + "abc": (_abc_unbound, _abc_bound("abc"), _abc_bound("cba")), +} + + +@pytest.mark.parametrize("config", [{"system_tags": True}, "all_info"]) +@pytest.mark.parametrize( + "makers", PREDICTION_CASES.values(), ids=PREDICTION_CASES.keys() +) +def test_the_predicted_schema_is_that_of_the_rows_bound_or_unbound( + makers: tuple[Callable[[], PipelineJob], ...], config: Any +) -> None: + """Unbound, declared, narrower, reordered and transformed binds predict alike. + + The prediction (system tags included: the record id is ``bytes``) equals the + Python schema of the populated rows, for the slot and the node after it. + """ + predictions = [] + for make in makers: + job = make() + predictions.append( + { + label: ( + job.nodes[label].output_schema(all_info=True) + if config == "all_info" + else job.nodes[label].output_schema(columns=config) + ) + for label in ("src", "renamed") + } + ) + if job.nodes["src"].bound_source is None: # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + continue + for label, (tag, data) in predictions[-1].items(): + node = job.nodes[label] + table = node.as_table(columns={"system_tags": True}) + rows = node.data_context.type_converter.arrow_schema_to_python_schema( + table.schema + ) + assert {**tag, **data} == dict(rows), label + assert all(p == predictions[0] for p in predictions), predictions diff --git a/tests/test_pipeline/test_graph_walk_cost.py b/tests/test_pipeline/test_graph_walk_cost.py index 7e820f802..715a43726 100644 --- a/tests/test_pipeline/test_graph_walk_cost.py +++ b/tests/test_pipeline/test_graph_walk_cost.py @@ -32,7 +32,7 @@ from orcapod.core.sources import DictSource from orcapod.core.streams.base import StreamBase from orcapod.databases import InMemoryArrowDatabase -from orcapod.errors import InputValidationError +from orcapod.errors import InputValidationError, SourceSpecMismatchError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator STAGES = 20 @@ -521,19 +521,42 @@ def _function_job() -> PipelineJob: class TestMemoInvalidation: """A memoized schema is predicted again when an input of the prediction changes.""" - def test_bind_to_a_source_of_another_column_type_changes_downstream_schemas( - self, + @pytest.mark.parametrize("how", ["bind", "bound_source"]) + def test_a_source_of_another_column_type_is_refused_and_moves_no_schema( + self, how: str ) -> None: - """``job.bind`` to a source with another column type changes every downstream schema.""" + """A source with another column type is refused; no downstream schema moves. + + The pipeline determines every node's schema (ITL-1112): ``job.bind`` and a + direct ``bound_source`` assignment refuse a source whose column types the + slot does not declare, so no memoized downstream schema can go stale. + """ job = _rename_job() - assert job.nodes["renamed"].output_schema()[1]["y"] is int + configs: list[dict[str, bool]] = [{}, {"system_tags": True}, {"meta": True}] + before = { + (label, str(config)): job.nodes[label].output_schema(columns=config) + for label in ("renamed", "renamed_again") + for config in configs + } assert job.nodes["renamed_again"].output_schema()[1]["z"] is int - assert job.nodes["renamed_again"].keys() == (("id",), ("z",)) - - job.bind(sources={"src": _one_row_source("a")}) - assert job.nodes["renamed"].output_schema()[1]["y"] is str - assert job.nodes["renamed_again"].output_schema()[1]["z"] is str + with pytest.raises(SourceSpecMismatchError, match="declared int"): + if how == "bind": + job.bind(sources={"src": _one_row_source("a")}) + else: + job.nodes["src"].bound_source = _one_row_source("a") # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + for label in ("renamed", "renamed_again"): + node = job.nodes[label] + assert isinstance(node, StreamBase) + # Predict afresh: the test is about the schemas, not the memo. + node._forget_output_schema() # pyright: ignore[reportPrivateUsage] + after = { + (label, str(config)): job.nodes[label].output_schema(columns=config) + for label in ("renamed", "renamed_again") + for config in configs + } + assert after == before def test_setting_an_operator_nodes_upstreams_changes_its_schema(self) -> None: """Setting an operator node's ``upstreams`` changes its own schema.""" @@ -558,25 +581,6 @@ def test_setting_a_function_nodes_upstreams_changes_its_schema(self) -> None: assert node.output_schema()[0]["id"] is str - @pytest.mark.xfail( - strict=True, - reason=( - "A direct bound_source assignment reaches no downstream node, so their " - "memoized schemas are not predicted again; use job.bind (ITL-1111). " - "Their pipeline hashes are equally stale today." - ), - ) - def test_assigning_bound_source_directly_changes_downstream_schemas( - self, - ) -> None: - """A direct ``bound_source`` assignment should change the downstream schemas.""" - job = _rename_job() - assert job.nodes["renamed_again"].output_schema()[1]["z"] is int - - job.nodes["src"].bound_source = _one_row_source("a") - - assert job.nodes["renamed_again"].output_schema()[1]["z"] is str - @pytest.mark.xfail( strict=True, reason=( From 47c5bad084e39ba8f44c49499f94659593f8167a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:55:44 -0800 Subject: [PATCH 20/26] fix(source_node): structural classes of one identity fit; a bound source's system tags are exactly one str/bytes pair (ITL-1112, ITL-1126) CodeRabbit round 2: - _fit and _is_narrower compare column types with equivalent_hints, so a structural class of the declared identity is accepted (exact classes stay distinct); the invariant test covers it (paths, table schemas, pipeline hashes). - A source with any system-tag column besides one source-id and one record-id column is refused. - The pair must be str (source id) and bytes (record id), the types the slot predicts. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 6 +- CLAUDE.md | 6 +- changelog.d/ITL-1112.md | 6 +- src/orcapod/core/nodes/source_node.py | 64 ++++--- tests/test_core/nodes/test_source_node.py | 14 +- .../test_bind_keeps_node_schemas.py | 59 +++++- .../test_bind_narrower_source.py | 173 +++++++++++++++++- 7 files changed, 297 insertions(+), 31 deletions(-) diff --git a/.zed/rules b/.zed/rules index 967eada73..19a7c4e3c 100644 --- a/.zed/rules +++ b/.zed/rules @@ -795,8 +795,10 @@ data. The rules: schema of any node's or stored table (column names with system tags, types, nullable flags, order); only values differ. SourceNodeBase.validate is the one check, used by PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and - PipelineJob.load: per column the declared Python type, or T where the slot declares - T | None; anything else raises SourceSpecMismatchError before anything is bound. The + PipelineJob.load: per column the declared Python type (structural classes of one identity + are one type, equivalent_hints), or T where the slot declares T | None; and exactly + one source-id (str) and one record-id (bytes) system-tag column, no other. Anything + else raises SourceSpecMismatchError before anything is bound. The source's rows are given as the declared schema at one place, _DeclaredForm (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's diff --git a/CLAUDE.md b/CLAUDE.md index 1e7dbdbc0..017277f6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -837,8 +837,10 @@ data. The rules: schema of any node's or stored table (column names with system tags, types, nullable flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and - `PipelineJob.load`: per column the declared Python type, or `T` where the slot declares - `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. The + `PipelineJob.load`: per column the declared Python type (structural classes of one identity + are one type, `equivalent_hints`), or `T` where the slot declares `T | None`; and exactly + one source-id (`str`) and one record-id (`bytes`) system-tag column, no other. Anything + else raises `SourceSpecMismatchError` before anything is bound. The source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 99d7d78e1..4932307a6 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -8,7 +8,8 @@ not checked at all. A source with other column types was accepted: every pipelin from the slot's declared schema, stayed the same, so a run wrote rows of the new types to the stored paths of the declared ones. Every way of binding now compares the source's tag and data columns with the slot's: names, role (tag or data) and Python type, the inputs of the pipeline -hash. A column must have the declared type, or `T` where the slot declares `T | None`. Anything +hash. A column must have the declared type (two `orcapod.dataclass` classes of one identity are +one type), or `T` where the slot declares `T | None`. Anything else raises `SourceSpecMismatchError` naming the slot, the column and both types, before anything is bound; a refused `bind` leaves the job as it was, its database included. @@ -41,7 +42,8 @@ before with its own system-tag columns and column order, so the schema of the sl and of every table after it, depended on what was bound. Every way of binding (`bind`, the `bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever -its column types. So does building a pipeline or job directly over such a stream +its column types; so is a stream with any other system-tag column, or whose source id is not +`str` or record id not `bytes`. So does building a pipeline or job directly over such a stream (`with job: op(ArrowTableStream(...))`), which binds it to the pipeline's input slot. Migration: wrap a raw stream's table as a source, diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index a5310ef47..95e6e13c0 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -31,6 +31,7 @@ check_head_size, ) from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.logical_types.user_class_identity import equivalent_hints from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import ColumnConfig, Schema @@ -60,13 +61,14 @@ def _is_narrower(declared: Any, given: Any) -> bool: """Return whether *given* is *declared* without its ``None`` member. ``int`` is narrower than ``int | None`` and ``int | str`` than ``int | str | None``; - nothing else is (developer, November 14, 2026, ITL-1112). + nothing else is (developer, November 14, 2026, ITL-1112). Structural classes of one + identity are one type here as everywhere (``equivalent_hints``). Args: declared: The slot's declared Python type of a column. given: The bound source's Python type of that column. """ - return declared != given and _without_none(declared) == given + return declared != given and equivalent_hints(_without_none(declared), given) @dataclasses.dataclass(frozen=True) @@ -139,8 +141,8 @@ def _declared_positions(self, names: list[str]) -> list[int]: def _system_tag_columns( source: StreamProtocol, tag_names: Collection[str] -) -> tuple[list[str], list[str]]: - """Return *source*'s source-id and record-id system-tag columns. +) -> tuple[list[str], list[str], list[str]]: + """Return *source*'s source-id, record-id and any other system-tag columns. A root source, or a stream that keeps its system tags (a filter, a rename), has one of each; a stream that joins or merges inputs has one pair per input; a stream over a @@ -150,6 +152,10 @@ def _system_tag_columns( Args: source: The stream being bound. tag_names: Its user tag columns. + + Returns: + The source-id columns, the record-id columns, and every other system-tag + column, which no slot declares. """ names = [ name @@ -158,35 +164,41 @@ def _system_tag_columns( ] source_id = f"{constants.SYSTEM_TAG_SOURCE_ID_PREFIX}{constants.BLOCK_SEPARATOR}" record_id = f"{constants.SYSTEM_TAG_RECORD_ID_PREFIX}{constants.BLOCK_SEPARATOR}" - return ( - [name for name in names if name.startswith(source_id)], - [name for name in names if name.startswith(record_id)], - ) + source_ids = [name for name in names if name.startswith(source_id)] + record_ids = [name for name in names if name.startswith(record_id)] + others = [name for name in names if name not in {*source_ids, *record_ids}] + return source_ids, record_ids, others -def _one_system_tag_pair_issue(source_ids: list[str], record_ids: list[str]) -> str: +def _one_system_tag_pair_issue( + source_ids: list[str], record_ids: list[str], others: list[str] +) -> str: """Return why a source without exactly one system-tag pair is refused, and the remedy. Args: source_ids: The source's source-id system-tag columns. record_ids: The source's record-id system-tag columns. + others: The source's other system-tag columns. """ - found = sorted(source_ids + record_ids) + found = sorted(source_ids + record_ids + others) + issue = ( + "a source must carry exactly one source-id and one record-id system-tag " + f"column and no other, not {found}" + ) if not found: remedy = ( "a stream without system-tag columns (a raw ArrowTableStream) cannot be " "bound or used as a pipeline's input; wrap its table as a source, e.g. " "ArrowTableSource(stream.as_table(), tag_columns=[...])" ) - else: + elif len(source_ids) > 1 or len(record_ids) > 1: remedy = ( "a joined or merged stream cannot be bound; make the join part of the " "pipeline, or store the joined stream and bind the stored source" ) - return ( - "a source must carry exactly one source-id and one record-id system-tag " - f"column, not {found}: {remedy} (binding such a stream is planned, ITL-1127)" - ) + else: + return issue + return f"{issue}: {remedy} (binding such a stream is planned, ITL-1127)" class SourceNodeBase(TraceableBase, ABC): @@ -433,7 +445,7 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] if extra: issues.append(f"unexpected {role} columns: {sorted(extra)}") for name in declared.keys(): - if name not in given or given[name] == declared[name]: + if name not in given or equivalent_hints(given[name], declared[name]): continue if _is_narrower(declared[name], given[name]): narrower.add(name) @@ -448,9 +460,19 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] # a source with exactly one pair can take. A joined or merged stream (one pair # per input) or a stream with none is refused (developer, November 14, 2026, # ITL-1126); binding one properly is ITL-1127. - source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if len(source_ids) != 1 or len(record_ids) != 1: - issues.append(_one_system_tag_pair_issue(source_ids, record_ids)) + source_ids, record_ids, others = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) != 1 or len(record_ids) != 1 or others: + issues.append(_one_system_tag_pair_issue(source_ids, record_ids, others)) + else: + # The pair must have the types the slot predicts for it (str, bytes). + given_tags = source.output_schema(columns={"system_tags": True})[0] + for name, declared_type in ((source_ids[0], str), (record_ids[0], bytes)): + if given_tags[name] != declared_type: + issues.append( + f"system-tag column {name!r} is declared " + f"{_type_text(declared_type)} but the source has " + f"{_type_text(given_tags[name])}" + ) if issues: raise SourceSpecMismatchError( @@ -814,8 +836,8 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: SourceSpecMismatchError: If *source* does not fit the declared schema. """ source_tag, source_data, narrower = self._fit(source) - # _fit has checked that the source has exactly one pair. - source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + # _fit has checked that the source has exactly one pair and nothing else. + source_ids, record_ids, _ = _system_tag_columns(source, source_tag.keys()) declared_source_id, declared_record_id = system_tag_column_names( self._schema_hash_str ) diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index e2c8dae9b..7c1cedbc8 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -8,7 +8,7 @@ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.errors import SourceSpecMismatchError, UnboundSourceError -from orcapod.types import Schema +from orcapod.types import ColumnConfig, Schema @pytest.fixture @@ -295,8 +295,18 @@ def _mock_source_with_the_slots_system_tags( column (ITL-1126); the slot's own names make the bind pass rows unchanged. """ tag_keys = node.keys(columns={"system_tags": True})[0] + with_system_tags = node.output_schema(columns={"system_tags": True}) + + def output_schema( + columns: ColumnConfig | dict[str, bool] | None = None, all_info: bool = False + ) -> tuple[Schema, Schema]: + """The slot's schema, with its system-tag pair when asked for.""" + if ColumnConfig.handle_config(columns, all_info=all_info).system_tags: + return with_system_tags + return tag_schema, data_schema + mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) + mock_source.output_schema.side_effect = output_schema mock_source.keys.return_value = (tag_keys, tuple(data_schema.keys())) return mock_source diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index f9cc4fee2..42931bcf7 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -16,10 +16,11 @@ import pathlib from collections.abc import Callable -from typing import Any +from typing import Any, cast import pyarrow as pa +import orcapod from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import MapData @@ -144,3 +145,59 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( # A loaded function node orders its meta columns unlike a fresh one, bind or not # (reported separately), so the loaded job is compared with itself. assert _snapshot(loaded, LOADED_NODES) == before + + +@orcapod.dataclass(name="KeepsPoint") +class _PointA: + """A structural class, identified by its name and fields.""" + + x: int + y: int + + +@orcapod.dataclass(name="KeepsPoint") +class _PointB: + """Another class object of the same structural identity as ``_PointA``.""" + + x: int + y: int + + +def test_a_structural_class_of_the_same_identity_changes_no_path_or_schema( + tmp_path: pathlib.Path, +) -> None: + """A source of another class object of the slot's structural identity moves nothing. + + The sources are built before the baseline hashes are taken: on main too, the + pipeline hash of a schema holding a structural class differs before and after a + source of that class is first built, bind or no bind (reported separately). + """ + sources = [ + DictSource( + cast(list[dict[str, Any]], [{"id": 0, "p": point_type(1, 2)}]), + tag_columns=["id"], + data_schema={"id": int, "p": point_type}, + source_id="points", + ) + for point_type in (_PointA, _PointB, _PointB) + ] + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"p": _PointA}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = MapData({"p": "q"})(slot, label="renamed") + hashes = {name: node.pipeline_hash() for name, node in job.nodes.items()} + + snapshots = [] + for source, bind in zip(sources, (_bind, _bind, _assign), strict=True): + bind(job, source) + assert {name: node.pipeline_hash() for name, node in job.nodes.items()} == ( + hashes + ) + assert job.run().ok + snapshots.append(_snapshot(job, ("src", "renamed"))) + assert snapshots[1] == snapshots[0] + assert snapshots[2] == snapshots[0] diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 883598913..030569b0b 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio +import dataclasses import logging import pathlib from collections.abc import Callable @@ -20,11 +21,16 @@ import pyarrow as pa import pytest +import orcapod from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.operators import Join, MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.core.streams.base import ZERO_ROW_INPUT_COLUMNS, typed_zero_row_table +from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, + StreamBase, + typed_zero_row_table, +) from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -812,3 +818,168 @@ def test_the_predicted_schema_is_that_of_the_rows_bound_or_unbound( ) assert {**tag, **data} == dict(rows), label assert all(p == predictions[0] for p in predictions), predictions + + +# --------------------------------------------------------------------------- +# CodeRabbit round 2: structural classes, extra system tags, system-tag types +# --------------------------------------------------------------------------- + + +@orcapod.dataclass(name="BindPoint") +class _PointA: + """A structural class, identified by its name and fields.""" + + x: int + y: int + + +@orcapod.dataclass(name="BindPoint") +class _PointB: + """Another class object of the same structural identity as ``_PointA``.""" + + x: int + y: int + + +@dataclasses.dataclass +class _ExactA: + """An exact class: one type only with itself.""" + + x: int + y: int + + +@dataclasses.dataclass +class _ExactB: + """An exact class with the same fields as ``_ExactA``, yet another type.""" + + x: int + y: int + + +def _rows(*rows: dict[str, Any]) -> list[dict[str, Any]]: + """Return *rows* as a ``DictSource`` takes them (user-class values included).""" + return list(rows) + + +def _point_job(point_type: Any) -> PipelineJob: + """Return an unbound job: slot ``src`` (``id: int``, ``p: point_type``) -> rename.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"p": point_type}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"p": "q"})(slot, label="renamed") + return job + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_structural_class_of_the_same_identity_is_accepted( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A column of another class object of the declared structural identity fits. + + Two structural classes with equal identity records are one type in the schema + check and in the pipeline hash, so the bind moves no hash or schema. + """ + job = _point_job(_PointA) + source = DictSource( + _rows({"id": 0, "p": _PointB(1, 2)}), + tag_columns=["id"], + data_schema={"id": int, "p": _PointB}, + source_id="b", + ) + assert source.output_schema()[1]["p"] is not _PointA + hashes = _pipeline_hashes(job) + schemas = { + label: node.output_schema(all_info=True) for label, node in job.nodes.items() + } + bind(job, source) + assert job.nodes["src"].bound_source is source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + assert _pipeline_hashes(job) == hashes + for label, node in job.nodes.items(): + if isinstance(node, StreamBase): + # Predict afresh: the test is about the schema, not the memo. + node._forget_output_schema() # pyright: ignore[reportPrivateUsage] + assert node.output_schema(all_info=True) == schemas[label], label + assert job.run().ok + assert job.nodes["renamed"].as_table().num_rows == 1 + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_an_exact_class_with_the_same_fields_is_refused( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """Two exact classes with the same fields are two types.""" + job = _point_job(_ExactA) + source = DictSource( + _rows({"id": 0, "p": _ExactB(1, 2)}), + tag_columns=["id"], + data_schema={"id": int, "p": _ExactB}, + ) + with pytest.raises(SourceSpecMismatchError, match="data column 'p'"): + bind(job, source) + + +def _stream_with_system_tags(columns: dict[str, pa.Array[Any]]) -> ArrowTableStream: + """Return a stream of tag ``id``, data ``x`` and the given system-tag columns.""" + table = pa.table( + { + "id": pa.array([0], pa.int64()), + "x": pa.array([1], pa.int64()), + **columns, + }, + schema=pa.schema( + [ + pa.field("id", pa.int64(), False), + pa.field("x", pa.int64(), False), + *(pa.field(name, array.type, False) for name, array in columns.items()), + ] + ), + ) + return ArrowTableStream(table, tag_columns=["id"]) + + +SOURCE_ID = "_tag_source_id::abc" +RECORD_ID = "_tag_record_id::abc" + + +@pytest.mark.parametrize("x_type", [int, int | None], ids=["declared", "narrower"]) +@pytest.mark.parametrize("bind", ALL_BINDS.values(), ids=ALL_BINDS.keys()) +def test_an_extra_system_tag_column_is_refused( + bind: Callable[[PipelineJob, Any], None], x_type: Any +) -> None: + """A system-tag column besides the one pair is not in the slot's schema.""" + stream = _stream_with_system_tags( + { + SOURCE_ID: pa.array(["s"], pa.large_string()), + RECORD_ID: pa.array([b"r"], pa.large_binary()), + "_tag_extra": pa.array(["e"], pa.large_string()), + } + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(x_type), label="renamed") + with pytest.raises(SourceSpecMismatchError, match="_tag_extra"): + bind(job, stream) + + +@pytest.mark.parametrize("x_type", [int, int | None], ids=["declared", "narrower"]) +@pytest.mark.parametrize("bind", ALL_BINDS.values(), ids=ALL_BINDS.keys()) +def test_a_system_tag_pair_of_other_types_is_refused( + bind: Callable[[PipelineJob, Any], None], x_type: Any +) -> None: + """The source id must be ``str`` and the record id ``bytes``, as the slot predicts.""" + stream = _stream_with_system_tags( + { + SOURCE_ID: pa.array(["s"], pa.large_string()), + RECORD_ID: pa.array(["r"], pa.large_string()), + } + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(x_type), label="renamed") + with pytest.raises(SourceSpecMismatchError, match=r"declared bytes but the source"): + bind(job, stream) From 4ffdef06cd40955bd77066b9ae5043d33c944e85 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:41:55 -0800 Subject: [PATCH 21/26] test(current_run): bind a source, not a raw stream, to a source node (ITL-1126) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_current_run_review_fixes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py index 429ccde6a..1789ba4fa 100644 --- a/tests/test_pipeline/test_current_run_review_fixes.py +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -425,7 +425,8 @@ def test_an_unrun_loaded_operator_without_its_operator_keeps_its_columns( def test_a_bound_source_with_no_modified_time_makes_its_slot_unknown() -> None: """A source slot bound to a stream with no modified time has none.""" - stream = ArrowTableStream(pa.table({"k": [0], "x": [1]}), tag_columns=["k"]) + # A table wrapped as a source: a bound stream needs one system-tag pair (ITL-1126). + stream = ArrowTableSource(pa.table({"k": [0], "x": [1]}), tag_columns=["k"]) slot = SourceJobNode(name="s", bound_source=stream) assert slot.last_modified is not None stream._set_modified_time(None) # pyright: ignore[reportPrivateUsage] # an unknown modified time is what this test is about From e5155e34c64f0db7e34875905e293316f0e0a069 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:31:54 -0800 Subject: [PATCH 22/26] test(large_normalization): a raw ArrowTableStream is refused as a job input; job sweeps take every other kind (ITL-1126) Co-Authored-By: Claude Opus 5.5 --- ...1-14-itl-1112-followup-merge-and-cr-392.md | 59 +++++++++++++++++++ tests/test_core/large_types_cases.py | 8 +++ .../operators/test_large_normalization.py | 15 +++-- .../test_large_normalization_sources.py | 27 ++++++--- 4 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md diff --git a/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md new file mode 100644 index 000000000..b3b80e611 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md @@ -0,0 +1,59 @@ +# ITL-1112 / ITL-1126 follow-up: merge with main, CodeRabbit findings from PR #392 + +Branch `eywalker/itl-1112-binding-a-source-with-a-different-column`, PR #386. Head before +this work: 4ffdef06. Main at 810d1b03. + +## 1. Merge origin/main (done: 5c9ac6cd, no conflicts) + +PR #389 (ITL-1122) added a large-normalization sweep whose source kinds include a raw +`ArrowTableStream`; its JOB tests use it as a pipeline input, which ITL-1126 refuses +(developer decision; ITL-1127 adds the wrapper). Fix, tests only: + +- `tests/test_core/operators/test_large_normalization.py`: `test_job_has_no_non_large_type_and_equals_plain` + sweeps every kind except `ArrowTableStream` (the plain-stream sweeps keep it). One new test: + a raw `ArrowTableStream` as a job input raises `SourceSpecMismatchError` naming the remedy + (wrap the table as a source). +- `tests/test_core/sources/test_large_normalization_sources.py`: same for + `test_job_nodes_have_no_non_large_type` (the table/head test keeps it), and the refusal test. +- Nothing else changes in either file; every other kind keeps its #389 assertions. + +## 2. CodeRabbit findings raised on PR #392 against #386's code + +### A (comment 4575906153) — CONFIRMED by reproduction + +A source whose system-tag pair comes record-id first (an `ArrowTableStream` over a table +whose `_tag_record_id::…` column precedes `_tag_source_id::…`) is accepted, and the bound +node's table (and zero-row table) keeps record-id first while the unbound node's +`output_schema(all_info=True)` predicts source-id first. With an exact-schema source +(`_declared_form_for` returns `None`) the node passes the source through, so even its +`output_schema` reports the reversed order: the bind changed the node's schema order. + +Fix: `_declared_form_for` also requires the system-tag pair in (source-id, record-id) order +for the pass-through, and `_DeclaredForm._declared_positions` orders the pair as a fourth +group (declared names, after renames), like the tag/data/source-info groups. This agrees with +main's row order (#390/ITL-1131: user tags, system tags, data, source info): the pair keeps +the positions it holds; only its internal order is fixed. Test first: a bound reversed-pair +source gives `as_table`, `head`, the zero-row table and `output_schema` in the unbound +node's order (exact and nullable-slot cases). + +### C (comment 4575906162) — NOT REPRODUCIBLE; propose to decline + +`Pipeline.compile` raises `InconsistentSourceError` for same-named slots whose schemas +differ, so same-named slots have equal declared tag and data schemas (and so equal +system-tag names). `validate`/`_fit` depend only on the name and those schemas, so a +rebuilt source gets the same verdict from every same-named slot: one accepting and the other +refusing cannot happen. Reproduction: two CSV sources with one source_id and differing +schemas are refused at compile; with equal schemas they deduplicate to one slot. + +## 3. Verification + +Targeted: tests/test_pipeline, tests/test_core/{nodes,sources,operators}, golden (3.12; +async files also 3.11), one process. Ruff, basedpyright. One push; CI on the merge. + +## Status notes + +- [x] merge +- [ ] part 1 tests +- [ ] plan for A/C sent to coordinator +- [ ] A fix (after approval) +- [ ] push, CI, CodeRabbit, replies on #392 threads diff --git a/tests/test_core/large_types_cases.py b/tests/test_core/large_types_cases.py index d7a67332c..0d5ae54b8 100644 --- a/tests/test_core/large_types_cases.py +++ b/tests/test_core/large_types_cases.py @@ -274,6 +274,14 @@ def _cached_source_delta(n_rows: int, tmp: Path) -> StreamProtocol: (name, n_rows) for name, (_, counts) in SOURCES.items() for n_rows in counts ] +# A raw ``ArrowTableStream`` has no system-tag columns, so it cannot be a pipeline's +# input (ITL-1126; binding one is ITL-1127): the job sweeps take every other kind, and +# ``test_large_normalization_sources`` asserts the refusal. +RAW_STREAM = "ArrowTableStream" +JOB_SOURCE_CASES = [ + (name, n_rows) for name, n_rows in SOURCE_CASES if name != RAW_STREAM +] + def build_source(name: str, n_rows: int, tmp: Path) -> StreamProtocol: """Build the source kind ``name`` with ``n_rows`` rows, its files under ``tmp``.""" diff --git a/tests/test_core/operators/test_large_normalization.py b/tests/test_core/operators/test_large_normalization.py index 5abc10c86..8fc110508 100644 --- a/tests/test_core/operators/test_large_normalization.py +++ b/tests/test_core/operators/test_large_normalization.py @@ -44,6 +44,7 @@ from tests.test_core.large_types_cases import ( ALL_COLUMNS, FULL_SCHEMA, + JOB_SOURCE_CASES, ROWS, SOURCE_CASES, SOURCES, @@ -98,11 +99,14 @@ def _other(n_rows: int, data_column: str) -> ArrowTableSource: } -def _cases(row_counts: tuple[int, ...]) -> list[tuple[str, str, int]]: - """Every (source kind, operator, rows) with rows among ``row_counts``.""" +def _cases( + row_counts: tuple[int, ...], source_cases: list[tuple[str, int]] = SOURCE_CASES +) -> list[tuple[str, str, int]]: + """Every (source kind, operator, rows) of ``source_cases`` with rows among + ``row_counts``.""" return [ (source, operator, n_rows) - for source, n_rows in SOURCE_CASES + for source, n_rows in source_cases if n_rows in row_counts for operator in OPERATORS ] @@ -154,7 +158,10 @@ def test_zero_row_schema_equals_populated( @pytest.mark.parametrize("orchestrator", ["sync", "async"]) -@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2, 0))) +# Every kind a job takes as input; a raw ArrowTableStream is refused (ITL-1126). +@pytest.mark.parametrize( + ("source", "operator", "n_rows"), _cases((2, 0), JOB_SOURCE_CASES) +) def test_job_has_no_non_large_type_and_equals_plain( source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path ) -> None: diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py index a878c41ce..2bb515851 100644 --- a/tests/test_core/sources/test_large_normalization_sources.py +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -26,6 +26,7 @@ ) from orcapod.core.streams import ArrowTableStream from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError from orcapod.operators import SelectTagColumns from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob @@ -37,6 +38,8 @@ from tests.test_core.large_types_cases import ( ALL_COLUMNS, FULL_SCHEMA, + JOB_SOURCE_CASES, + RAW_STREAM, ROWS, SOURCE_CASES, TAGS, @@ -71,13 +74,23 @@ def test_table_and_head_have_no_non_large_type( assert non_large_columns(table) == [] assert non_large_columns(source.head(columns=ALL_COLUMNS).table) == [] - @pytest.mark.parametrize("orchestrator", ["sync", "async"]) - def test_job_nodes_have_no_non_large_type( - self, name: str, n_rows: int, orchestrator: str, tmp_path: Path - ) -> None: - tables = _job_node_tables(build_source(name, n_rows, tmp_path), orchestrator) - for table in tables: - assert non_large_columns(table) == [] + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize(("name", "n_rows"), JOB_SOURCE_CASES) +def test_job_nodes_have_no_non_large_type( + name: str, n_rows: int, orchestrator: str, tmp_path: Path +) -> None: + tables = _job_node_tables(build_source(name, n_rows, tmp_path), orchestrator) + for table in tables: + assert non_large_columns(table) == [] + + +@pytest.mark.parametrize("n_rows", [2, 0]) +def test_raw_stream_is_refused_as_a_job_input(n_rows: int, tmp_path: Path) -> None: + """A raw ``ArrowTableStream`` has no system tags, so a job refuses it as an input + (ITL-1126) and says to wrap its table as a source; binding one is ITL-1127.""" + with pytest.raises(SourceSpecMismatchError, match="wrap its table as a source"): + _job_node_tables(build_source(RAW_STREAM, n_rows, tmp_path), "sync") def test_explicit_arrow_schema_is_normalized(tmp_path: Path) -> None: From ffcdcf654846f702dbab1754105553df514bf1eb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:37:47 -0800 Subject: [PATCH 23/26] fix(source_node): a bound source's system-tag pair is given source id first (ITL-1112) A source whose pair came record-id first was accepted and kept that order in the node's table, against the order every node predicts. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +- CLAUDE.md | 4 +- changelog.d/ITL-1112.md | 10 +- src/orcapod/core/nodes/source_node.py | 31 +++-- ...1-14-itl-1112-followup-merge-and-cr-392.md | 6 +- .../test_bind_keeps_node_schemas.py | 114 ++++++++++++++++++ 6 files changed, 151 insertions(+), 18 deletions(-) diff --git a/.zed/rules b/.zed/rules index 09759cc33..fc91d6f1b 100644 --- a/.zed/rules +++ b/.zed/rules @@ -827,8 +827,8 @@ data. The rules: else raises SourceSpecMismatchError before anything is bound. The source's rows are given as the declared schema at one place, _DeclaredForm (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, the VALUES the source's - own). Limits: the check compares Python types, so a difference in Arrow width only + declared schema hash, taken from the source's own pair by prefix, in the declared order + (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/CLAUDE.md b/CLAUDE.md index 8b8fe2ae1..71ba41269 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -869,8 +869,8 @@ data. The rules: else raises `SourceSpecMismatchError` before anything is bound. The source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, the VALUES the source's - own). Limits: the check compares Python types, so a difference in Arrow width only + declared schema hash, taken from the source's own pair by prefix, in the declared order + (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 4932307a6..2ad4b2651 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -65,8 +65,9 @@ source. Binding such a stream properly is planned (ITL-1127). stored path or the column names, types, nullable flags or order of any node's or stored table. A source whose columns come in another order than the slot declares gives them in the declared order, and a stream bound to a slot (a source passed through a filter or a - rename, with one system-tag pair) gives its system-tag columns the slot's names; their - values stay the stream's own. Both passed through as they were before. + rename, with one system-tag pair) gives its system-tag columns the slot's names, source id + before record id; their values stay the stream's own. Both passed through as they were + before. ### Fixed @@ -80,3 +81,8 @@ source. Binding such a stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). +- A stream bound to a slot that is not a source (a function pod's stream, with one system-tag + pair) keeps its own column layout in the source node's own table: its system-tag columns + before its data, and its own nullable flags for `_content_hash` and the source-info + columns. That table is not stored and no downstream node sees it: stored paths, stored + table schemas and every downstream node's table are those of a source bind. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index dcbe39ba8..8209bac19 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -79,8 +79,9 @@ class _DeclaredForm: The pipeline determines every node's schema (ITL-1112). A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows as rows of ``x: int | None``: the same arrays under nullable fields. The source's system-tag - columns take the slot's names (their values stay the source's own), and its tag and - data columns, with their source-info columns, the declared order. Nothing else + columns take the slot's names and order, source id then record id (their values stay + the source's own), and its tag and data columns, with their source-info columns, the + declared order. Nothing else differs: values, types, source-info tokens and meta columns pass unchanged, so the rows equal those of a source of the declared schema with the same source id. @@ -89,12 +90,14 @@ class _DeclaredForm: renames: The source's system-tag column names, mapped to the declared ones. tag_order: The declared tag columns, in declared order. data_order: The declared data columns, in declared order. + system_tag_order: The declared system-tag columns, source id then record id. """ nullable: frozenset[str] renames: Mapping[str, str] tag_order: tuple[str, ...] data_order: tuple[str, ...] + system_tag_order: tuple[str, str] def apply(self, table: pa.Table) -> pa.Table: """Return *table*'s arrays under the declared fields and system-tag names. @@ -107,8 +110,9 @@ def apply(self, table: pa.Table) -> pa.Table: Returns: The same arrays, with the columns in ``nullable`` flagged nullable, the - columns in ``renames`` renamed, and the tag, data and source-info columns in - declared order within the positions each group holds. + columns in ``renames`` renamed, and the tag, system-tag, data and + source-info columns in declared order within the positions each group + holds. """ schema = table.schema for position, field in enumerate(table.schema): @@ -124,14 +128,15 @@ def apply(self, table: pa.Table) -> pa.Table: def _declared_positions(self, names: list[str]) -> list[int]: """Return the column positions of *names* with each group in declared order. - The tag columns, the data columns and the data columns' source-info columns each - fill the positions their group holds, in declared order; every other column - keeps its position. + The tag columns, the system-tag pair, the data columns and the data columns' + source-info columns each fill the positions their group holds, in declared + order; every other column keeps its position. """ position = {name: index for index, name in enumerate(names)} order = list(range(len(names))) source_info = tuple(f"{constants.SOURCE_PREFIX}{n}" for n in self.data_order) - for group in (self.tag_order, self.data_order, source_info): + groups = (self.tag_order, self.system_tag_order, self.data_order, source_info) + for group in groups: present = [name for name in group if name in position] for slot, name in zip( sorted(position[n] for n in present), present, strict=True @@ -879,7 +884,8 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: Returns: ``None`` when the source's rows already are rows of the declared schema - (same types, column order and system-tag names), so they pass unchanged; + (same types, column order and system-tag names and order), so they pass + unchanged; otherwise the ``_DeclaredForm`` that makes them so. Raises: @@ -901,11 +907,17 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: } tag_order = tuple(self._tag_schema.keys()) data_order = tuple(self._data_schema.keys()) + source_system_tags = [ + name + for name in source.keys(columns={"system_tags": True})[0] + if name not in source_tag + ] if ( not narrower and not renames and tuple(source_tag.keys()) == tag_order and tuple(source_data.keys()) == data_order + and source_system_tags == [declared_source_id, declared_record_id] ): return None return _DeclaredForm( @@ -913,6 +925,7 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: renames=renames, tag_order=tag_order, data_order=data_order, + system_tag_order=(declared_source_id, declared_record_id), ) def _declared_rows( diff --git a/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md index b3b80e611..c018aad15 100644 --- a/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md +++ b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md @@ -53,7 +53,7 @@ async files also 3.11), one process. Ruff, basedpyright. One push; CI on the mer ## Status notes - [x] merge -- [ ] part 1 tests -- [ ] plan for A/C sent to coordinator -- [ ] A fix (after approval) +- [x] part 1 tests (e5155e34) +- [x] plan for A/C sent to coordinator: A approved, C decline approved; layout-by-kind: option 1 (known limit in PR body + changelog; coordinator files the issue) +- [x] A fix + tests - [ ] push, CI, CodeRabbit, replies on #392 threads diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index 42931bcf7..2f972f85b 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -24,8 +24,10 @@ from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream from orcapod.databases import DeltaTableDatabase from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.system_constants import constants from orcapod.types import Schema @@ -54,6 +56,22 @@ def _csv(tmp_path: pathlib.Path, name: str, text: str) -> CSVSource: return CSVSource(str(path), tag_columns=["id"], source_id=name) +def _record_id_first(source: ArrowTableSource) -> ArrowTableStream: + """Return a stream of *source*'s rows with its system-tag pair record-id first. + + The pair swaps places; every other column keeps its position. + """ + table = source.as_table( + columns={"system_tags": True, "source": True, "context": True} + ) + names = table.column_names + pair = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] + assert len(pair) == 2 and constants.SYSTEM_TAG_SOURCE_ID_PREFIX in pair[0] + swap = {pair[0]: pair[1], pair[1]: pair[0]} + reordered = table.select([swap.get(name, name) for name in names]) + return ArrowTableStream(reordered, tag_columns=["id"]) + + NODES = ("src", "renamed", "inc") # A loaded job reads stored results only; its uncached operator has none to read. LOADED_NODES = ("src", "inc") @@ -122,6 +140,16 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( DictSource([{"id": 10, "q": 10}], tag_columns=["id"], source_id="m") ), ), + ( + "system-tag pair record-id first", + _assign, + _record_id_first(_arrow([11, 12], [11, None], x_nullable=True)), + ), + ( + "narrower, system-tag pair record-id first", + _bind, + _record_id_first(_arrow([13], [13], x_nullable=False)), + ), ("narrower csv", _bind, _csv(tmp_path, "b.csv", "id,x\n9,9\n")), ] first: dict[str, Any] | None = None @@ -201,3 +229,89 @@ def test_a_structural_class_of_the_same_identity_changes_no_path_or_schema( snapshots.append(_snapshot(job, ("src", "renamed"))) assert snapshots[1] == snapshots[0] assert snapshots[2] == snapshots[0] + + +def test_a_record_id_first_pair_is_given_in_the_declared_order() -> None: + """A source whose system-tag pair comes record-id first is given source-id first. + + The order every node predicts (``output_schema``) and a source of the declared order + has; in the schema, the table, the head and the zero-row table, for a slot of the + source's own types (rows passed as they are otherwise) and for a ``T | None`` slot + the source is narrower than. + """ + for x_type, x_nullable in ((int, False), (int | None, True)): + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": x_type}), + ) + job = PipelineJob(name="j") + with job: + _ = MapData({"x": "y"})(slot, label="renamed") + node = job.nodes["src"] + declared = list(node.output_schema(all_info=True)[0]) + reference = _arrow([0], [0], x_nullable=x_nullable) + job.bind(sources={"src": reference}) + order = node.as_table(all_info=True).column_names + zero_row_order = node.head(0, all_info=True).table.column_names + + for rows in ([1, 2], []): + # A non-null ``x``: the slot's own type, or narrower than ``int | None``. + xs: list[int | None] = list(rows) + source = _record_id_first(_arrow(rows, xs, x_nullable=False)) + job.bind(sources={"src": source}) + assert list(node.output_schema(all_info=True)[0]) == declared, x_type + assert node.as_table(all_info=True).column_names == order, x_type + assert node.head(all_info=True).table.column_names == ( + order if rows else zero_row_order + ), x_type + + +@function_pod(output_keys="x") +def passes_on(w: int | None) -> int | None: + """Return ``w`` unchanged.""" + return w + + +def test_a_bound_function_stream_moves_no_stored_or_downstream_schema( + tmp_path: pathlib.Path, +) -> None: + """A function pod's stream (one system-tag pair) moves no path, stored or downstream schema. + + Known limit: the source node's own table keeps the bound stream's column layout (a + function pod's stream gives its system tags before its data) and the nullable flags + of its ``_content_hash`` and source-info columns. That table is not stored and no + downstream node sees it; its columns, and its user and system-tag fields, are the + same. + """ + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int | None}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = inc.pod(MapData({"x": "y"})(slot, label="renamed"), label="inc") + _bind(job, _arrow([0, 1], [1, None], x_nullable=True)) + assert job.run().ok + first = _snapshot(job) + + upstream = MapData({"x": "w"})(_arrow([5, 6], [5, None], x_nullable=True)) + _bind(job, passes_on.pod(upstream)) + assert job.run().ok + second = _snapshot(job) + assert second["paths"] == first["paths"] + assert second["stored"] == first["stored"] + for name in ("renamed", "inc"): + assert second["nodes"][name] == first["nodes"][name], name + src_first, src_second = first["nodes"]["src"], second["nodes"]["src"] + assert sorted(src_second.names) == sorted(src_first.names) + compared = [ + name + for name in src_first.names + if name in ("id", "x", constants.CONTEXT_KEY) + or name.startswith(constants.SYSTEM_TAG_PREFIX) + ] + assert len(compared) == 5 + for name in compared: + assert src_second.field(name) == src_first.field(name), name From f2aaf91f2455ff58f7c28ac0f0f1a0dc4abba00c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:48:11 -0800 Subject: [PATCH 24/26] docs(source_node): column order is not part of the bind invariant (ITL-1112) Developer correction, November 14, 2026: the invariant is each node's path and its columns (names, roles, types, nullability, system-tag names). The invariant test compares columns by name; the declared-order presentation is a convenience with its own tests; columns match by name within the tag and data groups, and the role decides. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 +++-- CLAUDE.md | 13 +++-- changelog.d/ITL-1112.md | 23 ++++----- design_issues/ITL-1112-BT1.md | 7 +-- src/orcapod/core/nodes/source_node.py | 7 +-- .../test_bind_keeps_node_schemas.py | 50 +++++++++++-------- .../test_bind_narrower_source.py | 6 ++- tests/test_pipeline/test_bind_type_check.py | 35 +++++++++++++ 8 files changed, 103 insertions(+), 51 deletions(-) diff --git a/.zed/rules b/.zed/rules index fc91d6f1b..fb455287d 100644 --- a/.zed/rules +++ b/.zed/rules @@ -818,17 +818,20 @@ data. The rules: - ArrowTableSource raises ValueError if any tag_columns are not in the table. - The pipeline determines every node's schema; a bound source never changes it (developer, November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the - schema of any node's or stored table (column names with system tags, types, nullable - flags, order); only values differ. SourceNodeBase.validate is the one check, used by + columns of any node's or stored table (names, tag or data roles, types, nullable flags, + system-tag column names); only values differ. Column order is not part of it: matching a + source to a slot, columns match by name within the tag group and within the data group, + and their order never decides acceptance (developer, November 14, 2026). Do not reorder + columns without a reason. SourceNodeBase.validate is the one check, used by PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and PipelineJob.load: per column the declared Python type (structural classes of one identity are one type, equivalent_hints), or T where the slot declares T | None; and exactly one source-id (str) and one record-id (bytes) system-tag column, no other. Anything else raises SourceSpecMismatchError before anything is bound. The source's rows are given as the declared schema at one place, _DeclaredForm (nullable - fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, in the declared order - (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only + fields over the same arrays; the system-tag column NAMES of the declared schema hash, + taken from the source's own pair by prefix, the VALUES the source's own; as a stable + convenience, not a guarantee, the declared column order, source id before record id). Limits: the check compares Python types, so a difference in Arrow width only (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/CLAUDE.md b/CLAUDE.md index 71ba41269..3d84ae935 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -860,17 +860,20 @@ data. The rules: - `ArrowTableSource.__init__` raises `ValueError` if any `tag_columns` are not in the table. - The pipeline determines every node's schema; a bound source never changes it (developer, November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the - schema of any node's or stored table (column names with system tags, types, nullable - flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by + columns of any node's or stored table (names, tag or data roles, types, nullable flags, + system-tag column names); only values differ. Column order is not part of it: matching a + source to a slot, columns match by name within the tag group and within the data group, + and their order never decides acceptance (developer, November 14, 2026). Do not reorder + columns without a reason. `SourceNodeBase.validate` is the one check, used by `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and `PipelineJob.load`: per column the declared Python type (structural classes of one identity are one type, `equivalent_hints`), or `T` where the slot declares `T | None`; and exactly one source-id (`str`) and one record-id (`bytes`) system-tag column, no other. Anything else raises `SourceSpecMismatchError` before anything is bound. The source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable - fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, in the declared order - (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only + fields over the same arrays; the system-tag column NAMES of the declared schema hash, + taken from the source's own pair by prefix, the VALUES the source's own; as a stable + convenience, not a guarantee, the declared column order, source id before record id). Limits: the check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 2ad4b2651..8405269c3 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -38,8 +38,8 @@ source's types, which gives it its own stored paths. A slot's rows carry one source-id and one record-id system-tag column, under the slot's names. A stream with one pair per input (the result of a `Join` or `MergeJoin`), or with none (an `ArrowTableStream` over a table without them), cannot give its rows that way; it was bound -before with its own system-tag columns and column order, so the schema of the slot's table, -and of every table after it, depended on what was bound. Every way of binding (`bind`, the +before with its own system-tag columns, so the system-tag columns of the slot's table, and +of every table after it, depended on what was bound. Every way of binding (`bind`, the `bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever its column types; so is a stream with any other system-tag column, or whose source id is not @@ -62,12 +62,14 @@ source. Binding such a stream properly is planned (ITL-1127). predicted schema is the declared one. The source node's `content_hash` and source id stay the bound source's own. - The pipeline determines every node's schema (ITL-1112): no accepted bind changes a node's - stored path or the column names, types, nullable flags or order of any node's or stored - table. A source whose columns come in another order than the slot declares gives them in - the declared order, and a stream bound to a slot (a source passed through a filter or a - rename, with one system-tag pair) gives its system-tag columns the slot's names, source id - before record id; their values stay the stream's own. Both passed through as they were - before. + stored path or the columns of any node's or stored table (names, tag or data roles, types, + nullable flags, system-tag column names). A stream bound to a slot (a source passed + through a filter or a rename, with one system-tag pair) gives its system-tag columns the + slot's names; their values stay the stream's own. It passed through as it was before. + Column order is not part of that: a source matches its slot by column name within the tag + group and within the data group, whatever their order. As a convenience, a source whose + columns come in another order is shown in the slot's declared order, its system tags + source id first. ### Fixed @@ -81,8 +83,3 @@ source. Binding such a stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A stream bound to a slot that is not a source (a function pod's stream, with one system-tag - pair) keeps its own column layout in the source node's own table: its system-tag columns - before its data, and its own nullable flags for `_content_hash` and the source-info - columns. That table is not stored and no downstream node sees it: stored paths, stored - table schemas and every downstream node's table are those of a source bind. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 0a42f71c3..59e10e395 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -31,9 +31,10 @@ hash to the declared one (their names carry the schema hash). `iter_data`, `asyn `as_table`, `head` and `output_schema` go through it, so the rows equal those of a nullable source with the same values and source id, row hashes and record ids included, and both append to one stored table. The node's `content_hash` and source id stay the source's own. The same -form gives a source's columns in the declared order, and a stream's one system-tag pair the -slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a -node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) +form gives a stream's one system-tag pair the slot's names (taken from the stream's own +columns by prefix), so no accepted bind changes a node's path or its columns (names, roles, +types, nullable flags, system-tag names; column order is not part of that, and the form's +declared order is a convenience). Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8209bac19..f11ca368f 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -79,9 +79,10 @@ class _DeclaredForm: The pipeline determines every node's schema (ITL-1112). A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows as rows of ``x: int | None``: the same arrays under nullable fields. The source's system-tag - columns take the slot's names and order, source id then record id (their values stay - the source's own), and its tag and data columns, with their source-info columns, the - declared order. Nothing else + columns take the slot's names (their values stay the source's own). As a stable + convenience, not a guarantee (column order is not part of a node's schema), the + columns come in the declared order: tags, data and source info as declared, the + system tags source id first. Nothing else differs: values, types, source-info tokens and meta columns pass unchanged, so the rows equal those of a source of the declared schema with the same source id. diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index 2f972f85b..641cbd17d 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -1,12 +1,14 @@ -"""No accepted bind changes a node's stored path or table schema (ITL-1112). +"""No accepted bind changes a node's stored path or its columns (ITL-1112). Developer, November 14, 2026: "for pipeline, all node's path/table schema should NOT change regardless of what source you bind to the source nodes in the pipeline". One job is run over every kind of accepted bind: a source of the declared schema, a narrower source into a ``T | None`` slot, sources of different kinds, through ``bind``, the ``bound_source`` setter and ``PipelineJob.load``. After each, the stored table paths, the Arrow schema of every stored -table and of every node's ``as_table(all_info=True)`` (column names with system tags, types, -nullable flags, order) are those of the first bind: only values differ. +table and of every node's ``as_table(all_info=True)`` (each column by name, system tags +included, with its type and nullable flag) are those of the first bind: only values differ. +Column order is not part of that (developer, November 14, 2026), so the snapshots map names to +fields; the order a slot presents is tested on its own (``test_order_convenience_*``). The bind check compares Python types, so two Arrow widths of one Python type (``int32`` and ``int64``) are not covered here (ITL-949). @@ -77,8 +79,13 @@ def _record_id_first(source: ArrowTableSource) -> ArrowTableStream: LOADED_NODES = ("src", "inc") +def _fields(schema: pa.Schema) -> dict[str, Any]: + """Return *schema*'s fields by name: the columns, whatever their order.""" + return {field.name: field for field in schema} + + def _snapshot(job: PipelineJob, nodes: tuple[str, ...] = NODES) -> dict[str, Any]: - """Return the stored paths, stored table schemas and *nodes*' table schemas of *job*.""" + """Return the stored paths, and the columns of the stored tables and *nodes*' tables.""" database = job.database assert database is not None paths = sorted(database.list_table_paths(())) @@ -86,12 +93,13 @@ def _snapshot(job: PipelineJob, nodes: tuple[str, ...] = NODES) -> dict[str, Any for path in paths: table = database.get_all_records(path) assert table is not None - stored[path] = table.schema + stored[path] = _fields(table.schema) return { "paths": paths, "stored": stored, "nodes": { - name: job.nodes[name].as_table(all_info=True).schema for name in nodes + name: _fields(job.nodes[name].as_table(all_info=True).schema) + for name in nodes }, } @@ -231,13 +239,13 @@ def test_a_structural_class_of_the_same_identity_changes_no_path_or_schema( assert snapshots[2] == snapshots[0] -def test_a_record_id_first_pair_is_given_in_the_declared_order() -> None: - """A source whose system-tag pair comes record-id first is given source-id first. +def test_order_convenience_a_record_id_first_pair_is_shown_source_id_first() -> None: + """A source whose system-tag pair comes record-id first is shown source-id first. - The order every node predicts (``output_schema``) and a source of the declared order - has; in the schema, the table, the head and the zero-row table, for a slot of the - source's own types (rows passed as they are otherwise) and for a ``T | None`` slot - the source is narrower than. + A convenience, not part of the bind invariant (column order is not): the slot shows + the order it predicts (``output_schema``) and a source of the declared order has, in + the schema, the table, the head and the zero-row table, for a slot of the source's + own types and for a ``T | None`` slot the source is narrower than. """ for x_type, x_nullable in ((int, False), (int | None, True)): slot = SourceNode( @@ -276,13 +284,13 @@ def passes_on(w: int | None) -> int | None: def test_a_bound_function_stream_moves_no_stored_or_downstream_schema( tmp_path: pathlib.Path, ) -> None: - """A function pod's stream (one system-tag pair) moves no path, stored or downstream schema. + """A function pod's stream (one system-tag pair) moves no path or column. - Known limit: the source node's own table keeps the bound stream's column layout (a - function pod's stream gives its system tags before its data) and the nullable flags - of its ``_content_hash`` and source-info columns. That table is not stored and no - downstream node sees it; its columns, and its user and system-tag fields, are the - same. + Stored paths, stored tables' columns and every downstream node's columns are those of + a source bind, as are the source node's user, system-tag and context columns. The + source node's own table keeps the nullable flags the stream gives its + ``_content_hash`` and source-info columns; that table is not stored and no downstream + node sees those flags. """ slot = SourceNode( name="src", @@ -305,13 +313,13 @@ def test_a_bound_function_stream_moves_no_stored_or_downstream_schema( for name in ("renamed", "inc"): assert second["nodes"][name] == first["nodes"][name], name src_first, src_second = first["nodes"]["src"], second["nodes"]["src"] - assert sorted(src_second.names) == sorted(src_first.names) + assert src_second.keys() == src_first.keys() compared = [ name - for name in src_first.names + for name in src_first if name in ("id", "x", constants.CONTEXT_KEY) or name.startswith(constants.SYSTEM_TAG_PREFIX) ] assert len(compared) == 5 for name in compared: - assert src_second.field(name) == src_first.field(name), name + assert src_second[name] == src_first[name], name diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 030569b0b..6201ff4a6 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -423,7 +423,11 @@ def _abc_job(a_type: Any) -> PipelineJob: @pytest.mark.parametrize("a_type", [int, int | None], ids=["equal", "narrower"]) def test_columns_follow_the_declared_order(a_type: Any) -> None: - """A source with its columns in another order gives them in the declared order.""" + """A source with its columns in another order is shown in the declared order. + + A convenience, not part of the bind invariant: column order is not (developer, + November 14, 2026). + """ job = _abc_job(a_type) job.bind( sources={ diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py index b5ab4d84d..106e089bb 100644 --- a/tests/test_pipeline/test_bind_type_check.py +++ b/tests/test_pipeline/test_bind_type_check.py @@ -184,3 +184,38 @@ def test_an_accepted_rebind_runs_and_stores_the_new_rows( _ = job.run() # Results of the first source (x 1, 2) would be [1, 2]. assert sorted(job.nodes["const"].as_table().column("z").to_pylist()) == [10, 11] + + +def _two_by_two_slot_job() -> PipelineJob: + """Return a job whose slot declares tags ``a``, ``b`` and data ``x``, ``y``, in order.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"a": int, "b": int}), + data_schema=Schema({"x": int, "y": int}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "x2"})(slot, label="renamed") + return job + + +def test_columns_match_by_name_within_each_group_whatever_their_order() -> None: + """Tags and data in another order than declared are accepted (developer, November 14, + 2026): within the tag group and the data group, columns match by name.""" + job = _two_by_two_slot_job() + reordered = DictSource( + [{"y": 2, "b": 1, "x": 3, "a": 0}], tag_columns=["b", "a"], source_id="r" + ) + job.bind(sources={"src": reordered}) + assert job.run().ok + + +def test_a_column_in_the_other_role_is_refused() -> None: + """A column that is a tag on one side and data on the other is refused: the role + (tag or data) is part of the match, its position is not.""" + job = _two_by_two_slot_job() + swapped = DictSource( + [{"a": 0, "b": 1, "x": 3, "y": 2}], tag_columns=["a", "x"], source_id="s" + ) + with pytest.raises(SourceSpecMismatchError, match="tag columns"): + job.bind(sources={"src": swapped}) From fd1278d3461e386b4c2f879b29bb6dce9f95b260 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 15:27:19 -0800 Subject: [PATCH 25/26] fix(source_node): a slot shows its declared types for a structural class of the declared identity (ITL-1112) After #391 a structural class hashes by its identity, so another class object of the declared identity gives the slot's system-tag names and was passed through, showing its own class in output_schema. Pass through only when the source's types are exactly the declared type objects. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index f11ca368f..9432c84de 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -885,8 +885,8 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: Returns: ``None`` when the source's rows already are rows of the declared schema - (same types, column order and system-tag names and order), so they pass - unchanged; + (the same type objects, column order and system-tag names and order), so + they pass unchanged; otherwise the ``_DeclaredForm`` that makes them so. Raises: @@ -913,8 +913,11 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: for name in source.keys(columns={"system_tags": True})[0] if name not in source_tag ] + # Exactly the declared types: a structural class of the declared identity is + # another class object, and the slot shows the declared one. if ( - not narrower + dict(source_tag) == dict(self._tag_schema) + and dict(source_data) == dict(self._data_schema) and not renames and tuple(source_tag.keys()) == tag_order and tuple(source_data.keys()) == data_order From af162ef1596ae4e9adf7d90d47635cf3646d1720 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 15:33:47 -0800 Subject: [PATCH 26/26] test(node_view): bind source slots to streams over a source, not a raw stream (ITL-1126) #388's load-only refusal tests bound streams over a raw ArrowTableStream, which a source slot refuses since ITL-1126. The base stream is a source of the same columns and types; every assertion is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_node_view_review_fixes.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index b3d90e9f9..05f6de37f 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -111,10 +111,14 @@ def flaky(x: int) -> int: def _base_stream() -> Any: - """Two rows, tag ``id`` and data ``x``, both non-nullable ``int64``.""" + """Two rows, tag ``id`` and data ``x``, both non-nullable ``int64``. + + A source, not a raw ``ArrowTableStream``: the streams built over it are bound to a + source slot, which needs one system-tag pair (ITL-1126). + """ import pyarrow as pa - from orcapod.core.streams import ArrowTableStream + from orcapod.core.sources import ArrowTableSource schema = pa.schema( [ @@ -122,8 +126,10 @@ def _base_stream() -> Any: pa.field("x", pa.int64(), nullable=False), ] ) - return ArrowTableStream( - pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"] + return ArrowTableSource( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), + tag_columns=["id"], + source_id="base", )