From 227f1b6c333818865700ba7f84152b883efcfa6e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:55:46 -0800 Subject: [PATCH 01/33] test(pipeline): bind refuses a source whose column types differ (failing, ITL-1112) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-1112-bind-refuses-type.md | 72 +++++++ tests/test_pipeline/test_bind_type_check.py | 181 ++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md create mode 100644 tests/test_pipeline/test_bind_type_check.py diff --git a/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md new file mode 100644 index 000000000..223ca6415 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md @@ -0,0 +1,72 @@ +# ITL-1112 bind refuses a source whose column types differ — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `PipelineJob.bind(sources=...)` and the `SourceJobNode.bound_source` setter refuse a +source whose column types differ from the slot's declared schema, so an accepted bind never +moves a pipeline hash and never sends rows of another type to a stored path. + +**Architecture:** `SourceNodeBase.validate` (the node owns the declared schema and its name) +compares, column by column, the source's `output_schema()` with the declared tag and data +schemas: names, role (tag/data), Python type. `bind` validates every slot before it changes +anything (sources and database); the setter calls the same `validate` before assigning. +`SourceSpecMismatchError` becomes a subclass of `InputValidationError` and `ValueError` (E1). + +**Tech Stack:** Python, pytest, `uv run`. + +--- + +## Decisions (coordinator, 2026-11-14) + +- Developer: "Definitely refuse". No recompute of downstream identities. +- A1: compare names, roles, Python types (the pipeline-hash inputs); `optional_fields` and + column order are not compared. +- E1: `SourceSpecMismatchError(InputValidationError, ValueError)` for name and type mismatches. +- Move `self._database = database` in `bind` after source validation. +- Setter calls `validate` (setting `None` unbinds, unchecked). +- **Nullability (T vs T | None): PENDING with the developer.** No tests or code may fix either + answer until the coordinator replies. Tests use int vs str only. +- ITL-949 (int32 vs int64) out of scope. + +## Reproduction (main a9b50350) + +Scratchpad scripts `repro.py`, `repro3.py`: slot `src` (id:int, x:int) -> MapData(x->y) -> +`const(y: int) -> int`. Binding a str source is accepted by bind and the setter, fresh and +loaded job; pipeline hashes unchanged (src 70bf0c37, renamed e777d53b); a run sends str values +into the int-declared paths (`schema:f289a35f.../pdb_v2`, `_result/.../rdb_v2`). + +## Files + +- Modify: `src/orcapod/errors.py` (`SourceSpecMismatchError` bases) +- Modify: `src/orcapod/core/nodes/source_node.py` (`validate`, `bound_source` setter) +- Modify: `src/orcapod/pipeline/job.py` (`bind`: database assignment after validation; docstring) +- Test: `tests/test_pipeline/test_bind_type_check.py` (new) +- Add: `changelog.d/ITL-1112.md`, `design_issues/ITL-1112-BT1.md` +- Docs: `docs/concepts/function-pods.md` near line 783 (one sentence on bind) + +## Tasks + +- [ ] 1. Failing tests (`tests/test_pipeline/test_bind_type_check.py`), public interface: + - bind with x: str in place of x: int raises `SourceSpecMismatchError`, which is an + `InputValidationError`; message names slot `src`, column `x`, `int` and `str`. + - same through `job.nodes["src"].bound_source = ...`. + - both on a fresh job and on a job from `PipelineJob.load`. + - after the refusal the previous binding is kept (`job.nodes["src"].bound_source is old`), + and `bind(database=new, sources={bad})` leaves `job.database` unchanged. + - a tag column given as str for a tag declared int is refused too. + - after an accepted rebind (same types, other values/source id) every node's + `pipeline_hash()` equals its value before the bind (fresh and loaded). + - an accepted bind still works and the run stores rows. + Run: `uv run pytest tests/test_pipeline/test_bind_type_check.py -v` -> FAIL. +- [ ] 2. Implement: errors.py bases; `validate` compares types after names; setter calls + `validate`; `bind` moves database assignment. Run tests -> PASS. Commit. +- [ ] 3. Nullability: after the coordinator's reply, add tests and the rule. +- [ ] 4. Changelog fragment (Breaking changes and migration), design_issues entry, docs + sentence; `uv run python scripts/assemble_changelog.py --check`, + `uv run python scripts/design_issues.py --check`. +- [ ] 5. ruff, basedpyright, full `uv run pytest tests/`; merge origin/main; push; PR. + +## Notes for a successor + +- Conflicts: PR #384 (ITL-1102) edits the `bound_source` setter (adds `_update_modified_time()`); + ITL-1111 adds a memo clear inside `bind`. Whoever merges second resolves. diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py new file mode 100644 index 000000000..77fc8ad37 --- /dev/null +++ b/tests/test_pipeline/test_bind_type_check.py @@ -0,0 +1,181 @@ +"""Binding a source whose column types differ from the slot's is refused (ITL-1112). + +A source slot's declared tag and data schemas are what every downstream pipeline hash is +built from, and the pipeline hash scopes the stored table paths. A source whose columns have +other types would send rows of those types to the paths of the declared ones, so both ways +of binding refuse it: ``PipelineJob.bind(sources=...)`` and the source node's +``bound_source`` setter, on a fresh job and on a loaded one. An accepted bind never moves a +pipeline hash. +""" + +from __future__ import annotations + +import pathlib +from collections.abc import Callable + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import MapData +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError, SourceSpecMismatchError +from orcapod.sources import DictSource +from orcapod.types import Schema + + +@function_pod(output_keys="z") +def const(y: int) -> int: + """Return 0 for any *y*.""" + return 0 + + +def _ints(source_id: str = "ints", start: int = 1) -> DictSource: + """Return a source with tag ``id: int`` and data ``x: int``.""" + return DictSource( + [{"id": 0, "x": start}, {"id": 1, "x": start + 1}], + tag_columns=["id"], + source_id=source_id, + ) + + +def _strs() -> DictSource: + """Return a source with tag ``id: int`` and data ``x: str``.""" + return DictSource( + [{"id": 0, "x": "a"}, {"id": 1, "x": "b"}], tag_columns=["id"], source_id="strs" + ) + + +def _str_tags() -> DictSource: + """Return a source with tag ``id: str`` and data ``x: int``.""" + return DictSource( + [{"id": "a", "x": 1}, {"id": "b", "x": 2}], tag_columns=["id"], source_id="stags" + ) + + +def _fresh_job(tmp_path: pathlib.Path) -> PipelineJob: + """Return a job with slot ``src`` (id: int, x: int) -> MapData -> ``const``, bound to ints.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = const.pod(MapData({"x": "y"})(slot, label="renamed"), label="const") + job.bind(sources={"src": _ints()}) + return job + + +def _loaded_job(tmp_path: pathlib.Path) -> PipelineJob: + """Return the fresh job, run and saved, loaded again from its file.""" + job = _fresh_job(tmp_path) + _ = job.run() + path = tmp_path / "job.json" + job.save(str(path)) + return PipelineJob.load(str(path)) + + +JOBS: dict[str, Callable[[pathlib.Path], PipelineJob]] = { + "fresh": _fresh_job, + "loaded": _loaded_job, +} + + +def _bind(job: PipelineJob, source: DictSource) -> None: + """Bind *source* to slot ``src`` with ``PipelineJob.bind``.""" + job.bind(sources={"src": source}) + + +def _assign(job: PipelineJob, source: DictSource) -> None: + """Bind *source* to slot ``src`` by assigning the node's ``bound_source``.""" + job.nodes["src"].bound_source = source # pyright: ignore[reportAttributeAccessIssue] # nodes is typed by the base node; "src" is a SourceJobNode + + +BINDS: dict[str, Callable[[PipelineJob, DictSource], None]] = { + "bind": _bind, + "bound_source": _assign, +} + + +def _pipeline_hashes(job: PipelineJob) -> dict[str, str]: + """Return every node's pipeline hash, by node label.""" + return {name: node.pipeline_hash().to_string() for name, node in job.nodes.items()} + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_data_column_of_another_type_is_refused( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, DictSource], None], +) -> None: + """A source with ``x: str`` for a slot declaring ``x: int`` is refused, naming both types.""" + job = make_job(tmp_path) + with pytest.raises(SourceSpecMismatchError) as excinfo: + bind(job, _strs()) + assert isinstance(excinfo.value, InputValidationError) + message = str(excinfo.value) + assert "'src'" in message + assert "'x'" in message + assert "int" in message and "str" in message + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_tag_column_of_another_type_is_refused( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, DictSource], None], +) -> None: + """A source with tag ``id: str`` for a slot declaring ``id: int`` is refused.""" + job = make_job(tmp_path) + with pytest.raises(SourceSpecMismatchError, match="'id'"): + bind(job, _str_tags()) + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_refused_source_leaves_the_binding_unchanged( + tmp_path: pathlib.Path, bind: Callable[[PipelineJob, DictSource], None] +) -> None: + """After a refusal the slot keeps its source, its schema and every pipeline hash.""" + job = _fresh_job(tmp_path) + before_source = job.nodes["src"].bound_source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + before_hashes = _pipeline_hashes(job) + with pytest.raises(SourceSpecMismatchError): + bind(job, _strs()) + assert job.nodes["src"].bound_source is before_source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + assert dict(job.nodes["renamed"].output_schema()[1]) == {"y": int} + assert _pipeline_hashes(job) == before_hashes + + +def test_a_refused_bind_leaves_the_database_unchanged(tmp_path: pathlib.Path) -> None: + """``bind(database=..., sources=...)`` with a refused source changes nothing.""" + job = _fresh_job(tmp_path) + database = job.database + with pytest.raises(SourceSpecMismatchError): + job.bind(sources={"src": _strs()}, database=InMemoryArrowDatabase()) + assert job.database is database + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_an_accepted_bind_moves_no_pipeline_hash( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, DictSource], None], +) -> None: + """Binding another source of the declared types leaves every pipeline hash as it was.""" + job = make_job(tmp_path) + before = _pipeline_hashes(job) + bind(job, _ints(source_id="other", start=10)) + assert _pipeline_hashes(job) == before + + +def test_an_accepted_rebind_runs_and_stores_the_new_rows(tmp_path: pathlib.Path) -> None: + """A rebind to a source of the declared types runs and stores its rows.""" + job = _fresh_job(tmp_path) + _ = job.run() + job.bind(sources={"src": _ints(source_id="other", start=10)}) + _ = job.run() + assert job.nodes["const"].as_table().column("z").to_pylist() == [0, 0] From 715b36295dce2a110f78ef80609def3d3c60913e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:59:06 -0800 Subject: [PATCH 02/33] fix(pipeline)!: bind and bound_source refuse a source whose column types differ from the slot's (ITL-1112) SourceNodeBase.validate compares Python types per column after names and roles; the bound_source setter validates; bind validates every source before it changes the database. SourceSpecMismatchError is now an InputValidationError as well as a ValueError. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 66 +++++++++++++++++---------- src/orcapod/errors.py | 16 +++---- src/orcapod/pipeline/job.py | 18 ++++++-- 3 files changed, 62 insertions(+), 38 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8d7cf8923..e98c015de 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -43,6 +43,11 @@ logger = logging.getLogger(__name__) +def _type_text(data_type: Any) -> str: + """Return *data_type* as it is written in an annotation (``int``, ``int | None``).""" + return data_type.__name__ if isinstance(data_type, type) else str(data_type) + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -237,43 +242,45 @@ def keys( # ------------------------------------------------------------------ def validate(self, source: "StreamProtocol") -> None: - """Check that *source* is schema-compatible with this node's declared schema. + """Check that *source* has this slot's declared schema. + + The source's tag and data columns must have the declared names, roles and + Python types (ITL-1112): those are what every downstream pipeline hash, and so + every stored path, is built from, so a source that passes never moves one. + Column order and ``optional_fields`` are not compared; neither enters the + pipeline hash. Args: source: A concrete stream to validate. Raises: - SourceSpecMismatchError: If schema columns don't match. + SourceSpecMismatchError: If a column is missing, unexpected, or of another + type than declared. """ source_tag, source_data = source.output_schema() - tag_issues: list[str] = [] - data_issues: list[str] = [] - - spec_tag_cols = set(self._tag_schema.keys()) - src_tag_cols = set(source_tag.keys()) - if spec_tag_cols != src_tag_cols: - missing = spec_tag_cols - src_tag_cols - extra = src_tag_cols - spec_tag_cols - if missing: - tag_issues.append(f"missing tag columns: {sorted(missing)}") - if extra: - tag_issues.append(f"unexpected tag columns: {sorted(extra)}") - - spec_data_cols = set(self._data_schema.keys()) - src_data_cols = set(source_data.keys()) - if spec_data_cols != src_data_cols: - missing = spec_data_cols - src_data_cols - extra = src_data_cols - spec_data_cols + issues: list[str] = [] + for role, declared, given in ( + ("tag", self._tag_schema, source_tag), + ("data", self._data_schema, source_data), + ): + missing = set(declared.keys()) - set(given.keys()) + extra = set(given.keys()) - set(declared.keys()) if missing: - data_issues.append(f"missing data columns: {sorted(missing)}") + issues.append(f"missing {role} columns: {sorted(missing)}") if extra: - data_issues.append(f"unexpected data columns: {sorted(extra)}") + issues.append(f"unexpected {role} columns: {sorted(extra)}") + issues.extend( + f"{role} column {name!r} is declared {_type_text(declared[name])} " + f"but the source has {_type_text(given[name])}" + for name in declared.keys() + if name in given and given[name] != declared[name] + ) - if tag_issues or data_issues: + if issues: raise SourceSpecMismatchError( f"SourceNode '{self._name}' is not compatible with the provided source. " - + "; ".join(tag_issues + data_issues) + + "; ".join(issues) ) def as_table( @@ -597,7 +604,16 @@ def bound_source(self) -> StreamProtocol | None: @bound_source.setter def bound_source(self, value: StreamProtocol | None) -> None: - """Bind *value* as the concrete source and invalidate both hash caches.""" + """Bind *value* as the concrete source and invalidate both hash caches. + + A source is checked against the slot's declared schema first (``validate``), so + one of other column types is refused before anything changes. ``None`` unbinds. + + Raises: + SourceSpecMismatchError: If *value*'s schema differs from the declared one. + """ + if value is not None: + self.validate(value) self._bound_source = value self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index d776adff9..fb73eb656 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -213,14 +213,14 @@ class UnboundSourceError(RuntimeError): """ -class SourceSpecMismatchError(ValueError): - """Raised when a concrete source's schema is incompatible with a SourceNode slot. - - The class name ``SourceSpecMismatchError`` is preserved for compatibility - with any code that catches it by name. - - Contains the slot name and a description of the incompatible field(s). - Raised at ``bind()`` time — schema mismatches are rejected before execution. +class SourceSpecMismatchError(InputValidationError, ValueError): + """Raised when a concrete source's schema differs from a source slot's declared schema. + + A source fits a slot when its tag and data columns have the slot's names, roles and + Python types (ITL-1112): those are what every downstream pipeline hash, and so every + stored path, is built from. The message names the slot and the column, and for a type + difference both types. Raised by ``PipelineJob.bind(sources=...)`` and by assigning a + source node's ``bound_source``, before anything is bound. """ diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c38146edc..f015f1c51 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -417,7 +417,11 @@ def bind( When *sources* is provided, each concrete source is validated against its matching ``SourceJobNode`` slot schema, then the corresponding - ``SourceJobNode.bound_source`` is updated in-place. + ``SourceJobNode.bound_source`` is updated in-place. A source fits its slot + when its tag and data columns have the declared names, roles and Python + types, so a bind never changes a pipeline hash (ITL-1112). Every source is + validated before anything is changed: a refused source leaves the job's + bindings and database as they were. When *database* is provided and differs from the current database, ``_distribute_databases()`` is called so that all job nodes receive @@ -429,16 +433,14 @@ def bind( execution_context: Replaces the current execution context. Raises: - SourceSpecMismatchError: If any source's schema is incompatible. + SourceSpecMismatchError: If any source's columns differ from its slot's + declared names, roles or types. ValueError: If a source key has no matching ``SourceJobNode`` slot. """ from orcapod.core.nodes.source_node import SourceNodeBase database_changed = database is not None and database is not self._database - if database is not None: - self._database = database - if sources is not None: source_names = { node.name @@ -456,6 +458,12 @@ def bind( if isinstance(node, SourceNodeBase) and node.name in sources: node.validate(sources[node.name]) + # Every source is validated before anything changes: a refused source leaves + # the job as it was, its database included (ITL-1112). + if database is not None: + self._database = database + + if sources is not None: # Update bound_source on SourceJobNodes and maintain both dicts. for h, job_node in (self._persistent_node_map or {}).items(): if isinstance(job_node, SourceJobNode) and job_node.name in sources: From e024d40bab614775bc8722ec13ff9f58508eac8d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:11:06 -0800 Subject: [PATCH 03/33] docs: changelog fragment, design issue BT1 and bind docs sentence (ITL-1112) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1112.md | 25 +++++++++++++++++++++++++ design_issues/ITL-1112-BT1.md | 24 ++++++++++++++++++++++++ docs/concepts/function-pods.md | 5 ++++- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ITL-1112.md create mode 100644 design_issues/ITL-1112-BT1.md diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md new file mode 100644 index 000000000..f96eefd64 --- /dev/null +++ b/changelog.d/ITL-1112.md @@ -0,0 +1,25 @@ +### Breaking changes and migration + +#### Binding refuses a source whose column types differ from the slot's (ITL-1112) + +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and +assigning `job.nodes[name].bound_source` checked nothing. A source with other column types was +accepted: every pipeline hash, built from the slot's declared schema, stayed the same, so a run +wrote rows of the new types to the stored paths of the declared ones. Both ways of binding now +compare the source's tag and data columns with the slot's: names, role (tag or data) and Python +type, the inputs of the pipeline hash. Any difference raises `SourceSpecMismatchError` naming +the slot, the column and both types, before anything is bound; a refused `bind` leaves the job +as it was, its database included. A bind that is accepted never changes a pipeline hash. + +```python +slot = SourceNode(name="src", tag_schema={"id": int}, data_schema={"x": int}) +job.bind(sources={"src": DictSource([{"id": 0, "x": "a"}], tag_columns=["id"])}) +# v0.2.0: accepted; rows with x: str were stored under the path of x: int +# now: SourceSpecMismatchError: ... data column 'x' is declared int but the source has str +``` + +`SourceSpecMismatchError` is now an `InputValidationError` as well as a `ValueError`. +Migration: bind a source whose columns have the slot's types, converting them before the bind, +or record the pipeline with a slot of the source's types (which gives it its own stored paths). +Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart +(ITL-949). diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md new file mode 100644 index 000000000..76757682d --- /dev/null +++ b/design_issues/ITL-1112-BT1.md @@ -0,0 +1,24 @@ +## `src/orcapod/core/nodes/source_node.py` + +### BT1 — Binding a source of other column types left every pipeline hash unchanged +**Status:** resolved +**Severity:** high +**Issue:** ITL-1112 + +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and +assigning a source node's `bound_source` checked nothing. A source whose `x` is `str`, bound to +a slot declaring `x: int`, was accepted on a fresh and on a loaded job. Every pipeline hash is +built from the slot's declared schema, so none moved, while `output_schema()` passed the bound +source through: a downstream `MapData` reported `y: str` under the pipeline hash of `y: int`. A +run then sent `str` values to a function typed `int` and wrote its rows to the stored paths of +the declared types (`schema:/pdb_v2` and the pod's result table). + +**Fix:** `SourceNodeBase.validate` compares, column by column, the source's tag and data schemas +with the declared ones: names, role (tag or data) and Python type, the inputs of the pipeline +hash, so an accepted bind never moves one. Column order and `optional_fields` are not compared; +neither enters the pipeline hash. The `bound_source` setter calls the same check before it +assigns, and `bind` checks every source before it changes anything, its database included. +`SourceSpecMismatchError` names the slot, the column and both types and is now an +`InputValidationError` as well as a `ValueError`. Two Arrow types that are one Python type +(`int32` and `int64`) are still not told apart (ITL-949). No hash, golden value or stored path +changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index c04e4e8cc..6cbaaf03c 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -780,7 +780,10 @@ rows under that key is an empty result, not an error. paths. In-memory sources (`DictSource`, `ListSource`, `ArrowTableSource`, `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs this at INFO level. This does not affect reading the other nodes. To read such a source - node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. + node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. The source must + have the slot's tag and data columns with the same Python types; `bind` refuses any other + (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source never changes a + stored path. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each From 4a92072285d841d37fa0dfaa50e2c4e60affd627 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:14:59 -0800 Subject: [PATCH 04/33] style(tests): ruff format the bind type check tests Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_bind_type_check.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py index 77fc8ad37..044f8314f 100644 --- a/tests/test_pipeline/test_bind_type_check.py +++ b/tests/test_pipeline/test_bind_type_check.py @@ -49,7 +49,9 @@ def _strs() -> DictSource: def _str_tags() -> DictSource: """Return a source with tag ``id: str`` and data ``x: int``.""" return DictSource( - [{"id": "a", "x": 1}, {"id": "b", "x": 2}], tag_columns=["id"], source_id="stags" + [{"id": "a", "x": 1}, {"id": "b", "x": 2}], + tag_columns=["id"], + source_id="stags", ) @@ -172,7 +174,9 @@ def test_an_accepted_bind_moves_no_pipeline_hash( assert _pipeline_hashes(job) == before -def test_an_accepted_rebind_runs_and_stores_the_new_rows(tmp_path: pathlib.Path) -> None: +def test_an_accepted_rebind_runs_and_stores_the_new_rows( + tmp_path: pathlib.Path, +) -> None: """A rebind to a source of the declared types runs and stores its rows.""" job = _fresh_job(tmp_path) _ = job.run() From 582049aa2a6250f852ddeeed2969d0956fa9734c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:17:23 -0800 Subject: [PATCH 05/33] docs(plan): ITL-1112 round-2 decisions (narrower bind, cast, load) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-1112-bind-refuses-type.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md index 223ca6415..e488167f0 100644 --- a/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md +++ b/superpowers/plans/2026-11-14-itl-1112-bind-refuses-type.md @@ -70,3 +70,32 @@ into the int-declared paths (`schema:f289a35f.../pdb_v2`, `_result/.../rdb_v2`). - Conflicts: PR #384 (ITL-1102) edits the `bound_source` setter (adds `_update_modified_time()`); ITL-1111 adds a memo clear inside `bind`. Whoever merges second resolves. + +## Decisions, round 2 (coordinator/developer, 2026-11-14) + +- Nullability: a slot accepts a STRICTER source. Declared `T | None` accepts a source with `T`; + declared `T` refuses `T | None`. Nothing else is compatible (no other union, subclass, width). +- The node presents the DECLARED schema: `output_schema()`/`keys()` and its rows. Rows are cast + to the declared form at one helper: same arrays re-wrapped under nullable fields (no cast) and + the two system-tag columns renamed from the source's schema-hash suffix to the declared one + (N1; names from `system_tag_column_names` on both hashes). Row hashes, record ids, stored + schema are those of a `T | None` source (the flag reaches the row hash today: measured). +- `content_hash` and `source_id` stay the source's own. +- A narrower source whose system-tag columns are not exactly the two of its schema hash is + refused at bind (never mid-run). +- Entry points: `iter_data`, `async_iter_data` (per row: Tag/Data rebuilt with the row's data + context), `as_table` and `head` (an `ArrowTableStream` over the cast table, so content-hash + column, sort and column selection are computed on the cast rows), `output_schema`. +- L1: `SourceJobNode.__init__` binds through the setter; `PipelineJob.load` handles a rebuilt + source the slot refuses like one that could not be rebuilt (warning, slot unbound). +- PR as draft; CodeRabbit reviews by itself (do not request). + +## Tasks, round 2 + +- [ ] 6. Tests `tests/test_pipeline/test_bind_narrower_source.py`: accepted narrower bind (bind, + setter; fresh, loaded), reverse refused, downstream schema and pipeline hashes unchanged, + rows table-equal to a nullable-flag source (names, flags, row hashes, record ids), sync and + async, same-path append (narrower run then nullable run with a null), zero rows, head. + Loaded job: CSV changed type (warning, unbound, run fails as unbound, rebind works); CSV now + narrower (bound, cast). +- [ ] 7. Implement `_DeclaredForm` + entry points + `__init__` through setter + load handling. From 0ced55c252305ad46bd7afe43be2387274e19607 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:19:03 -0800 Subject: [PATCH 06/33] test(pipeline): a slot of T | None accepts a source of T, read as declared (failing, ITL-1112) Co-Authored-By: Claude Opus 5.5 --- .../test_bind_narrower_source.py | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 tests/test_pipeline/test_bind_narrower_source.py diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py new file mode 100644 index 000000000..737df30d4 --- /dev/null +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -0,0 +1,353 @@ +"""A slot declared ``T | None`` accepts a source of ``T``, presented as ``T | None`` (ITL-1112). + +Developer decision, November 14, 2026: a source slot accepts a source that is stricter than +it declares (a column ``x: int`` for a slot declaring ``x: int | None``) and refuses the +reverse. The slot gives the rows of its declared schema: the bound source is treated as +``x: int | None``. Its rows are exactly those of a source with a nullable ``x``, the same +values and the same source id: Arrow fields, system-tag column names, row hashes and record +ids alike, so both write to one stored path and append to one table. Nothing downstream +moves: every pipeline hash and predicted schema is the declared one. +""" + +from __future__ import annotations + +import asyncio +import logging +import pathlib +from collections.abc import Callable +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import MapData +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.types import Schema + + +@function_pod(output_keys="z") +def inc(y: int | None) -> int: + """Return ``y + 1``, or 0 for a missing ``y``.""" + return 0 if y is None else y + 1 + + +def _arrow_source( + xs: list[int | None], *, x_nullable: bool, ids: list[int] | None = None +) -> ArrowTableSource: + """Return a source of tag ``id`` (non-null) and data ``x`` with the given flag.""" + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), x_nullable)] + ) + ids = list(range(len(xs))) if ids is None else ids + return ArrowTableSource( + pa.table({"id": ids, "x": xs}, schema=schema), + tag_columns=["id"], + source_id="same", + ) + + +def _slot(x_type: Any) -> SourceNode: + """Return slot ``src`` declaring tag ``id: int`` and data ``x`` of ``x_type``.""" + return SourceNode( + name="src", tag_schema=Schema({"id": int}), data_schema=Schema({"x": x_type}) + ) + + +def _job(tmp_path: pathlib.Path, x_type: Any = int | None) -> PipelineJob: + """Return an unbound job: slot ``src`` -> MapData x->y ``renamed`` -> ``inc``.""" + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = inc.pod(MapData({"x": "y"})(_slot(x_type), label="renamed"), label="inc") + return job + + +def _fresh(tmp_path: pathlib.Path) -> PipelineJob: + """Return the job bound to a nullable source, as recorded.""" + job = _job(tmp_path) + job.bind(sources={"src": _arrow_source([1, None], x_nullable=True)}) + return job + + +def _loaded(tmp_path: pathlib.Path) -> PipelineJob: + """Return the job, run with a nullable source and saved, loaded again.""" + job = _fresh(tmp_path) + _ = job.run() + path = tmp_path / "job.json" + job.save(str(path)) + return PipelineJob.load(str(path)) + + +JOBS: dict[str, Callable[[pathlib.Path], PipelineJob]] = { + "fresh": _fresh, + "loaded": _loaded, +} + + +def _bind(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` with ``PipelineJob.bind``.""" + job.bind(sources={"src": source}) + + +def _assign(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` by assigning the node's ``bound_source``.""" + job.nodes["src"].bound_source = source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +BINDS: dict[str, Callable[[PipelineJob, Any], None]] = { + "bind": _bind, + "bound_source": _assign, +} + + +def _pipeline_hashes(job: PipelineJob) -> dict[str, str]: + """Return every node's pipeline hash, by node label.""" + return {name: node.pipeline_hash().to_string() for name, node in job.nodes.items()} + + +def _row_hashes(pairs: Any) -> list[tuple[str, str]]: + """Return the tag and data content hashes of each ``(tag, data)`` pair.""" + return [ + (t.content_hash().to_string(), d.content_hash().to_string()) for t, d in pairs + ] + + +async def _collect(node: Any) -> list[Any]: + """Return every pair of ``node.async_iter_data()``.""" + return [pair async for pair in node.async_iter_data()] + + +# --------------------------------------------------------------------------- +# Accepted and refused +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_narrower_source_is_accepted_and_moves_nothing( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, Any], None], +) -> None: + """``x: int`` binds to a slot of ``x: int | None``; schemas and pipeline hashes stay.""" + job = make_job(tmp_path) + before = _pipeline_hashes(job) + bind(job, DictSource([{"id": 0, "x": 1}], tag_columns=["id"], source_id="d")) + assert _pipeline_hashes(job) == before + assert dict(job.nodes["src"].output_schema()[1]) == {"x": int | None} + assert dict(job.nodes["renamed"].output_schema()[1]) == {"y": int | None} + assert job.nodes["src"].as_table().schema.field("x").nullable + + +@pytest.mark.parametrize("make_job", JOBS.values(), ids=JOBS.keys()) +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_nullable_source_is_refused_by_a_non_null_slot( + tmp_path: pathlib.Path, + make_job: Callable[[pathlib.Path], PipelineJob], + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A source with ``x: int | None`` is refused by a slot declaring ``x: int``.""" + job = _job(tmp_path, x_type=int) + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + if make_job is _loaded: + _ = job.run() + job.save(str(tmp_path / "job.json")) + job = PipelineJob.load(str(tmp_path / "job.json")) + with pytest.raises( + SourceSpecMismatchError, + match=r"'x' is declared int but the source has int \| None", + ): + bind(job, _arrow_source([1, None], x_nullable=True)) + + +def test_a_narrower_tag_column_is_accepted(tmp_path: pathlib.Path) -> None: + """A tag ``id: int`` binds to a slot declaring ``id: int | None`` and reads as one.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int | None}), + data_schema=Schema({"x": int}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(slot, label="renamed") + job.bind(sources={"src": DictSource([{"id": 0, "x": 1}], tag_columns=["id"])}) + assert dict(job.nodes["src"].output_schema()[0]) == {"id": int | None} + assert job.nodes["src"].as_table().schema.field("id").nullable + + +@pytest.mark.parametrize( + "declared, given", + [(int | None, str), (int | str | None, int), (int | None, float)], + ids=["other-type", "other-union-member", "numeric"], +) +def test_only_the_none_member_may_be_missing( + tmp_path: pathlib.Path, declared: Any, given: type +) -> None: + """``T | None`` accepts ``T`` only: no other type, union member or numeric type.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(declared), label="renamed") + value: Any = {str: "a", int: 1, float: 1.5}[given] + with pytest.raises(SourceSpecMismatchError, match="'x'"): + job.bind( + sources={"src": DictSource([{"id": 0, "x": value}], tag_columns=["id"])} + ) + + +# --------------------------------------------------------------------------- +# The rows are those of the declared schema +# --------------------------------------------------------------------------- + + +def _bound_slot(tmp_path: pathlib.Path, source: ArrowTableSource) -> Any: + """Return slot ``src`` of a fresh ``x: int | None`` job, bound to *source*.""" + job = _job(tmp_path) + job.bind(sources={"src": source}) + return job.nodes["src"] + + +def test_rows_equal_those_of_a_nullable_source(tmp_path: pathlib.Path) -> None: + """Narrower rows equal a nullable source's: fields, system tags, hashes, record ids.""" + narrower = _bound_slot(tmp_path / "a", _arrow_source([1, 2], x_nullable=False)) + nullable = _bound_slot(tmp_path / "b", _arrow_source([1, 2], x_nullable=True)) + assert narrower.as_table(all_info=True).equals( + nullable.as_table(all_info=True), check_metadata=True + ) + assert _row_hashes(narrower.iter_data()) == _row_hashes(nullable.iter_data()) + assert _row_hashes(asyncio.run(_collect(narrower))) == _row_hashes( + nullable.iter_data() + ) + assert narrower.keys(all_info=True) == nullable.keys(all_info=True) + + +def test_rows_of_each_read_carry_the_declared_flags(tmp_path: pathlib.Path) -> None: + """``iter_data``, ``async_iter_data`` and ``head`` give nullable ``x``.""" + slot = _bound_slot(tmp_path, _arrow_source([1, 2], x_nullable=False)) + for _, data in slot.iter_data(): + assert data.as_table().schema.field("x").nullable + for _, data in asyncio.run(_collect(slot)): + assert data.as_table().schema.field("x").nullable + head = slot.head(1, all_info=True).table + assert head.schema.field("x").nullable + assert head.equals(slot.as_table(all_info=True).slice(0, 1)) + + +def test_zero_rows_have_the_types_of_rows(tmp_path: pathlib.Path) -> None: + """A narrower source with no rows reads with the types it has with rows.""" + + @function_pod(output_keys="x") + def same(v: int) -> int: + """Return ``v``.""" + return v + + origin = PipelineJob(name="origin", database=InMemoryArrowDatabase()) + with origin: + _ = same.pod( + DictSource([{"id": 0, "v": 1}], tag_columns=["id"], source_id="o"), + label="same", + ) + empty = origin.nodes["same"].as_source() # pyright: ignore[reportAttributeAccessIssue] # a function job node; as_source before a run has no rows + job = _job(tmp_path) + job.bind(sources={"src": empty}) + with_rows = _bound_slot(tmp_path / "rows", _arrow_source([1], x_nullable=False)) + table = job.nodes["src"].as_table() + assert table.num_rows == 0 + assert table.schema == with_rows.as_table().schema + + +# --------------------------------------------------------------------------- +# One stored path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +def test_narrower_then_nullable_runs_append_to_one_path( + tmp_path: pathlib.Path, orchestrator: type +) -> None: + """A run of a narrower source, then of a nullable one, append under the same path.""" + job = _job(tmp_path) + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + first = job.run(orchestrator=orchestrator()) + assert first.ok + paths = set(job.database.list_table_paths(())) # pyright: ignore[reportOptionalMemberAccess] # the job has a database + job.bind(sources={"src": _arrow_source([None, 3], x_nullable=True, ids=[2, 3])}) + second = job.run(orchestrator=orchestrator()) + assert second.ok + assert set(job.database.list_table_paths(())) == paths # pyright: ignore[reportOptionalMemberAccess] # the job has a database + assert sorted(job.nodes["inc"].as_table().column("z").to_pylist()) == [0, 4] + stored = [ + job.database.get_all_records(path) # pyright: ignore[reportOptionalMemberAccess] # the job has a database + for path in paths + if path[-1] == "pdb_v2" + ] + assert [table.num_rows for table in stored] == [4] + + +# --------------------------------------------------------------------------- +# A loaded job's rebuilt file source +# --------------------------------------------------------------------------- + + +def _csv_job(tmp_path: pathlib.Path, text: str) -> str: + """Save a job over a CSV source of *text*; return the job file's path.""" + csv = tmp_path / "data.csv" + _ = csv.write_text(text) + job = PipelineJob(name="c", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = MapData({"x": "y"})( + CSVSource(str(csv), tag_columns=["id"], source_id="csv"), label="renamed" + ) + _ = job.run() + path = tmp_path / "job.json" + job.save(str(path)) + return str(path) + + +def _csv_slot(job: PipelineJob) -> Any: + """Return the CSV source's slot of *job*.""" + return job.nodes["csv"] + + +def test_loaded_csv_of_another_type_is_left_unbound( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """A CSV whose column changed type loads unbound, with a warning naming the slot.""" + path = _csv_job(tmp_path, "id,x\n0,1\n1,2\n") + _ = (tmp_path / "data.csv").write_text("id,x\n0,a\n1,b\n") + with caplog.at_level(logging.WARNING): + job = PipelineJob.load(path) + assert any( + "'csv'" in r.getMessage() and "declared int" in r.getMessage() + for r in caplog.records + ) + assert job.unbound_sources == ["csv"] + with pytest.raises(UnboundSourceError): + _ = _csv_slot(job).as_table() + _ = (tmp_path / "fixed.csv").write_text("id,x\n0,7\n") + job.bind( + sources={ + "csv": CSVSource( + str(tmp_path / "fixed.csv"), tag_columns=["id"], source_id="csv" + ) + } + ) + assert _csv_slot(job).as_table().column("x").to_pylist() == [7] + + +def test_loaded_csv_now_narrower_is_bound_as_declared(tmp_path: pathlib.Path) -> None: + """A CSV saved with an empty cell (``x: int | None``) and now without loads bound.""" + path = _csv_job(tmp_path, "id,x\n0,1\n1,\n") + _ = (tmp_path / "data.csv").write_text("id,x\n0,1\n1,2\n") + job = PipelineJob.load(path) + slot = _csv_slot(job) + assert dict(slot.output_schema()[1]) == {"x": int | None} + assert slot.as_table().schema.field("x").nullable + assert slot.as_table().column("x").to_pylist() == [1, 2] From 0484c23131e9c1652afd8b282f7a443e1b2a7c76 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:21:55 -0800 Subject: [PATCH 07/33] feat(pipeline): a slot of T | None accepts a source of T and gives its rows as declared (ITL-1112) A narrower bound source is cast to the slot's declared schema at one place (_DeclaredForm): the same arrays under nullable fields and the two system-tag columns renamed after the declared schema hash. Applied on iter_data, async_iter_data, as_table, head and output_schema. The reverse (a nullable source for a non-null slot) is refused. SourceJobNode.__init__ binds through the setter; PipelineJob.load treats a rebuilt source its slot refuses as one that could not be rebuilt. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 262 +++++++++++++++++++++++--- src/orcapod/pipeline/job.py | 10 + 2 files changed, 242 insertions(+), 30 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index e98c015de..7b7fd9550 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -7,15 +7,19 @@ from __future__ import annotations +import dataclasses import functools import logging from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any from orcapod import contexts from orcapod.config import OrcapodConfig from orcapod.core.base import TraceableBase +from orcapod.core.datagrams import Data, Tag +from orcapod.core.operators.drop_nulls import _without_none # pyright: ignore[reportPrivateUsage] # DropNulls' rule for removing None from a type, shared, never copied +from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import schema_requires_io from orcapod.core.streams.view import ( ViewStatus, @@ -31,6 +35,7 @@ from orcapod.types import ColumnConfig, Schema from orcapod.utils import arrow_utils from orcapod.utils.arrow_utils import system_tag_column_names +from orcapod.utils.lazy_module import LazyModule from orcapod.utils.schema_utils import compute_source_schema_hash if TYPE_CHECKING: @@ -39,6 +44,8 @@ from orcapod.channels import WritableChannel from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol +else: + pa = LazyModule("pyarrow") logger = logging.getLogger(__name__) @@ -48,6 +55,63 @@ def _type_text(data_type: Any) -> str: return data_type.__name__ if isinstance(data_type, type) else str(data_type) +def _is_narrower(declared: Any, given: Any) -> bool: + """Return whether *given* is *declared* without its ``None`` member. + + ``int`` is narrower than ``int | None`` and ``int | str`` than ``int | str | None``; + nothing else is (developer, November 14, 2026, ITL-1112). + + Args: + declared: The slot's declared Python type of a column. + given: The bound source's Python type of that column. + """ + return declared != given and _without_none(declared) == given + + +@dataclasses.dataclass(frozen=True) +class _DeclaredForm: + """How the rows of a narrower bound source become rows of the declared schema. + + A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows + as rows of ``x: int | None`` (ITL-1112): the same arrays under nullable fields, and + the two system-tag columns, named after the source's schema hash, renamed after the + declared schema's. Nothing else differs: values, types, source-info columns and + their tokens pass unchanged, so the rows equal those of a source of the declared + schema with the same source id. + + Attributes: + nullable: The columns declared ``T | None`` that the source gives as ``T``. + renames: The source's system-tag column names, mapped to the declared ones. + """ + + nullable: frozenset[str] + renames: Mapping[str, str] + + def apply(self, table: pa.Table) -> pa.Table: + """Return *table*'s arrays under the declared fields and system-tag names. + + Serves a whole stream table and a single row's tag or data table: columns that + are not in the table are skipped. + + Args: + table: Rows read from the bound source. + + Returns: + The same arrays, with the columns in ``nullable`` flagged nullable and the + columns in ``renames`` renamed. + """ + fields = [] + for field in table.schema: + if field.name in self.nullable: + field = field.with_nullable(True) + if field.name in self.renames: + field = field.with_name(self.renames[field.name]) + fields.append(field) + schema = pa.schema(fields, metadata=table.schema.metadata) + # Same arrays under the declared fields: no cast, so no value or type changes. + return pa.Table.from_arrays(table.columns, schema=schema) + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -242,24 +306,40 @@ def keys( # ------------------------------------------------------------------ def validate(self, source: "StreamProtocol") -> None: - """Check that *source* has this slot's declared schema. + """Check that *source* fits this slot's declared schema. - The source's tag and data columns must have the declared names, roles and - Python types (ITL-1112): those are what every downstream pipeline hash, and so - every stored path, is built from, so a source that passes never moves one. - Column order and ``optional_fields`` are not compared; neither enters the - pipeline hash. + The source's tag and data columns must have the declared names and roles, and + each the declared Python type, or, where the slot declares ``T | None``, ``T`` + (ITL-1112). Those are what every downstream pipeline hash, and so every stored + path, is built from, so a source that fits never moves one. Column order and + ``optional_fields`` are not compared; neither enters the pipeline hash. Args: source: A concrete stream to validate. Raises: - SourceSpecMismatchError: If a column is missing, unexpected, or of another - type than declared. + SourceSpecMismatchError: If a column is missing, unexpected, or of a type + the declared one does not accept. + """ + _ = self._fit(source) + + def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str]]: + """Check *source* against the declared schema, as ``validate`` describes. + + Args: + source: A concrete stream to check. + + Returns: + The source's tag and data schemas, and the names of the columns it gives as + ``T`` where the slot declares ``T | None``. + + Raises: + SourceSpecMismatchError: If *source* does not fit. """ source_tag, source_data = source.output_schema() issues: list[str] = [] + narrower: set[str] = set() for role, declared, given in ( ("tag", self._tag_schema, source_tag), ("data", self._data_schema, source_data), @@ -270,18 +350,24 @@ def validate(self, source: "StreamProtocol") -> None: issues.append(f"missing {role} columns: {sorted(missing)}") if extra: issues.append(f"unexpected {role} columns: {sorted(extra)}") - issues.extend( - f"{role} column {name!r} is declared {_type_text(declared[name])} " - f"but the source has {_type_text(given[name])}" - for name in declared.keys() - if name in given and given[name] != declared[name] - ) + for name in declared.keys(): + if name not in given or given[name] == declared[name]: + continue + if _is_narrower(declared[name], given[name]): + narrower.add(name) + else: + issues.append( + f"{role} column {name!r} is declared " + f"{_type_text(declared[name])} but the source has " + f"{_type_text(given[name])}" + ) if issues: raise SourceSpecMismatchError( f"SourceNode '{self._name}' is not compatible with the provided source. " + "; ".join(issues) ) + return source_tag, source_data, frozenset(narrower) def as_table( self, @@ -588,10 +674,13 @@ def __init__( data_schema=data_schema, data_context=data_context, ) - # Direct assignment to the backing attribute — super().__init__() has - # already initialised _content_hash_cache, so the property setter is - # safe to use here; we bypass it only to make the init path explicit. - self._bound_source: StreamProtocol | None = bound_source + self._bound_source: StreamProtocol | None = None + self._declared_form: _DeclaredForm | None = None + if bound_source is not None: + # Through the setter, which checks the source and, for a narrower one, + # records how its rows become rows of the declared schema (ITL-1112). + # super().__init__() has initialised the hash caches it invalidates. + self.bound_source = bound_source # ------------------------------------------------------------------ # bound_source property — explicit binding with cache invalidation @@ -607,17 +696,89 @@ def bound_source(self, value: StreamProtocol | None) -> None: """Bind *value* as the concrete source and invalidate both hash caches. A source is checked against the slot's declared schema first (``validate``), so - one of other column types is refused before anything changes. ``None`` unbinds. + one that does not fit is refused before anything changes. A narrower source + (``x: int`` for a slot declaring ``x: int | None``) is accepted, and the node + gives its rows as rows of the declared schema (ITL-1112). ``None`` unbinds. Raises: - SourceSpecMismatchError: If *value*'s schema differs from the declared one. + SourceSpecMismatchError: If *value* does not fit the declared schema. """ - if value is not None: - self.validate(value) + form = None if value is None else self._declared_form_for(value) self._bound_source = value + self._declared_form = form self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: + """Check *source* and return how its rows become rows of the declared schema. + + Args: + source: The stream being bound. + + Returns: + ``None`` when the source's schema equals the declared one, so its rows pass + unchanged; otherwise the ``_DeclaredForm`` of its narrower columns. + + Raises: + SourceSpecMismatchError: If *source* does not fit the declared schema. + """ + source_tag, source_data, narrower = self._fit(source) + if not narrower: + return None + source_schema_hash = compute_source_schema_hash( + source_tag, source_data, self.data_context, self.orcapod_config + ) + return _DeclaredForm( + nullable=narrower, + renames=dict( + zip( + system_tag_column_names(source_schema_hash), + system_tag_column_names(self._schema_hash_str), + strict=True, + ) + ), + ) + + def _declared_rows( + self, rows: Iterator[tuple[TagProtocol, DataProtocol]], form: _DeclaredForm + ) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield each row of a narrower source as a row of the declared schema.""" + for tag, data in rows: + yield self._declared_row(tag, data, form) + + @staticmethod + def _declared_row( + tag: TagProtocol, data: DataProtocol, form: _DeclaredForm + ) -> tuple[TagProtocol, DataProtocol]: + """Return one row of a narrower source as a row of the declared schema. + + The row's tag (with its system tags) and data (with its source info) Arrow + tables go through ``form``, and the row is rebuilt from them with its own + data context. + """ + tag_table = form.apply(tag.as_table(columns={"system_tags": True})) + data_table = form.apply(data.as_table(columns={"source": True, "meta": True})) + new_tag = ( + Tag(tag_table, data_context=tag.data_context_key) + if tag_table.num_columns + else tag + ) + return new_tag, Data(data_table, data_context=data.data_context_key) + + def _declared_stream( + self, source: StreamProtocol, form: _DeclaredForm, table: pa.Table + ) -> ArrowTableStream: + """Return a stream over *table*, read from *source*, as the declared schema. + + *table* must hold the system-tag, source-info and context columns, so the + stream selects, sorts and hashes the declared rows as any stream does. + """ + return ArrowTableStream( + form.apply(table), + tag_columns=tuple(self._tag_schema.keys()), + data_context=source.data_context, # pyright: ignore[reportAttributeAccessIssue] # StreamProtocol omits data_context, which every concrete stream provides (ITL-844) + ) + # ------------------------------------------------------------------ # Identity — delegate to bound source when set # ------------------------------------------------------------------ @@ -648,7 +809,9 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before running." ) - return self._bound_source.iter_data() + if self._declared_form is None: + return self._bound_source.iter_data() + return self._declared_rows(self._bound_source.iter_data(), self._declared_form) def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll: as for the bound source.""" @@ -671,8 +834,33 @@ def head( ViewStatus.UNAVAILABLE, note="no source bound", ) - return head_of( - self._bound_source.head(n, columns=columns, all_info=all_info), self + if self._declared_form is None: + return head_of( + self._bound_source.head(n, columns=columns, all_info=all_info), self + ) + # The source's own head, read with every column group the declared stream + # needs; the declared stream then selects the asked-for groups and computes + # the content hashes of the declared rows. Nothing is computed. + config = ColumnConfig.handle_config(columns, all_info=all_info) + view = head_of( + self._bound_source.head( + n, + columns=ColumnConfig( + source=True, + system_tags=True, + context=True, + sort_by_tags=config.sort_by_tags, + ), + ), + self, + ) + table = self._declared_stream( + self._bound_source, self._declared_form, view.table + ).as_table(columns=config) + return dataclasses.replace( + view, + table=table, + tag_columns=tuple(c for c in view.tag_columns if c in table.column_names), ) async def async_iter_data( @@ -694,8 +882,9 @@ async def async_iter_data( f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before running." ) - async for pair in self._bound_source.async_iter_data(): - yield pair + form = self._declared_form + async for tag, data in self._bound_source.async_iter_data(): + yield (tag, data) if form is None else self._declared_row(tag, data, form) def output_schema( self, @@ -719,7 +908,8 @@ def output_schema( Returns: Tuple of ``(tag_schema, data_schema)``. """ - if self._bound_source is None: + if self._bound_source is None or self._declared_form is not None: + # Unbound, or bound to a narrower source given as the declared schema. return super().output_schema(columns=columns, all_info=all_info) return self._bound_source.output_schema(columns=columns, all_info=all_info) @@ -743,7 +933,19 @@ def as_table( f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before calling as_table()." ) - return self._bound_source.as_table(columns=columns, all_info=all_info) + if self._declared_form is None: + return self._bound_source.as_table(columns=columns, all_info=all_info) + table = self._bound_source.as_table( + columns={ + "source": True, + "system_tags": True, + "context": True, + "sort_by_tags": False, + } + ) + return self._declared_stream( + self._bound_source, self._declared_form, table + ).as_table(columns=columns, all_info=all_info) @classmethod def from_stream( diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index f015f1c51..f89297e3d 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1437,6 +1437,7 @@ def load( import tempfile from pathlib import Path as _Path + from orcapod.core.nodes.source_node import SourceNodeBase from orcapod.pipeline.graph import Pipeline from orcapod.pipeline.serialization import ( SUPPORTED_JOB_FORMAT_VERSIONS, @@ -1464,6 +1465,11 @@ def load( os.unlink(tmp_path) # Reconstruct sources (best-effort) + slots = { + node.name: node + for node in pipeline._persistent_node_map.values() + if isinstance(node, SourceNodeBase) + } sources: dict[str, cp.StreamProtocol] = {} for spec_name, src_config in ( data.get("bindings", {}).get("sources", {}).items() @@ -1472,6 +1478,10 @@ def load( try: source = resolve_source_from_config(src_config) if source is not None: + # A rebuilt source its slot refuses (a file whose column types + # changed) is one that could not be rebuilt (ITL-1112). + if spec_name in slots: + slots[spec_name].validate(source) sources[spec_name] = source except NotImplementedError: # In-memory sources (DictSource, ListSource, ArrowTableSource, From acc423bb1f8303bf95c5bdbdb3769e4ea35c2f41 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:29:27 -0800 Subject: [PATCH 08/33] docs: changelog, design issue and bind docs for the narrower-source rule (ITL-1112) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1112.md | 48 +++++++++++++------ design_issues/ITL-1112-BT1.md | 39 +++++++++------ docs/concepts/function-pods.md | 7 +-- src/orcapod/core/nodes/source_node.py | 7 ++- .../test_bind_narrower_source.py | 16 +++---- 5 files changed, 74 insertions(+), 43 deletions(-) diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index f96eefd64..ede494318 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -1,15 +1,16 @@ ### Breaking changes and migration -#### Binding refuses a source whose column types differ from the slot's (ITL-1112) +#### Binding refuses a source whose column types the slot does not declare (ITL-1112) -`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and -assigning `job.nodes[name].bound_source` checked nothing. A source with other column types was -accepted: every pipeline hash, built from the slot's declared schema, stayed the same, so a run -wrote rows of the new types to the stored paths of the declared ones. Both ways of binding now -compare the source's tag and data columns with the slot's: names, role (tag or data) and Python -type, the inputs of the pipeline hash. Any difference raises `SourceSpecMismatchError` naming -the slot, the column and both types, before anything is bound; a refused `bind` leaves the job -as it was, its database included. A bind that is accepted never changes a pipeline hash. +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only; +assigning `job.nodes[name].bound_source` and the file sources `PipelineJob.load` rebuilds were +not checked at all. A source with other column types was accepted: every pipeline hash, built +from the slot's declared schema, stayed the same, so a run wrote rows of the new types to the +stored paths of the declared ones. Every way of binding now compares the source's tag and data +columns with the slot's: names, role (tag or data) and Python type, the inputs of the pipeline +hash. A column must have the declared type, or `T` where the slot declares `T | None`. Anything +else raises `SourceSpecMismatchError` naming the slot, the column and both types, before +anything is bound; a refused `bind` leaves the job as it was, its database included. ```python slot = SourceNode(name="src", tag_schema={"id": int}, data_schema={"x": int}) @@ -18,8 +19,27 @@ job.bind(sources={"src": DictSource([{"id": 0, "x": "a"}], tag_columns=["id"])}) # now: SourceSpecMismatchError: ... data column 'x' is declared int but the source has str ``` -`SourceSpecMismatchError` is now an `InputValidationError` as well as a `ValueError`. -Migration: bind a source whose columns have the slot's types, converting them before the bind, -or record the pipeline with a slot of the source's types (which gives it its own stored paths). -Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart -(ITL-949). +Refused now, accepted before: a column of another type, and a nullable column (`int | None`) +for a slot that declares it non-null (`int`). A slot recorded from data without a null (a +`DictSource`, `ListSource`, `CSVSource` or `DataFrameSource` infers `int` then) refuses a +later source whose data has one; a `pa.table` and a Delta table written from one are nullable +(`int | None`) in every column, tags included, unless their schema says otherwise. +`SourceSpecMismatchError` is now an `InputValidationError` as well as a `ValueError`. A file +source that `PipelineJob.load` rebuilds and its slot refuses is handled like one that could not +be rebuilt: a warning names the slot and the slot stays unbound. + +Migration: bind a source whose columns have the slot's types (convert them, or give the +Arrow table a schema with the slot's nullability), or record the pipeline with a slot of the +source's types, which gives it its own stored paths. Two Arrow types that are one Python type +(`int32` and `int64`) are still not told apart (ITL-949). + +### Changed + +- A slot that declares `x: int | None` accepts a source whose `x` is `int` (ITL-1112). The slot + gives that source's rows as rows of its declared schema: `output_schema()`, `as_table()`, + `iter_data()`, `head()` and the async path show `x` nullable, and the system-tag columns + carry the declared schema's names. The rows are exactly those of a source with a nullable + `x`, the same values and the same source id (row hashes and record ids included), so both + write to the same stored tables. Nothing downstream changes: every pipeline hash and + predicted schema is the declared one. The source node's `content_hash` and source id stay + the bound source's own. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 76757682d..c5318d82f 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -5,20 +5,31 @@ **Severity:** high **Issue:** ITL-1112 -`PipelineJob.bind(sources=...)` checked a source against its slot by column names only, and -assigning a source node's `bound_source` checked nothing. A source whose `x` is `str`, bound to -a slot declaring `x: int`, was accepted on a fresh and on a loaded job. Every pipeline hash is -built from the slot's declared schema, so none moved, while `output_schema()` passed the bound -source through: a downstream `MapData` reported `y: str` under the pipeline hash of `y: int`. A -run then sent `str` values to a function typed `int` and wrote its rows to the stored paths of -the declared types (`schema:/pdb_v2` and the pod's result table). +`PipelineJob.bind(sources=...)` checked a source against its slot by column names only; +assigning a source node's `bound_source`, and the file sources `PipelineJob.load` rebuilds, were +not checked at all. A source whose `x` is `str`, bound to a slot declaring `x: int`, was +accepted on a fresh and on a loaded job. Every pipeline hash is built from the slot's declared +schema, so none moved, while `output_schema()` passed the bound source through: a downstream +`MapData` reported `y: str` under the pipeline hash of `y: int`. A run then sent `str` values to +a function typed `int` and wrote its rows to the stored paths of the declared types +(`schema:/pdb_v2` and the pod's result table). The same held for a +nullability difference (`int` against `int | None`), which has its own pipeline hash. **Fix:** `SourceNodeBase.validate` compares, column by column, the source's tag and data schemas with the declared ones: names, role (tag or data) and Python type, the inputs of the pipeline -hash, so an accepted bind never moves one. Column order and `optional_fields` are not compared; -neither enters the pipeline hash. The `bound_source` setter calls the same check before it -assigns, and `bind` checks every source before it changes anything, its database included. -`SourceSpecMismatchError` names the slot, the column and both types and is now an -`InputValidationError` as well as a `ValueError`. Two Arrow types that are one Python type -(`int32` and `int64`) are still not told apart (ITL-949). No hash, golden value or stored path -changed. +hash. A column must have the declared type or, where the slot declares `T | None`, `T` +(developer, November 14, 2026); nothing else fits. Column order and `optional_fields` are not +compared; neither enters the pipeline hash. The `bound_source` setter (which `__init__` now goes +through) calls the same check before it assigns, `bind` checks every source before it changes +anything, its database included, and `load` handles a rebuilt source its slot refuses like one +it could not rebuild. `SourceSpecMismatchError` names the slot, the column and both types and +is now an `InputValidationError` as well as a `ValueError`. + +A narrower source is given as the declared schema at one place, `_DeclaredForm`: the same +arrays under nullable fields, and the two system-tag columns renamed from the source's schema +hash to the declared one (their names carry the schema hash). `iter_data`, `async_iter_data`, +`as_table`, `head` and `output_schema` go through it, so the rows equal those of a nullable +source with the same values and source id, row hashes and record ids included, and both append +to one stored table. The node's `content_hash` and source id stay the source's own. Two Arrow +types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949). No +hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2d420a771..050fc0e20 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -833,9 +833,10 @@ rows under that key is an empty result, not an error. `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs this at INFO level. This does not affect reading the other nodes. To read such a source node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. The source must - have the slot's tag and data columns with the same Python types; `bind` refuses any other - (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source never changes a - stored path. + have the slot's tag and data columns with the same Python types, except that a column the + slot declares `T | None` may be `T` (the slot then reads it as `T | None`); `bind` refuses + any other (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source + never changes a stored path. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 7b7fd9550..e22ad7210 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -100,14 +100,13 @@ def apply(self, table: pa.Table) -> pa.Table: The same arrays, with the columns in ``nullable`` flagged nullable and the columns in ``renames`` renamed. """ - fields = [] - for field in table.schema: + schema = table.schema + for position, field in enumerate(table.schema): if field.name in self.nullable: field = field.with_nullable(True) if field.name in self.renames: field = field.with_name(self.renames[field.name]) - fields.append(field) - schema = pa.schema(fields, metadata=table.schema.metadata) + schema = schema.set(position, field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized # Same arrays under the declared fields: no cast, so no value or type changes. return pa.Table.from_arrays(table.columns, schema=schema) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 737df30d4..4d25ff5b5 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -275,20 +275,18 @@ def test_narrower_then_nullable_runs_append_to_one_path( """A run of a narrower source, then of a nullable one, append under the same path.""" job = _job(tmp_path) job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + database = job.database + assert database is not None first = job.run(orchestrator=orchestrator()) assert first.ok - paths = set(job.database.list_table_paths(())) # pyright: ignore[reportOptionalMemberAccess] # the job has a database + paths = set(database.list_table_paths(())) job.bind(sources={"src": _arrow_source([None, 3], x_nullable=True, ids=[2, 3])}) second = job.run(orchestrator=orchestrator()) assert second.ok - assert set(job.database.list_table_paths(())) == paths # pyright: ignore[reportOptionalMemberAccess] # the job has a database + assert set(database.list_table_paths(())) == paths assert sorted(job.nodes["inc"].as_table().column("z").to_pylist()) == [0, 4] - stored = [ - job.database.get_all_records(path) # pyright: ignore[reportOptionalMemberAccess] # the job has a database - for path in paths - if path[-1] == "pdb_v2" - ] - assert [table.num_rows for table in stored] == [4] + stored = [database.get_all_records(path) for path in paths if path[-1] == "pdb_v2"] + assert [None if table is None else table.num_rows for table in stored] == [4] # --------------------------------------------------------------------------- @@ -331,6 +329,8 @@ def test_loaded_csv_of_another_type_is_left_unbound( assert job.unbound_sources == ["csv"] with pytest.raises(UnboundSourceError): _ = _csv_slot(job).as_table() + with pytest.raises(RuntimeError, match="cannot execute a loaded job"): + _ = job.run() _ = (tmp_path / "fixed.csv").write_text("id,x\n0,7\n") job.bind( sources={ From 091bf52d7ca2dd2e847da3a5cb57d605595b718c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:29:52 -0800 Subject: [PATCH 09/33] docs(source_node): docstrings of the read paths name the declared-schema cast (ITL-1112) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index e22ad7210..769c9fd3b 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -825,7 +825,10 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the bound source's head, or no rows when no source is bound.""" + """Return the bound source's head, or no rows when no source is bound. + + A narrower bound source's rows are shown as rows of the declared schema. + """ if self._bound_source is None: check_head_size(n) return self._head_without_rows( @@ -893,9 +896,11 @@ def output_schema( ) -> tuple[Schema, Schema]: """Return ``(tag_schema, data_schema)``, delegating to ``bound_source`` when set. - When ``bound_source`` is present, this is a transparent pass-through to - ``bound_source.output_schema(columns=columns, all_info=all_info)`` so callers - get the same result as querying the source directly. + When ``bound_source`` has the declared schema, this is a transparent + pass-through to ``bound_source.output_schema(columns=columns, all_info=all_info)`` + so callers get the same result as querying the source directly. A narrower + bound source (``x: int`` for a declared ``x: int | None``) is given as the + declared schema (ITL-1112). When unbound, delegates to ``SourceNodeBase.output_schema()`` which includes system-tag schema entries derived from the declared schemas. @@ -920,6 +925,9 @@ def as_table( ) -> pa.Table: """Materialize the concrete source as a PyArrow Table. + A narrower bound source's rows are given as rows of the declared schema + (ITL-1112). + Args: columns: Column selection config. all_info: If True, include all metadata columns. From 5c9cfb96711beffc9b786a8e21de1a8d32642299 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:33:43 -0800 Subject: [PATCH 10/33] test(pipeline): no accepted bind changes a node's path or table schema; system-tag values are the source's own (ITL-1112) Adds the invariant to the docs, CLAUDE.md and .zed/rules, with its limit (Arrow width only, ITL-949). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 11 ++ CLAUDE.md | 11 ++ docs/concepts/function-pods.md | 6 +- .../test_bind_keeps_node_schemas.py | 138 ++++++++++++++++++ .../test_bind_narrower_source.py | 40 +++++ 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 tests/test_pipeline/test_bind_keeps_node_schemas.py diff --git a/.zed/rules b/.zed/rules index af3ce036a..ebdde7893 100644 --- a/.zed/rules +++ b/.zed/rules @@ -771,6 +771,17 @@ data. The rules: ### Important implementation details - ArrowTableSource raises ValueError if any tag_columns are not in the table. +- The pipeline determines every node's schema; a bound source never changes it (developer, + November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the + schema of any node's or stored table (column names with system tags, types, nullable + flags, order); only values differ. SourceNodeBase.validate is the one check, used by + PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and + PipelineJob.load: per column the declared Python type, or T where the slot declares + T | None; anything else raises SourceSpecMismatchError before anything is bound. A + narrower source is given as the declared schema at one place, _DeclaredForm (nullable + fields over the same arrays; the system-tag column NAMES of the declared schema hash, the + VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow + width only (int32 against int64) is not yet refused (ITL-949). - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index d66e4d9c6..387d5ada9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -813,6 +813,17 @@ data. The rules: ### Important implementation details - `ArrowTableSource.__init__` raises `ValueError` if any `tag_columns` are not in the table. +- The pipeline determines every node's schema; a bound source never changes it (developer, + November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the + schema of any node's or stored table (column names with system tags, types, nullable + flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by + `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and + `PipelineJob.load`: per column the declared Python type, or `T` where the slot declares + `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. A + narrower source is given as the declared schema at one place, `_DeclaredForm` (nullable + fields over the same arrays; the system-tag column NAMES of the declared schema hash, the + VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow + width only (`int32` against `int64`) is not yet refused (ITL-949). - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 050fc0e20..eafed3358 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -835,8 +835,10 @@ rows under that key is an empty result, not an error. node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. The source must have the slot's tag and data columns with the same Python types, except that a column the slot declares `T | None` may be `T` (the slot then reads it as `T | None`); `bind` refuses - any other (`SourceSpecMismatchError`, an `InputValidationError`), so binding a source - never changes a stored path. + any other (`SourceSpecMismatchError`, an `InputValidationError`). The pipeline determines + every node's schema: binding a source never changes a node's stored path or the schema of + any node's or stored table, only the values. The check compares Python types, so a + difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py new file mode 100644 index 000000000..c01b037b8 --- /dev/null +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -0,0 +1,138 @@ +"""No accepted bind changes a node's stored path or table schema (ITL-1112). + +Developer, November 14, 2026: "for pipeline, all node's path/table schema should NOT change +regardless of what source you bind to the source nodes in the pipeline". One job is run over +every kind of accepted bind: a source of the declared schema, a narrower source into a +``T | None`` slot, sources of different kinds, through ``bind``, the ``bound_source`` setter +and ``PipelineJob.load``. After each, the stored table paths, the Arrow schema of every stored +table and of every node's ``as_table(all_info=True)`` (column names with system tags, types, +nullable flags, order) are those of the first bind: only values differ. + +The bind check compares Python types, so two Arrow widths of one Python type (``int32`` and +``int64``) are not covered here (ITL-949). +""" + +from __future__ import annotations + +import pathlib +from collections.abc import Callable +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.operators import MapData +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.types import Schema + + +@function_pod(output_keys="z") +def inc(y: int | None) -> int: + """Return ``y + 1``, or 0 for a missing ``y``.""" + return 0 if y is None else y + 1 + + +def _arrow( + ids: list[int], xs: list[int | None], *, x_nullable: bool +) -> ArrowTableSource: + """Return an Arrow source of tag ``id`` (non-null) and data ``x``.""" + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), x_nullable)] + ) + return ArrowTableSource( + pa.table({"id": ids, "x": xs}, schema=schema), tag_columns=["id"] + ) + + +def _csv(tmp_path: pathlib.Path, name: str, text: str) -> CSVSource: + """Return a CSV source over *text*, written to *name* in *tmp_path*.""" + path = tmp_path / name + _ = path.write_text(text) + return CSVSource(str(path), tag_columns=["id"], source_id=name) + + +NODES = ("src", "renamed", "inc") +# A loaded job reads stored results only; its uncached operator has none to read. +LOADED_NODES = ("src", "inc") + + +def _snapshot(job: PipelineJob, nodes: tuple[str, ...] = NODES) -> dict[str, Any]: + """Return the stored paths, stored table schemas and *nodes*' table schemas of *job*.""" + database = job.database + assert database is not None + paths = sorted(database.list_table_paths(())) + stored = {} + for path in paths: + table = database.get_all_records(path) + assert table is not None + stored[path] = table.schema + return { + "paths": paths, + "stored": stored, + "nodes": { + name: job.nodes[name].as_table(all_info=True).schema for name in nodes + }, + } + + +def _assign(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` by assigning the node's ``bound_source``.""" + job.nodes["src"].bound_source = source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +def _bind(job: PipelineJob, source: Any) -> None: + """Bind *source* to slot ``src`` with ``PipelineJob.bind``.""" + job.bind(sources={"src": source}) + + +def test_no_accepted_bind_changes_a_path_or_a_table_schema( + tmp_path: pathlib.Path, +) -> None: + """Every accepted bind leaves paths and every stored and node table schema as they were.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int | None}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = inc.pod(MapData({"x": "y"})(slot, label="renamed"), label="inc") + + binds: list[tuple[str, Callable[[PipelineJob, Any], None], Any]] = [ + ("declared schema", _bind, _arrow([0, 1], [1, None], x_nullable=True)), + ("narrower arrow", _bind, _arrow([2, 3], [2, 3], x_nullable=False)), + ( + "narrower dict", + _assign, + DictSource([{"id": 4, "x": 4}], tag_columns=["id"], source_id="d"), + ), + ( + "dict with a null", + _assign, + DictSource([{"id": 5, "x": None}, {"id": 6, "x": 6}], tag_columns=["id"]), + ), + ("csv with an empty cell", _bind, _csv(tmp_path, "a.csv", "id,x\n7,\n8,8\n")), + ("narrower csv", _bind, _csv(tmp_path, "b.csv", "id,x\n9,9\n")), + ] + first: dict[str, Any] | None = None + for label, bind, source in binds: + bind(job, source) + assert job.run().ok, label + snapshot = _snapshot(job) + if first is None: + first = snapshot + assert snapshot == first, label + + # The last bound source, the narrower CSV, is rebuilt from its file on load. + job.save(str(tmp_path / "job.json")) + loaded = PipelineJob.load(str(tmp_path / "job.json")) + before = _snapshot(loaded, LOADED_NODES) + assert before["paths"] == first["paths"] + assert before["stored"] == first["stored"] + assert before["nodes"]["src"] == first["nodes"]["src"] + _bind(loaded, _arrow([0, 1], [1, None], x_nullable=True)) + # A loaded function node orders its meta columns unlike a fresh one, bind or not + # (reported separately), so the loaded job is compared with itself. + assert _snapshot(loaded, LOADED_NODES) == before diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 4d25ff5b5..3fc6208b7 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -351,3 +351,43 @@ def test_loaded_csv_now_narrower_is_bound_as_declared(tmp_path: pathlib.Path) -> assert dict(slot.output_schema()[1]) == {"x": int | None} assert slot.as_table().schema.field("x").nullable assert slot.as_table().column("x").to_pylist() == [1, 2] + + +# --------------------------------------------------------------------------- +# System-tag values are the source's own +# --------------------------------------------------------------------------- + + +def test_system_tag_values_and_source_info_are_the_sources_own( + tmp_path: pathlib.Path, +) -> None: + """Names follow the slot; source id, record ids and source-info tokens the source. + + The source is given no source id, so its default hash-based id is computed from its + true table (``x: int``) and is carried unchanged. + """ + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), False)] + ) + source = ArrowTableSource( + pa.table({"id": [0, 1], "x": [1, 2]}, schema=schema), tag_columns=["id"] + ) + direct = source.as_table(all_info=True) + slot = _bound_slot(tmp_path, source) + bound = slot.as_table(all_info=True) + + def system_tag(table: pa.Table, prefix: str) -> list[Any]: + """Return the values of the one column of *table* whose name starts with *prefix*.""" + (name,) = [c for c in table.column_names if c.startswith(prefix)] + return table.column(name).to_pylist() + + for prefix in ("_tag_source_id::", "_tag_record_id::"): + assert system_tag(bound, prefix) == system_tag(direct, prefix) + assert system_tag(bound, "_tag_source_id::") == [source.source_id] * 2 + assert ( + bound.column("_source_x").to_pylist() == direct.column("_source_x").to_pylist() + ) + # The names are the slot's, not the source's. + names = [c for c in bound.column_names if c.startswith("_tag_")] + assert names == list(slot.keys(columns={"system_tags": True})[0][1:]) + assert not set(names) & set(direct.column_names) From 493252c15ca1958c051e2963c11fad89f3f85094 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:57:45 -0800 Subject: [PATCH 11/33] test(pipeline): narrow the first snapshot before the loaded-job checks (basedpyright) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_bind_keeps_node_schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index c01b037b8..75718f7bb 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -125,6 +125,7 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( first = snapshot assert snapshot == first, label + assert first is not None # The last bound source, the narrower CSV, is rebuilt from its file on load. job.save(str(tmp_path / "job.json")) loaded = PipelineJob.load(str(tmp_path / "job.json")) From b5e69147507f740eab2437bfdd17269a3b909863 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 04:34:53 -0800 Subject: [PATCH 12/33] fix(source_node): columns in declared order, a stream's system tags take the slot's names, meta kept; narrower joined stream refused (ITL-1112) CodeRabbit round 1 on #386: _DeclaredForm also orders tag, data and source-info columns as declared and renames a stream's one system-tag pair (found by prefix) to the slot's names, on every bind. A joined or merged stream of the declared schema passes through (known gap); a narrower one is refused at bind. The narrower as_table and head paths read meta columns too, and the head view lists the renamed system tags. The rebind test's pod now returns its input. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 +- CLAUDE.md | 13 +- changelog.d/ITL-1112.md | 17 +- design_issues/ITL-1112-BT1.md | 10 +- docs/concepts/function-pods.md | 3 +- src/orcapod/core/nodes/source_node.py | 134 ++++++++++++--- .../test_bind_keeps_node_schemas.py | 7 + .../test_bind_narrower_source.py | 155 +++++++++++++++++- tests/test_pipeline/test_bind_type_check.py | 7 +- 9 files changed, 314 insertions(+), 45 deletions(-) diff --git a/.zed/rules b/.zed/rules index ebdde7893..b7f4d8285 100644 --- a/.zed/rules +++ b/.zed/rules @@ -777,11 +777,14 @@ data. The rules: flags, order); only values differ. SourceNodeBase.validate is the one check, used by PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and PipelineJob.load: per column the declared Python type, or T where the slot declares - T | None; anything else raises SourceSpecMismatchError before anything is bound. A - narrower source is given as the declared schema at one place, _DeclaredForm (nullable - fields over the same arrays; the system-tag column NAMES of the declared schema hash, the - VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow - width only (int32 against int64) is not yet refused (ITL-949). + T | None; anything else raises SourceSpecMismatchError before anything is bound. The + source's rows are given as the declared schema at one place, _DeclaredForm (nullable + fields over the same arrays; the declared column order; the system-tag column NAMES of the + declared schema hash, taken from the source's own pair by prefix, the VALUES the source's + own). Limits: the check compares Python types, so a difference in Arrow width only + (int32 against int64) is not yet refused (ITL-949); a joined or merged stream (several + system-tag pairs) of the declared schema passes through with its own system tags, and a + narrower one is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 387d5ada9..1c9d867e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -819,11 +819,14 @@ data. The rules: flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and `PipelineJob.load`: per column the declared Python type, or `T` where the slot declares - `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. A - narrower source is given as the declared schema at one place, `_DeclaredForm` (nullable - fields over the same arrays; the system-tag column NAMES of the declared schema hash, the - VALUES the source's own). Limit: the check compares Python types, so a difference in Arrow - width only (`int32` against `int64`) is not yet refused (ITL-949). + `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. The + source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable + fields over the same arrays; the declared column order; the system-tag column NAMES of the + declared schema hash, taken from the source's own pair by prefix, the VALUES the source's + own). Limits: the check compares Python types, so a difference in Arrow width only + (`int32` against `int64`) is not yet refused (ITL-949); a joined or merged stream (several + system-tag pairs) of the declared schema passes through with its own system tags, and a + narrower one is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index ede494318..726d036d1 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -30,8 +30,7 @@ be rebuilt: a warning names the slot and the slot stays unbound. Migration: bind a source whose columns have the slot's types (convert them, or give the Arrow table a schema with the slot's nullability), or record the pipeline with a slot of the -source's types, which gives it its own stored paths. Two Arrow types that are one Python type -(`int32` and `int64`) are still not told apart (ITL-949). +source's types, which gives it its own stored paths. ### Changed @@ -43,3 +42,17 @@ source's types, which gives it its own stored paths. Two Arrow types that are on write to the same stored tables. Nothing downstream changes: every pipeline hash and predicted schema is the declared one. The source node's `content_hash` and source id stay the bound source's own. +- The pipeline determines every node's schema (ITL-1112): no accepted bind changes a node's + stored path or the column names, types, nullable flags or order of any node's or stored + table. A source whose columns come in another order than the slot declares gives them in + the declared order, and a stream bound to a slot (a source passed through a filter or a + rename, with one system-tag pair) gives its system-tag columns the slot's names; their + values stay the stream's own. Both passed through as they were before. + +### Known limitations + +- The bind check compares Python types, so a source that differs from its slot only in + Arrow width (`int32` against `int64`) is still accepted (ITL-949). +- A joined or merged stream (one system-tag pair per input) of the slot's declared schema + is bound as before, with its own system-tag columns, which the slot does not predict; a + narrower one is refused. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index c5318d82f..747d25b9d 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -30,6 +30,10 @@ arrays under nullable fields, and the two system-tag columns renamed from the so hash to the declared one (their names carry the schema hash). `iter_data`, `async_iter_data`, `as_table`, `head` and `output_schema` go through it, so the rows equal those of a nullable source with the same values and source id, row hashes and record ids included, and both append -to one stored table. The node's `content_hash` and source id stay the source's own. Two Arrow -types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949). No -hash, golden value or stored path changed. +to one stored table. The node's `content_hash` and source id stay the source's own. The same +form gives a source's columns in the declared order, and a stream's one system-tag pair the +slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a +node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) +are still not told apart (ITL-949), and a joined or merged stream of the declared schema +(several system-tag pairs) is still bound with its own system tags; a narrower one is +refused. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index eafed3358..960391f85 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -838,7 +838,8 @@ rows under that key is an empty result, not an error. any other (`SourceSpecMismatchError`, an `InputValidationError`). The pipeline determines every node's schema: binding a source never changes a node's stored path or the schema of any node's or stored table, only the values. The check compares Python types, so a - difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). + difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949), and + a joined stream of the slot's schema is bound with its own system-tag columns. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 769c9fd3b..7bd8b43a6 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -11,7 +11,7 @@ import functools import logging from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Collection, Iterator, Mapping from typing import TYPE_CHECKING, Any from orcapod import contexts @@ -31,6 +31,7 @@ check_head_size, ) from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import ColumnConfig, Schema from orcapod.utils import arrow_utils @@ -70,22 +71,27 @@ def _is_narrower(declared: Any, given: Any) -> bool: @dataclasses.dataclass(frozen=True) class _DeclaredForm: - """How the rows of a narrower bound source become rows of the declared schema. + """How the rows of a bound source become rows of the declared schema. - A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows - as rows of ``x: int | None`` (ITL-1112): the same arrays under nullable fields, and - the two system-tag columns, named after the source's schema hash, renamed after the - declared schema's. Nothing else differs: values, types, source-info columns and - their tokens pass unchanged, so the rows equal those of a source of the declared - schema with the same source id. + The pipeline determines every node's schema (ITL-1112). A slot declaring + ``x: int | None`` accepts a source of ``x: int`` and gives its rows as rows of + ``x: int | None``: the same arrays under nullable fields. The source's system-tag + columns take the slot's names (their values stay the source's own), and its tag and + data columns, with their source-info columns, the declared order. Nothing else + differs: values, types, source-info tokens and meta columns pass unchanged, so the + rows equal those of a source of the declared schema with the same source id. Attributes: nullable: The columns declared ``T | None`` that the source gives as ``T``. renames: The source's system-tag column names, mapped to the declared ones. + tag_order: The declared tag columns, in declared order. + data_order: The declared data columns, in declared order. """ nullable: frozenset[str] renames: Mapping[str, str] + tag_order: tuple[str, ...] + data_order: tuple[str, ...] def apply(self, table: pa.Table) -> pa.Table: """Return *table*'s arrays under the declared fields and system-tag names. @@ -97,8 +103,9 @@ def apply(self, table: pa.Table) -> pa.Table: table: Rows read from the bound source. Returns: - The same arrays, with the columns in ``nullable`` flagged nullable and the - columns in ``renames`` renamed. + The same arrays, with the columns in ``nullable`` flagged nullable, the + columns in ``renames`` renamed, and the tag, data and source-info columns in + declared order within the positions each group holds. """ schema = table.schema for position, field in enumerate(table.schema): @@ -108,7 +115,52 @@ def apply(self, table: pa.Table) -> pa.Table: field = field.with_name(self.renames[field.name]) schema = schema.set(position, field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized # Same arrays under the declared fields: no cast, so no value or type changes. - return pa.Table.from_arrays(table.columns, schema=schema) + table = pa.Table.from_arrays(table.columns, schema=schema) + return table.select(self._declared_positions(table.column_names)) + + def _declared_positions(self, names: list[str]) -> list[int]: + """Return the column positions of *names* with each group in declared order. + + The tag columns, the data columns and the data columns' source-info columns each + fill the positions their group holds, in declared order; every other column + keeps its position. + """ + position = {name: index for index, name in enumerate(names)} + order = list(range(len(names))) + source_info = tuple(f"{constants.SOURCE_PREFIX}{n}" for n in self.data_order) + for group in (self.tag_order, self.data_order, source_info): + present = [name for name in group if name in position] + for slot, name in zip( + sorted(position[n] for n in present), present, strict=True + ): + order[slot] = position[name] + return order + + +def _system_tag_columns( + source: StreamProtocol, tag_names: Collection[str] +) -> tuple[list[str], list[str]]: + """Return *source*'s source-id and record-id system-tag columns. + + A root source, or a stream that keeps its system tags (a filter, a rename), has one + of each; a stream that joins or merges inputs has one pair per input; a source with + no rows may have none. + + Args: + source: The stream being bound. + tag_names: Its user tag columns. + """ + names = [ + name + for name in source.keys(columns={"system_tags": True})[0] + if name not in tag_names + ] + source_id = f"{constants.SYSTEM_TAG_SOURCE_ID_PREFIX}{constants.BLOCK_SEPARATOR}" + record_id = f"{constants.SYSTEM_TAG_RECORD_ID_PREFIX}{constants.BLOCK_SEPARATOR}" + return ( + [name for name in names if name.startswith(source_id)], + [name for name in names if name.startswith(record_id)], + ) class SourceNodeBase(TraceableBase, ABC): @@ -361,6 +413,16 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] f"{_type_text(given[name])}" ) + if narrower and not issues: + # A narrower source's rows are given with the slot's one system-tag pair, + # which a joined or merged stream's several pairs cannot become. + source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) > 1 or len(record_ids) > 1: + issues.append( + "a narrower source must carry one source-id and one record-id " + f"system-tag column, not {sorted(source_ids + record_ids)}" + ) + if issues: raise SourceSpecMismatchError( f"SourceNode '{self._name}' is not compatible with the provided source. " @@ -715,27 +777,47 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: source: The stream being bound. Returns: - ``None`` when the source's schema equals the declared one, so its rows pass - unchanged; otherwise the ``_DeclaredForm`` of its narrower columns. + ``None`` when the source's rows already are rows of the declared schema + (same types, column order and system-tag names), or are a joined or merged + stream of the declared schema, so its rows pass unchanged; otherwise the + ``_DeclaredForm`` that makes them so. Raises: SourceSpecMismatchError: If *source* does not fit the declared schema. """ source_tag, source_data, narrower = self._fit(source) - if not narrower: + source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) > 1 or len(record_ids) > 1: + # A joined or merged stream of the declared schema passes through, its + # system tags as they are: a known gap (ITL-1112). A narrower one was + # refused by _fit. return None - source_schema_hash = compute_source_schema_hash( - source_tag, source_data, self.data_context, self.orcapod_config + declared_source_id, declared_record_id = system_tag_column_names( + self._schema_hash_str ) + renames = { + name: declared + for names, declared in ( + (source_ids, declared_source_id), + (record_ids, declared_record_id), + ) + for name in names + if name != declared + } + tag_order = tuple(self._tag_schema.keys()) + data_order = tuple(self._data_schema.keys()) + if ( + not narrower + and not renames + and tuple(source_tag.keys()) == tag_order + and tuple(source_data.keys()) == data_order + ): + return None return _DeclaredForm( nullable=narrower, - renames=dict( - zip( - system_tag_column_names(source_schema_hash), - system_tag_column_names(self._schema_hash_str), - strict=True, - ) - ), + renames=renames, + tag_order=tag_order, + data_order=data_order, ) def _declared_rows( @@ -848,6 +930,7 @@ def head( self._bound_source.head( n, columns=ColumnConfig( + meta=True, source=True, system_tags=True, context=True, @@ -859,10 +942,12 @@ def head( table = self._declared_stream( self._bound_source, self._declared_form, view.table ).as_table(columns=config) + renames = self._declared_form.renames + tag_columns = {renames.get(c, c) for c in view.tag_columns} return dataclasses.replace( view, table=table, - tag_columns=tuple(c for c in view.tag_columns if c in table.column_names), + tag_columns=tuple(c for c in table.column_names if c in tag_columns), ) async def async_iter_data( @@ -944,6 +1029,7 @@ def as_table( return self._bound_source.as_table(columns=columns, all_info=all_info) table = self._bound_source.as_table( columns={ + "meta": True, "source": True, "system_tags": True, "context": True, diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index 75718f7bb..f9cc4fee2 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -114,6 +114,13 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( DictSource([{"id": 5, "x": None}, {"id": 6, "x": 6}], tag_columns=["id"]), ), ("csv with an empty cell", _bind, _csv(tmp_path, "a.csv", "id,x\n7,\n8,8\n")), + ( + "stream over a source", + _assign, + MapData({"q": "x"})( + DictSource([{"id": 10, "q": 10}], tag_columns=["id"], source_id="m") + ), + ), ("narrower csv", _bind, _csv(tmp_path, "b.csv", "id,x\n9,9\n")), ] first: dict[str, Any] | None = None diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 3fc6208b7..31093cb0c 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -22,7 +22,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode -from orcapod.core.operators import MapData +from orcapod.core.operators import Join, MapData from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -231,8 +231,10 @@ def test_rows_of_each_read_carry_the_declared_flags(tmp_path: pathlib.Path) -> N assert data.as_table().schema.field("x").nullable for _, data in asyncio.run(_collect(slot)): assert data.as_table().schema.field("x").nullable - head = slot.head(1, all_info=True).table + view = slot.head(1, all_info=True) + head = view.table assert head.schema.field("x").nullable + assert view.tag_columns == slot.keys(columns={"system_tags": True})[0] assert head.equals(slot.as_table(all_info=True).slice(0, 1)) @@ -391,3 +393,152 @@ def system_tag(table: pa.Table, prefix: str) -> list[Any]: names = [c for c in bound.column_names if c.startswith("_tag_")] assert names == list(slot.keys(columns={"system_tags": True})[0][1:]) assert not set(names) & set(direct.column_names) + + +# --------------------------------------------------------------------------- +# Column order, transformed streams and meta columns (CodeRabbit round 1) +# --------------------------------------------------------------------------- + + +def _abc_job(a_type: Any) -> PipelineJob: + """Return a job whose slot declares tag ``id`` and data ``a``, ``b``, ``c`` in order.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"a": a_type, "b": str, "c": float}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"a": "a2"})(slot, label="renamed") + return job + + +@pytest.mark.parametrize("a_type", [int, int | None], ids=["equal", "narrower"]) +def test_columns_follow_the_declared_order(a_type: Any) -> None: + """A source with its columns in another order gives them in the declared order.""" + job = _abc_job(a_type) + job.bind( + sources={ + "src": DictSource( + [{"id": 0, "c": 1.5, "b": "s", "a": 1}], tag_columns=["id"] + ) + } + ) + node = job.nodes["src"] + assert list(node.output_schema()[1]) == ["a", "b", "c"] + table = node.as_table(columns={"source": True}) + assert table.column_names == [ + "id", + "a", + "b", + "c", + "_source_a", + "_source_b", + "_source_c", + ] + for _, data in node.iter_data(): + assert data.as_table().column_names == ["a", "b", "c"] + for _, data in asyncio.run(_collect(node)): + assert data.as_table().column_names == ["a", "b", "c"] + assert node.head(1).table.column_names == ["id", "a", "b", "c"] + assert job.nodes["renamed"].as_table().column_names == ["id", "a2", "b", "c"] + + +def _mapped_stream() -> Any: + """Return a stream (not a root source): a DictSource with ``q`` renamed to ``x``.""" + source = DictSource([{"id": 0, "q": 1}], tag_columns=["id"], source_id="m") + return MapData({"q": "x"})(source) + + +@pytest.mark.parametrize("x_type", [int, int | None], ids=["equal", "narrower"]) +def test_a_transformed_stream_gets_the_slots_system_tag_names( + tmp_path: pathlib.Path, x_type: Any +) -> None: + """A stream's one system-tag pair takes the slot's names; its values stay its own.""" + stream = _mapped_stream() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(x_type), label="renamed") + job.bind(sources={"src": stream}) + node = job.nodes["src"] + predicted = node.keys(columns={"system_tags": True})[0][1:] + table = node.as_table(columns={"system_tags": True}) + names = [c for c in table.column_names if c.startswith("_tag_")] + assert names == list(predicted) + direct = stream.as_table(columns={"system_tags": True}) + direct_names = [c for c in direct.column_names if c.startswith("_tag_")] + assert not set(direct_names) & set(names) + for mine, theirs in zip(names, direct_names, strict=True): + assert table.column(mine).to_pylist() == direct.column(theirs).to_pylist() + + +def _joined_stream(x_nullable: bool) -> Any: + """Return a Join of two sources: two system-tag pairs, data ``x`` and ``w``.""" + left = _arrow_source([1, 2], x_nullable=x_nullable) + right = DictSource([{"id": 0, "w": "p"}, {"id": 1, "w": "q"}], tag_columns=["id"]) + return Join()(left, right) + + +def _xw_job() -> PipelineJob: + """Return a job whose slot declares tag ``id`` and data ``x: int | None``, ``w``.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int | None, "w": str}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(slot, label="renamed") + return job + + +def test_a_joined_stream_of_the_declared_schema_passes_through() -> None: + """An equal-schema stream with two system-tag pairs is bound as it is (known gap).""" + stream = _joined_stream(x_nullable=True) + job = _xw_job() + job.bind(sources={"src": stream}) + assert job.nodes["src"].keys(all_info=True) == stream.keys(all_info=True) + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_narrower_joined_stream_is_refused( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A narrower stream with two system-tag pairs cannot take the slot's one pair.""" + job = _xw_job() + with pytest.raises(SourceSpecMismatchError, match="system-tag"): + bind(job, _joined_stream(x_nullable=False)) + assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +def test_meta_columns_of_a_narrower_stream_are_kept() -> None: + """The ``__`` columns a job node's rows carry reach the rows of a narrower bind.""" + + @function_pod(output_keys="x") + def same(v: int) -> int: + """Return ``v``.""" + return v + + origin = PipelineJob(name="origin", database=InMemoryArrowDatabase()) + with origin: + _ = same.pod( + DictSource([{"id": 0, "v": 1}], tag_columns=["id"], source_id="o"), + label="same", + ) + _ = origin.run() + stream = origin.nodes["same"] + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + job.bind(sources={"src": stream}) + node = job.nodes["src"] + own = [ + sorted(data.as_table(columns={"meta": True}).column_names) + for _, data in stream.iter_data() + ] + assert any("__computed" in names for names in own) + for rows in (list(node.iter_data()), asyncio.run(_collect(node))): + assert [ + sorted(data.as_table(columns={"meta": True}).column_names) + for _, data in rows + ] == own diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py index 044f8314f..b5ab4d84d 100644 --- a/tests/test_pipeline/test_bind_type_check.py +++ b/tests/test_pipeline/test_bind_type_check.py @@ -26,8 +26,8 @@ @function_pod(output_keys="z") def const(y: int) -> int: - """Return 0 for any *y*.""" - return 0 + """Return *y*, so stored results show which source they came from.""" + return y def _ints(source_id: str = "ints", start: int = 1) -> DictSource: @@ -182,4 +182,5 @@ def test_an_accepted_rebind_runs_and_stores_the_new_rows( _ = job.run() job.bind(sources={"src": _ints(source_id="other", start=10)}) _ = job.run() - assert job.nodes["const"].as_table().column("z").to_pylist() == [0, 0] + # Results of the first source (x 1, 2) would be [1, 2]. + assert sorted(job.nodes["const"].as_table().column("z").to_pylist()) == [10, 11] From 390c5d204b76fdee88c18eb24b25e1024fdbeb59 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:01:29 -0800 Subject: [PATCH 13/33] fix(source_node): a narrower bind needs exactly one system-tag pair; pass-through kept in one function (ITL-1112) CodeRabbit round 2 on #386: a narrower stream without system tags is refused at bind like a narrower joined one. The equal-schema pass-through of streams without exactly one pair is the known gap pending the developer, in _keeps_its_own_system_tags. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 ++- CLAUDE.md | 5 ++- changelog.d/ITL-1112.md | 6 +-- design_issues/ITL-1112-BT1.md | 4 +- docs/concepts/function-pods.md | 3 +- src/orcapod/core/nodes/source_node.py | 41 +++++++++++++------ .../test_bind_narrower_source.py | 20 +++++++++ 7 files changed, 62 insertions(+), 22 deletions(-) diff --git a/.zed/rules b/.zed/rules index b7f4d8285..075fdefff 100644 --- a/.zed/rules +++ b/.zed/rules @@ -783,8 +783,9 @@ data. The rules: declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (int32 against int64) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) of the declared schema passes through with its own system tags, and a - narrower one is refused. + system-tag pairs) or a stream with none, of the declared schema, passes through with its + own system tags (_keeps_its_own_system_tags, pending the developer), and a narrower one + is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 1c9d867e0..9d6b344a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -825,8 +825,9 @@ data. The rules: declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) of the declared schema passes through with its own system tags, and a - narrower one is refused. + system-tag pairs) or a stream with none, of the declared schema, passes through with its + own system tags (`_keeps_its_own_system_tags`, pending the developer), and a narrower one + is refused. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 726d036d1..bfe75979f 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -53,6 +53,6 @@ source's types, which gives it its own stored paths. - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A joined or merged stream (one system-tag pair per input) of the slot's declared schema - is bound as before, with its own system-tag columns, which the slot does not predict; a - narrower one is refused. +- A joined or merged stream (one system-tag pair per input), or a stream with no system-tag + columns, of the slot's declared schema is bound as before, with its own system-tag + columns, which the slot does not predict; a narrower one is refused. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 747d25b9d..1f1d542b6 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -35,5 +35,5 @@ form gives a source's columns in the declared order, and a stream's one system-t slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949), and a joined or merged stream of the declared schema -(several system-tag pairs) is still bound with its own system tags; a narrower one is -refused. No hash, golden value or stored path changed. +(several system-tag pairs), or one with no system tags, is still bound with its own system +tags; a narrower one is refused. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 960391f85..b8638263c 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -839,7 +839,8 @@ rows under that key is an empty result, not an error. every node's schema: binding a source never changes a node's stored path or the schema of any node's or stored table, only the values. The check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949), and - a joined stream of the slot's schema is bound with its own system-tag columns. + a joined stream, or one without system tags, of the slot's schema is bound with its own + system-tag columns. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 7bd8b43a6..1b4764ac1 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -163,6 +163,24 @@ def _system_tag_columns( ) +def _keeps_its_own_system_tags(source_ids: list[str], record_ids: list[str]) -> bool: + """Return whether a bound stream's system tags stay as they are, not the slot's. + + The slot's rows carry one system-tag pair, under the slot's names. A stream with + exactly one pair takes those names; a joined or merged stream (one pair per input) + or a stream with none cannot. A narrower such stream is refused at bind; one of the + declared schema passes through with its own system tags, a known gap against the + rule that the pipeline determines every node's schema. Pending the developer's + decision (ITL-1112, "S1"): refusing these binds too is a change here and in + ``SourceNodeBase._fit``. + + Args: + source_ids: The stream's source-id system-tag columns. + record_ids: The stream's record-id system-tag columns. + """ + return len(source_ids) != 1 or len(record_ids) != 1 + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -415,12 +433,13 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] if narrower and not issues: # A narrower source's rows are given with the slot's one system-tag pair, - # which a joined or merged stream's several pairs cannot become. + # which only a source with exactly one pair can take. source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if len(source_ids) > 1 or len(record_ids) > 1: + if _keeps_its_own_system_tags(source_ids, record_ids): issues.append( - "a narrower source must carry one source-id and one record-id " - f"system-tag column, not {sorted(source_ids + record_ids)}" + "a narrower source must carry exactly one source-id and one " + "record-id system-tag column, not " + f"{sorted(source_ids + record_ids)}" ) if issues: @@ -787,21 +806,19 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: """ source_tag, source_data, narrower = self._fit(source) source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if len(source_ids) > 1 or len(record_ids) > 1: - # A joined or merged stream of the declared schema passes through, its - # system tags as they are: a known gap (ITL-1112). A narrower one was - # refused by _fit. + if _keeps_its_own_system_tags(source_ids, record_ids): + # Of the declared schema (a narrower one was refused by _fit): passes + # through as it is. return None declared_source_id, declared_record_id = system_tag_column_names( self._schema_hash_str ) renames = { name: declared - for names, declared in ( - (source_ids, declared_source_id), - (record_ids, declared_record_id), + for name, declared in ( + (source_ids[0], declared_source_id), + (record_ids[0], declared_record_id), ) - for name in names if name != declared } tag_order = tuple(self._tag_schema.keys()) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 31093cb0c..9647e7ba4 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -23,6 +23,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join, MapData +from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -542,3 +543,22 @@ def same(v: int) -> int: sorted(data.as_table(columns={"meta": True}).column_names) for _, data in rows ] == own + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_narrower_stream_without_system_tags_is_refused( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A narrower stream with no system-tag pair cannot be given the slot's pair.""" + schema = pa.schema( + [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), False)] + ) + stream = ArrowTableStream( + pa.table({"id": [0], "x": [1]}, schema=schema), tag_columns=["id"] + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + with pytest.raises(SourceSpecMismatchError, match="system-tag"): + bind(job, stream) + assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode From de1ed2587174024a0f5ac08a6e65b6cb1d10c2f5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:31:11 -0800 Subject: [PATCH 14/33] fix(source_node): a slot's zero-row table is its bound source's in the declared form (ITL-1112, ITL-912) The typed zero-row table #375 added for a bound source node returned the source's own (its nullable flags, column order and system-tag names), so a downstream node before a run read with other types than after it. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 12 ++- .../test_bind_narrower_source.py | 84 +++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 0008b5e02..79a2ca8ec 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -912,11 +912,17 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: return self._declared_rows(self._bound_source.iter_data(), self._declared_form) def _typed_zero_row_table(self) -> "pa.Table | None": - """The bound source's zero-row table; ``None`` when unbound or unknown. + """The bound source's zero-row table in the declared form; ``None`` if unknown. - See ``typed_zero_row_table``. + A source given in the declared form (``_DeclaredForm``) has its zero-row table + given in that form too, so it has exactly the schema of the node's populated + table (ITL-1112, ITL-912). ``None`` when unbound or when the source cannot type + its rows without rows. See ``typed_zero_row_table``. """ - return typed_zero_row_table(self._bound_source) + table = typed_zero_row_table(self._bound_source) + if table is None or self._declared_form is None: + return table + return self._declared_form.apply(table) def _schema_requires_io(self, config: ColumnConfig) -> bool: """Whether reading the schema with ``config`` would poll: as for the bound source.""" diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 9647e7ba4..b37a15e5e 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -24,6 +24,7 @@ from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join, MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream +from orcapod.core.streams.base import ZERO_ROW_INPUT_COLUMNS, typed_zero_row_table from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -562,3 +563,86 @@ def test_a_narrower_stream_without_system_tags_is_refused( with pytest.raises(SourceSpecMismatchError, match="system-tag"): bind(job, stream) assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + +# --------------------------------------------------------------------------- +# Zero-row tables (ITL-912): the declared form, as the populated table has it +# --------------------------------------------------------------------------- + + +def _declared_bind() -> PipelineJob: + """Return the ``x: int`` job bound to a source of exactly the declared form.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int), label="renamed") + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + return job + + +def _narrower_bind() -> PipelineJob: + """Return the ``x: int | None`` job bound to a source of ``x: int``.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + job.bind(sources={"src": _arrow_source([1, 2], x_nullable=False)}) + return job + + +def _reordered_bind() -> PipelineJob: + """Return the ``a, b, c`` job bound to a source whose data come as ``c, b, a``.""" + job = _abc_job(int) + source = DictSource([{"id": 0, "c": 1.5, "b": "s", "a": 1}], tag_columns=["id"]) + job.bind(sources={"src": source}) + return job + + +def _transformed_bind() -> PipelineJob: + """Return the ``x: int`` job bound to a stream whose system tags have other names.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int), label="renamed") + job.bind(sources={"src": _mapped_stream()}) + return job + + +DECLARED_FORM_BINDS: dict[str, Callable[[], PipelineJob]] = { + "declared": _declared_bind, + "narrower": _narrower_bind, + "reordered": _reordered_bind, + "transformed": _transformed_bind, +} + + +@pytest.mark.parametrize( + "make_job", DECLARED_FORM_BINDS.values(), ids=DECLARED_FORM_BINDS.keys() +) +def test_the_zero_row_table_of_a_slot_is_its_populated_declared_form( + make_job: Callable[[], PipelineJob], +) -> None: + """A slot's zero-row table has exactly the schema of its populated table.""" + job = make_job() + node = job.nodes["src"] + populated = node.as_table(columns=ZERO_ROW_INPUT_COLUMNS) + assert populated.num_rows > 0 + table = typed_zero_row_table(node) + assert table is not None + assert table.num_rows == 0 + assert table.schema == populated.schema + + +@pytest.mark.parametrize( + "make_job", DECLARED_FORM_BINDS.values(), ids=DECLARED_FORM_BINDS.keys() +) +def test_a_downstream_node_before_a_run_has_the_types_of_its_rows( + make_job: Callable[[], PipelineJob], +) -> None: + """The node after the slot reads, before a run, with the schema it has after one.""" + job = make_job() + before = job.nodes["renamed"].as_table(all_info=True) + head = job.nodes["renamed"].head(all_info=True).table + assert before.num_rows == 0 + _ = job.run() + after = job.nodes["renamed"].as_table(all_info=True) + assert after.num_rows > 0 + assert before.schema == after.schema + assert head.schema == after.schema From c6dfdc1ba491631b91dfcac82bb8e96bc647288b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:36:34 -0800 Subject: [PATCH 15/33] fix(source_node)!: every bind refuses a source without exactly one system-tag pair (ITL-1126) A joined or merged stream (one system-tag pair per input) or a raw stream with none was bound with its own system-tag columns and column order when its schema equalled the slot's, so the slot's table schema depended on what was bound. Developer, November 14, 2026: refuse it for now; binding one properly is ITL-1127. _fit refuses it on every bind (bind, bound_source setter, construction, load) with SourceSpecMismatchError naming the slot and the columns found; the pass-through function is removed. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 8 +- CLAUDE.md | 8 +- changelog.d/ITL-1112.md | 18 ++- design_issues/ITL-1112-BT1.md | 7 +- docs/concepts/function-pods.md | 7 +- src/orcapod/core/nodes/source_node.py | 61 ++++----- .../test_bind_narrower_source.py | 123 ++++++++++++------ 7 files changed, 137 insertions(+), 95 deletions(-) diff --git a/.zed/rules b/.zed/rules index f6d052c8c..64193bd06 100644 --- a/.zed/rules +++ b/.zed/rules @@ -801,10 +801,10 @@ data. The rules: fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only - (int32 against int64) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) or a stream with none, of the declared schema, passes through with its - own system tags (_keeps_its_own_system_tags, pending the developer), and a narrower one - is refused. + (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns + are not exactly one source-id and one record-id column (a joined or merged stream, a raw + stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); + binding one properly is ITL-1127. Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 4cfc94195..b1575bb8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -843,10 +843,10 @@ data. The rules: fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only - (`int32` against `int64`) is not yet refused (ITL-949); a joined or merged stream (several - system-tag pairs) or a stream with none, of the declared schema, passes through with its - own system tags (`_keeps_its_own_system_tags`, pending the developer), and a narrower one - is refused. + (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns + are not exactly one source-id and one record-id column (a joined or merged stream, a raw + stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); + binding one properly is ITL-1127. Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index bfe75979f..79e783e91 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -32,6 +32,21 @@ Migration: bind a source whose columns have the slot's types (convert them, or g Arrow table a schema with the slot's nullability), or record the pipeline with a slot of the source's types, which gives it its own stored paths. +#### Binding refuses a joined or merged stream, and a stream without system tags (ITL-1126) + +A slot's rows carry one source-id and one record-id system-tag column, under the slot's names. +A stream with one pair per input (the result of a `Join` or `MergeJoin`), or with none (an +`ArrowTableStream` over a table without them), cannot give its rows that way; it was bound +before with its own system-tag columns and column order, so the schema of the slot's table, +and of every table after it, depended on what was bound. Every way of binding (`bind`, the +`bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses +it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever +its column types. + +Migration: make the join part of the pipeline (bind its inputs to slots of their own), or +store the joined stream and bind the stored source (a source has one pair). Binding such a +stream properly is planned (ITL-1127). + ### Changed - A slot that declares `x: int | None` accepts a source whose `x` is `int` (ITL-1112). The slot @@ -53,6 +68,3 @@ source's types, which gives it its own stored paths. - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A joined or merged stream (one system-tag pair per input), or a stream with no system-tag - columns, of the slot's declared schema is bound as before, with its own system-tag - columns, which the slot does not predict; a narrower one is refused. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 1f1d542b6..d9957b835 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -34,6 +34,7 @@ to one stored table. The node's `content_hash` and source id stay the source's o form gives a source's columns in the declared order, and a stream's one system-tag pair the slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) -are still not told apart (ITL-949), and a joined or merged stream of the declared schema -(several system-tag pairs), or one with no system tags, is still bound with its own system -tags; a narrower one is refused. No hash, golden value or stored path changed. +are still not told apart (ITL-949). A source whose system-tag columns are not exactly one +source-id and one record-id column (a joined or merged stream, a raw stream with none) is +refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is +ITL-1127). No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8638263c..d44e5c2be 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -838,9 +838,10 @@ rows under that key is an empty result, not an error. any other (`SourceSpecMismatchError`, an `InputValidationError`). The pipeline determines every node's schema: binding a source never changes a node's stored path or the schema of any node's or stored table, only the values. The check compares Python types, so a - difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949), and - a joined stream, or one without system tags, of the slot's schema is bound with its own - system-tag columns. + difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A + joined or merged stream (one pair of system-tag columns per input), or one with no + system-tag columns, is refused too: make the join part of the pipeline, or store the joined + stream and bind the stored source. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 79a2ca8ec..ca2ac73a1 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -143,8 +143,9 @@ def _system_tag_columns( """Return *source*'s source-id and record-id system-tag columns. A root source, or a stream that keeps its system tags (a filter, a rename), has one - of each; a stream that joins or merges inputs has one pair per input; a source with - no rows may have none. + of each; a stream that joins or merges inputs has one pair per input; a stream over a + table without them (a raw ``ArrowTableStream``) has none. A slot accepts only the + first (ITL-1126). Args: source: The stream being bound. @@ -163,24 +164,6 @@ def _system_tag_columns( ) -def _keeps_its_own_system_tags(source_ids: list[str], record_ids: list[str]) -> bool: - """Return whether a bound stream's system tags stay as they are, not the slot's. - - The slot's rows carry one system-tag pair, under the slot's names. A stream with - exactly one pair takes those names; a joined or merged stream (one pair per input) - or a stream with none cannot. A narrower such stream is refused at bind; one of the - declared schema passes through with its own system tags, a known gap against the - rule that the pipeline determines every node's schema. Pending the developer's - decision (ITL-1112, "S1"): refusing these binds too is a change here and in - ``SourceNodeBase._fit``. - - Args: - source_ids: The stream's source-id system-tag columns. - record_ids: The stream's record-id system-tag columns. - """ - return len(source_ids) != 1 or len(record_ids) != 1 - - class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -381,14 +364,18 @@ def validate(self, source: "StreamProtocol") -> None: each the declared Python type, or, where the slot declares ``T | None``, ``T`` (ITL-1112). Those are what every downstream pipeline hash, and so every stored path, is built from, so a source that fits never moves one. Column order and - ``optional_fields`` are not compared; neither enters the pipeline hash. + ``optional_fields`` are not compared; neither enters the pipeline hash. The + source must also carry exactly one source-id and one record-id system-tag + column, which take the slot's names; a joined or merged stream, or one with no + system tags, is refused (ITL-1126). Args: source: A concrete stream to validate. Raises: SourceSpecMismatchError: If a column is missing, unexpected, or of a type - the declared one does not accept. + the declared one does not accept, or the source's system-tag columns + are not exactly one source-id and one record-id column. """ _ = self._fit(source) @@ -431,16 +418,16 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] f"{_type_text(given[name])}" ) - if narrower and not issues: - # A narrower source's rows are given with the slot's one system-tag pair, - # which only a source with exactly one pair can take. - source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if _keeps_its_own_system_tags(source_ids, record_ids): - issues.append( - "a narrower source must carry exactly one source-id and one " - "record-id system-tag column, not " - f"{sorted(source_ids + record_ids)}" - ) + # The slot's rows carry one system-tag pair under the slot's names, which only + # a source with exactly one pair can take. A joined or merged stream (one pair + # per input) or a stream with none is refused (developer, November 14, 2026, + # ITL-1126); binding one properly is ITL-1127. + source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) != 1 or len(record_ids) != 1: + issues.append( + "a source must carry exactly one source-id and one record-id " + f"system-tag column, not {sorted(source_ids + record_ids)}" + ) if issues: raise SourceSpecMismatchError( @@ -797,19 +784,15 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: Returns: ``None`` when the source's rows already are rows of the declared schema - (same types, column order and system-tag names), or are a joined or merged - stream of the declared schema, so its rows pass unchanged; otherwise the - ``_DeclaredForm`` that makes them so. + (same types, column order and system-tag names), so they pass unchanged; + otherwise the ``_DeclaredForm`` that makes them so. Raises: SourceSpecMismatchError: If *source* does not fit the declared schema. """ source_tag, source_data, narrower = self._fit(source) + # _fit has checked that the source has exactly one pair. source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if _keeps_its_own_system_tags(source_ids, record_ids): - # Of the declared schema (a narrower one was refused by _fit): passes - # through as it is. - return None declared_source_id, declared_record_id = system_tag_column_names( self._schema_hash_str ) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index b37a15e5e..dd2ff2768 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -21,7 +21,7 @@ import pytest from orcapod import PipelineJob, function_pod -from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.operators import Join, MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ZERO_ROW_INPUT_COLUMNS, typed_zero_row_table @@ -494,25 +494,6 @@ def _xw_job() -> PipelineJob: return job -def test_a_joined_stream_of_the_declared_schema_passes_through() -> None: - """An equal-schema stream with two system-tag pairs is bound as it is (known gap).""" - stream = _joined_stream(x_nullable=True) - job = _xw_job() - job.bind(sources={"src": stream}) - assert job.nodes["src"].keys(all_info=True) == stream.keys(all_info=True) - - -@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) -def test_a_narrower_joined_stream_is_refused( - bind: Callable[[PipelineJob, Any], None], -) -> None: - """A narrower stream with two system-tag pairs cannot take the slot's one pair.""" - job = _xw_job() - with pytest.raises(SourceSpecMismatchError, match="system-tag"): - bind(job, _joined_stream(x_nullable=False)) - assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode - - def test_meta_columns_of_a_narrower_stream_are_kept() -> None: """The ``__`` columns a job node's rows carry reach the rows of a narrower bind.""" @@ -546,25 +527,6 @@ def same(v: int) -> int: ] == own -@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) -def test_a_narrower_stream_without_system_tags_is_refused( - bind: Callable[[PipelineJob, Any], None], -) -> None: - """A narrower stream with no system-tag pair cannot be given the slot's pair.""" - schema = pa.schema( - [pa.field("id", pa.int64(), False), pa.field("x", pa.int64(), False)] - ) - stream = ArrowTableStream( - pa.table({"id": [0], "x": [1]}, schema=schema), tag_columns=["id"] - ) - job = PipelineJob(name="j", database=InMemoryArrowDatabase()) - with job: - _ = MapData({"x": "y"})(_slot(int | None), label="renamed") - with pytest.raises(SourceSpecMismatchError, match="system-tag"): - bind(job, stream) - assert job.nodes["src"].bound_source is None # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode - - # --------------------------------------------------------------------------- # Zero-row tables (ITL-912): the declared form, as the populated table has it # --------------------------------------------------------------------------- @@ -646,3 +608,86 @@ def test_a_downstream_node_before_a_run_has_the_types_of_its_rows( assert after.num_rows > 0 assert before.schema == after.schema assert head.schema == after.schema + + +# --------------------------------------------------------------------------- +# Not exactly one system-tag pair: refused (developer, November 14, 2026, ITL-1126) +# --------------------------------------------------------------------------- + + +def _raw_stream(x_nullable: bool) -> Any: + """Return an ``ArrowTableStream`` of tag ``id`` and data ``x``, ``w``: no system tags.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), False), + pa.field("x", pa.int64(), x_nullable), + pa.field("w", pa.large_string(), False), + ] + ) + return ArrowTableStream( + pa.table({"id": [0], "x": [1], "w": ["p"]}, schema=schema), tag_columns=["id"] + ) + + +def _construct(job: PipelineJob, source: Any) -> None: + """Build a node for slot ``src`` with *source* bound at construction.""" + slot = job.nodes["src"] + _ = SourceJobNode( + name="src", + tag_schema=slot.output_schema()[0], + data_schema=slot.output_schema()[1], + bound_source=source, + ) + + +ALL_BINDS: dict[str, Callable[[PipelineJob, Any], None]] = { + **BINDS, + "construction": _construct, +} + +ONE_PAIR_REQUIRED: dict[str, Callable[[bool], Any]] = { + "joined": _joined_stream, + "no_system_tags": _raw_stream, +} + + +@pytest.mark.parametrize("x_nullable", [True, False], ids=["declared", "narrower"]) +@pytest.mark.parametrize( + "make_stream", ONE_PAIR_REQUIRED.values(), ids=ONE_PAIR_REQUIRED.keys() +) +@pytest.mark.parametrize("bind", ALL_BINDS.values(), ids=ALL_BINDS.keys()) +def test_a_stream_without_exactly_one_system_tag_pair_is_refused( + bind: Callable[[PipelineJob, Any], None], + make_stream: Callable[[bool], Any], + x_nullable: bool, +) -> None: + """A joined stream, or one with no system tags, cannot take the slot's one pair. + + The slot's rows carry one source-id and one record-id column, under the slot's + names. Refused whatever the column types; the error names the slot and the + system-tag columns found. + """ + stream = make_stream(x_nullable) + found = [ + name + for name in stream.keys(columns={"system_tags": True})[0] + if name.startswith("_tag") + ] + job = _xw_job() + with pytest.raises(SourceSpecMismatchError) as raised: + bind(job, stream) + message = str(raised.value) + assert "'src'" in message + assert "exactly one source-id and one record-id" in message + assert str(sorted(found)) in message + assert job.nodes["src"].bound_source is None + + +def test_a_refused_joined_stream_keeps_the_previous_binding() -> None: + """After the refusal the slot keeps the source it had.""" + job = _xw_job() + before = DictSource([{"id": 0, "x": 1, "w": "p"}], tag_columns=["id"]) + job.bind(sources={"src": before}) + with pytest.raises(SourceSpecMismatchError): + job.bind(sources={"src": _joined_stream(x_nullable=True)}) + assert job.nodes["src"].bound_source is before From 9b842de2605287cf283fa84015b6bc0d572ce62a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:41:37 -0800 Subject: [PATCH 16/33] fix(source_node): a pipeline's own leaf at compile is taken as it is, pending the developer (ITL-1126) Pipeline.compile binds each leaf through SourceJobNode.from_stream, so the ITL-1126 refusal broke a pipeline built directly over a raw ArrowTableStream. Coordinator decision (option B): compile accepts its leaf as on main, in one place (_accept_pipeline_leaf); every other bind refuses it. Refusing at compile too is the developer's decision. Two MagicMock fixtures carry the slot's system-tag pair. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 +- CLAUDE.md | 5 +- changelog.d/ITL-1112.md | 4 ++ design_issues/ITL-1112-BT1.md | 3 +- docs/concepts/function-pods.md | 4 +- src/orcapod/core/nodes/source_node.py | 47 ++++++++++++++++--- tests/test_core/nodes/test_source_node.py | 46 +++++++++++++----- .../test_bind_narrower_source.py | 22 +++++++++ 8 files changed, 113 insertions(+), 23 deletions(-) diff --git a/.zed/rules b/.zed/rules index 64193bd06..3613fe731 100644 --- a/.zed/rules +++ b/.zed/rules @@ -804,7 +804,10 @@ data. The rules: (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through. + binding one properly is ITL-1127. Never pass a bound stream's own system tags through, + with one exception pending the developer: Pipeline.compile binds each leaf through + SourceJobNode._accept_pipeline_leaf, which takes a leaf without exactly one pair (a raw + ArrowTableStream) as it is, so a pipeline built directly over a raw stream still compiles. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index b1575bb8d..5e6d8ca2f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -846,7 +846,10 @@ data. The rules: (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through. + binding one properly is ITL-1127. Never pass a bound stream's own system tags through, + with one exception pending the developer: `Pipeline.compile` binds each leaf through + `SourceJobNode._accept_pipeline_leaf`, which takes a leaf without exactly one pair (a raw + `ArrowTableStream`) as it is, so a pipeline built directly over a raw stream still compiles. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 79e783e91..ceab9bd41 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -68,3 +68,7 @@ stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). +- A pipeline built directly over a raw `ArrowTableStream` (one without system-tag columns) + still compiles: its source node takes the stream as it is, so that node's rows carry no + system-tag columns although the slot predicts a pair. Binding such a stream to a slot + afterwards is refused (ITL-1126). diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index d9957b835..baf49f2c4 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -37,4 +37,5 @@ node's path or table schema. Two Arrow types that are one Python type (`int32` a are still not told apart (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is -ITL-1127). No hash, golden value or stored path changed. +ITL-1127), except a pipeline's own leaf at compile (`_accept_pipeline_leaf`), which a +pipeline built directly over a raw stream still passes through, pending the developer. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index d44e5c2be..a7bf9e70b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -841,7 +841,9 @@ rows under that key is an empty result, not an error. difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A joined or merged stream (one pair of system-tag columns per input), or one with no system-tag columns, is refused too: make the join part of the pipeline, or store the joined - stream and bind the stored source. + stream and bind the stored source. A pipeline built directly over a raw `ArrowTableStream` + still compiles, for now: its source node's rows carry no system-tag columns, although the + slot predicts a pair. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index ca2ac73a1..06e20fd6e 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -776,6 +776,34 @@ def bound_source(self, value: StreamProtocol | None) -> None: self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + def _accept_pipeline_leaf(self, stream: StreamProtocol) -> None: + """Bind *stream*, a leaf of the pipeline this slot was built from. + + ``Pipeline.compile`` makes each leaf stream a slot through ``from_stream``, with + the slot's schema taken from that same stream. A leaf with exactly one + source-id and one record-id system-tag column is bound as any source is + (``bound_source``). A leaf without exactly one pair (a raw ``ArrowTableStream`` + has none) is bound as it is, as before ITL-1126, so a pipeline built directly + over a raw stream still compiles: its rows carry the stream's own system-tag + columns, not the pair the slot predicts. This is the only place a source + without exactly one pair is bound. PENDING the developer's decision (option + "A"): refuse such a leaf here too, as every other way of binding does. + + Args: + stream: The leaf stream, whose schema the slot was built from. + + Raises: + SourceSpecMismatchError: If a leaf with one pair does not fit the slot. + """ + source_ids, record_ids = _system_tag_columns(stream, self._tag_schema.keys()) + if len(source_ids) == 1 and len(record_ids) == 1: + self.bound_source = stream + return + self._bound_source = stream + self._declared_form = None + self._invalidate_content_hash_cache() + self._invalidate_pipeline_hash_cache() + def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: """Check *source* and return how its rows become rows of the declared schema. @@ -1074,6 +1102,10 @@ def from_stream( **bound** ``SourceJobNode`` whose ``content_hash()`` delegates to the concrete stream. + Used by ``Pipeline.compile`` for each leaf of the pipeline: the stream is + bound through ``_accept_pipeline_leaf``, which also accepts a leaf without + exactly one system-tag pair (a raw ``ArrowTableStream``), as before ITL-1126. + Args: stream: The stream to wrap. name: Optional explicit slot name. In all three cases, when *name* @@ -1089,12 +1121,14 @@ def from_stream( match stream: case SourceJobNode(): # Case 1: copy — preserve bound_source, do NOT wrap the SJN itself. - return cls( + node = cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, - bound_source=stream.bound_source, ) + if stream.bound_source is not None: + node._accept_pipeline_leaf(stream.bound_source) + return node case SourceNode(): # Case 2: unbound — schema placeholder; SJN hash = SourceNode hash. return cls( @@ -1121,12 +1155,11 @@ def from_stream( slot_name = stream.source_id else: slot_name = f"{stream.label}:{stream.content_hash().to_hex()}" - return cls( - name=slot_name, - tag_schema=tag_schema, - data_schema=data_schema, - bound_source=stream, + node = cls( + name=slot_name, tag_schema=tag_schema, data_schema=data_schema ) + node._accept_pipeline_leaf(stream) + return node def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 50532f33b..94470c1a8 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -2,8 +2,11 @@ from __future__ import annotations +from unittest.mock import MagicMock + import pytest +from orcapod.core.nodes.source_node import SourceJobNode from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.types import Schema @@ -283,6 +286,21 @@ def test_source_job_node_as_table_delegates_to_concrete( assert isinstance(table, pa.Table) +def _mock_source_with_the_slots_system_tags( + node: SourceJobNode, tag_schema: Schema, data_schema: Schema +) -> MagicMock: + """Return a mock source of the slot's schema carrying the slot's system-tag pair. + + A bound source must carry exactly one source-id and one record-id system-tag + column (ITL-1126); the slot's own names make the bind pass rows unchanged. + """ + tag_keys = node.keys(columns={"system_tags": True})[0] + mock_source = MagicMock() + mock_source.output_schema.return_value = (tag_schema, data_schema) + mock_source.keys.return_value = (tag_keys, tuple(data_schema.keys())) + return mock_source + + class TestSourceNodeColumnConfig: """SourceNodeBase must honour column_config / all_info in keys() and output_schema().""" @@ -385,15 +403,17 @@ def test_unbound_system_tag_names_match_bound(self, tag_schema, data_schema): def test_bound_keys_delegates_to_source(self, tag_schema, data_schema): """Bound SourceJobNode.keys() delegates to bound_source.output_schema().""" - from unittest.mock import MagicMock from orcapod.core.nodes.source_node import SourceJobNode from orcapod.types import ColumnConfig - mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) - - job_node = SourceJobNode(name="x", bound_source=mock_source) - mock_source.output_schema.reset_mock() # clear init-time schema derivation call + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema + ) + mock_source = _mock_source_with_the_slots_system_tags( + job_node, tag_schema, data_schema + ) + job_node.bound_source = mock_source + mock_source.output_schema.reset_mock() # clear bind-time schema check call cfg = ColumnConfig(system_tags=True) job_node.keys(columns=cfg, all_info=False) @@ -414,15 +434,17 @@ def test_bound_keys_all_info_matches_source_directly(self, tag_schema, data_sche def test_bound_output_schema_delegates_to_source(self, tag_schema, data_schema): """Bound SourceJobNode.output_schema() delegates to bound_source.output_schema().""" - from unittest.mock import MagicMock from orcapod.core.nodes.source_node import SourceJobNode from orcapod.types import ColumnConfig - mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) - - job_node = SourceJobNode(name="x", bound_source=mock_source) - # Reset call history accumulated during __init__ (schema derivation). + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema + ) + mock_source = _mock_source_with_the_slots_system_tags( + job_node, tag_schema, data_schema + ) + job_node.bound_source = mock_source + # Reset call history accumulated during the bind (schema check). mock_source.output_schema.reset_mock() cfg = ColumnConfig(system_tags=True) job_node.output_schema(columns=cfg, all_info=False) diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index dd2ff2768..87485b722 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -691,3 +691,25 @@ def test_a_refused_joined_stream_keeps_the_previous_binding() -> None: with pytest.raises(SourceSpecMismatchError): job.bind(sources={"src": _joined_stream(x_nullable=True)}) assert job.nodes["src"].bound_source is before + + +def test_a_pipeline_built_over_a_raw_stream_still_compiles() -> None: + """A raw stream as a pipeline's leaf is accepted as before; binding it is refused. + + The one exception to ITL-1126, pending the developer's decision: the slot built + from the leaf takes the leaf as it is (its rows carry no system tags, although + the slot predicts a pair). Binding the same stream to the slot afterwards goes + through the bind check and is refused. + """ + raw = _raw_stream(x_nullable=True) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(raw, label="renamed") + (slot,) = [name for name in job.nodes if name != "renamed"] + assert job.nodes[slot].bound_source is raw + assert job.nodes["renamed"].as_table().num_rows == 0 + _ = job.run() + assert job.nodes["renamed"].as_table().column("y").to_pylist() == [1] + with pytest.raises(SourceSpecMismatchError, match="exactly one source-id"): + job.bind(sources={slot: raw}) + assert job.nodes[slot].bound_source is raw From 0d9fb60b8c53ba15a7996d17dc1af9bd35a67ff9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:54:21 -0800 Subject: [PATCH 17/33] fix(source_node)!: a pipeline built over a raw stream is refused too (ITL-1126) Developer, November 14, 2026 (option A): until a raw stream is captured into a source wrapper when the pipeline is built (ITL-1127), building a pipeline or job directly over a stream without exactly one system-tag pair is refused like every other bind. _accept_pipeline_leaf is removed. The error names the found columns and the remedy: ArrowTableSource(stream.as_table(), tag_columns=[...]) for a raw stream; the join in the pipeline, or a stored source, for a joined one. Tests that bound a raw stream as a stand-in for a source now use a source; #375's meta-column zero-row test keeps its point outside a job. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 8 +- CLAUDE.md | 8 +- changelog.d/ITL-1112.md | 14 ++-- design_issues/ITL-1112-BT1.md | 4 +- docs/concepts/function-pods.md | 4 +- src/orcapod/core/nodes/source_node.py | 82 +++++++++---------- .../function_pod/test_ephemeral_result.py | 11 ++- tests/test_core/nodes/test_source_node.py | 8 +- tests/test_core/test_tracker.py | 41 ++++++---- .../test_bind_narrower_source.py | 37 +++++---- .../test_operator_node_empty_table_types.py | 24 +++--- 11 files changed, 133 insertions(+), 108 deletions(-) diff --git a/.zed/rules b/.zed/rules index 3613fe731..967eada73 100644 --- a/.zed/rules +++ b/.zed/rules @@ -804,10 +804,10 @@ data. The rules: (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through, - with one exception pending the developer: Pipeline.compile binds each leaf through - SourceJobNode._accept_pipeline_leaf, which takes a leaf without exactly one pair (a raw - ArrowTableStream) as it is, so a pipeline built directly over a raw stream still compiles. + binding one properly is ITL-1127. That includes a pipeline's input: Pipeline.compile + binds each leaf through SourceJobNode.from_stream, so a pipeline built directly over a + raw ArrowTableStream is refused (developer, November 14, 2026; wrap the table as a + source). Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is :: and is built only by core/sources/source_ids.hash_source_id: never format one by hand, and never write a label diff --git a/CLAUDE.md b/CLAUDE.md index 5e6d8ca2f..1e7dbdbc0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -846,10 +846,10 @@ data. The rules: (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); - binding one properly is ITL-1127. Never pass a bound stream's own system tags through, - with one exception pending the developer: `Pipeline.compile` binds each leaf through - `SourceJobNode._accept_pipeline_leaf`, which takes a leaf without exactly one pair (a raw - `ArrowTableStream`) as it is, so a pipeline built directly over a raw stream still compiles. + binding one properly is ITL-1127. That includes a pipeline's input: `Pipeline.compile` + binds each leaf through `SourceJobNode.from_stream`, so a pipeline built directly over a + raw `ArrowTableStream` is refused (developer, November 14, 2026; wrap the table as a + source). Never pass a bound stream's own system tags through. - A source id is the source's name (developer, November 11, 2026; ITL-957). A default id built from a hash is `::` and is built only by `core/sources/source_ids.hash_source_id`: never format one by hand, and never write a label diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index ceab9bd41..971019738 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -41,11 +41,13 @@ before with its own system-tag columns and column order, so the schema of the sl and of every table after it, depended on what was bound. Every way of binding (`bind`, the `bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever -its column types. +its column types. So does building a pipeline or job directly over such a stream +(`with job: op(ArrowTableStream(...))`), which binds it to the pipeline's input slot. -Migration: make the join part of the pipeline (bind its inputs to slots of their own), or -store the joined stream and bind the stored source (a source has one pair). Binding such a -stream properly is planned (ITL-1127). +Migration: wrap a raw stream's table as a source, +`ArrowTableSource(stream.as_table(), tag_columns=[...])`; make a join part of the pipeline +(bind its inputs to slots of their own), or store the joined stream and bind the stored +source. Binding such a stream properly is planned (ITL-1127). ### Changed @@ -68,7 +70,3 @@ stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A pipeline built directly over a raw `ArrowTableStream` (one without system-tag columns) - still compiles: its source node takes the stream as it is, so that node's rows carry no - system-tag columns although the slot predicts a pair. Binding such a stream to a slot - afterwards is refused (ITL-1126). diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index baf49f2c4..49c2e0a0c 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -37,5 +37,5 @@ node's path or table schema. Two Arrow types that are one Python type (`int32` a are still not told apart (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is -ITL-1127), except a pipeline's own leaf at compile (`_accept_pipeline_leaf`), which a -pipeline built directly over a raw stream still passes through, pending the developer. No hash, golden value or stored path changed. +ITL-1127), also as a pipeline's input: a pipeline built directly over a raw stream is +refused at compile. No hash, golden value or stored path changed. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index a7bf9e70b..d7ef8e4c6 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -842,8 +842,8 @@ rows under that key is an empty result, not an error. joined or merged stream (one pair of system-tag columns per input), or one with no system-tag columns, is refused too: make the join part of the pipeline, or store the joined stream and bind the stored source. A pipeline built directly over a raw `ArrowTableStream` - still compiles, for now: its source node's rows carry no system-tag columns, although the - slot predicts a pair. + is refused the same way: wrap its table as a source, + `ArrowTableSource(stream.as_table(), tag_columns=[...])`. - The results are read from the store recorded in the file. Pass `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 06e20fd6e..5767ec866 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -145,7 +145,7 @@ def _system_tag_columns( A root source, or a stream that keeps its system tags (a filter, a rename), has one of each; a stream that joins or merges inputs has one pair per input; a stream over a table without them (a raw ``ArrowTableStream``) has none. A slot accepts only the - first (ITL-1126). + first, also as a pipeline's input (ITL-1126). Args: source: The stream being bound. @@ -164,6 +164,31 @@ def _system_tag_columns( ) +def _one_system_tag_pair_issue(source_ids: list[str], record_ids: list[str]) -> str: + """Return why a source without exactly one system-tag pair is refused, and the remedy. + + Args: + source_ids: The source's source-id system-tag columns. + record_ids: The source's record-id system-tag columns. + """ + found = sorted(source_ids + record_ids) + if not found: + remedy = ( + "a stream without system-tag columns (a raw ArrowTableStream) cannot be " + "bound or used as a pipeline's input; wrap its table as a source, e.g. " + "ArrowTableSource(stream.as_table(), tag_columns=[...])" + ) + else: + remedy = ( + "a joined or merged stream cannot be bound; make the join part of the " + "pipeline, or store the joined stream and bind the stored source" + ) + return ( + "a source must carry exactly one source-id and one record-id system-tag " + f"column, not {found}: {remedy} (binding such a stream is planned, ITL-1127)" + ) + + class SourceNodeBase(TraceableBase, ABC): """Abstract base for SourceNode and SourceJobNode. @@ -424,10 +449,7 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] # ITL-1126); binding one properly is ITL-1127. source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) if len(source_ids) != 1 or len(record_ids) != 1: - issues.append( - "a source must carry exactly one source-id and one record-id " - f"system-tag column, not {sorted(source_ids + record_ids)}" - ) + issues.append(_one_system_tag_pair_issue(source_ids, record_ids)) if issues: raise SourceSpecMismatchError( @@ -776,34 +798,6 @@ def bound_source(self, value: StreamProtocol | None) -> None: self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() - def _accept_pipeline_leaf(self, stream: StreamProtocol) -> None: - """Bind *stream*, a leaf of the pipeline this slot was built from. - - ``Pipeline.compile`` makes each leaf stream a slot through ``from_stream``, with - the slot's schema taken from that same stream. A leaf with exactly one - source-id and one record-id system-tag column is bound as any source is - (``bound_source``). A leaf without exactly one pair (a raw ``ArrowTableStream`` - has none) is bound as it is, as before ITL-1126, so a pipeline built directly - over a raw stream still compiles: its rows carry the stream's own system-tag - columns, not the pair the slot predicts. This is the only place a source - without exactly one pair is bound. PENDING the developer's decision (option - "A"): refuse such a leaf here too, as every other way of binding does. - - Args: - stream: The leaf stream, whose schema the slot was built from. - - Raises: - SourceSpecMismatchError: If a leaf with one pair does not fit the slot. - """ - source_ids, record_ids = _system_tag_columns(stream, self._tag_schema.keys()) - if len(source_ids) == 1 and len(record_ids) == 1: - self.bound_source = stream - return - self._bound_source = stream - self._declared_form = None - self._invalidate_content_hash_cache() - self._invalidate_pipeline_hash_cache() - def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: """Check *source* and return how its rows become rows of the declared schema. @@ -1102,9 +1096,10 @@ def from_stream( **bound** ``SourceJobNode`` whose ``content_hash()`` delegates to the concrete stream. - Used by ``Pipeline.compile`` for each leaf of the pipeline: the stream is - bound through ``_accept_pipeline_leaf``, which also accepts a leaf without - exactly one system-tag pair (a raw ``ArrowTableStream``), as before ITL-1126. + Used by ``Pipeline.compile`` for each leaf of the pipeline. The stream is + bound as any source is, so a leaf without exactly one source-id and one + record-id system-tag column (a raw ``ArrowTableStream``) is refused + (ITL-1126). Args: stream: The stream to wrap. @@ -1121,14 +1116,12 @@ def from_stream( match stream: case SourceJobNode(): # Case 1: copy — preserve bound_source, do NOT wrap the SJN itself. - node = cls( + return cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, + bound_source=stream.bound_source, ) - if stream.bound_source is not None: - node._accept_pipeline_leaf(stream.bound_source) - return node case SourceNode(): # Case 2: unbound — schema placeholder; SJN hash = SourceNode hash. return cls( @@ -1155,11 +1148,12 @@ def from_stream( slot_name = stream.source_id else: slot_name = f"{stream.label}:{stream.content_hash().to_hex()}" - node = cls( - name=slot_name, tag_schema=tag_schema, data_schema=data_schema + return cls( + name=slot_name, + tag_schema=tag_schema, + data_schema=data_schema, + bound_source=stream, ) - node._accept_pipeline_leaf(stream) - return node def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. diff --git a/tests/test_core/function_pod/test_ephemeral_result.py b/tests/test_core/function_pod/test_ephemeral_result.py index b7fb8dbeb..89244c18d 100644 --- a/tests/test_core/function_pod/test_ephemeral_result.py +++ b/tests/test_core/function_pod/test_ephemeral_result.py @@ -650,7 +650,11 @@ def test_pipeline_job_set_ephemeral_store_propagates(self): """PipelineJob.set_ephemeral_store propagates to all compiled nodes.""" from orcapod.pipeline.job import PipelineJob - stream = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) + # A job's input must be a source (ITL-1126: a raw stream has no system tags). + stream = ArrowTableSource( + _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]).as_table(), + tag_columns=["id"], + ) pod = _make_pod() job = PipelineJob(name="test_pipeline") @@ -670,7 +674,10 @@ def test_pipeline_job_set_ephemeral_store_none_detaches(self): """PipelineJob.set_ephemeral_store(None) detaches the store from all nodes.""" from orcapod.pipeline.job import PipelineJob - stream = _make_stream([{"id": 0, "x": 10}]) + # A job's input must be a source (ITL-1126: a raw stream has no system tags). + stream = ArrowTableSource( + _make_stream([{"id": 0, "x": 10}]).as_table(), tag_columns=["id"] + ) pod = _make_pod() job = PipelineJob(name="test_pipeline") diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 94470c1a8..f1d9bd396 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -501,7 +501,13 @@ def test_source_job_node_from_stream_slot_name_uses_full_hash(self): """SourceJobNode.from_stream(non-RootSource) builds name as label + full content hash.""" from orcapod.core.nodes.source_node import SourceJobNode - stream = self._make_arrow_stream() + # A stream derived from a source: not a RootSource, and it carries the one + # system-tag pair a bound stream must have (ITL-1126; a raw stream has none). + from orcapod.core.operators import MapData + from orcapod.core.sources import DictSource + + source = DictSource([{"id": 1, "value": 1.0}], tag_columns=["id"]) + stream = MapData({"value": "v"})(source) job_node = SourceJobNode.from_stream(stream) expected_name = f"{stream.label}:{stream.content_hash().to_hex()}" diff --git a/tests/test_core/test_tracker.py b/tests/test_core/test_tracker.py index f28e63f1e..a5da5864b 100644 --- a/tests/test_core/test_tracker.py +++ b/tests/test_core/test_tracker.py @@ -105,6 +105,15 @@ def _make_stream(n: int = 3) -> ArrowTableStream: return ArrowTableStream(table, tag_columns=["id"]) +def _make_source(n: int = 3) -> ArrowTableSource: + """The rows of ``_make_stream`` as a source: one system-tag pair, so it can be bound. + + A raw ``ArrowTableStream`` carries no system-tag columns, and a source node refuses + it (ITL-1126). + """ + return ArrowTableSource(_make_stream(n).as_table(), tag_columns=["id"]) + + def _make_two_col_stream(n: int = 3) -> ArrowTableStream: """Stream with tag=id, data={a, b} for binary operator tests. Uses nullable=False schema.""" schema = pa.schema( @@ -154,7 +163,7 @@ class TestSourceNode: def _make_job_node(self, stream=None, name="test_source"): """Create a SourceJobNode wrapping an optional concrete stream.""" if stream is None: - stream = _make_stream() + stream = _make_source() tag_schema = Schema({"id": int}) data_schema = Schema({"x": int}) return SourceJobNode( @@ -165,7 +174,7 @@ def _make_job_node(self, stream=None, name="test_source"): ) def test_construction(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) assert node.bound_source is stream assert node.node_type == "source" @@ -173,32 +182,32 @@ def test_construction(self): assert node.upstreams == () def test_label_from_argument(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="my_source") # label resolves to the name via computed_label() assert node.label == "my_source" def test_label_set_explicitly(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="my_source") node._label = "explicit" assert node.label == "explicit" def test_label_defaults_to_name(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="my_source") # No explicit label → computed_label() returns the slot name assert node.label == "my_source" def test_repr(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream, name="test") r = repr(node) assert "SourceJobNode" in r assert "test" in r def test_content_hash_delegates_to_concrete(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) # SourceJobNode with concrete delegates content_hash to the concrete source assert node.content_hash() == stream.content_hash() @@ -209,28 +218,28 @@ def test_upstreams_setter_rejects_nonempty(self): node.upstreams = (_make_stream(),) def test_output_schema(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) tag_s, data_s = node.output_schema() assert set(tag_s.keys()) == {"id"} assert set(data_s.keys()) == {"x"} def test_keys(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) tag_keys, data_keys = node.keys() assert set(tag_keys) == {"id"} assert set(data_keys) == {"x"} def test_delegates_as_table(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) node_table = node.as_table() stream_table = stream.as_table() assert node_table.equals(stream_table) def test_delegates_iter_data(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) node_data = list(node.iter_data()) stream_data = list(stream.iter_data()) @@ -238,19 +247,19 @@ def test_delegates_iter_data(self): def test_run_equivalent(self): """execute() on SourceJobNode returns all data rows.""" - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) result = node.execute() assert len(result) == stream.as_table().num_rows def test_data_context_key(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) # SourceJobNode uses its own data context (not the concrete stream's) assert node.data_context_key is not None def test_data_context(self): - stream = _make_stream() + stream = _make_source() node = self._make_job_node(stream) assert node.data_context is not None assert node.data_context.context_key is not None @@ -265,7 +274,7 @@ class TestNodeContextDelegation: """All node types delegate data_context to their wrapped entity.""" def test_source_node_context_matches_stream(self): - stream = _make_stream() + stream = _make_source() node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), @@ -292,7 +301,7 @@ def test_operator_node_context_matches_operator(self): assert node.data_context.context_key == op.data_context_key def test_source_node_hash_consistent_with_stream(self): - stream = _make_stream() + stream = _make_source() node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 87485b722..edabdb5d9 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -693,23 +693,32 @@ def test_a_refused_joined_stream_keeps_the_previous_binding() -> None: assert job.nodes["src"].bound_source is before -def test_a_pipeline_built_over_a_raw_stream_still_compiles() -> None: - """A raw stream as a pipeline's leaf is accepted as before; binding it is refused. +def test_a_pipeline_built_over_a_raw_stream_is_refused() -> None: + """A raw stream as a pipeline's input is refused, telling the user what to do. - The one exception to ITL-1126, pending the developer's decision: the slot built - from the leaf takes the leaf as it is (its rows carry no system tags, although - the slot predicts a pair). Binding the same stream to the slot afterwards goes - through the bind check and is refused. + Developer, November 14, 2026: refuse a pipeline built directly over a stream + without system tags until a raw stream is captured into a source (ITL-1127). """ raw = _raw_stream(x_nullable=True) job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with pytest.raises(SourceSpecMismatchError) as raised: + with job: + _ = MapData({"x": "y"})(raw, label="renamed") + message = str(raised.value) + assert "exactly one source-id and one record-id" in message + assert "ArrowTableSource(stream.as_table(), tag_columns=[...])" in message + assert "ITL-1127" in message + + +def test_a_raw_streams_table_wrapped_as_the_error_advises_runs() -> None: + """``ArrowTableSource(stream.as_table(), tag_columns=[...])`` builds and runs.""" + raw = _raw_stream(x_nullable=True) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: - _ = MapData({"x": "y"})(raw, label="renamed") - (slot,) = [name for name in job.nodes if name != "renamed"] - assert job.nodes[slot].bound_source is raw - assert job.nodes["renamed"].as_table().num_rows == 0 + _ = MapData({"x": "y"})( + ArrowTableSource(raw.as_table(), tag_columns=["id"]), label="renamed" + ) _ = job.run() - assert job.nodes["renamed"].as_table().column("y").to_pylist() == [1] - with pytest.raises(SourceSpecMismatchError, match="exactly one source-id"): - job.bind(sources={slot: raw}) - assert job.nodes[slot].bound_source is raw + table = job.nodes["renamed"].as_table() + assert table.column("y").to_pylist() == [1] + assert table.column("w").to_pylist() == ["p"] diff --git a/tests/test_pipeline/test_operator_node_empty_table_types.py b/tests/test_pipeline/test_operator_node_empty_table_types.py index 2608f8651..e42a3462f 100644 --- a/tests/test_pipeline/test_operator_node_empty_table_types.py +++ b/tests/test_pipeline/test_operator_node_empty_table_types.py @@ -642,8 +642,12 @@ def test_head_of_a_filter_has_the_large_types_its_run_gives() -> None: def test_an_input_with_a_meta_data_column_gives_a_zero_row_table() -> None: - """A raw stream lists a ``__`` column as data; an operator node over it still - reads as zero rows before a run, without that column.""" + """A raw stream lists a ``__`` column as data; an operator over it still reads as + zero rows before it computes, without that column, typed as its computed rows. + + A raw stream cannot be a job's input (ITL-1126: it has no system-tag pair), so + the operator is applied to it directly, outside a job. + """ raw = ArrowTableStream( pa.table( { @@ -654,18 +658,16 @@ def test_an_input_with_a_meta_data_column_gives_a_zero_row_table() -> None: ), tag_columns=["id"], ) - job = PipelineJob(name="itl912", database=InMemoryArrowDatabase()) - with job: - _ = SelectDataColumns(["x"])(raw, label="op") - node = _op(job) + stream = SelectDataColumns(["x"])(raw) - head = node.head(all_info=True).table - table = node.as_table(all_info=True) + head = stream.head(all_info=True).table + table = stream.as_table(all_info=True) - assert head.num_rows == table.num_rows == 0 + assert head.num_rows == 0 + assert table.num_rows == 2 assert head.schema == table.schema - assert table.schema.field("x").type == pa.int8() - assert not any("__debug" in name for name in table.column_names) + assert head.schema.field("x").type == pa.int8() + assert not any("__debug" in name for name in head.column_names) # Both branches share the data columns, which SemiJoin ignores: not under test here. From 1ee849088f951fab20f1a0cc6c71738770c86f64 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:01:54 -0800 Subject: [PATCH 18/33] test(polling_source): bind a source, not a raw stream, to a source node (ITL-1126) Co-Authored-By: Claude Opus 5.5 --- tests/test_channels/test_polling_source.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_channels/test_polling_source.py b/tests/test_channels/test_polling_source.py index eccc69523..4cae55f42 100644 --- a/tests/test_channels/test_polling_source.py +++ b/tests/test_channels/test_polling_source.py @@ -12,6 +12,7 @@ from orcapod.channels import Channel from orcapod.core.nodes.source_node import SourceJobNode +from orcapod.core.sources import ArrowTableSource from orcapod.core.sources.polling_source import PollingSource from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.errors import CursorInvalidatedError @@ -244,7 +245,8 @@ async def test_source_node_async_execute_uses_async_iter(self): """SourceJobNode.async_execute routes through async_iter_data — no regression.""" from orcapod.types import Schema - stream = _make_arrow_stream(2) + # A bound source needs one system-tag pair (ITL-1126); a raw stream has none. + stream = ArrowTableSource(_make_arrow_stream(2).as_table(), tag_columns=["id"]) node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), @@ -262,7 +264,8 @@ async def test_source_node_async_execute_uses_async_iter(self): async def test_source_node_closes_channel_after_exhaustion(self): from orcapod.types import Schema - stream = _make_arrow_stream(1) + # A bound source needs one system-tag pair (ITL-1126); a raw stream has none. + stream = ArrowTableSource(_make_arrow_stream(1).as_table(), tag_columns=["id"]) node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), From e83bcb5609e1687e6144cd73d6586cc584201bbf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:16:38 -0800 Subject: [PATCH 19/33] fix(source_node): a slot predicts its record id as bytes; #387's rebind test asserts the refusal (ITL-1112) - test_graph_walk_cost: the bind of another column type is refused and moves no downstream schema (bind and bound_source; the strict xfail for a direct assignment is folded in, since the setter refuses). ITL-1111 fragment and GW1 corrected. - SourceNodeBase.output_schema typed the record-id system tag str; rows have large_binary (bytes). A source bound as declared passed bytes through while an unbound slot and a narrower bind said str, so a bind changed predictions. Now bytes: predictions are equal unbound and under every accepted bind, and equal to the rows' Python schema (test). #387's memo clear is kept with a comment. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1111.md | 2 +- changelog.d/ITL-1112.md | 8 ++ design_issues/ITL-1111-GW1.md | 3 +- design_issues/ITL-1112-BT1.md | 4 +- src/orcapod/core/nodes/source_node.py | 9 +- src/orcapod/pipeline/job.py | 8 +- tests/test_core/nodes/test_source_node.py | 16 +++- .../test_bind_narrower_source.py | 90 +++++++++++++++++++ tests/test_pipeline/test_graph_walk_cost.py | 62 +++++++------ 9 files changed, 161 insertions(+), 41 deletions(-) diff --git a/changelog.d/ITL-1111.md b/changelog.d/ITL-1111.md index 745a398f0..a84d4c6f5 100644 --- a/changelog.d/ITL-1111.md +++ b/changelog.d/ITL-1111.md @@ -1,6 +1,6 @@ ### Fixed -- Defining a pipeline whose stages split a stream into two branches and join them again no longer takes time exponential in the number of stages (about 4x per stage before: 9 stages took 13 seconds to build, with no row computed; 20 stages now build in well under a second, as streams and inside a `PipelineJob`) (ITL-1111). Three graph walks asked a shared upstream once per path: every operator stream, function pod stream and node predicted its output schema again from its upstreams on each call, and a join asks each input several times; `head()` and the notebook preview walked every path to check whether reading the schema would poll a source; and each operator or function pod call formatted its input streams, whose description nests every upstream, into a debug log message even with debug logging off. A stream or node now keeps its predicted output schema per column selection (`output_schema()`, `keys()`), the poll check visits each upstream once, and the log message is formatted only when debug logging is on (see Changed for the description). Job compilation, `save`, `PipelineJob.load` and a run's schema queries are linear in the nodes as well: each node is asked a bounded number of times. A schema that includes the system tags is still as wide as the joins make it, because each join keeps both inputs' system-tag columns: stacked split-and-join stages over one source double their number per stage (ITL-1114). `head()` and the notebook preview of a stream that has not run build their table of no rows from a table that carries the system tags whatever columns are shown, so over such stacked stages they are as wide as the joins make them (ITL-1132). No hash, stored table or golden value changes. `PipelineJob.bind(sources=...)` makes every node of the job predict its schema again, so binding a source whose column has another type still changes the downstream schemas; assigning `bound_source` on a source node directly, or setting a node's `upstreams`, does not reach the nodes downstream of it, so use `bind`. +- Defining a pipeline whose stages split a stream into two branches and join them again no longer takes time exponential in the number of stages (about 4x per stage before: 9 stages took 13 seconds to build, with no row computed; 20 stages now build in well under a second, as streams and inside a `PipelineJob`) (ITL-1111). Three graph walks asked a shared upstream once per path: every operator stream, function pod stream and node predicted its output schema again from its upstreams on each call, and a join asks each input several times; `head()` and the notebook preview walked every path to check whether reading the schema would poll a source; and each operator or function pod call formatted its input streams, whose description nests every upstream, into a debug log message even with debug logging off. A stream or node now keeps its predicted output schema per column selection (`output_schema()`, `keys()`), the poll check visits each upstream once, and the log message is formatted only when debug logging is on (see Changed for the description). Job compilation, `save`, `PipelineJob.load` and a run's schema queries are linear in the nodes as well: each node is asked a bounded number of times. A schema that includes the system tags is still as wide as the joins make it, because each join keeps both inputs' system-tag columns: stacked split-and-join stages over one source double their number per stage (ITL-1114). `head()` and the notebook preview of a stream that has not run build their table of no rows from a table that carries the system tags whatever columns are shown, so over such stacked stages they are as wide as the joins make them (ITL-1132). No hash, stored table or golden value changes. `PipelineJob.bind(sources=...)` makes every node of the job predict its schema again (a source whose column has another type is refused, ITL-1112); assigning `bound_source` on a source node directly, or setting a node's `upstreams`, does not reach the nodes downstream of it, so use `bind`. ### Changed diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 971019738..99d7d78e1 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -66,6 +66,14 @@ source. Binding such a stream properly is planned (ITL-1127). rename, with one system-tag pair) gives its system-tag columns the slot's names; their values stay the stream's own. Both passed through as they were before. +### Fixed + +- A source slot's predicted schema with system tags (`output_schema(columns={"system_tags": True})`, + `all_info=True`) gave the record-id column as `str`; every source's rows have it as `bytes` + (Arrow `large_binary`). It is `bytes` now, for an unbound slot and for every bind, so the + slot's prediction and every prediction after it are what the rows are, whatever is bound + (ITL-1112). + ### Known limitations - The bind check compares Python types, so a source that differs from its slot only in diff --git a/design_issues/ITL-1111-GW1.md b/design_issues/ITL-1111-GW1.md index e96c27484..7746e42a2 100644 --- a/design_issues/ITL-1111-GW1.md +++ b/design_issues/ITL-1111-GW1.md @@ -27,7 +27,8 @@ Call-count tests over 20 stages: `tests/test_pipeline/test_graph_walk_cost.py`. Left as they are: assigning `SourceJobNode.bound_source` directly, or setting a node's `upstreams`, clears no memo of the nodes downstream of it, which keep their schema (their -pipeline hashes are equally stale today; strict xfail tests record both). The risk is bounded: +pipeline hashes are equally stale today; a strict xfail test records the `upstreams` case, and a +`bound_source` of another column type is refused, ITL-1112). The risk is bounded: nothing in `src` calls the `upstreams` setter (the loaders build a node with `__new__` and set `_input_streams` before any schema is asked; only tests call it), and after ITL-1112 a bind cannot change a slot's schema. Cascading the clear (a registry of dependents, or a setter that diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 49c2e0a0c..0a42f71c3 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -38,4 +38,6 @@ are still not told apart (ITL-949). A source whose system-tag columns are not ex source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is ITL-1127), also as a pipeline's input: a pipeline built directly over a raw stream is -refused at compile. No hash, golden value or stored path changed. +refused at compile. The slot's own prediction types the record-id system-tag column `bytes`, +as every source's rows have it (it said `str`), so a slot's prediction is the same unbound +and under every accepted bind. No hash, golden value or stored path changed. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 5767ec866..a5310ef47 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -329,9 +329,10 @@ def output_schema( When ``columns.system_tags`` is ``True`` (or ``all_info=True``), the returned tag schema is extended with the two system-tag entries for this - node's schema (both typed as ``str``). Their names are deterministic from - the declared schemas and match what a concrete source with the same schema - would produce. + node's schema: the source id as ``str`` and the record id as ``bytes``, the + types every source's rows have (Arrow ``large_string`` and ``large_binary``). + Their names are deterministic from the declared schemas and match what a + concrete source with the same schema would produce. Other ``ColumnConfig`` flags (``meta``, ``context``, ``source``, ``content_hash``, ``sort_by_tags``) are no-ops at the node level — @@ -352,7 +353,7 @@ def output_schema( self._schema_hash_str ) tag_schema = Schema( - {**dict(tag_schema), source_id_col: str, record_id_col: str} + {**dict(tag_schema), source_id_col: str, record_id_col: bytes} ) return tag_schema, self._data_schema diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 5dbad8e9d..632d06084 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -474,8 +474,12 @@ def bind( schema_hash = job_node.as_node().content_hash().to_string() self._sources_by_hash[schema_hash] = sources[job_node.name] self._sources.update(sources) - # ``validate`` checks column names only, so a rebound source may give a - # column another type: every node predicts its schema again (ITL-1111). + # Every node predicts its schema again (ITL-1111). Since ITL-1112 a bind + # should never change a prediction: ``validate`` refuses a source of other + # column names or types, and the slot gives the declared schema, so this + # clear is expected to find nothing to change. It is kept so that a bind + # can never leave a stale prediction if a bound source's schema ever + # differs in a way the check does not see. for job_node in (self._persistent_node_map or {}).values(): if isinstance(job_node, StreamBase): job_node._forget_output_schema() # pyright: ignore[reportPrivateUsage] # the job owns its nodes' bindings diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index f1d9bd396..e2c8dae9b 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -340,10 +340,13 @@ def test_unbound_keys_all_info_same_as_system_tags(self, tag_schema, data_schema via_all_info = node.keys(all_info=True) assert via_system_tags == via_all_info - def test_unbound_output_schema_system_tags_adds_str_entries( + def test_unbound_output_schema_system_tags_adds_typed_entries( self, tag_schema, data_schema ): - """output_schema(system_tags=True) tag schema includes two str-typed system-tag entries.""" + """output_schema(system_tags=True) adds the source id (str) and record id (bytes). + + The types a source's rows have (ITL-1112): ``large_string`` and ``large_binary``. + """ from orcapod.core.nodes.source_node import SourceNode from orcapod.system_constants import constants from orcapod.types import ColumnConfig @@ -361,7 +364,14 @@ def test_unbound_output_schema_system_tags_adds_str_entries( if k.startswith(constants.SYSTEM_TAG_PREFIX) } assert len(system_tag_entries) == 2 - assert all(v is str for v in system_tag_entries.values()) + types = { + k.split(constants.BLOCK_SEPARATOR)[0]: v + for k, v in system_tag_entries.items() + } + assert types == { + constants.SYSTEM_TAG_SOURCE_ID_PREFIX: str, + constants.SYSTEM_TAG_RECORD_ID_PREFIX: bytes, + } def test_unbound_output_schema_default_unchanged(self, tag_schema, data_schema): """Regression: output_schema() with no args still returns plain schemas.""" diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index edabdb5d9..883598913 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -722,3 +722,93 @@ def test_a_raw_streams_table_wrapped_as_the_error_advises_runs() -> None: table = job.nodes["renamed"].as_table() assert table.column("y").to_pylist() == [1] assert table.column("w").to_pylist() == ["p"] + + +# --------------------------------------------------------------------------- +# The slot's predicted schema says what its rows are, bound or unbound +# --------------------------------------------------------------------------- + + +def _x_job() -> PipelineJob: + """Return an unbound job: slot ``src`` (``id: int``, ``x: int | None``) -> rename.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(int | None), label="renamed") + return job + + +def _abc_unbound() -> PipelineJob: + """Return the unbound ``a, b, c`` job.""" + return _abc_job(int) + + +def _abc_bound(order: str) -> Callable[[], PipelineJob]: + """Return a maker of the ``a, b, c`` job bound to a source with data in *order*.""" + + def make() -> PipelineJob: + job = _abc_job(int) + values = {"a": 1, "b": "s", "c": 1.5} + row = {"id": 0, **{name: values[name] for name in order}} + job.bind(sources={"src": DictSource([row], tag_columns=["id"])}) + return job + + return make + + +def _x_bound(source: Callable[[], Any]) -> Callable[[], PipelineJob]: + """Return a maker of the ``x: int | None`` job bound to ``source()``.""" + + def make() -> PipelineJob: + job = _x_job() + job.bind(sources={"src": source()}) + return job + + return make + + +PREDICTION_CASES: dict[str, tuple[Callable[[], PipelineJob], ...]] = { + "x": ( + _x_job, + _x_bound(lambda: _arrow_source([1, None], x_nullable=True)), + _x_bound(lambda: _arrow_source([1, 2], x_nullable=False)), + _x_bound(_mapped_stream), + ), + "abc": (_abc_unbound, _abc_bound("abc"), _abc_bound("cba")), +} + + +@pytest.mark.parametrize("config", [{"system_tags": True}, "all_info"]) +@pytest.mark.parametrize( + "makers", PREDICTION_CASES.values(), ids=PREDICTION_CASES.keys() +) +def test_the_predicted_schema_is_that_of_the_rows_bound_or_unbound( + makers: tuple[Callable[[], PipelineJob], ...], config: Any +) -> None: + """Unbound, declared, narrower, reordered and transformed binds predict alike. + + The prediction (system tags included: the record id is ``bytes``) equals the + Python schema of the populated rows, for the slot and the node after it. + """ + predictions = [] + for make in makers: + job = make() + predictions.append( + { + label: ( + job.nodes[label].output_schema(all_info=True) + if config == "all_info" + else job.nodes[label].output_schema(columns=config) + ) + for label in ("src", "renamed") + } + ) + if job.nodes["src"].bound_source is None: # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + continue + for label, (tag, data) in predictions[-1].items(): + node = job.nodes[label] + table = node.as_table(columns={"system_tags": True}) + rows = node.data_context.type_converter.arrow_schema_to_python_schema( + table.schema + ) + assert {**tag, **data} == dict(rows), label + assert all(p == predictions[0] for p in predictions), predictions diff --git a/tests/test_pipeline/test_graph_walk_cost.py b/tests/test_pipeline/test_graph_walk_cost.py index 7e820f802..715a43726 100644 --- a/tests/test_pipeline/test_graph_walk_cost.py +++ b/tests/test_pipeline/test_graph_walk_cost.py @@ -32,7 +32,7 @@ from orcapod.core.sources import DictSource from orcapod.core.streams.base import StreamBase from orcapod.databases import InMemoryArrowDatabase -from orcapod.errors import InputValidationError +from orcapod.errors import InputValidationError, SourceSpecMismatchError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator STAGES = 20 @@ -521,19 +521,42 @@ def _function_job() -> PipelineJob: class TestMemoInvalidation: """A memoized schema is predicted again when an input of the prediction changes.""" - def test_bind_to_a_source_of_another_column_type_changes_downstream_schemas( - self, + @pytest.mark.parametrize("how", ["bind", "bound_source"]) + def test_a_source_of_another_column_type_is_refused_and_moves_no_schema( + self, how: str ) -> None: - """``job.bind`` to a source with another column type changes every downstream schema.""" + """A source with another column type is refused; no downstream schema moves. + + The pipeline determines every node's schema (ITL-1112): ``job.bind`` and a + direct ``bound_source`` assignment refuse a source whose column types the + slot does not declare, so no memoized downstream schema can go stale. + """ job = _rename_job() - assert job.nodes["renamed"].output_schema()[1]["y"] is int + configs: list[dict[str, bool]] = [{}, {"system_tags": True}, {"meta": True}] + before = { + (label, str(config)): job.nodes[label].output_schema(columns=config) + for label in ("renamed", "renamed_again") + for config in configs + } assert job.nodes["renamed_again"].output_schema()[1]["z"] is int - assert job.nodes["renamed_again"].keys() == (("id",), ("z",)) - - job.bind(sources={"src": _one_row_source("a")}) - assert job.nodes["renamed"].output_schema()[1]["y"] is str - assert job.nodes["renamed_again"].output_schema()[1]["z"] is str + with pytest.raises(SourceSpecMismatchError, match="declared int"): + if how == "bind": + job.bind(sources={"src": _one_row_source("a")}) + else: + job.nodes["src"].bound_source = _one_row_source("a") # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + + for label in ("renamed", "renamed_again"): + node = job.nodes[label] + assert isinstance(node, StreamBase) + # Predict afresh: the test is about the schemas, not the memo. + node._forget_output_schema() # pyright: ignore[reportPrivateUsage] + after = { + (label, str(config)): job.nodes[label].output_schema(columns=config) + for label in ("renamed", "renamed_again") + for config in configs + } + assert after == before def test_setting_an_operator_nodes_upstreams_changes_its_schema(self) -> None: """Setting an operator node's ``upstreams`` changes its own schema.""" @@ -558,25 +581,6 @@ def test_setting_a_function_nodes_upstreams_changes_its_schema(self) -> None: assert node.output_schema()[0]["id"] is str - @pytest.mark.xfail( - strict=True, - reason=( - "A direct bound_source assignment reaches no downstream node, so their " - "memoized schemas are not predicted again; use job.bind (ITL-1111). " - "Their pipeline hashes are equally stale today." - ), - ) - def test_assigning_bound_source_directly_changes_downstream_schemas( - self, - ) -> None: - """A direct ``bound_source`` assignment should change the downstream schemas.""" - job = _rename_job() - assert job.nodes["renamed_again"].output_schema()[1]["z"] is int - - job.nodes["src"].bound_source = _one_row_source("a") - - assert job.nodes["renamed_again"].output_schema()[1]["z"] is str - @pytest.mark.xfail( strict=True, reason=( From 47c5bad084e39ba8f44c49499f94659593f8167a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:55:44 -0800 Subject: [PATCH 20/33] fix(source_node): structural classes of one identity fit; a bound source's system tags are exactly one str/bytes pair (ITL-1112, ITL-1126) CodeRabbit round 2: - _fit and _is_narrower compare column types with equivalent_hints, so a structural class of the declared identity is accepted (exact classes stay distinct); the invariant test covers it (paths, table schemas, pipeline hashes). - A source with any system-tag column besides one source-id and one record-id column is refused. - The pair must be str (source id) and bytes (record id), the types the slot predicts. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 6 +- CLAUDE.md | 6 +- changelog.d/ITL-1112.md | 6 +- src/orcapod/core/nodes/source_node.py | 64 ++++--- tests/test_core/nodes/test_source_node.py | 14 +- .../test_bind_keeps_node_schemas.py | 59 +++++- .../test_bind_narrower_source.py | 173 +++++++++++++++++- 7 files changed, 297 insertions(+), 31 deletions(-) diff --git a/.zed/rules b/.zed/rules index 967eada73..19a7c4e3c 100644 --- a/.zed/rules +++ b/.zed/rules @@ -795,8 +795,10 @@ data. The rules: schema of any node's or stored table (column names with system tags, types, nullable flags, order); only values differ. SourceNodeBase.validate is the one check, used by PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and - PipelineJob.load: per column the declared Python type, or T where the slot declares - T | None; anything else raises SourceSpecMismatchError before anything is bound. The + PipelineJob.load: per column the declared Python type (structural classes of one identity + are one type, equivalent_hints), or T where the slot declares T | None; and exactly + one source-id (str) and one record-id (bytes) system-tag column, no other. Anything + else raises SourceSpecMismatchError before anything is bound. The source's rows are given as the declared schema at one place, _DeclaredForm (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's diff --git a/CLAUDE.md b/CLAUDE.md index 1e7dbdbc0..017277f6e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -837,8 +837,10 @@ data. The rules: schema of any node's or stored table (column names with system tags, types, nullable flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and - `PipelineJob.load`: per column the declared Python type, or `T` where the slot declares - `T | None`; anything else raises `SourceSpecMismatchError` before anything is bound. The + `PipelineJob.load`: per column the declared Python type (structural classes of one identity + are one type, `equivalent_hints`), or `T` where the slot declares `T | None`; and exactly + one source-id (`str`) and one record-id (`bytes`) system-tag column, no other. Anything + else raises `SourceSpecMismatchError` before anything is bound. The source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the declared schema hash, taken from the source's own pair by prefix, the VALUES the source's diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 99d7d78e1..4932307a6 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -8,7 +8,8 @@ not checked at all. A source with other column types was accepted: every pipelin from the slot's declared schema, stayed the same, so a run wrote rows of the new types to the stored paths of the declared ones. Every way of binding now compares the source's tag and data columns with the slot's: names, role (tag or data) and Python type, the inputs of the pipeline -hash. A column must have the declared type, or `T` where the slot declares `T | None`. Anything +hash. A column must have the declared type (two `orcapod.dataclass` classes of one identity are +one type), or `T` where the slot declares `T | None`. Anything else raises `SourceSpecMismatchError` naming the slot, the column and both types, before anything is bound; a refused `bind` leaves the job as it was, its database included. @@ -41,7 +42,8 @@ before with its own system-tag columns and column order, so the schema of the sl and of every table after it, depended on what was bound. Every way of binding (`bind`, the `bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever -its column types. So does building a pipeline or job directly over such a stream +its column types; so is a stream with any other system-tag column, or whose source id is not +`str` or record id not `bytes`. So does building a pipeline or job directly over such a stream (`with job: op(ArrowTableStream(...))`), which binds it to the pipeline's input slot. Migration: wrap a raw stream's table as a source, diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index a5310ef47..95e6e13c0 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -31,6 +31,7 @@ check_head_size, ) from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.logical_types.user_class_identity import equivalent_hints from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import ColumnConfig, Schema @@ -60,13 +61,14 @@ def _is_narrower(declared: Any, given: Any) -> bool: """Return whether *given* is *declared* without its ``None`` member. ``int`` is narrower than ``int | None`` and ``int | str`` than ``int | str | None``; - nothing else is (developer, November 14, 2026, ITL-1112). + nothing else is (developer, November 14, 2026, ITL-1112). Structural classes of one + identity are one type here as everywhere (``equivalent_hints``). Args: declared: The slot's declared Python type of a column. given: The bound source's Python type of that column. """ - return declared != given and _without_none(declared) == given + return declared != given and equivalent_hints(_without_none(declared), given) @dataclasses.dataclass(frozen=True) @@ -139,8 +141,8 @@ def _declared_positions(self, names: list[str]) -> list[int]: def _system_tag_columns( source: StreamProtocol, tag_names: Collection[str] -) -> tuple[list[str], list[str]]: - """Return *source*'s source-id and record-id system-tag columns. +) -> tuple[list[str], list[str], list[str]]: + """Return *source*'s source-id, record-id and any other system-tag columns. A root source, or a stream that keeps its system tags (a filter, a rename), has one of each; a stream that joins or merges inputs has one pair per input; a stream over a @@ -150,6 +152,10 @@ def _system_tag_columns( Args: source: The stream being bound. tag_names: Its user tag columns. + + Returns: + The source-id columns, the record-id columns, and every other system-tag + column, which no slot declares. """ names = [ name @@ -158,35 +164,41 @@ def _system_tag_columns( ] source_id = f"{constants.SYSTEM_TAG_SOURCE_ID_PREFIX}{constants.BLOCK_SEPARATOR}" record_id = f"{constants.SYSTEM_TAG_RECORD_ID_PREFIX}{constants.BLOCK_SEPARATOR}" - return ( - [name for name in names if name.startswith(source_id)], - [name for name in names if name.startswith(record_id)], - ) + source_ids = [name for name in names if name.startswith(source_id)] + record_ids = [name for name in names if name.startswith(record_id)] + others = [name for name in names if name not in {*source_ids, *record_ids}] + return source_ids, record_ids, others -def _one_system_tag_pair_issue(source_ids: list[str], record_ids: list[str]) -> str: +def _one_system_tag_pair_issue( + source_ids: list[str], record_ids: list[str], others: list[str] +) -> str: """Return why a source without exactly one system-tag pair is refused, and the remedy. Args: source_ids: The source's source-id system-tag columns. record_ids: The source's record-id system-tag columns. + others: The source's other system-tag columns. """ - found = sorted(source_ids + record_ids) + found = sorted(source_ids + record_ids + others) + issue = ( + "a source must carry exactly one source-id and one record-id system-tag " + f"column and no other, not {found}" + ) if not found: remedy = ( "a stream without system-tag columns (a raw ArrowTableStream) cannot be " "bound or used as a pipeline's input; wrap its table as a source, e.g. " "ArrowTableSource(stream.as_table(), tag_columns=[...])" ) - else: + elif len(source_ids) > 1 or len(record_ids) > 1: remedy = ( "a joined or merged stream cannot be bound; make the join part of the " "pipeline, or store the joined stream and bind the stored source" ) - return ( - "a source must carry exactly one source-id and one record-id system-tag " - f"column, not {found}: {remedy} (binding such a stream is planned, ITL-1127)" - ) + else: + return issue + return f"{issue}: {remedy} (binding such a stream is planned, ITL-1127)" class SourceNodeBase(TraceableBase, ABC): @@ -433,7 +445,7 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] if extra: issues.append(f"unexpected {role} columns: {sorted(extra)}") for name in declared.keys(): - if name not in given or given[name] == declared[name]: + if name not in given or equivalent_hints(given[name], declared[name]): continue if _is_narrower(declared[name], given[name]): narrower.add(name) @@ -448,9 +460,19 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] # a source with exactly one pair can take. A joined or merged stream (one pair # per input) or a stream with none is refused (developer, November 14, 2026, # ITL-1126); binding one properly is ITL-1127. - source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) - if len(source_ids) != 1 or len(record_ids) != 1: - issues.append(_one_system_tag_pair_issue(source_ids, record_ids)) + source_ids, record_ids, others = _system_tag_columns(source, source_tag.keys()) + if len(source_ids) != 1 or len(record_ids) != 1 or others: + issues.append(_one_system_tag_pair_issue(source_ids, record_ids, others)) + else: + # The pair must have the types the slot predicts for it (str, bytes). + given_tags = source.output_schema(columns={"system_tags": True})[0] + for name, declared_type in ((source_ids[0], str), (record_ids[0], bytes)): + if given_tags[name] != declared_type: + issues.append( + f"system-tag column {name!r} is declared " + f"{_type_text(declared_type)} but the source has " + f"{_type_text(given_tags[name])}" + ) if issues: raise SourceSpecMismatchError( @@ -814,8 +836,8 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: SourceSpecMismatchError: If *source* does not fit the declared schema. """ source_tag, source_data, narrower = self._fit(source) - # _fit has checked that the source has exactly one pair. - source_ids, record_ids = _system_tag_columns(source, source_tag.keys()) + # _fit has checked that the source has exactly one pair and nothing else. + source_ids, record_ids, _ = _system_tag_columns(source, source_tag.keys()) declared_source_id, declared_record_id = system_tag_column_names( self._schema_hash_str ) diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index e2c8dae9b..7c1cedbc8 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -8,7 +8,7 @@ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.errors import SourceSpecMismatchError, UnboundSourceError -from orcapod.types import Schema +from orcapod.types import ColumnConfig, Schema @pytest.fixture @@ -295,8 +295,18 @@ def _mock_source_with_the_slots_system_tags( column (ITL-1126); the slot's own names make the bind pass rows unchanged. """ tag_keys = node.keys(columns={"system_tags": True})[0] + with_system_tags = node.output_schema(columns={"system_tags": True}) + + def output_schema( + columns: ColumnConfig | dict[str, bool] | None = None, all_info: bool = False + ) -> tuple[Schema, Schema]: + """The slot's schema, with its system-tag pair when asked for.""" + if ColumnConfig.handle_config(columns, all_info=all_info).system_tags: + return with_system_tags + return tag_schema, data_schema + mock_source = MagicMock() - mock_source.output_schema.return_value = (tag_schema, data_schema) + mock_source.output_schema.side_effect = output_schema mock_source.keys.return_value = (tag_keys, tuple(data_schema.keys())) return mock_source diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index f9cc4fee2..42931bcf7 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -16,10 +16,11 @@ import pathlib from collections.abc import Callable -from typing import Any +from typing import Any, cast import pyarrow as pa +import orcapod from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import MapData @@ -144,3 +145,59 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( # A loaded function node orders its meta columns unlike a fresh one, bind or not # (reported separately), so the loaded job is compared with itself. assert _snapshot(loaded, LOADED_NODES) == before + + +@orcapod.dataclass(name="KeepsPoint") +class _PointA: + """A structural class, identified by its name and fields.""" + + x: int + y: int + + +@orcapod.dataclass(name="KeepsPoint") +class _PointB: + """Another class object of the same structural identity as ``_PointA``.""" + + x: int + y: int + + +def test_a_structural_class_of_the_same_identity_changes_no_path_or_schema( + tmp_path: pathlib.Path, +) -> None: + """A source of another class object of the slot's structural identity moves nothing. + + The sources are built before the baseline hashes are taken: on main too, the + pipeline hash of a schema holding a structural class differs before and after a + source of that class is first built, bind or no bind (reported separately). + """ + sources = [ + DictSource( + cast(list[dict[str, Any]], [{"id": 0, "p": point_type(1, 2)}]), + tag_columns=["id"], + data_schema={"id": int, "p": point_type}, + source_id="points", + ) + for point_type in (_PointA, _PointB, _PointB) + ] + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"p": _PointA}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = MapData({"p": "q"})(slot, label="renamed") + hashes = {name: node.pipeline_hash() for name, node in job.nodes.items()} + + snapshots = [] + for source, bind in zip(sources, (_bind, _bind, _assign), strict=True): + bind(job, source) + assert {name: node.pipeline_hash() for name, node in job.nodes.items()} == ( + hashes + ) + assert job.run().ok + snapshots.append(_snapshot(job, ("src", "renamed"))) + assert snapshots[1] == snapshots[0] + assert snapshots[2] == snapshots[0] diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 883598913..030569b0b 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio +import dataclasses import logging import pathlib from collections.abc import Callable @@ -20,11 +21,16 @@ import pyarrow as pa import pytest +import orcapod from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.operators import Join, MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.core.streams.base import ZERO_ROW_INPUT_COLUMNS, typed_zero_row_table +from orcapod.core.streams.base import ( + ZERO_ROW_INPUT_COLUMNS, + StreamBase, + typed_zero_row_table, +) from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -812,3 +818,168 @@ def test_the_predicted_schema_is_that_of_the_rows_bound_or_unbound( ) assert {**tag, **data} == dict(rows), label assert all(p == predictions[0] for p in predictions), predictions + + +# --------------------------------------------------------------------------- +# CodeRabbit round 2: structural classes, extra system tags, system-tag types +# --------------------------------------------------------------------------- + + +@orcapod.dataclass(name="BindPoint") +class _PointA: + """A structural class, identified by its name and fields.""" + + x: int + y: int + + +@orcapod.dataclass(name="BindPoint") +class _PointB: + """Another class object of the same structural identity as ``_PointA``.""" + + x: int + y: int + + +@dataclasses.dataclass +class _ExactA: + """An exact class: one type only with itself.""" + + x: int + y: int + + +@dataclasses.dataclass +class _ExactB: + """An exact class with the same fields as ``_ExactA``, yet another type.""" + + x: int + y: int + + +def _rows(*rows: dict[str, Any]) -> list[dict[str, Any]]: + """Return *rows* as a ``DictSource`` takes them (user-class values included).""" + return list(rows) + + +def _point_job(point_type: Any) -> PipelineJob: + """Return an unbound job: slot ``src`` (``id: int``, ``p: point_type``) -> rename.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"p": point_type}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"p": "q"})(slot, label="renamed") + return job + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_a_structural_class_of_the_same_identity_is_accepted( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """A column of another class object of the declared structural identity fits. + + Two structural classes with equal identity records are one type in the schema + check and in the pipeline hash, so the bind moves no hash or schema. + """ + job = _point_job(_PointA) + source = DictSource( + _rows({"id": 0, "p": _PointB(1, 2)}), + tag_columns=["id"], + data_schema={"id": int, "p": _PointB}, + source_id="b", + ) + assert source.output_schema()[1]["p"] is not _PointA + hashes = _pipeline_hashes(job) + schemas = { + label: node.output_schema(all_info=True) for label, node in job.nodes.items() + } + bind(job, source) + assert job.nodes["src"].bound_source is source # pyright: ignore[reportAttributeAccessIssue] # "src" is a SourceJobNode + assert _pipeline_hashes(job) == hashes + for label, node in job.nodes.items(): + if isinstance(node, StreamBase): + # Predict afresh: the test is about the schema, not the memo. + node._forget_output_schema() # pyright: ignore[reportPrivateUsage] + assert node.output_schema(all_info=True) == schemas[label], label + assert job.run().ok + assert job.nodes["renamed"].as_table().num_rows == 1 + + +@pytest.mark.parametrize("bind", BINDS.values(), ids=BINDS.keys()) +def test_an_exact_class_with_the_same_fields_is_refused( + bind: Callable[[PipelineJob, Any], None], +) -> None: + """Two exact classes with the same fields are two types.""" + job = _point_job(_ExactA) + source = DictSource( + _rows({"id": 0, "p": _ExactB(1, 2)}), + tag_columns=["id"], + data_schema={"id": int, "p": _ExactB}, + ) + with pytest.raises(SourceSpecMismatchError, match="data column 'p'"): + bind(job, source) + + +def _stream_with_system_tags(columns: dict[str, pa.Array[Any]]) -> ArrowTableStream: + """Return a stream of tag ``id``, data ``x`` and the given system-tag columns.""" + table = pa.table( + { + "id": pa.array([0], pa.int64()), + "x": pa.array([1], pa.int64()), + **columns, + }, + schema=pa.schema( + [ + pa.field("id", pa.int64(), False), + pa.field("x", pa.int64(), False), + *(pa.field(name, array.type, False) for name, array in columns.items()), + ] + ), + ) + return ArrowTableStream(table, tag_columns=["id"]) + + +SOURCE_ID = "_tag_source_id::abc" +RECORD_ID = "_tag_record_id::abc" + + +@pytest.mark.parametrize("x_type", [int, int | None], ids=["declared", "narrower"]) +@pytest.mark.parametrize("bind", ALL_BINDS.values(), ids=ALL_BINDS.keys()) +def test_an_extra_system_tag_column_is_refused( + bind: Callable[[PipelineJob, Any], None], x_type: Any +) -> None: + """A system-tag column besides the one pair is not in the slot's schema.""" + stream = _stream_with_system_tags( + { + SOURCE_ID: pa.array(["s"], pa.large_string()), + RECORD_ID: pa.array([b"r"], pa.large_binary()), + "_tag_extra": pa.array(["e"], pa.large_string()), + } + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(x_type), label="renamed") + with pytest.raises(SourceSpecMismatchError, match="_tag_extra"): + bind(job, stream) + + +@pytest.mark.parametrize("x_type", [int, int | None], ids=["declared", "narrower"]) +@pytest.mark.parametrize("bind", ALL_BINDS.values(), ids=ALL_BINDS.keys()) +def test_a_system_tag_pair_of_other_types_is_refused( + bind: Callable[[PipelineJob, Any], None], x_type: Any +) -> None: + """The source id must be ``str`` and the record id ``bytes``, as the slot predicts.""" + stream = _stream_with_system_tags( + { + SOURCE_ID: pa.array(["s"], pa.large_string()), + RECORD_ID: pa.array(["r"], pa.large_string()), + } + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "y"})(_slot(x_type), label="renamed") + with pytest.raises(SourceSpecMismatchError, match=r"declared bytes but the source"): + bind(job, stream) From 4ffdef06cd40955bd77066b9ae5043d33c944e85 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:41:55 -0800 Subject: [PATCH 21/33] test(current_run): bind a source, not a raw stream, to a source node (ITL-1126) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_current_run_review_fixes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py index 429ccde6a..1789ba4fa 100644 --- a/tests/test_pipeline/test_current_run_review_fixes.py +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -425,7 +425,8 @@ def test_an_unrun_loaded_operator_without_its_operator_keeps_its_columns( def test_a_bound_source_with_no_modified_time_makes_its_slot_unknown() -> None: """A source slot bound to a stream with no modified time has none.""" - stream = ArrowTableStream(pa.table({"k": [0], "x": [1]}), tag_columns=["k"]) + # A table wrapped as a source: a bound stream needs one system-tag pair (ITL-1126). + stream = ArrowTableSource(pa.table({"k": [0], "x": [1]}), tag_columns=["k"]) slot = SourceJobNode(name="s", bound_source=stream) assert slot.last_modified is not None stream._set_modified_time(None) # pyright: ignore[reportPrivateUsage] # an unknown modified time is what this test is about From a423bc713f89cdd38bc373d582740c887de5f675 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:55:16 -0800 Subject: [PATCH 22/33] test(pipeline): a job loaded before its user class is registered (ITL-1138, failing) Co-Authored-By: Claude Opus 5.5 --- ...11-14-itl-1138-loaded-slot-pending-type.md | 34 ++++ .../test_loaded_slot_unregistered_class.py | 187 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md create mode 100644 tests/test_pipeline/test_loaded_slot_unregistered_class.py diff --git a/superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md b/superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md new file mode 100644 index 000000000..37811cdcc --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md @@ -0,0 +1,34 @@ +# ITL-1138: a loaded source slot whose user class is not registered at load + +Branch `eywalker/itl-1138-a-job-loaded-before-its-user-class-is`, from PR #386's head +(`4ffdef06`), or `origin/main` if #386 has merged. Draft PR against `main`: +"Stacked on #386: merge right after it. Fixes ITL-1138". + +## Cause (measured for ITL-1137, sequence 4) + +`Pipeline.load` (`pipeline/graph.py`, `source_type == "node"`) restores a slot's saved schema +with `deserialize_schema`, which returns the raw saved text (`extension`) for a +column whose extension type is not registered in the loading process. `PipelineJob.load` copies +that schema into the `SourceJobNode`. Its pipeline hash is then the hash of the text; on #386 +the bind check compares the text with the source's class and refuses, and #386 names the +system-tag columns from the slot's pipeline hash. + +## Plan + +1. Failing fresh-process test first (public interface): sequence-4 job (s1 over `C`, s2; + `Join` -> `g(p: C, r: int)`; `MapData` -> `h(pp: C)`), exact and structural `C`; save; fresh + process loads without registering `C`, builds the sources, binds, `run(load_only=True)`; + every node's pipeline hash, content hash, `as_table(all_info=True)` schema (system-tag + names), rows and stored rows equal the saving process. Plus: pydantic; a class never + registered -> clear error at bind; a file saved before this change (the job file format + is unchanged, so an existing file is that file: write it with the base commit? no: the + saved descriptor already records `pipeline_hash`, so a file from before the change is the + same file; cover a descriptor without the saved hash if the format allows one). +2. Loaded slot keeps saved `pipeline_hash` (and `content_hash` while unbound) from its + descriptor, like the function and operator stubs. +3. Pending type text: the slot keeps a column it could not restore as text; at bind (the + `_fit` check) it restores it again through the data context's converter (the source has + registered the class by then). Restored -> the slot's schema holds the class and the check + compares types as usual. Not restored -> `SourceSpecMismatchError` naming the column and the + saved class identity (name and kind from the extension metadata) and what to do. +4. No hash, golden or stored-table change. diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py new file mode 100644 index 000000000..37450649a --- /dev/null +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -0,0 +1,187 @@ +"""A job loaded before its user class is registered reads like the job that saved it (ITL-1138). + +A job over a column of a user class is saved; a fresh process loads it before anything has +registered the class (no source of it built, no function pod naming it defined). The saved type +of such a column cannot be restored to a class at load, so the loaded source slot keeps it as +pending text. The slot keeps its saved pipeline hash and content hash, and restores the type when +a source is bound, by which time the source has registered the class. Every node then has the +saving process's hashes, table schema (with its system-tag column names), rows and stored rows. +A slot whose class is still not registered at bind refuses the source with an error that names +the column and the saved class. + +Each process runs in a fresh interpreter: what a class's registration changes is per process. +""" + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +_CLASSES = textwrap.dedent( + """\ + import dataclasses + + import orcapod + import pydantic + + + @dataclasses.dataclass + class Exact: + u: int + v: str + + + @dataclasses.dataclass + class OtherExact: + u: int + v: str + + + @orcapod.dataclass + class Structural: + u: int + v: str + + + class Model(pydantic.BaseModel): + u: int + v: str + """ +) + +_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + mode, work, class_name = sys.argv[2], sys.argv[3], sys.argv[4] + + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.operators import Join, MapData + from orcapod.sources import DictSource + + + def sources(cls): + s1 = DictSource( + [{"id": 1, "p": cls(u=1, v="a")}, {"id": 2, "p": cls(u=2, v="b")}], + tag_columns=["id"], + data_schema={"id": int, "p": cls}, + source_id="s1", + ) + s2 = DictSource([{"id": 1, "r": 5}, {"id": 2, "r": 6}], tag_columns=["id"], source_id="s2") + return s1, s2 + + + def describe(job): + out = {} + for label, node in sorted(job.nodes.items()): + table = node.as_table(all_info=True) + # A source node stores nothing; a function or operator node has a stored view. + stored_view = getattr(node, "as_source", None) + stored = None if stored_view is None else stored_view().as_table(all_info=True) + out[label] = { + "pipeline_hash": node.pipeline_hash().to_string(), + "content_hash": node.content_hash().to_string(), + "schema": [f"{field.name}: {field.type}" for field in table.schema], + "rows": table.num_rows, + "stored_rows": None if stored is None else stored.num_rows, + } + return out + + + if mode == "save": + import classes + + cls = getattr(classes, class_name) + s1, s2 = sources(cls) + + @function_pod(output_keys="q") + def add(p: cls, r: int) -> int: + return p.u + r + + @function_pod(output_keys="w") + def read_u(pp: cls) -> int: + return pp.u + + job = PipelineJob(name="demo", database=DeltaTableDatabase(work + "/store")) + with job: + joined = Join()(s1, s2, label="joined") + add.pod(joined, label="add") + renamed = MapData({"p": "pp"})(s1, label="renamed") + read_u.pod(renamed, label="read_u") + job.run() + job.save(work + "/job.json") + print(json.dumps(describe(job))) + else: + # Nothing registers the class before the load. + job = PipelineJob.load(work + "/job.json") + import classes + + bound_class = sys.argv[5] if len(sys.argv) > 5 else class_name + s1, s2 = sources(getattr(classes, bound_class)) # registers the class now + try: + job.bind(sources={"s1": s1, "s2": s2}) + except Exception as exc: + print(json.dumps({"bind_error": f"{type(exc).__name__}: {exc}"})) + sys.exit(0) + job.run(load_only=True) + out = describe(job) + out["_reused"] = {label: n.reused for label, n in job.last_run.nodes.items()} + print(json.dumps(out)) + """ +) + + +def _run(tmp_path: Path, *args: str) -> dict: + """Run the script in a fresh interpreter and return the JSON it prints last.""" + result = subprocess.run( + [sys.executable, str(tmp_path / "script.py"), str(tmp_path), *args], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-3000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@pytest.fixture +def saved_job(tmp_path: Path): + """Return a function that saves the job over a class and returns the saving view.""" + _ = (tmp_path / "classes.py").write_text(_CLASSES) + _ = (tmp_path / "script.py").write_text(_SCRIPT) + + def save(class_name: str) -> dict: + return _run(tmp_path, "save", str(tmp_path / "work"), class_name) + + return save + + +@pytest.mark.parametrize("class_name", ["Exact", "Structural", "Model"]) +def test_a_job_loaded_before_its_class_is_registered_reads_like_the_saved_job( + tmp_path: Path, saved_job, class_name: str +) -> None: + """Hashes, schemas with system-tag names, rows and stored rows equal the saving process.""" + saved = saved_job(class_name) + loaded = _run(tmp_path, "load", str(tmp_path / "work"), class_name) + assert "bind_error" not in loaded, loaded.get("bind_error") + assert loaded.pop("_reused") == {"add": 2, "read_u": 2} + assert loaded == saved + + +def test_a_slot_whose_class_is_never_registered_refuses_the_source_clearly( + tmp_path: Path, saved_job +) -> None: + """Binding a source of a look-alike class names the column and the saved class.""" + _ = saved_job("Exact") + loaded = _run(tmp_path, "load", str(tmp_path / "work"), "Exact", "OtherExact") + error = loaded["bind_error"] + assert error.startswith("SourceSpecMismatchError: ") + assert "'p'" in error + assert "classes.Exact" in error + assert "not registered" in error From eae0ef147c9f39bee78953a0b5090d9c700cc593 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:05:43 -0800 Subject: [PATCH 23/33] fix(pipeline): a loaded source slot keeps its saved hashes and restores a pending user-class type at bind (ITL-1138) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1138.md | 3 + design_issues/ITL-1138-LJ2.md | 21 ++ src/orcapod/core/nodes/source_node.py | 219 +++++++++++++++++- src/orcapod/pipeline/graph.py | 8 + src/orcapod/pipeline/job.py | 8 +- .../test_loaded_slot_unregistered_class.py | 24 ++ 6 files changed, 274 insertions(+), 9 deletions(-) create mode 100644 changelog.d/ITL-1138.md create mode 100644 design_issues/ITL-1138-LJ2.md diff --git a/changelog.d/ITL-1138.md b/changelog.d/ITL-1138.md new file mode 100644 index 000000000..19efff50b --- /dev/null +++ b/changelog.d/ITL-1138.md @@ -0,0 +1,3 @@ +### Fixed + +- A job over a column of a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model) loaded with `PipelineJob.load` in a process where nothing has registered the class yet reads like the job that saved it (ITL-1138). The load cannot restore such a column's saved type to a class, so the source slot held the saved type text: its pipeline hash was the hash of that text, and binding the same source was refused ("data column 'p' is declared extension<…> but the source has A") unless the class had been registered before the load. A loaded source slot now keeps its saved pipeline hash, and its saved content hash while no source is bound, as loaded function and operator nodes keep theirs, and restores such a column's type when a source is bound (building the source registers the class). Every node then has the saving process's hashes, schemas, system-tag column names and stored rows, and `run(load_only=True)` finds every stored result. A slot whose class is still not registered at bind refuses the source with an error that names the column and the saved class and says how to register it. No hash, golden value or stored table changes; the job file format is unchanged, and a file without a slot's saved hashes still loads. diff --git a/design_issues/ITL-1138-LJ2.md b/design_issues/ITL-1138-LJ2.md new file mode 100644 index 000000000..cabf52336 --- /dev/null +++ b/design_issues/ITL-1138-LJ2.md @@ -0,0 +1,21 @@ +## `src/orcapod/pipeline/job.py` — reading a loaded `PipelineJob` + +### LJ2 — a job loaded before its user class is registered held the class as text in its source slot +**Status:** resolved +**Severity:** high +**Issue:** ITL-1138 + +`Pipeline.load` restores a source slot's saved schema with `deserialize_schema`, which keeps the +saved type text (`extension`) for a column whose extension type the loading process +has not registered: a user class nothing has registered yet (no source of it built, no function +pod naming it defined). The loaded slot then hashed that text, so its pipeline hash differed from +the saved one, and with the bind check of ITL-1112 binding the same source was refused ("declared +extension<…> but the source has A"); a source node names its system-tag columns from the slot's +schema, so an accepted bind would have named them differently from the saving run. + +**Fix:** a slot rebuilt from a saved job keeps its saved pipeline hash, and its saved content hash +while unbound, as loaded function and operator nodes do (`SourceNodeBase._keep_saved`). A column +whose type was not restored is kept as pending and restored when a source is bound +(`_restore_pending_types`, called by the bind check), by which time the source has registered the +class; a column still not restorable makes the bind raise `SourceSpecMismatchError` naming the +column and the saved class. Tests: `tests/test_pipeline/test_loaded_slot_unregistered_class.py`. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index dcbe39ba8..6b9e71f46 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -7,8 +7,10 @@ from __future__ import annotations +import contextlib import dataclasses import functools +import json import logging from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Collection, Iterator, Mapping @@ -35,7 +37,7 @@ from orcapod.logical_types.user_class_identity import equivalent_hints from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -from orcapod.types import ColumnConfig, Schema +from orcapod.types import ColumnConfig, ContentHash, Schema from orcapod.utils import arrow_utils from orcapod.utils.arrow_utils import system_tag_column_names from orcapod.utils.lazy_module import LazyModule @@ -58,6 +60,44 @@ def _type_text(data_type: Any) -> str: return data_type.__name__ if isinstance(data_type, type) else str(data_type) +@dataclasses.dataclass(frozen=True) +class _PendingType: + """A column of a loaded slot whose saved type could not be restored at load (ITL-1138). + + Attributes: + role: ``"tag"`` or ``"data"``. + text: The saved type text, as the job file holds it. + nullable: Whether the file records the column as nullable (``T | None``). + """ + + role: str + text: str + nullable: bool + + +def _saved_type_description(text: str) -> str: + """Name the type a saved type text stands for, for a message. + + A user class is named by its identity record (``user class 'mod.A' (dataclass)``), + any other extension type by its extension name; other text is quoted as it is. + + Args: + text: A saved type text, e.g. ``extension;>``. + """ + start = text.find("extension<") + if start < 0: + return f"type {text!r}" + name, _, rest = text[start + len("extension<") :].partition(";") + try: + metadata = json.loads(bytes.fromhex(rest.split(">", 1)[0])) + except ValueError: # not hex, or not JSON (UnicodeDecodeError is a ValueError) + metadata = None + identity = metadata.get("identity") if isinstance(metadata, dict) else None + if isinstance(identity, dict) and isinstance(identity.get("name"), str): + return f"user class {identity['name']!r} ({identity.get('kind', 'class')})" + return f"extension type {name!r}" + + def _is_narrower(declared: Any, given: Any) -> bool: """Return whether *given* is *declared* without its ``None`` member. @@ -235,6 +275,129 @@ def __init__( self._name = name self._tag_schema = tag_schema self._data_schema = data_schema + # A slot rebuilt from a saved job keeps the saved hashes, and the columns whose + # saved type could not be restored at load (ITL-1138); see ``_keep_saved``. + self._saved_pipeline_hash: ContentHash | None = None + self._saved_content_hash: ContentHash | None = None + self._pending_types: dict[str, _PendingType] = {} + + # ------------------------------------------------------------------ + # Rebuilt from a saved job + # ------------------------------------------------------------------ + + def _keep_saved( + self, + *, + pipeline_hash: str | None, + content_hash: str | None, + saved_types: Mapping[str, Mapping[str, str]], + nullable: Mapping[str, Collection[str]] | None, + ) -> None: + """Keep what a saved job recorded for this slot (ITL-1138). + + The slot returns the saved pipeline hash, and the saved content hash while no + source is bound, as loaded function and operator nodes return theirs, so its + stored address does not depend on what the loading process had registered. A + column whose saved type the load could not restore to a Python type (a user + class nothing in this process has registered yet) holds its saved text; it is + kept as pending and restored when a source is bound (``_fit``). + + Args: + pipeline_hash: The saved pipeline hash text, or ``None`` if not saved. + content_hash: The saved content hash text, or ``None`` if not saved. + saved_types: The saved type texts, ``{"tag": {...}, "data": {...}}``. + nullable: The saved nullable column names per role, or ``None`` when the + file does not record them. + """ + if pipeline_hash is not None: + self._saved_pipeline_hash = ContentHash.from_string(pipeline_hash) + if content_hash is not None: + self._saved_content_hash = ContentHash.from_string(content_hash) + for role, schema in (("tag", self._tag_schema), ("data", self._data_schema)): + for name, value in schema.items(): + text = saved_types.get(role, {}).get(name) + if isinstance(value, str) and text is not None: + self._pending_types[name] = _PendingType( + role=role, + text=text, + nullable=nullable is not None and name in nullable[role], + ) + + def _take_saved_state_of(self, other: "SourceNodeBase") -> None: + """Keep the saved hashes and pending types of *other*, a slot of the same schema.""" + self._saved_pipeline_hash = other._saved_pipeline_hash + self._saved_content_hash = other._saved_content_hash + self._pending_types = dict(other._pending_types) + + def _restore_pending_types(self) -> dict[str, str]: + """Restore the pending columns whose type this process can now restore. + + A source being bound has registered the class it holds, so a column saved with + that class restores now. The slot's schemas then hold the class, as they would + had the class been registered before the load. + + Returns: + For each column still not restorable, the description of its saved type. + """ + if not self._pending_types: + return {} + # Local import: orcapod.pipeline imports the node modules. + from orcapod.pipeline.serialization import deserialize_schema + + converter = self.data_context.type_converter + schemas = {"tag": dict(self._tag_schema), "data": dict(self._data_schema)} + still_pending: dict[str, _PendingType] = {} + for name, pending in self._pending_types.items(): + restored = deserialize_schema( + {name: pending.text}, + converter, + nullable=[name] if pending.nullable else [], + )[name] + if isinstance(restored, str): + still_pending[name] = pending + else: + schemas[pending.role][name] = restored + if len(still_pending) < len(self._pending_types): + self._tag_schema = Schema( + schemas["tag"], optional_fields=self._tag_schema.optional_fields + ) + self._data_schema = Schema( + schemas["data"], optional_fields=self._data_schema.optional_fields + ) + self._pending_types = still_pending + # The system-tag names come from the schema hash, now of the restored types. + with contextlib.suppress(AttributeError): # not computed yet + del self._schema_hash_str + self._invalidate_content_hash_cache() + return { + name: _saved_type_description(pending.text) + for name, pending in still_pending.items() + } + + def _bound_stream(self) -> "StreamProtocol | None": + """The concrete stream bound to this slot; a declaration has none.""" + return None + + def content_hash(self, hasher=None) -> ContentHash: + """Return the content hash; a slot rebuilt from a saved job returns the saved one. + + The saved value stands while no source is bound (ITL-1138); a bound source gives + its own identity, as for any slot. + """ + if self._saved_content_hash is not None and self._bound_stream() is None: + return self._saved_content_hash + return super().content_hash(hasher) + + def pipeline_hash(self, hasher=None) -> ContentHash: + """Return the pipeline hash; a slot rebuilt from a saved job returns the saved one. + + A slot's saved type of a user class this process has not registered is text + until a source is bound, so recomputing would give another value than the run's + (ITL-1138), as for loaded function and operator nodes. + """ + if self._saved_pipeline_hash is not None: + return self._saved_pipeline_hash + return super().pipeline_hash(hasher) # ------------------------------------------------------------------ # Identity @@ -434,6 +597,15 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] source_tag, source_data = source.output_schema() issues: list[str] = [] + unrestored = self._restore_pending_types() + for name, saved in unrestored.items(): + role = self._pending_types[name].role + issues.append( + f"{role} column {name!r} was saved as {saved}, which is not registered " + "in this process, so the source cannot be checked against it: bind a " + "source of that class (building one registers it) or define a function " + "pod that uses it before binding" + ) narrower: set[str] = set() for role, declared, given in ( ("tag", self._tag_schema, source_tag), @@ -446,6 +618,8 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] if extra: issues.append(f"unexpected {role} columns: {sorted(extra)}") for name in declared.keys(): + if name in unrestored: + continue if name not in given or equivalent_hints(given[name], declared[name]): continue if _is_narrower(declared[name], given[name]): @@ -836,6 +1010,10 @@ def bound_source(self, value: StreamProtocol | None) -> None: self._bind(value) self._update_modified_time() + def _bound_stream(self) -> StreamProtocol | None: + """The concrete stream bound to this slot, or ``None``.""" + return self._bound_source + def _bind(self, value: StreamProtocol | None) -> None: """Check *value* and bind it, invalidating both hash caches. @@ -1189,20 +1367,24 @@ def from_stream( match stream: case SourceJobNode(): # Case 1: copy — preserve bound_source, do NOT wrap the SJN itself. - return cls( + node = cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, bound_source=stream.bound_source, ) + node._take_saved_state_of(stream) + return node case SourceNode(): # Case 2: unbound — schema placeholder; SJN hash = SourceNode hash. - return cls( + node = cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, bound_source=None, ) + node._take_saved_state_of(stream) + return node case _: # Case 3: concrete stream — bound SJN. # Slot name priority: @@ -1228,14 +1410,43 @@ def from_stream( bound_source=stream, ) + @classmethod + def _from_loaded_slot( + cls, slot: SourceNode, bound_source: StreamProtocol | None + ) -> "SourceJobNode": + """Build the job node of a slot rebuilt from a saved job (``PipelineJob.load``). + + The node takes the slot's saved hashes and pending types before *bound_source* + is checked, so a column saved with a class this process registers only now + (by building the source) is restored for the check (ITL-1138). + + Args: + slot: The slot ``Pipeline.load`` rebuilt from the job file. + bound_source: The source rebuilt from the saved config, if any. + + Raises: + SourceSpecMismatchError: If *bound_source* does not fit the slot. + """ + node = cls( + name=slot.name, + tag_schema=slot.tag_schema, + data_schema=slot.data_schema, + ) + node._take_saved_state_of(slot) + if bound_source is not None: + node._bind(bound_source) + return node + def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. Returns: A new ``SourceNode`` with the same name and schemas. """ - return SourceNode( + node = SourceNode( name=self._name, tag_schema=self._tag_schema, data_schema=self._data_schema, ) + node._take_saved_state_of(self) + return node diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 3d82e45ec..3d9f06e9d 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -394,6 +394,14 @@ def load(cls, path: str | Path) -> "Pipeline": descriptor.get("data_context_key") ), ) + # The saved hashes, and the saved type text of a column whose + # class this process has not registered yet (ITL-1138). + node._keep_saved( # pyright: ignore[reportPrivateUsage] # the loader is the one caller + pipeline_hash=descriptor.get("pipeline_hash"), + content_hash=descriptor.get("content_hash"), + saved_types={"tag": saved_tag, "data": saved_data}, + nullable=saved_nullable, + ) # Restore label from descriptor if set explicitly stored_label = descriptor.get("label") if stored_label and stored_label != node_name: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 4a930f940..d46888995 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1688,11 +1688,9 @@ def load( continue if isinstance(bp_node, _SourceNode): - sjn = SourceJobNode( - name=bp_node.name, - tag_schema=bp_node.tag_schema, - data_schema=bp_node.data_schema, - bound_source=sources.get(bp_node.name), + # Keeps the loaded slot's saved hashes and pending types (ITL-1138). + sjn = SourceJobNode._from_loaded_slot( # pyright: ignore[reportPrivateUsage] # the loader is the one caller + bp_node, bound_source=sources.get(bp_node.name) ) job._persistent_node_map[node_hash] = sjn if bp_node.name in sources: diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index 37450649a..74e41f906 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -120,6 +120,19 @@ def read_u(pp: cls) -> int: job.save(work + "/job.json") print(json.dumps(describe(job))) else: + if mode == "load_old_file": + # A file as an earlier Orcapod wrote it: the format before orcapod_version, + # and no saved hashes on the source slots. + with open(work + "/job.json") as f: + data = json.load(f) + data["orcapod_pipeline_job_version"] = "0.1.0" + data.pop("orcapod_version", None) + for descriptor in data["pipeline"]["nodes"].values(): + if descriptor["node_type"] == "source": + descriptor.pop("pipeline_hash", None) + descriptor.pop("content_hash", None) + with open(work + "/job.json", "w") as f: + json.dump(data, f) # Nothing registers the class before the load. job = PipelineJob.load(work + "/job.json") import classes @@ -185,3 +198,14 @@ def test_a_slot_whose_class_is_never_registered_refuses_the_source_clearly( assert "'p'" in error assert "classes.Exact" in error assert "not registered" in error + + +def test_a_file_without_the_slots_saved_hashes_loads_and_binds( + tmp_path: Path, saved_job +) -> None: + """With no saved hashes, the slot restores its type at bind and hashes as saved.""" + saved = saved_job("Exact") + loaded = _run(tmp_path, "load_old_file", str(tmp_path / "work"), "Exact") + assert "bind_error" not in loaded, loaded.get("bind_error") + assert loaded.pop("_reused") == {"add": 2, "read_u": 2} + assert loaded == saved From 4b1a074e6350fa4b1e3787d77030acc836ec53cb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:07:11 -0800 Subject: [PATCH 24/33] test(pipeline): type the helpers of the loaded-slot test (ITL-1138) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_loaded_slot_unregistered_class.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index 74e41f906..03a744ea8 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -17,6 +17,7 @@ import sys import textwrap from pathlib import Path +from typing import Any import pytest @@ -152,7 +153,7 @@ def read_u(pp: cls) -> int: ) -def _run(tmp_path: Path, *args: str) -> dict: +def _run(tmp_path: Path, *args: str) -> dict[str, Any]: """Run the script in a fresh interpreter and return the JSON it prints last.""" result = subprocess.run( [sys.executable, str(tmp_path / "script.py"), str(tmp_path), *args], @@ -169,7 +170,7 @@ def saved_job(tmp_path: Path): _ = (tmp_path / "classes.py").write_text(_CLASSES) _ = (tmp_path / "script.py").write_text(_SCRIPT) - def save(class_name: str) -> dict: + def save(class_name: str) -> dict[str, Any]: return _run(tmp_path, "save", str(tmp_path / "work"), class_name) return save From e5155e34c64f0db7e34875905e293316f0e0a069 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:31:54 -0800 Subject: [PATCH 25/33] test(large_normalization): a raw ArrowTableStream is refused as a job input; job sweeps take every other kind (ITL-1126) Co-Authored-By: Claude Opus 5.5 --- ...1-14-itl-1112-followup-merge-and-cr-392.md | 59 +++++++++++++++++++ tests/test_core/large_types_cases.py | 8 +++ .../operators/test_large_normalization.py | 15 +++-- .../test_large_normalization_sources.py | 27 ++++++--- 4 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md diff --git a/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md new file mode 100644 index 000000000..b3b80e611 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md @@ -0,0 +1,59 @@ +# ITL-1112 / ITL-1126 follow-up: merge with main, CodeRabbit findings from PR #392 + +Branch `eywalker/itl-1112-binding-a-source-with-a-different-column`, PR #386. Head before +this work: 4ffdef06. Main at 810d1b03. + +## 1. Merge origin/main (done: 5c9ac6cd, no conflicts) + +PR #389 (ITL-1122) added a large-normalization sweep whose source kinds include a raw +`ArrowTableStream`; its JOB tests use it as a pipeline input, which ITL-1126 refuses +(developer decision; ITL-1127 adds the wrapper). Fix, tests only: + +- `tests/test_core/operators/test_large_normalization.py`: `test_job_has_no_non_large_type_and_equals_plain` + sweeps every kind except `ArrowTableStream` (the plain-stream sweeps keep it). One new test: + a raw `ArrowTableStream` as a job input raises `SourceSpecMismatchError` naming the remedy + (wrap the table as a source). +- `tests/test_core/sources/test_large_normalization_sources.py`: same for + `test_job_nodes_have_no_non_large_type` (the table/head test keeps it), and the refusal test. +- Nothing else changes in either file; every other kind keeps its #389 assertions. + +## 2. CodeRabbit findings raised on PR #392 against #386's code + +### A (comment 4575906153) — CONFIRMED by reproduction + +A source whose system-tag pair comes record-id first (an `ArrowTableStream` over a table +whose `_tag_record_id::…` column precedes `_tag_source_id::…`) is accepted, and the bound +node's table (and zero-row table) keeps record-id first while the unbound node's +`output_schema(all_info=True)` predicts source-id first. With an exact-schema source +(`_declared_form_for` returns `None`) the node passes the source through, so even its +`output_schema` reports the reversed order: the bind changed the node's schema order. + +Fix: `_declared_form_for` also requires the system-tag pair in (source-id, record-id) order +for the pass-through, and `_DeclaredForm._declared_positions` orders the pair as a fourth +group (declared names, after renames), like the tag/data/source-info groups. This agrees with +main's row order (#390/ITL-1131: user tags, system tags, data, source info): the pair keeps +the positions it holds; only its internal order is fixed. Test first: a bound reversed-pair +source gives `as_table`, `head`, the zero-row table and `output_schema` in the unbound +node's order (exact and nullable-slot cases). + +### C (comment 4575906162) — NOT REPRODUCIBLE; propose to decline + +`Pipeline.compile` raises `InconsistentSourceError` for same-named slots whose schemas +differ, so same-named slots have equal declared tag and data schemas (and so equal +system-tag names). `validate`/`_fit` depend only on the name and those schemas, so a +rebuilt source gets the same verdict from every same-named slot: one accepting and the other +refusing cannot happen. Reproduction: two CSV sources with one source_id and differing +schemas are refused at compile; with equal schemas they deduplicate to one slot. + +## 3. Verification + +Targeted: tests/test_pipeline, tests/test_core/{nodes,sources,operators}, golden (3.12; +async files also 3.11), one process. Ruff, basedpyright. One push; CI on the merge. + +## Status notes + +- [x] merge +- [ ] part 1 tests +- [ ] plan for A/C sent to coordinator +- [ ] A fix (after approval) +- [ ] push, CI, CodeRabbit, replies on #392 threads diff --git a/tests/test_core/large_types_cases.py b/tests/test_core/large_types_cases.py index d7a67332c..0d5ae54b8 100644 --- a/tests/test_core/large_types_cases.py +++ b/tests/test_core/large_types_cases.py @@ -274,6 +274,14 @@ def _cached_source_delta(n_rows: int, tmp: Path) -> StreamProtocol: (name, n_rows) for name, (_, counts) in SOURCES.items() for n_rows in counts ] +# A raw ``ArrowTableStream`` has no system-tag columns, so it cannot be a pipeline's +# input (ITL-1126; binding one is ITL-1127): the job sweeps take every other kind, and +# ``test_large_normalization_sources`` asserts the refusal. +RAW_STREAM = "ArrowTableStream" +JOB_SOURCE_CASES = [ + (name, n_rows) for name, n_rows in SOURCE_CASES if name != RAW_STREAM +] + def build_source(name: str, n_rows: int, tmp: Path) -> StreamProtocol: """Build the source kind ``name`` with ``n_rows`` rows, its files under ``tmp``.""" diff --git a/tests/test_core/operators/test_large_normalization.py b/tests/test_core/operators/test_large_normalization.py index 5abc10c86..8fc110508 100644 --- a/tests/test_core/operators/test_large_normalization.py +++ b/tests/test_core/operators/test_large_normalization.py @@ -44,6 +44,7 @@ from tests.test_core.large_types_cases import ( ALL_COLUMNS, FULL_SCHEMA, + JOB_SOURCE_CASES, ROWS, SOURCE_CASES, SOURCES, @@ -98,11 +99,14 @@ def _other(n_rows: int, data_column: str) -> ArrowTableSource: } -def _cases(row_counts: tuple[int, ...]) -> list[tuple[str, str, int]]: - """Every (source kind, operator, rows) with rows among ``row_counts``.""" +def _cases( + row_counts: tuple[int, ...], source_cases: list[tuple[str, int]] = SOURCE_CASES +) -> list[tuple[str, str, int]]: + """Every (source kind, operator, rows) of ``source_cases`` with rows among + ``row_counts``.""" return [ (source, operator, n_rows) - for source, n_rows in SOURCE_CASES + for source, n_rows in source_cases if n_rows in row_counts for operator in OPERATORS ] @@ -154,7 +158,10 @@ def test_zero_row_schema_equals_populated( @pytest.mark.parametrize("orchestrator", ["sync", "async"]) -@pytest.mark.parametrize(("source", "operator", "n_rows"), _cases((2, 0))) +# Every kind a job takes as input; a raw ArrowTableStream is refused (ITL-1126). +@pytest.mark.parametrize( + ("source", "operator", "n_rows"), _cases((2, 0), JOB_SOURCE_CASES) +) def test_job_has_no_non_large_type_and_equals_plain( source: str, operator: str, n_rows: int, orchestrator: str, tmp_path: Path ) -> None: diff --git a/tests/test_core/sources/test_large_normalization_sources.py b/tests/test_core/sources/test_large_normalization_sources.py index a878c41ce..2bb515851 100644 --- a/tests/test_core/sources/test_large_normalization_sources.py +++ b/tests/test_core/sources/test_large_normalization_sources.py @@ -26,6 +26,7 @@ ) from orcapod.core.streams import ArrowTableStream from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError from orcapod.operators import SelectTagColumns from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob @@ -37,6 +38,8 @@ from tests.test_core.large_types_cases import ( ALL_COLUMNS, FULL_SCHEMA, + JOB_SOURCE_CASES, + RAW_STREAM, ROWS, SOURCE_CASES, TAGS, @@ -71,13 +74,23 @@ def test_table_and_head_have_no_non_large_type( assert non_large_columns(table) == [] assert non_large_columns(source.head(columns=ALL_COLUMNS).table) == [] - @pytest.mark.parametrize("orchestrator", ["sync", "async"]) - def test_job_nodes_have_no_non_large_type( - self, name: str, n_rows: int, orchestrator: str, tmp_path: Path - ) -> None: - tables = _job_node_tables(build_source(name, n_rows, tmp_path), orchestrator) - for table in tables: - assert non_large_columns(table) == [] + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize(("name", "n_rows"), JOB_SOURCE_CASES) +def test_job_nodes_have_no_non_large_type( + name: str, n_rows: int, orchestrator: str, tmp_path: Path +) -> None: + tables = _job_node_tables(build_source(name, n_rows, tmp_path), orchestrator) + for table in tables: + assert non_large_columns(table) == [] + + +@pytest.mark.parametrize("n_rows", [2, 0]) +def test_raw_stream_is_refused_as_a_job_input(n_rows: int, tmp_path: Path) -> None: + """A raw ``ArrowTableStream`` has no system tags, so a job refuses it as an input + (ITL-1126) and says to wrap its table as a source; binding one is ITL-1127.""" + with pytest.raises(SourceSpecMismatchError, match="wrap its table as a source"): + _job_node_tables(build_source(RAW_STREAM, n_rows, tmp_path), "sync") def test_explicit_arrow_schema_is_normalized(tmp_path: Path) -> None: From 09ea1fb179477bfb2fc05fdc4d80e599ee1de111 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:32:48 -0800 Subject: [PATCH 26/33] test(pipeline): compare loaded slots' unbound hashes and row values with the saving process (ITL-1138, review round 1) Co-Authored-By: Claude Opus 5.5 --- .../test_loaded_slot_unregistered_class.py | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index 03a744ea8..d36edcd7a 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -80,19 +80,26 @@ def sources(cls): return s1, s2 + def rows(table): + # The table's rows as sorted JSON text, so row order does not matter. + return sorted(json.dumps(row, sort_keys=True, default=str) for row in table.to_pylist()) + + def describe(job): out = {} for label, node in sorted(job.nodes.items()): - table = node.as_table(all_info=True) + # The schema with every column group: the system-tag names are compared. + schema = node.as_table(all_info=True).schema # A source node stores nothing; a function or operator node has a stored view. stored_view = getattr(node, "as_source", None) - stored = None if stored_view is None else stored_view().as_table(all_info=True) out[label] = { "pipeline_hash": node.pipeline_hash().to_string(), "content_hash": node.content_hash().to_string(), - "schema": [f"{field.name}: {field.type}" for field in table.schema], - "rows": table.num_rows, - "stored_rows": None if stored is None else stored.num_rows, + "schema": [f"{field.name}: {field.type}" for field in schema], + # Values of the user columns: a load-only run's status and execution + # columns differ from the saving run's by design. + "rows": rows(node.as_table()), + "stored_rows": None if stored_view is None else rows(stored_view().as_table()), } return out @@ -136,6 +143,14 @@ def read_u(pp: cls) -> int: json.dump(data, f) # Nothing registers the class before the load. job = PipelineJob.load(work + "/job.json") + # The source slots' hashes while unbound, before the class is imported. + from orcapod.core.nodes.source_node import SourceJobNode + + unbound_slots = { + label: [node.pipeline_hash().to_string(), node.content_hash().to_string()] + for label, node in job.nodes.items() + if isinstance(node, SourceJobNode) + } import classes bound_class = sys.argv[5] if len(sys.argv) > 5 else class_name @@ -148,6 +163,7 @@ def read_u(pp: cls) -> int: job.run(load_only=True) out = describe(job) out["_reused"] = {label: n.reused for label, n in job.last_run.nodes.items()} + out["_unbound_slots"] = unbound_slots print(json.dumps(out)) """ ) @@ -185,6 +201,19 @@ def test_a_job_loaded_before_its_class_is_registered_reads_like_the_saved_job( loaded = _run(tmp_path, "load", str(tmp_path / "work"), class_name) assert "bind_error" not in loaded, loaded.get("bind_error") assert loaded.pop("_reused") == {"add": 2, "read_u": 2} + # Each source slot has its saved hashes as soon as it is loaded, before any bind: + # the saving run's pipeline hash, and the content hash the job file records for + # the slot (a bound slot's content hash is its source's, compared below). + job_file = json.loads((tmp_path / "work" / "job.json").read_text()) + file_content_hashes = { + descriptor["label"]: descriptor["content_hash"] + for descriptor in job_file["pipeline"]["nodes"].values() + if descriptor["node_type"] == "source" + } + assert loaded.pop("_unbound_slots") == { + label: [saved[label]["pipeline_hash"], file_content_hashes[label]] + for label in ("s1", "s2") + } assert loaded == saved @@ -209,4 +238,5 @@ def test_a_file_without_the_slots_saved_hashes_loads_and_binds( loaded = _run(tmp_path, "load_old_file", str(tmp_path / "work"), "Exact") assert "bind_error" not in loaded, loaded.get("bind_error") assert loaded.pop("_reused") == {"add": 2, "read_u": 2} + _ = loaded.pop("_unbound_slots") # no saved hashes: recomputed once bound assert loaded == saved From ffcdcf654846f702dbab1754105553df514bf1eb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:37:47 -0800 Subject: [PATCH 27/33] fix(source_node): a bound source's system-tag pair is given source id first (ITL-1112) A source whose pair came record-id first was accepted and kept that order in the node's table, against the order every node predicts. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +- CLAUDE.md | 4 +- changelog.d/ITL-1112.md | 10 +- src/orcapod/core/nodes/source_node.py | 31 +++-- ...1-14-itl-1112-followup-merge-and-cr-392.md | 6 +- .../test_bind_keeps_node_schemas.py | 114 ++++++++++++++++++ 6 files changed, 151 insertions(+), 18 deletions(-) diff --git a/.zed/rules b/.zed/rules index 09759cc33..fc91d6f1b 100644 --- a/.zed/rules +++ b/.zed/rules @@ -827,8 +827,8 @@ data. The rules: else raises SourceSpecMismatchError before anything is bound. The source's rows are given as the declared schema at one place, _DeclaredForm (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, the VALUES the source's - own). Limits: the check compares Python types, so a difference in Arrow width only + declared schema hash, taken from the source's own pair by prefix, in the declared order + (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/CLAUDE.md b/CLAUDE.md index 8b8fe2ae1..71ba41269 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -869,8 +869,8 @@ data. The rules: else raises `SourceSpecMismatchError` before anything is bound. The source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, the VALUES the source's - own). Limits: the check compares Python types, so a difference in Arrow width only + declared schema hash, taken from the source's own pair by prefix, in the declared order + (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 4932307a6..2ad4b2651 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -65,8 +65,9 @@ source. Binding such a stream properly is planned (ITL-1127). stored path or the column names, types, nullable flags or order of any node's or stored table. A source whose columns come in another order than the slot declares gives them in the declared order, and a stream bound to a slot (a source passed through a filter or a - rename, with one system-tag pair) gives its system-tag columns the slot's names; their - values stay the stream's own. Both passed through as they were before. + rename, with one system-tag pair) gives its system-tag columns the slot's names, source id + before record id; their values stay the stream's own. Both passed through as they were + before. ### Fixed @@ -80,3 +81,8 @@ source. Binding such a stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). +- A stream bound to a slot that is not a source (a function pod's stream, with one system-tag + pair) keeps its own column layout in the source node's own table: its system-tag columns + before its data, and its own nullable flags for `_content_hash` and the source-info + columns. That table is not stored and no downstream node sees it: stored paths, stored + table schemas and every downstream node's table are those of a source bind. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index dcbe39ba8..8209bac19 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -79,8 +79,9 @@ class _DeclaredForm: The pipeline determines every node's schema (ITL-1112). A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows as rows of ``x: int | None``: the same arrays under nullable fields. The source's system-tag - columns take the slot's names (their values stay the source's own), and its tag and - data columns, with their source-info columns, the declared order. Nothing else + columns take the slot's names and order, source id then record id (their values stay + the source's own), and its tag and data columns, with their source-info columns, the + declared order. Nothing else differs: values, types, source-info tokens and meta columns pass unchanged, so the rows equal those of a source of the declared schema with the same source id. @@ -89,12 +90,14 @@ class _DeclaredForm: renames: The source's system-tag column names, mapped to the declared ones. tag_order: The declared tag columns, in declared order. data_order: The declared data columns, in declared order. + system_tag_order: The declared system-tag columns, source id then record id. """ nullable: frozenset[str] renames: Mapping[str, str] tag_order: tuple[str, ...] data_order: tuple[str, ...] + system_tag_order: tuple[str, str] def apply(self, table: pa.Table) -> pa.Table: """Return *table*'s arrays under the declared fields and system-tag names. @@ -107,8 +110,9 @@ def apply(self, table: pa.Table) -> pa.Table: Returns: The same arrays, with the columns in ``nullable`` flagged nullable, the - columns in ``renames`` renamed, and the tag, data and source-info columns in - declared order within the positions each group holds. + columns in ``renames`` renamed, and the tag, system-tag, data and + source-info columns in declared order within the positions each group + holds. """ schema = table.schema for position, field in enumerate(table.schema): @@ -124,14 +128,15 @@ def apply(self, table: pa.Table) -> pa.Table: def _declared_positions(self, names: list[str]) -> list[int]: """Return the column positions of *names* with each group in declared order. - The tag columns, the data columns and the data columns' source-info columns each - fill the positions their group holds, in declared order; every other column - keeps its position. + The tag columns, the system-tag pair, the data columns and the data columns' + source-info columns each fill the positions their group holds, in declared + order; every other column keeps its position. """ position = {name: index for index, name in enumerate(names)} order = list(range(len(names))) source_info = tuple(f"{constants.SOURCE_PREFIX}{n}" for n in self.data_order) - for group in (self.tag_order, self.data_order, source_info): + groups = (self.tag_order, self.system_tag_order, self.data_order, source_info) + for group in groups: present = [name for name in group if name in position] for slot, name in zip( sorted(position[n] for n in present), present, strict=True @@ -879,7 +884,8 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: Returns: ``None`` when the source's rows already are rows of the declared schema - (same types, column order and system-tag names), so they pass unchanged; + (same types, column order and system-tag names and order), so they pass + unchanged; otherwise the ``_DeclaredForm`` that makes them so. Raises: @@ -901,11 +907,17 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: } tag_order = tuple(self._tag_schema.keys()) data_order = tuple(self._data_schema.keys()) + source_system_tags = [ + name + for name in source.keys(columns={"system_tags": True})[0] + if name not in source_tag + ] if ( not narrower and not renames and tuple(source_tag.keys()) == tag_order and tuple(source_data.keys()) == data_order + and source_system_tags == [declared_source_id, declared_record_id] ): return None return _DeclaredForm( @@ -913,6 +925,7 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: renames=renames, tag_order=tag_order, data_order=data_order, + system_tag_order=(declared_source_id, declared_record_id), ) def _declared_rows( diff --git a/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md index b3b80e611..c018aad15 100644 --- a/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md +++ b/superpowers/plans/2026-11-14-itl-1112-followup-merge-and-cr-392.md @@ -53,7 +53,7 @@ async files also 3.11), one process. Ruff, basedpyright. One push; CI on the mer ## Status notes - [x] merge -- [ ] part 1 tests -- [ ] plan for A/C sent to coordinator -- [ ] A fix (after approval) +- [x] part 1 tests (e5155e34) +- [x] plan for A/C sent to coordinator: A approved, C decline approved; layout-by-kind: option 1 (known limit in PR body + changelog; coordinator files the issue) +- [x] A fix + tests - [ ] push, CI, CodeRabbit, replies on #392 threads diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index 42931bcf7..2f972f85b 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -24,8 +24,10 @@ from orcapod import PipelineJob, function_pod from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream from orcapod.databases import DeltaTableDatabase from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.system_constants import constants from orcapod.types import Schema @@ -54,6 +56,22 @@ def _csv(tmp_path: pathlib.Path, name: str, text: str) -> CSVSource: return CSVSource(str(path), tag_columns=["id"], source_id=name) +def _record_id_first(source: ArrowTableSource) -> ArrowTableStream: + """Return a stream of *source*'s rows with its system-tag pair record-id first. + + The pair swaps places; every other column keeps its position. + """ + table = source.as_table( + columns={"system_tags": True, "source": True, "context": True} + ) + names = table.column_names + pair = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] + assert len(pair) == 2 and constants.SYSTEM_TAG_SOURCE_ID_PREFIX in pair[0] + swap = {pair[0]: pair[1], pair[1]: pair[0]} + reordered = table.select([swap.get(name, name) for name in names]) + return ArrowTableStream(reordered, tag_columns=["id"]) + + NODES = ("src", "renamed", "inc") # A loaded job reads stored results only; its uncached operator has none to read. LOADED_NODES = ("src", "inc") @@ -122,6 +140,16 @@ def test_no_accepted_bind_changes_a_path_or_a_table_schema( DictSource([{"id": 10, "q": 10}], tag_columns=["id"], source_id="m") ), ), + ( + "system-tag pair record-id first", + _assign, + _record_id_first(_arrow([11, 12], [11, None], x_nullable=True)), + ), + ( + "narrower, system-tag pair record-id first", + _bind, + _record_id_first(_arrow([13], [13], x_nullable=False)), + ), ("narrower csv", _bind, _csv(tmp_path, "b.csv", "id,x\n9,9\n")), ] first: dict[str, Any] | None = None @@ -201,3 +229,89 @@ def test_a_structural_class_of_the_same_identity_changes_no_path_or_schema( snapshots.append(_snapshot(job, ("src", "renamed"))) assert snapshots[1] == snapshots[0] assert snapshots[2] == snapshots[0] + + +def test_a_record_id_first_pair_is_given_in_the_declared_order() -> None: + """A source whose system-tag pair comes record-id first is given source-id first. + + The order every node predicts (``output_schema``) and a source of the declared order + has; in the schema, the table, the head and the zero-row table, for a slot of the + source's own types (rows passed as they are otherwise) and for a ``T | None`` slot + the source is narrower than. + """ + for x_type, x_nullable in ((int, False), (int | None, True)): + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": x_type}), + ) + job = PipelineJob(name="j") + with job: + _ = MapData({"x": "y"})(slot, label="renamed") + node = job.nodes["src"] + declared = list(node.output_schema(all_info=True)[0]) + reference = _arrow([0], [0], x_nullable=x_nullable) + job.bind(sources={"src": reference}) + order = node.as_table(all_info=True).column_names + zero_row_order = node.head(0, all_info=True).table.column_names + + for rows in ([1, 2], []): + # A non-null ``x``: the slot's own type, or narrower than ``int | None``. + xs: list[int | None] = list(rows) + source = _record_id_first(_arrow(rows, xs, x_nullable=False)) + job.bind(sources={"src": source}) + assert list(node.output_schema(all_info=True)[0]) == declared, x_type + assert node.as_table(all_info=True).column_names == order, x_type + assert node.head(all_info=True).table.column_names == ( + order if rows else zero_row_order + ), x_type + + +@function_pod(output_keys="x") +def passes_on(w: int | None) -> int | None: + """Return ``w`` unchanged.""" + return w + + +def test_a_bound_function_stream_moves_no_stored_or_downstream_schema( + tmp_path: pathlib.Path, +) -> None: + """A function pod's stream (one system-tag pair) moves no path, stored or downstream schema. + + Known limit: the source node's own table keeps the bound stream's column layout (a + function pod's stream gives its system tags before its data) and the nullable flags + of its ``_content_hash`` and source-info columns. That table is not stored and no + downstream node sees it; its columns, and its user and system-tag fields, are the + same. + """ + slot = SourceNode( + name="src", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int | None}), + ) + job = PipelineJob(name="j", database=DeltaTableDatabase(str(tmp_path / "store"))) + with job: + _ = inc.pod(MapData({"x": "y"})(slot, label="renamed"), label="inc") + _bind(job, _arrow([0, 1], [1, None], x_nullable=True)) + assert job.run().ok + first = _snapshot(job) + + upstream = MapData({"x": "w"})(_arrow([5, 6], [5, None], x_nullable=True)) + _bind(job, passes_on.pod(upstream)) + assert job.run().ok + second = _snapshot(job) + assert second["paths"] == first["paths"] + assert second["stored"] == first["stored"] + for name in ("renamed", "inc"): + assert second["nodes"][name] == first["nodes"][name], name + src_first, src_second = first["nodes"]["src"], second["nodes"]["src"] + assert sorted(src_second.names) == sorted(src_first.names) + compared = [ + name + for name in src_first.names + if name in ("id", "x", constants.CONTEXT_KEY) + or name.startswith(constants.SYSTEM_TAG_PREFIX) + ] + assert len(compared) == 5 + for name in compared: + assert src_second.field(name) == src_first.field(name), name From f2aaf91f2455ff58f7c28ac0f0f1a0dc4abba00c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:48:11 -0800 Subject: [PATCH 28/33] docs(source_node): column order is not part of the bind invariant (ITL-1112) Developer correction, November 14, 2026: the invariant is each node's path and its columns (names, roles, types, nullability, system-tag names). The invariant test compares columns by name; the declared-order presentation is a convenience with its own tests; columns match by name within the tag and data groups, and the role decides. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 +++-- CLAUDE.md | 13 +++-- changelog.d/ITL-1112.md | 23 ++++----- design_issues/ITL-1112-BT1.md | 7 +-- src/orcapod/core/nodes/source_node.py | 7 +-- .../test_bind_keeps_node_schemas.py | 50 +++++++++++-------- .../test_bind_narrower_source.py | 6 ++- tests/test_pipeline/test_bind_type_check.py | 35 +++++++++++++ 8 files changed, 103 insertions(+), 51 deletions(-) diff --git a/.zed/rules b/.zed/rules index fc91d6f1b..fb455287d 100644 --- a/.zed/rules +++ b/.zed/rules @@ -818,17 +818,20 @@ data. The rules: - ArrowTableSource raises ValueError if any tag_columns are not in the table. - The pipeline determines every node's schema; a bound source never changes it (developer, November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the - schema of any node's or stored table (column names with system tags, types, nullable - flags, order); only values differ. SourceNodeBase.validate is the one check, used by + columns of any node's or stored table (names, tag or data roles, types, nullable flags, + system-tag column names); only values differ. Column order is not part of it: matching a + source to a slot, columns match by name within the tag group and within the data group, + and their order never decides acceptance (developer, November 14, 2026). Do not reorder + columns without a reason. SourceNodeBase.validate is the one check, used by PipelineJob.bind, the bound_source setter (and SourceJobNode.__init__) and PipelineJob.load: per column the declared Python type (structural classes of one identity are one type, equivalent_hints), or T where the slot declares T | None; and exactly one source-id (str) and one record-id (bytes) system-tag column, no other. Anything else raises SourceSpecMismatchError before anything is bound. The source's rows are given as the declared schema at one place, _DeclaredForm (nullable - fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, in the declared order - (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only + fields over the same arrays; the system-tag column NAMES of the declared schema hash, + taken from the source's own pair by prefix, the VALUES the source's own; as a stable + convenience, not a guarantee, the declared column order, source id before record id). Limits: the check compares Python types, so a difference in Arrow width only (int32 against int64) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/CLAUDE.md b/CLAUDE.md index 71ba41269..3d84ae935 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -860,17 +860,20 @@ data. The rules: - `ArrowTableSource.__init__` raises `ValueError` if any `tag_columns` are not in the table. - The pipeline determines every node's schema; a bound source never changes it (developer, November 14, 2026, ITL-1112). No accepted bind changes any node's stored path or the - schema of any node's or stored table (column names with system tags, types, nullable - flags, order); only values differ. `SourceNodeBase.validate` is the one check, used by + columns of any node's or stored table (names, tag or data roles, types, nullable flags, + system-tag column names); only values differ. Column order is not part of it: matching a + source to a slot, columns match by name within the tag group and within the data group, + and their order never decides acceptance (developer, November 14, 2026). Do not reorder + columns without a reason. `SourceNodeBase.validate` is the one check, used by `PipelineJob.bind`, the `bound_source` setter (and `SourceJobNode.__init__`) and `PipelineJob.load`: per column the declared Python type (structural classes of one identity are one type, `equivalent_hints`), or `T` where the slot declares `T | None`; and exactly one source-id (`str`) and one record-id (`bytes`) system-tag column, no other. Anything else raises `SourceSpecMismatchError` before anything is bound. The source's rows are given as the declared schema at one place, `_DeclaredForm` (nullable - fields over the same arrays; the declared column order; the system-tag column NAMES of the - declared schema hash, taken from the source's own pair by prefix, in the declared order - (source id, then record id), the VALUES the source's own). Limits: the check compares Python types, so a difference in Arrow width only + fields over the same arrays; the system-tag column NAMES of the declared schema hash, + taken from the source's own pair by prefix, the VALUES the source's own; as a stable + convenience, not a guarantee, the declared column order, source id before record id). Limits: the check compares Python types, so a difference in Arrow width only (`int32` against `int64`) is not yet refused (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126); diff --git a/changelog.d/ITL-1112.md b/changelog.d/ITL-1112.md index 2ad4b2651..8405269c3 100644 --- a/changelog.d/ITL-1112.md +++ b/changelog.d/ITL-1112.md @@ -38,8 +38,8 @@ source's types, which gives it its own stored paths. A slot's rows carry one source-id and one record-id system-tag column, under the slot's names. A stream with one pair per input (the result of a `Join` or `MergeJoin`), or with none (an `ArrowTableStream` over a table without them), cannot give its rows that way; it was bound -before with its own system-tag columns and column order, so the schema of the slot's table, -and of every table after it, depended on what was bound. Every way of binding (`bind`, the +before with its own system-tag columns, so the system-tag columns of the slot's table, and +of every table after it, depended on what was bound. Every way of binding (`bind`, the `bound_source` setter, `SourceJobNode(..., bound_source=...)`, `PipelineJob.load`) now refuses it with `SourceSpecMismatchError`, naming the slot and the system-tag columns found, whatever its column types; so is a stream with any other system-tag column, or whose source id is not @@ -62,12 +62,14 @@ source. Binding such a stream properly is planned (ITL-1127). predicted schema is the declared one. The source node's `content_hash` and source id stay the bound source's own. - The pipeline determines every node's schema (ITL-1112): no accepted bind changes a node's - stored path or the column names, types, nullable flags or order of any node's or stored - table. A source whose columns come in another order than the slot declares gives them in - the declared order, and a stream bound to a slot (a source passed through a filter or a - rename, with one system-tag pair) gives its system-tag columns the slot's names, source id - before record id; their values stay the stream's own. Both passed through as they were - before. + stored path or the columns of any node's or stored table (names, tag or data roles, types, + nullable flags, system-tag column names). A stream bound to a slot (a source passed + through a filter or a rename, with one system-tag pair) gives its system-tag columns the + slot's names; their values stay the stream's own. It passed through as it was before. + Column order is not part of that: a source matches its slot by column name within the tag + group and within the data group, whatever their order. As a convenience, a source whose + columns come in another order is shown in the slot's declared order, its system tags + source id first. ### Fixed @@ -81,8 +83,3 @@ source. Binding such a stream properly is planned (ITL-1127). - The bind check compares Python types, so a source that differs from its slot only in Arrow width (`int32` against `int64`) is still accepted (ITL-949). -- A stream bound to a slot that is not a source (a function pod's stream, with one system-tag - pair) keeps its own column layout in the source node's own table: its system-tag columns - before its data, and its own nullable flags for `_content_hash` and the source-info - columns. That table is not stored and no downstream node sees it: stored paths, stored - table schemas and every downstream node's table are those of a source bind. diff --git a/design_issues/ITL-1112-BT1.md b/design_issues/ITL-1112-BT1.md index 0a42f71c3..59e10e395 100644 --- a/design_issues/ITL-1112-BT1.md +++ b/design_issues/ITL-1112-BT1.md @@ -31,9 +31,10 @@ hash to the declared one (their names carry the schema hash). `iter_data`, `asyn `as_table`, `head` and `output_schema` go through it, so the rows equal those of a nullable source with the same values and source id, row hashes and record ids included, and both append to one stored table. The node's `content_hash` and source id stay the source's own. The same -form gives a source's columns in the declared order, and a stream's one system-tag pair the -slot's names (taken from the stream's own columns by prefix), so no accepted bind changes a -node's path or table schema. Two Arrow types that are one Python type (`int32` and `int64`) +form gives a stream's one system-tag pair the slot's names (taken from the stream's own +columns by prefix), so no accepted bind changes a node's path or its columns (names, roles, +types, nullable flags, system-tag names; column order is not part of that, and the form's +declared order is a convenience). Two Arrow types that are one Python type (`int32` and `int64`) are still not told apart (ITL-949). A source whose system-tag columns are not exactly one source-id and one record-id column (a joined or merged stream, a raw stream with none) is refused by the same check (developer, November 14, 2026, ITL-1126; binding one properly is diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8209bac19..f11ca368f 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -79,9 +79,10 @@ class _DeclaredForm: The pipeline determines every node's schema (ITL-1112). A slot declaring ``x: int | None`` accepts a source of ``x: int`` and gives its rows as rows of ``x: int | None``: the same arrays under nullable fields. The source's system-tag - columns take the slot's names and order, source id then record id (their values stay - the source's own), and its tag and data columns, with their source-info columns, the - declared order. Nothing else + columns take the slot's names (their values stay the source's own). As a stable + convenience, not a guarantee (column order is not part of a node's schema), the + columns come in the declared order: tags, data and source info as declared, the + system tags source id first. Nothing else differs: values, types, source-info tokens and meta columns pass unchanged, so the rows equal those of a source of the declared schema with the same source id. diff --git a/tests/test_pipeline/test_bind_keeps_node_schemas.py b/tests/test_pipeline/test_bind_keeps_node_schemas.py index 2f972f85b..641cbd17d 100644 --- a/tests/test_pipeline/test_bind_keeps_node_schemas.py +++ b/tests/test_pipeline/test_bind_keeps_node_schemas.py @@ -1,12 +1,14 @@ -"""No accepted bind changes a node's stored path or table schema (ITL-1112). +"""No accepted bind changes a node's stored path or its columns (ITL-1112). Developer, November 14, 2026: "for pipeline, all node's path/table schema should NOT change regardless of what source you bind to the source nodes in the pipeline". One job is run over every kind of accepted bind: a source of the declared schema, a narrower source into a ``T | None`` slot, sources of different kinds, through ``bind``, the ``bound_source`` setter and ``PipelineJob.load``. After each, the stored table paths, the Arrow schema of every stored -table and of every node's ``as_table(all_info=True)`` (column names with system tags, types, -nullable flags, order) are those of the first bind: only values differ. +table and of every node's ``as_table(all_info=True)`` (each column by name, system tags +included, with its type and nullable flag) are those of the first bind: only values differ. +Column order is not part of that (developer, November 14, 2026), so the snapshots map names to +fields; the order a slot presents is tested on its own (``test_order_convenience_*``). The bind check compares Python types, so two Arrow widths of one Python type (``int32`` and ``int64``) are not covered here (ITL-949). @@ -77,8 +79,13 @@ def _record_id_first(source: ArrowTableSource) -> ArrowTableStream: LOADED_NODES = ("src", "inc") +def _fields(schema: pa.Schema) -> dict[str, Any]: + """Return *schema*'s fields by name: the columns, whatever their order.""" + return {field.name: field for field in schema} + + def _snapshot(job: PipelineJob, nodes: tuple[str, ...] = NODES) -> dict[str, Any]: - """Return the stored paths, stored table schemas and *nodes*' table schemas of *job*.""" + """Return the stored paths, and the columns of the stored tables and *nodes*' tables.""" database = job.database assert database is not None paths = sorted(database.list_table_paths(())) @@ -86,12 +93,13 @@ def _snapshot(job: PipelineJob, nodes: tuple[str, ...] = NODES) -> dict[str, Any for path in paths: table = database.get_all_records(path) assert table is not None - stored[path] = table.schema + stored[path] = _fields(table.schema) return { "paths": paths, "stored": stored, "nodes": { - name: job.nodes[name].as_table(all_info=True).schema for name in nodes + name: _fields(job.nodes[name].as_table(all_info=True).schema) + for name in nodes }, } @@ -231,13 +239,13 @@ def test_a_structural_class_of_the_same_identity_changes_no_path_or_schema( assert snapshots[2] == snapshots[0] -def test_a_record_id_first_pair_is_given_in_the_declared_order() -> None: - """A source whose system-tag pair comes record-id first is given source-id first. +def test_order_convenience_a_record_id_first_pair_is_shown_source_id_first() -> None: + """A source whose system-tag pair comes record-id first is shown source-id first. - The order every node predicts (``output_schema``) and a source of the declared order - has; in the schema, the table, the head and the zero-row table, for a slot of the - source's own types (rows passed as they are otherwise) and for a ``T | None`` slot - the source is narrower than. + A convenience, not part of the bind invariant (column order is not): the slot shows + the order it predicts (``output_schema``) and a source of the declared order has, in + the schema, the table, the head and the zero-row table, for a slot of the source's + own types and for a ``T | None`` slot the source is narrower than. """ for x_type, x_nullable in ((int, False), (int | None, True)): slot = SourceNode( @@ -276,13 +284,13 @@ def passes_on(w: int | None) -> int | None: def test_a_bound_function_stream_moves_no_stored_or_downstream_schema( tmp_path: pathlib.Path, ) -> None: - """A function pod's stream (one system-tag pair) moves no path, stored or downstream schema. + """A function pod's stream (one system-tag pair) moves no path or column. - Known limit: the source node's own table keeps the bound stream's column layout (a - function pod's stream gives its system tags before its data) and the nullable flags - of its ``_content_hash`` and source-info columns. That table is not stored and no - downstream node sees it; its columns, and its user and system-tag fields, are the - same. + Stored paths, stored tables' columns and every downstream node's columns are those of + a source bind, as are the source node's user, system-tag and context columns. The + source node's own table keeps the nullable flags the stream gives its + ``_content_hash`` and source-info columns; that table is not stored and no downstream + node sees those flags. """ slot = SourceNode( name="src", @@ -305,13 +313,13 @@ def test_a_bound_function_stream_moves_no_stored_or_downstream_schema( for name in ("renamed", "inc"): assert second["nodes"][name] == first["nodes"][name], name src_first, src_second = first["nodes"]["src"], second["nodes"]["src"] - assert sorted(src_second.names) == sorted(src_first.names) + assert src_second.keys() == src_first.keys() compared = [ name - for name in src_first.names + for name in src_first if name in ("id", "x", constants.CONTEXT_KEY) or name.startswith(constants.SYSTEM_TAG_PREFIX) ] assert len(compared) == 5 for name in compared: - assert src_second.field(name) == src_first.field(name), name + assert src_second[name] == src_first[name], name diff --git a/tests/test_pipeline/test_bind_narrower_source.py b/tests/test_pipeline/test_bind_narrower_source.py index 030569b0b..6201ff4a6 100644 --- a/tests/test_pipeline/test_bind_narrower_source.py +++ b/tests/test_pipeline/test_bind_narrower_source.py @@ -423,7 +423,11 @@ def _abc_job(a_type: Any) -> PipelineJob: @pytest.mark.parametrize("a_type", [int, int | None], ids=["equal", "narrower"]) def test_columns_follow_the_declared_order(a_type: Any) -> None: - """A source with its columns in another order gives them in the declared order.""" + """A source with its columns in another order is shown in the declared order. + + A convenience, not part of the bind invariant: column order is not (developer, + November 14, 2026). + """ job = _abc_job(a_type) job.bind( sources={ diff --git a/tests/test_pipeline/test_bind_type_check.py b/tests/test_pipeline/test_bind_type_check.py index b5ab4d84d..106e089bb 100644 --- a/tests/test_pipeline/test_bind_type_check.py +++ b/tests/test_pipeline/test_bind_type_check.py @@ -184,3 +184,38 @@ def test_an_accepted_rebind_runs_and_stores_the_new_rows( _ = job.run() # Results of the first source (x 1, 2) would be [1, 2]. assert sorted(job.nodes["const"].as_table().column("z").to_pylist()) == [10, 11] + + +def _two_by_two_slot_job() -> PipelineJob: + """Return a job whose slot declares tags ``a``, ``b`` and data ``x``, ``y``, in order.""" + slot = SourceNode( + name="src", + tag_schema=Schema({"a": int, "b": int}), + data_schema=Schema({"x": int, "y": int}), + ) + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _ = MapData({"x": "x2"})(slot, label="renamed") + return job + + +def test_columns_match_by_name_within_each_group_whatever_their_order() -> None: + """Tags and data in another order than declared are accepted (developer, November 14, + 2026): within the tag group and the data group, columns match by name.""" + job = _two_by_two_slot_job() + reordered = DictSource( + [{"y": 2, "b": 1, "x": 3, "a": 0}], tag_columns=["b", "a"], source_id="r" + ) + job.bind(sources={"src": reordered}) + assert job.run().ok + + +def test_a_column_in_the_other_role_is_refused() -> None: + """A column that is a tag on one side and data on the other is refused: the role + (tag or data) is part of the match, its position is not.""" + job = _two_by_two_slot_job() + swapped = DictSource( + [{"a": 0, "b": 1, "x": 3, "y": 2}], tag_columns=["a", "x"], source_id="s" + ) + with pytest.raises(SourceSpecMismatchError, match="tag columns"): + job.bind(sources={"src": swapped}) From fd1278d3461e386b4c2f879b29bb6dce9f95b260 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 15:27:19 -0800 Subject: [PATCH 29/33] fix(source_node): a slot shows its declared types for a structural class of the declared identity (ITL-1112) After #391 a structural class hashes by its identity, so another class object of the declared identity gives the slot's system-tag names and was passed through, showing its own class in output_schema. Pass through only when the source's types are exactly the declared type objects. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index f11ca368f..9432c84de 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -885,8 +885,8 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: Returns: ``None`` when the source's rows already are rows of the declared schema - (same types, column order and system-tag names and order), so they pass - unchanged; + (the same type objects, column order and system-tag names and order), so + they pass unchanged; otherwise the ``_DeclaredForm`` that makes them so. Raises: @@ -913,8 +913,11 @@ def _declared_form_for(self, source: StreamProtocol) -> _DeclaredForm | None: for name in source.keys(columns={"system_tags": True})[0] if name not in source_tag ] + # Exactly the declared types: a structural class of the declared identity is + # another class object, and the slot shows the declared one. if ( - not narrower + dict(source_tag) == dict(self._tag_schema) + and dict(source_data) == dict(self._data_schema) and not renames and tuple(source_tag.keys()) == tag_order and tuple(source_data.keys()) == data_order From af162ef1596ae4e9adf7d90d47635cf3646d1720 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 15:33:47 -0800 Subject: [PATCH 30/33] test(node_view): bind source slots to streams over a source, not a raw stream (ITL-1126) #388's load-only refusal tests bound streams over a raw ArrowTableStream, which a source slot refuses since ITL-1126. The base stream is a source of the same columns and types; every assertion is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_node_view_review_fixes.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index b3d90e9f9..05f6de37f 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -111,10 +111,14 @@ def flaky(x: int) -> int: def _base_stream() -> Any: - """Two rows, tag ``id`` and data ``x``, both non-nullable ``int64``.""" + """Two rows, tag ``id`` and data ``x``, both non-nullable ``int64``. + + A source, not a raw ``ArrowTableStream``: the streams built over it are bound to a + source slot, which needs one system-tag pair (ITL-1126). + """ import pyarrow as pa - from orcapod.core.streams import ArrowTableStream + from orcapod.core.sources import ArrowTableSource schema = pa.schema( [ @@ -122,8 +126,10 @@ def _base_stream() -> Any: pa.field("x", pa.int64(), nullable=False), ] ) - return ArrowTableStream( - pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"] + return ArrowTableSource( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), + tag_columns=["id"], + source_id="base", ) From a7b56675ce7041ce2f2c211a6b55e3a964bdb3e4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 16:13:51 -0800 Subject: [PATCH 31/33] test(pipeline): compare column nullability of loaded and saved schemas (ITL-1138, review round 2) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_loaded_slot_unregistered_class.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index d36edcd7a..00861e006 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -95,7 +95,9 @@ def describe(job): out[label] = { "pipeline_hash": node.pipeline_hash().to_string(), "content_hash": node.content_hash().to_string(), - "schema": [f"{field.name}: {field.type}" for field in schema], + "schema": [ + f"{field.name}: {field.type} nullable={field.nullable}" for field in schema + ], # Values of the user columns: a load-only run's status and execution # columns differ from the saving run's by design. "rows": rows(node.as_table()), From 2a220ccacd32e5b0979fa5a8f40f7c422615eb4e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 17:02:38 -0800 Subject: [PATCH 32/33] fix(pipeline): a saved-again loaded job keeps a pending column's nullability; copies keep pending types and name-bound hashes (ITL-1138, review round 3) job.save() wrote the blueprint's pending column (saved type text) as non-nullable, because a type text gives no nullability: the source-node descriptor now adds the pending columns' saved nullability. SourceNode.from_stream keeps a wrapped slot's pending types, and a copy of a loaded slot keeps its saved content hash only under the same name. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1138.md | 2 +- src/orcapod/core/nodes/source_node.py | 29 +++- src/orcapod/pipeline/graph.py | 41 ++++++ .../test_loaded_slot_unregistered_class.py | 137 ++++++++++++++++++ 4 files changed, 205 insertions(+), 4 deletions(-) diff --git a/changelog.d/ITL-1138.md b/changelog.d/ITL-1138.md index 19efff50b..f29c3ed43 100644 --- a/changelog.d/ITL-1138.md +++ b/changelog.d/ITL-1138.md @@ -1,3 +1,3 @@ ### Fixed -- A job over a column of a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model) loaded with `PipelineJob.load` in a process where nothing has registered the class yet reads like the job that saved it (ITL-1138). The load cannot restore such a column's saved type to a class, so the source slot held the saved type text: its pipeline hash was the hash of that text, and binding the same source was refused ("data column 'p' is declared extension<…> but the source has A") unless the class had been registered before the load. A loaded source slot now keeps its saved pipeline hash, and its saved content hash while no source is bound, as loaded function and operator nodes keep theirs, and restores such a column's type when a source is bound (building the source registers the class). Every node then has the saving process's hashes, schemas, system-tag column names and stored rows, and `run(load_only=True)` finds every stored result. A slot whose class is still not registered at bind refuses the source with an error that names the column and the saved class and says how to register it. No hash, golden value or stored table changes; the job file format is unchanged, and a file without a slot's saved hashes still loads. +- A job over a column of a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model) loaded with `PipelineJob.load` in a process where nothing has registered the class yet reads like the job that saved it (ITL-1138). The load cannot restore such a column's saved type to a class, so the source slot held the saved type text: its pipeline hash was the hash of that text, and binding the same source was refused ("data column 'p' is declared extension<…> but the source has A") unless the class had been registered before the load. A loaded source slot now keeps its saved pipeline hash, and its saved content hash while no source is bound, as loaded function and operator nodes keep theirs, and restores such a column's type when a source is bound (building the source registers the class). Every node then has the saving process's hashes, schemas, system-tag column names and stored rows, and `run(load_only=True)` finds every stored result. A slot whose class is still not registered at bind refuses the source with an error that names the column and the saved class and says how to register it. Saving such a loaded job again (`job.save`), with or without a bind, keeps each such column's saved nullability: the type text cannot give it, so before this fix a nullable user-class column (`A | None`) was written as non-nullable and a later load restored it as `A`. A job file saved on a build before this fix from a job loaded before its class was registered may record a nullable user-class column as non-nullable; save it again from the code that defines the job. No hash, golden value or stored table changes; the job file format is unchanged, and a file without a slot's saved hashes still loads. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 789f7c125..2d9f23cab 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -330,11 +330,29 @@ class nothing in this process has registered yet) holds its saved text; it is ) def _take_saved_state_of(self, other: "SourceNodeBase") -> None: - """Keep the saved hashes and pending types of *other*, a slot of the same schema.""" + """Keep the saved state of *other*, a slot of the same schema. + + The pending types and the saved pipeline hash, which does not depend on the + slot's name, are always kept. The saved content hash covers the name too, so + it is kept only when this slot has *other*'s name. + """ self._saved_pipeline_hash = other._saved_pipeline_hash - self._saved_content_hash = other._saved_content_hash + if self._name == other._name: + self._saved_content_hash = other._saved_content_hash self._pending_types = dict(other._pending_types) + def _pending_nullable_columns(self) -> dict[str, list[str]]: + """The pending columns the saved job records as nullable, by role. + + A column whose type is still the saved text has no Python type to read its + nullability from, so a job saved again records the saved nullability (ITL-1138). + """ + names: dict[str, list[str]] = {"tag": [], "data": []} + for name, pending in self._pending_types.items(): + if pending.nullable: + names[pending.role].append(name) + return names + def _restore_pending_types(self) -> dict[str, str]: """Restore the pending columns whose type this process can now restore. @@ -900,11 +918,16 @@ def from_stream( slot_name = stream.source_id else: slot_name = f"{stream.label}:{stream.content_hash().to_hex()}" - return cls( + node = cls( name=slot_name, tag_schema=tag_schema, data_schema=data_schema, ) + if isinstance(stream, SourceNodeBase): + # A loaded job's slot keeps its pending types (ITL-1138); its saved + # hashes stay behind, since the new slot has another name. + node._pending_types = dict(stream._pending_types) + return node class SourceJobNode(SourceNodeBase): diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index f5befa938..f101d1628 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -35,6 +35,33 @@ # --------------------------------------------------------------------------- +def _with_pending_nullability( + record: dict[str, list[str]], + pending: dict[str, list[str]], + slot: "cp.StreamProtocol", +) -> dict[str, list[str]]: + """Add a slot's pending nullable columns to a saved nullability record (ITL-1138). + + Args: + record: The nullable column names per role, as ``serialize_nullability`` + writes them. + pending: The slot's pending columns that its saved job records as nullable. + slot: The source slot, whose tag and data schemas give the column order. + + Returns: + The record with the pending names added, in the slot's column order. + """ + tag_schema, data_schema = slot.output_schema() + return { + role: [ + name + for name in schema + if name in record.get(role, []) or name in pending[role] + ] + for role, schema in (("tag", tag_schema), ("data", data_schema)) + } + + class Pipeline(AbstractPipelineBase[GraphNode]): """A pure computational blueprint recording operator and function pod invocations. @@ -211,6 +238,20 @@ def save(self, path: str | Path) -> None: ), } descriptor["reconstructable"] = True + # A column whose saved type is still text keeps its saved + # nullability, which the text cannot give (ITL-1138). + pending = node._pending_nullable_columns() # pyright: ignore[reportPrivateUsage] # the saver is the one caller + output_nullable = descriptor["output_schema"].get("nullable") + if output_nullable is not None: + descriptor["output_schema"]["nullable"] = ( + _with_pending_nullability(output_nullable, pending, node) + ) + source_nullable = serialize_nullability( + node.tag_schema, node.data_schema, type_converter + ) + descriptor["source_config"]["nullable"] = _with_pending_nullability( + source_nullable, pending, node + ) case FunctionNode(): if node._function_pod is not None: diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index 00861e006..0dc6a0a08 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -242,3 +242,140 @@ def test_a_file_without_the_slots_saved_hashes_loads_and_binds( assert loaded.pop("_reused") == {"add": 2, "read_u": 2} _ = loaded.pop("_unbound_slots") # no saved hashes: recomputed once bound assert loaded == saved + + +# --------------------------------------------------------------------------- +# A loaded slot's pending column: saving the job again, and copies of the slot +# --------------------------------------------------------------------------- + +_NULLABLE_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + mode, work = sys.argv[2], sys.argv[3] + + from orcapod import PipelineJob, function_pod + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.databases import DeltaTableDatabase + from orcapod.sources import DictSource + + + def source(cls): + return DictSource( + [{"id": 1, "p": cls(u=1, v="a")}, {"id": 2, "p": None}], + tag_columns=["id"], + data_schema={"id": int, "p": cls | None}, + source_id="s", + ) + + + def slot_view(job): + node = job.nodes["s"] + return { + "data_schema": {k: str(v) for k, v in node.output_schema()[1].items()}, + "pipeline_hash": node.pipeline_hash().to_string(), + } + + + if mode == "save": + from classes import Exact + + @function_pod(output_keys="q") + def u_or_zero(p: Exact | None) -> int: + return 0 if p is None else p.u + + job = PipelineJob(name="demo", database=DeltaTableDatabase(work + "/store")) + with job: + u_or_zero.pod(source(Exact), label="u_or_zero") + job.run() + job.save(work + "/saved.json") + print(json.dumps(slot_view(job))) + elif mode in ("load_bind_save", "load_save"): + # Nothing registers the class before the load; the job is saved again. + job = PipelineJob.load(work + "/saved.json") + if mode == "load_bind_save": + from classes import Exact + + job.bind(sources={"s": source(Exact)}) + job.save(work + "/resaved.json") + print(json.dumps({})) + elif mode == "reload": + from classes import Exact + + _ = source(Exact) # registers the class before the load + job = PipelineJob.load(work + "/" + sys.argv[4]) + print(json.dumps(slot_view(job))) + else: # copies of a loaded slot whose class is not registered + job = PipelineJob.load(work + "/saved.json") + slot = job.nodes["s"] + renamed = SourceJobNode.from_stream(slot, name="renamed") + same_name = SourceJobNode.from_stream(slot) + wrapped = SourceNode.from_stream(slot) + from classes import Exact + + exact = source(Exact) # registers the class now + bound = SourceJobNode.from_stream(wrapped) + bound.bound_source = exact + print(json.dumps({ + "renamed_keeps_content_hash": renamed.content_hash() == slot.content_hash(), + "same_name_keeps_content_hash": same_name.content_hash() == slot.content_hash(), + "renamed_keeps_pipeline_hash": renamed.pipeline_hash() == slot.pipeline_hash(), + "wrapped_then_bound_schema": str(bound.output_schema()[1]["p"]), + })) + """ +) + + +@pytest.fixture +def nullable_job(tmp_path: Path): + """Save a job over ``p: Exact | None`` and return how to run the other modes.""" + _ = (tmp_path / "classes.py").write_text(_CLASSES) + _ = (tmp_path / "nullable.py").write_text(_NULLABLE_SCRIPT) + work = tmp_path / "work" + + def run(*args: str) -> dict[str, Any]: + result = subprocess.run( + [ + sys.executable, + str(tmp_path / "nullable.py"), + str(tmp_path), + args[0], + str(work), + *args[1:], + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-3000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + saved = run("save") + return saved, run + + +@pytest.mark.parametrize("mode", ["load_bind_save", "load_save"]) +def test_saving_a_loaded_job_again_keeps_a_pending_columns_nullability( + nullable_job, mode: str +) -> None: + """A job loaded before its class is registered and saved again reloads as saved.""" + saved, run = nullable_job + _ = run(mode) + assert run("reload", "resaved.json") == run("reload", "saved.json") == saved + + +def test_copies_of_a_loaded_slot_keep_its_pending_types_and_name_bound_hashes( + nullable_job, +) -> None: + """A copy keeps pending types; only a copy of the same name keeps its content hash.""" + _, run = nullable_job + copies = run("copies") + assert copies == { + "renamed_keeps_content_hash": False, + "same_name_keeps_content_hash": True, + "renamed_keeps_pipeline_hash": True, + "wrapped_then_bound_schema": "classes.Exact | None", + } From e8c213e26127fa0ee18d9a0f494a12aeb573309d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 17:17:54 -0800 Subject: [PATCH 33/33] fix(pipeline): a wrapped loaded slot keeps its saved pipeline hash; restoring a type rehashes the slot (ITL-1138, review round 4) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/source_node.py | 8 ++++-- .../test_loaded_slot_unregistered_class.py | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 2d9f23cab..c73d531bf 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -393,6 +393,8 @@ def _restore_pending_types(self) -> dict[str, str]: with contextlib.suppress(AttributeError): # not computed yet del self._schema_hash_str self._invalidate_content_hash_cache() + # A slot without a saved pipeline hash computed one from the type text. + self._invalidate_pipeline_hash_cache() return { name: _saved_type_description(pending.text) for name, pending in still_pending.items() @@ -924,9 +926,11 @@ def from_stream( data_schema=data_schema, ) if isinstance(stream, SourceNodeBase): - # A loaded job's slot keeps its pending types (ITL-1138); its saved - # hashes stay behind, since the new slot has another name. + # A loaded job's slot keeps its pending types and its saved pipeline hash, + # which does not depend on the name (ITL-1138); its saved content hash + # covers the name, and the new slot has another name. node._pending_types = dict(stream._pending_types) + node._saved_pipeline_hash = stream._saved_pipeline_hash return node diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index 0dc6a0a08..6661ffdfe 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -309,6 +309,24 @@ def u_or_zero(p: Exact | None) -> int: _ = source(Exact) # registers the class before the load job = PipelineJob.load(work + "/" + sys.argv[4]) print(json.dumps(slot_view(job))) + elif mode == "legacy_validate": + # A file without the slots' saved hashes: the slot hashes the type text until + # the type is restored, here by validate() without a bind. + with open(work + "/saved.json") as f: + data = json.load(f) + for descriptor in data["pipeline"]["nodes"].values(): + if descriptor["node_type"] == "source": + descriptor.pop("pipeline_hash", None) + descriptor.pop("content_hash", None) + with open(work + "/legacy.json", "w") as f: + json.dump(data, f) + job = PipelineJob.load(work + "/legacy.json") + slot = job.nodes["s"] + _ = slot.pipeline_hash() # computed, and kept, from the type text + from classes import Exact + + slot.validate(source(Exact)) + print(json.dumps({"pipeline_hash": slot.pipeline_hash().to_string()})) else: # copies of a loaded slot whose class is not registered job = PipelineJob.load(work + "/saved.json") slot = job.nodes["s"] @@ -324,6 +342,7 @@ def u_or_zero(p: Exact | None) -> int: "renamed_keeps_content_hash": renamed.content_hash() == slot.content_hash(), "same_name_keeps_content_hash": same_name.content_hash() == slot.content_hash(), "renamed_keeps_pipeline_hash": renamed.pipeline_hash() == slot.pipeline_hash(), + "wrapped_keeps_pipeline_hash": wrapped.pipeline_hash() == slot.pipeline_hash(), "wrapped_then_bound_schema": str(bound.output_schema()[1]["p"]), })) """ @@ -377,5 +396,14 @@ def test_copies_of_a_loaded_slot_keep_its_pending_types_and_name_bound_hashes( "renamed_keeps_content_hash": False, "same_name_keeps_content_hash": True, "renamed_keeps_pipeline_hash": True, + "wrapped_keeps_pipeline_hash": True, "wrapped_then_bound_schema": "classes.Exact | None", } + + +def test_restoring_a_type_by_validation_rehashes_a_slot_without_saved_hashes( + nullable_job, +) -> None: + """A slot that hashed its type text hashes the restored class after validate().""" + saved, run = nullable_job + assert run("legacy_validate") == {"pipeline_hash": saved["pipeline_hash"]}