From 1fbb2a689dbe679409fe3b2a67687219e3ed6932 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:32:03 -0800 Subject: [PATCH 1/6] docs(plans): ITL-807 function pod auto-select plan Co-Authored-By: Claude Fable 5.2 --- ...-11-12-itl-807-function-pod-auto-select.md | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-807-function-pod-auto-select.md diff --git a/superpowers/plans/2026-11-12-itl-807-function-pod-auto-select.md b/superpowers/plans/2026-11-12-itl-807-function-pod-auto-select.md new file mode 100644 index 000000000..939672662 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-807-function-pod-auto-select.md @@ -0,0 +1,127 @@ +# Function pod `__call__` drops unused data columns (ITL-807, ITL-193) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `pod(a, b)` on a function pod drops the data columns the function does not take, by +inserting one automatic `SelectDataColumns` after the automatic join; `pod(..., strict=True)` +and `pod.process(...)` stay exact. + +**Architecture:** One method, `_FunctionPodBase.__call__(*streams, label=None, strict=False)`. +It is graph construction only: it joins the inputs as `process` does today +(`handle_input_streams`), and when the joined stream carries data columns that are not +parameters it applies `SelectDataColumns(columns=)` (the select's default +`strict=True`, because the select's `strict` is in its identity) and hands the result to +`process`. `process` is not changed. No hashing code is touched. + +**Tech Stack:** Python, pytest, `uv run`. + +**Branch:** `eywalker/itl-807-docs-say-a-function-pod-extracts-the`. PR: `Fixes ITL-807`, `Fixes ITL-193`. + +--- + +## Rulings (coordinator, relaying the developer, November 12, 2026) + +1. Select goes AFTER the join: inputs -> automatic Join (several inputs) -> ONE automatic + `SelectDataColumns` -> pod. This supersedes "per input before the join". +2. Any data column shared by two inputs fails at the Join as today, used or not ("that is on + user"). No new handling or message. +3. If the function accepts arbitrary keywords (`**kwargs`), no automatic select. Today no + data function can (`PythonDataFunction` rejects variadics at construction). Expressed as + a property on the data function, `accepts_arbitrary_keywords` (base: `False`; wrapper + delegates), read by the pod; no signature inspection in the pod. Do NOT add `**kwargs` + support. +4. No select when the joined stream's data columns already match (including a parameter + with a default that is absent). `pod(a, b)` with nothing to drop must have the same + hashes as before this PR. +5. A stream passed for its tags: it is joined and its data columns are then dropped. Works + with no new rule; test and document it. +6. `SideEffectPod` / `sink_pod` / `tap_pod`: unchanged. Operators: unchanged. +7. One info log per call naming the dropped columns of the joined stream. Not a warning. +8. Changelog: `### Changed` only, with a sentence that code relying on `pod(wide)` raising + must use `strict=True` or `process`. +9. Docs / CLAUDE.md / .zed wording: "one automatic SelectDataColumns after the automatic join". +10. Out of scope, report only: `CachedFunctionPod.process` does not record the invocation + with the tracker. Give a minimal reproduction in worker_done. + +## Measured on main (25650c42) + +- `SelectDataColumns` identity is `('SelectDataColumns', [sorted columns], strict)`. +- `SelectDataColumns(columns=[])(b)` -> `ValueError: No data columns found in the table`. +- Parameter with default: column present = used, absent = default. +- Hand-written `select -> pod` in a `PipelineJob`: nodes `{source, SelectDataColumns + (OperatorJobNode), add (FunctionJobNode)}`; a changed unused column (source_id given) is a + cache hit. + +## Design of `__call__` + +```python +def __call__(self, *streams, label=None, strict=False): + # strict, or a function that takes arbitrary keywords (developer, Nov 12 2026): exact call + if strict or self.data_function.accepts_arbitrary_keywords: + return self.process(*streams, label=label) + input_stream = self.handle_input_streams(*streams) # the automatic join, as in process + _, data_schema = input_stream.output_schema() + parameters = self.input_data_schema # ctx argument already excluded + dropped = [c for c in data_schema if c not in parameters] + kept = [c for c in data_schema if c in parameters] + if dropped and kept: + # missing column / type errors are raised before a step is added + self._validate_input_schema(Schema({c: data_schema[c] for c in kept})) + logger.info(...) # one message per call + input_stream = SelectDataColumns(columns=kept)(input_stream) + # nothing kept: no stream without data columns exists; process raises its exact error + return self.process(input_stream, label=label) +``` + +`FunctionPod.__call__` stays as a typed override forwarding `strict`. +`process(input_stream)` with one stream does not join again, so the Join is recorded once. + +## Tasks + +### Task 1: failing tests +- [ ] Create `tests/test_core/function_pod/test_function_pod_auto_select.py` (public interface + only). Cases: wide single input equals hand-written select (pipeline hash, content hash, + `as_table(all_info=True)`); two inputs equal hand-written `Join` + select; nothing to drop: + `pod(a, b)`, `pod(a, b, strict=True)`, `pod.process(a, b)` equal, hashes pinned to the values + main gives; default parameter absent = no select; `strict=True` and `process` raise on an + extra column; missing required column raises in all three modes; tags untouched; + tags-only input works; shared data column (used and unused) raises at the Join; misspelt + column for a defaulted parameter: dropped, default used, info log; exactly one INFO record + per dropping call, none otherwise, no WARNING; ctx_arg pod; `CachedFunctionPod`; + `WrappedFunctionPod`; `accepts_arbitrary_keywords` true -> no select (a test-local data + function wrapper overriding the property; comment that it is about that property); + `SideEffectPod.__call__` and operator `__call__` take no `strict`. +- [ ] PipelineJob cases: job with `pod(wide)` vs job with hand-written select over one + database: same node labels and types, same function node hashes, same + `db.list_table_paths()`, second job makes zero function calls; changed unused column = + zero new calls; no select node when the input already matches; sync vs async orchestrator + give equal tables; saved job files equal apart from run id. +- [ ] Run, confirm they fail (`TypeError: unexpected keyword 'strict'` / `ValueError`). Commit (WIP). + +### Task 2: implementation +- [ ] `protocols/core_protocols/data_function.py`: `accepts_arbitrary_keywords` property. +- [ ] `core/data_function.py`: `DataFunctionBase.accepts_arbitrary_keywords -> False`; + `DataFunctionWrapper` delegates. +- [ ] `core/function_pod.py`: `_FunctionPodBase.__call__` as above; `FunctionPod.__call__` + forwards `strict`. +- [ ] Move `tests/test_core/test_newcomer_papercuts.py::test_extra_column_error_names_the_column_and_logs_nothing` + to `pod.process(...)`; add a `strict=True` twin. +- [ ] Tests pass; ruff; basedpyright exit code 0. Commit. + +### Task 3: docs and records +- [ ] `docs/concepts/function-pods.md`: tip and section "Input columns" rewritten; check + `getting-started.md`, `operators.md`, `README.md`, `docs/index.md` for the old claim. +- [ ] `changelog.d/ITL-807.md` (`### Changed`), `design_issues/ITL-807-FP1.md` (resolved), + CLAUDE.md + `.zed/rules` paragraph. +- [ ] Doc tests, mkdocs build, `scripts/design_issues.py --check`, + `scripts/assemble_changelog.py --check`. Commit. + +### Task 4: finish +- [ ] Full `uv run pytest tests/`, golden unchanged, ruff check/format, basedpyright. +- [ ] Merge origin/main if moved, push, PR, `orca linear attach`, `orca worktree set --pr`. +- [ ] CodeRabbit rounds (bodies too), CI green, worker_done with the CachedFunctionPod + reproduction. + +## Progress notes + +- (keep updated) From 24ea3a9d3836a6dc0dc6ad532358828202a73958 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:33:49 -0800 Subject: [PATCH 2/6] test(function_pod): failing tests for dropping unused columns on __call__ (ITL-807) Co-Authored-By: Claude Fable 5.2 --- .../test_function_pod_auto_select.py | 685 ++++++++++++++++++ 1 file changed, 685 insertions(+) create mode 100644 tests/test_core/function_pod/test_function_pod_auto_select.py diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py new file mode 100644 index 000000000..72c74721e --- /dev/null +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -0,0 +1,685 @@ +"""Calling a function pod drops the data columns its function does not take (ITL-807, ITL-193). + +``pod(a, b)`` is the convenient call: the inputs are joined as before and, when the joined +stream carries data columns that are not parameters of the function, one automatic +``SelectDataColumns`` is placed between the join and the pod. ``pod(a, b, strict=True)`` and +``pod.process(a, b)`` are exact: an extra column is an error. + +Every test drives the public interface. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +import pytest + +from orcapod import FunctionPod, PipelineJob, function_pod +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.data_function import DataFunctionWrapper, PythonDataFunction +from orcapod.core.function_pod import WrappedFunctionPod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.invocation import InvocationContext +from orcapod.operators import Join, SelectDataColumns +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.side_effects import side_effect_pod +from orcapod.sources import DictSource + +_POD_LOGGER = "orcapod.core.function_pod" + + +@function_pod(output_keys="total") +def add(x: int, y: int) -> int: + """Add two integers. + + Args: + x: First addend. + y: Second addend. + + Returns: + The sum. + """ + return x + y + + +@function_pod(output_keys="total") +def add_default(x: int, y: int = 10) -> int: + """Add two integers; ``y`` has a default. + + Args: + x: First addend. + y: Second addend. + + Returns: + The sum. + """ + return x + y + + +def _wide(extra: int = 9, source_id: str = "wide") -> DictSource: + """Return a source with the columns of ``add`` plus one the function does not take. + + Args: + extra: Value of the unused column. + source_id: Name of the source. + + Returns: + A two-row source tagged by ``k``. + """ + return DictSource( + [ + {"k": 1, "x": 2, "y": 3, "extra": extra}, + {"k": 2, "x": 4, "y": 5, "extra": extra}, + ], + tag_columns=["k"], + source_id=source_id, + ) + + +def _xs(source_id: str = "xs", **more: int) -> DictSource: + """Return a source carrying ``x`` (and any further constant columns). + + Args: + source_id: Name of the source. + **more: Further data columns, one constant value each. + + Returns: + A two-row source tagged by ``k``. + """ + return DictSource( + [{"k": 1, "x": 2, **more}, {"k": 2, "x": 4, **more}], + tag_columns=["k"], + source_id=source_id, + ) + + +def _ys(source_id: str = "ys", **more: int) -> DictSource: + """Return a source carrying ``y`` (and any further constant columns). + + Args: + source_id: Name of the source. + **more: Further data columns, one constant value each. + + Returns: + A two-row source tagged by ``k``. + """ + return DictSource( + [{"k": 1, "y": 3, **more}, {"k": 2, "y": 5, **more}], + tag_columns=["k"], + source_id=source_id, + ) + + +def _rows(stream: StreamProtocol) -> list[dict[str, Any]]: + """Return a stream's rows with every column group, ordered by tag. + + Args: + stream: The stream to read. + + Returns: + One dict per row, including system tags, source info and content hashes. + """ + return stream.as_table(all_info=True).sort_by("k").to_pylist() + + +def _assert_same_stream(actual: StreamProtocol, expected: StreamProtocol) -> None: + """Assert two streams have the same identity, schema and rows. + + Args: + actual: The stream under test. + expected: The stream it must equal. + """ + assert actual.pipeline_hash() == expected.pipeline_hash() + assert actual.content_hash() == expected.content_hash() + assert actual.output_schema(all_info=True) == expected.output_schema(all_info=True) + assert actual.as_table(all_info=True).schema == ( + expected.as_table(all_info=True).schema + ) + assert _rows(actual) == _rows(expected) + + +def _drop_messages(caplog: pytest.LogCaptureFixture) -> list[logging.LogRecord]: + """Return the records the function pod module logged at INFO or above. + + Args: + caplog: The pytest log-capture fixture of the running test. + + Returns: + The matching records, in emission order. + """ + return [ + record + for record in caplog.records + if record.name == _POD_LOGGER and record.levelno >= logging.INFO + ] + + +# --------------------------------------------------------------------------- +# The convenient call +# --------------------------------------------------------------------------- + + +def test_call_drops_unused_columns_like_a_hand_written_select() -> None: + """``pod(wide)`` is the pipeline ``pod(SelectDataColumns([...])(wide))``.""" + automatic = add.pod(_wide()) + by_hand = add.pod.process(SelectDataColumns(columns=["x", "y"])(_wide())) + + _assert_same_stream(automatic, by_hand) + assert [row["total"] for row in _rows(automatic)] == [5, 9] + + +def test_call_selects_once_after_the_join() -> None: + """With several inputs the select comes after the join, as written by hand.""" + automatic = add.pod(_xs(a=1), _ys(b=2)) + by_hand = add.pod.process( + SelectDataColumns(columns=["x", "y"])(Join()(_xs(a=1), _ys(b=2))) + ) + + _assert_same_stream(automatic, by_hand) + assert [row["total"] for row in _rows(automatic)] == [5, 9] + + +def test_tags_are_untouched() -> None: + """Only data columns are dropped; every tag column stays.""" + source = DictSource( + [{"k": 1, "session": "a", "x": 2, "y": 3, "extra": 9}], + tag_columns=["k", "session"], + source_id="two_tags", + ) + + tag_keys, data_keys = add.pod(source).keys() + + assert tuple(tag_keys) == ("k", "session") + assert tuple(data_keys) == ("total",) + + +def test_stream_passed_for_its_tags_restricts_the_rows() -> None: + """A stream whose data columns are all unused is joined, then its columns are dropped.""" + full = DictSource( + [{"k": 1, "x": 2, "y": 3}, {"k": 2, "x": 4, "y": 5}], + tag_columns=["k"], + source_id="full", + ) + chosen = DictSource([{"k": 2, "note": 0}], tag_columns=["k"], source_id="chosen") + + result = add.pod(full, chosen) + + assert [(row["k"], row["total"]) for row in _rows(result)] == [(2, 9)] + + +# --------------------------------------------------------------------------- +# Nothing to drop: no step is added +# --------------------------------------------------------------------------- + + +# The hashes ``add.pod(_xs(), _ys())`` had on main before ITL-807 (commit 25650c42). +_JOINED_PIPELINE_HASH = "3202203f55b6da5b8a255f526d2a2e04792e88bde45b48588df7c31bd7c98b17" +_JOINED_CONTENT_HASH = "07a4093d77ff0f2d0267123f8403b860a4ce332857f84e02dbf8f35ed6492f46" + + +def test_matching_inputs_build_the_pipeline_they_built_before() -> None: + """With nothing to drop, all three calls give the stream ``pod(a, b)`` gave before.""" + convenient = add.pod(_xs(), _ys()) + strict = add.pod(_xs(), _ys(), strict=True) + exact = add.pod.process(_xs(), _ys()) + + _assert_same_stream(convenient, exact) + _assert_same_stream(strict, exact) + assert convenient.pipeline_hash().to_hex() == _JOINED_PIPELINE_HASH + assert convenient.content_hash().to_hex() == _JOINED_CONTENT_HASH + + +def test_absent_defaulted_parameter_adds_no_step( + caplog: pytest.LogCaptureFixture, +) -> None: + """A stream lacking only a parameter that has a default already matches.""" + with caplog.at_level(logging.DEBUG): + convenient = add_default.pod(_xs()) + exact = add_default.pod.process(_xs()) + + _assert_same_stream(convenient, exact) + assert [row["total"] for row in _rows(convenient)] == [12, 14] + assert _drop_messages(caplog) == [] + + +def test_matching_input_adds_no_select_node_to_a_job() -> None: + """Inside a job, a matching input records the source and the function only.""" + job = PipelineJob(name="narrow", store=InMemoryArrowDatabase()) + with job: + add.pod(SelectDataColumns(columns=["x", "y"])(_wide()), label="add") + by_hand = sorted(type(node).__name__ for node in job.nodes.values()) + + job = PipelineJob(name="narrow", store=InMemoryArrowDatabase()) + with job: + narrow = DictSource( + [{"k": 1, "x": 2, "y": 3}], tag_columns=["k"], source_id="narrow" + ) + add.pod(narrow, label="add") + + assert sorted(type(node).__name__ for node in job.nodes.values()) == [ + "FunctionJobNode", + "SourceJobNode", + ] + assert by_hand == ["FunctionJobNode", "OperatorJobNode", "SourceJobNode"] + + +# --------------------------------------------------------------------------- +# The exact calls +# --------------------------------------------------------------------------- + + +def test_strict_call_rejects_an_extra_column() -> None: + """``strict=True`` raises the error ``process`` raises.""" + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + add.pod(_wide(), strict=True) + + +def test_process_rejects_an_extra_column() -> None: + """``process`` is the exact primitive: it never drops a column.""" + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + add.pod.process(_wide()) + + +def test_process_takes_no_strict_argument() -> None: + """``process`` has no ``strict`` argument: it is always strict.""" + with pytest.raises(TypeError): + add.pod.process(_wide(), strict=False) # pyright: ignore[reportCallIssue] + + +@pytest.mark.parametrize("mode", ["convenient", "strict", "process"]) +def test_missing_required_column_is_an_error_in_every_mode(mode: str) -> None: + """A parameter without a default must have a column, however the pod is applied. + + Args: + mode: Which of the three calls is made. + """ + with pytest.raises(ValueError, match="'y'"): + if mode == "convenient": + add.pod(_xs(extra=1)) + elif mode == "strict": + add.pod(_xs(), strict=True) + else: + add.pod.process(_xs()) + + +def test_missing_required_column_adds_no_step_to_a_job() -> None: + """The missing-column error is raised before a select is recorded.""" + job = PipelineJob(name="missing", store=InMemoryArrowDatabase()) + with job: + with pytest.raises(ValueError, match="'y'"): + add.pod(_xs(extra=1)) + + assert sorted(type(node).__name__ for node in job.nodes.values()) == [ + "SourceJobNode" + ] + + +def test_input_with_no_parameter_column_is_rejected() -> None: + """A stream cannot be left without data columns: the exact error is raised.""" + only_extra = DictSource( + [{"k": 1, "extra": 9}], tag_columns=["k"], source_id="only_extra" + ) + + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + add_default.pod(only_extra) + + +@pytest.mark.parametrize("shared", ["x", "extra"]) +def test_data_column_shared_by_two_inputs_fails_at_the_join(shared: str) -> None: + """Two inputs carrying the same data column fail at the join, used or not. + + Args: + shared: The data column both inputs carry. + """ + left = DictSource( + [{"k": 1, "x": 2, "extra": 1}], tag_columns=["k"], source_id="left" + ) + right = DictSource( + [{"k": 1, "y": 3, shared: 7}], tag_columns=["k"], source_id="right" + ) + + with pytest.raises(InputValidationError, match="overlapping"): + add.pod(left, right) + + +# --------------------------------------------------------------------------- +# The info message +# --------------------------------------------------------------------------- + + +def test_dropping_logs_one_info_message_naming_the_columns( + caplog: pytest.LogCaptureFixture, +) -> None: + """One INFO record per call names the dropped columns; nothing is a warning.""" + with caplog.at_level(logging.DEBUG): + add.pod(_xs(a=1), _ys(b=2)) + + records = _drop_messages(caplog) + assert [record.levelno for record in records] == [logging.INFO] + assert "'a'" in records[0].getMessage() + assert "'b'" in records[0].getMessage() + assert "'x'" not in records[0].getMessage() + + +def test_nothing_dropped_logs_nothing(caplog: pytest.LogCaptureFixture) -> None: + """A call that drops nothing logs nothing at INFO or above.""" + with caplog.at_level(logging.DEBUG): + add.pod(_xs(), _ys()) + + assert _drop_messages(caplog) == [] + + +def test_misspelt_column_for_a_defaulted_parameter_is_dropped_and_logged( + caplog: pytest.LogCaptureFixture, +) -> None: + """A misspelt column is dropped and the default used; the INFO record is the trace.""" + misspelt = DictSource( + [{"k": 1, "x": 2, "Y": 3}], tag_columns=["k"], source_id="misspelt" + ) + + with caplog.at_level(logging.DEBUG): + result = add_default.pod(misspelt) + + assert [row["total"] for row in _rows(result)] == [12] + records = _drop_messages(caplog) + assert len(records) == 1 + assert "'Y'" in records[0].getMessage() + with pytest.raises(ValueError, match="column 'Y' is not a parameter"): + add_default.pod(misspelt, strict=True) + + +# --------------------------------------------------------------------------- +# Every kind of function pod +# --------------------------------------------------------------------------- + + +def test_function_with_invocation_context_drops_unused_columns() -> None: + """The context argument is not a data column and is never selected.""" + + @function_pod(output_keys="total", ctx_arg="ctx") + def add_with_ctx(x: int, y: int, ctx: InvocationContext) -> int: + """Add two integers, ignoring the invocation context. + + Args: + x: First addend. + y: Second addend. + ctx: The invocation context. + + Returns: + The sum. + """ + return x + y + + automatic = add_with_ctx.pod(_wide()) + by_hand = add_with_ctx.pod.process(SelectDataColumns(columns=["x", "y"])(_wide())) + + _assert_same_stream(automatic, by_hand) + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + add_with_ctx.pod(_wide(), strict=True) + + +def test_cached_function_pod_drops_unused_columns() -> None: + """A pod-cached function pod has the same convenient and strict calls.""" + cached = CachedFunctionPod( + function_pod=add.pod, result_database=InMemoryArrowDatabase() + ) + + automatic = cached(_wide()) + by_hand = cached.process(SelectDataColumns(columns=["x", "y"])(_wide())) + + _assert_same_stream(automatic, by_hand) + assert [row["total"] for row in _rows(automatic)] == [5, 9] + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + cached(_wide(), strict=True) + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + cached.process(_wide()) + + +def test_decorator_cached_pod_drops_unused_columns() -> None: + """The ``.pod`` of a function decorated with a pod cache database does the same.""" + + @function_pod(output_keys="total", pod_cache_database=InMemoryArrowDatabase()) + def cached_add(x: int, y: int) -> int: + """Add two integers. + + Args: + x: First addend. + y: Second addend. + + Returns: + The sum. + """ + return x + y + + assert [row["total"] for row in _rows(cached_add.pod(_wide()))] == [5, 9] + + +def test_wrapped_function_pod_drops_unused_columns() -> None: + """A wrapper around a function pod has the same convenient and strict calls.""" + wrapped = WrappedFunctionPod(add.pod) + + automatic = wrapped(_wide()) + by_hand = wrapped.process(SelectDataColumns(columns=["x", "y"])(_wide())) + + _assert_same_stream(automatic, by_hand) + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + wrapped(_wide(), strict=True) + + +def test_function_taking_arbitrary_keywords_gets_no_select() -> None: + """No select is added for a function that accepts arbitrary keyword arguments. + + No shipped data function accepts them (``PythonDataFunction`` rejects ``**kwargs``), + so this test is about the ``accepts_arbitrary_keywords`` property the pod reads: a + data function reporting ``True`` is applied exactly as ``process`` applies it. + """ + + class TakesAnything(DataFunctionWrapper): + """A data function that reports it accepts arbitrary keywords.""" + + @property + def accepts_arbitrary_keywords(self) -> bool: + """Report that any keyword is accepted.""" + return True + + def plain_add(x: int, y: int) -> int: + """Add two integers. + + Args: + x: First addend. + y: Second addend. + + Returns: + The sum. + """ + return x + y + + pod = FunctionPod( + data_function=TakesAnything(PythonDataFunction(plain_add, output_keys="total")) + ) + + assert add.pod.data_function.accepts_arbitrary_keywords is False + with pytest.raises(ValueError, match="column 'extra' is not a parameter"): + pod(_wide()) + + +def test_other_pods_take_no_strict_argument() -> None: + """Operators and side-effect pods keep ``__call__`` as an alias of ``process``.""" + + @side_effect_pod + def record(x: int, y: int, extra: int, ctx: InvocationContext) -> None: + """Do nothing with a row. + + Args: + x: A value. + y: A value. + extra: A value. + ctx: The invocation context. + """ + + with pytest.raises(TypeError): + record(_wide(), strict=False) # pyright: ignore[reportCallIssue] + with pytest.raises(TypeError): + Join()(_xs(), _ys(), strict=False) + assert tuple(record(_wide()).keys()[1]) == ("x", "y", "extra") + + +# --------------------------------------------------------------------------- +# Inside a PipelineJob: tracker, stored tables, saved file +# --------------------------------------------------------------------------- + + +def _counting_pod(calls: list[int]) -> Any: + """Return a decorated ``add`` function that records each call. + + Args: + calls: Receives the ``x`` of every invocation. + + Returns: + The decorated function; its ``pod`` attribute is the function pod. + """ + + @function_pod(output_keys="total", function_name="counted_add") + def counted_add(x: int, y: int) -> int: + """Add two integers and record the call. + + Args: + x: First addend. + y: Second addend. + + Returns: + The sum. + """ + calls.append(x) + return x + y + + return counted_add + + +def _saved_pipeline(job: PipelineJob, path: Path) -> dict[str, Any]: + """Save a job and return the saved file without its run id. + + Args: + job: The job to save. + path: Where to write the file. + + Returns: + The saved document, with ``run.run_id`` removed. + """ + job.save(path) + document = json.loads(path.read_text()) + document["run"].pop("run_id") + return document + + +def test_job_equals_the_job_with_a_hand_written_select(tmp_path: Path) -> None: + """Graph, hashes, stored tables and saved file equal the hand-written pipeline.""" + calls: list[int] = [] + counted = _counting_pod(calls) + db = InMemoryArrowDatabase() + + automatic = PipelineJob(name="job", store=db) + with automatic: + counted.pod(_wide(), label="add") + automatic.run() + automatic_paths = sorted(db.list_table_paths()) + assert len(calls) == 2 + + by_hand = PipelineJob(name="job", store=db) + with by_hand: + counted.pod.process( + SelectDataColumns(columns=["x", "y"])(_wide()), label="add" + ) + by_hand.run() + + assert len(calls) == 2 # every row came from the tables the automatic job wrote + assert sorted(db.list_table_paths()) == automatic_paths + assert { + label: type(node).__name__ for label, node in automatic.nodes.items() + } == { + "wide": "SourceJobNode", + "SelectDataColumns": "OperatorJobNode", + "add": "FunctionJobNode", + } + for label, node in by_hand.nodes.items(): + assert automatic.nodes[label].pipeline_hash() == node.pipeline_hash() + assert automatic.nodes[label].content_hash() == node.content_hash() + assert _rows(automatic.nodes["add"]) == _rows(by_hand.nodes["add"]) + assert _saved_pipeline(automatic, tmp_path / "automatic.json") == ( + _saved_pipeline(by_hand, tmp_path / "by_hand.json") + ) + + +def test_job_with_two_inputs_equals_hand_written_join_and_select() -> None: + """With two inputs the job has the join, then one select, then the function.""" + calls: list[int] = [] + counted = _counting_pod(calls) + db = InMemoryArrowDatabase() + + automatic = PipelineJob(name="job", store=db) + with automatic: + counted.pod(_xs(a=1), _ys(b=2), label="add") + automatic.run() + + by_hand = PipelineJob(name="job", store=db) + with by_hand: + joined = Join()(_xs(a=1), _ys(b=2)) + counted.pod.process(SelectDataColumns(columns=["x", "y"])(joined), label="add") + by_hand.run() + + assert len(calls) == 2 + assert sorted(automatic.nodes) == sorted(by_hand.nodes) + assert sorted(type(node).__name__ for node in automatic.nodes.values()) == [ + "FunctionJobNode", + "OperatorJobNode", + "OperatorJobNode", + "SourceJobNode", + "SourceJobNode", + ] + for label, node in by_hand.nodes.items(): + assert automatic.nodes[label].pipeline_hash() == node.pipeline_hash() + assert automatic.nodes[label].content_hash() == node.content_hash() + assert _rows(automatic.nodes["add"]) == _rows(by_hand.nodes["add"]) + + +def test_change_in_an_unused_column_is_a_cache_hit() -> None: + """A different value in a dropped column computes nothing again.""" + calls: list[int] = [] + counted = _counting_pod(calls) + db = InMemoryArrowDatabase() + + first = PipelineJob(name="job", store=db) + with first: + counted.pod(_wide(extra=9), label="add") + first.run() + assert len(calls) == 2 + + second = PipelineJob(name="job", store=db) + with second: + counted.pod(_wide(extra=100), label="add") + second.run() + + assert len(calls) == 2 + assert first.nodes["add"].pipeline_hash() == second.nodes["add"].pipeline_hash() + assert [row["total"] for row in _rows(second.nodes["add"])] == [5, 9] + + +def test_sync_and_async_runs_give_the_same_rows() -> None: + """The automatic select is graph construction: both orchestrators run the same graph.""" + tables = [] + for orchestrator in (SyncPipelineOrchestrator(), AsyncPipelineOrchestrator()): + job = PipelineJob(name="job", store=InMemoryArrowDatabase()) + with job: + add.pod(_xs(a=1), _ys(b=2), label="add") + job.run(orchestrator=orchestrator) + tables.append(job.nodes["add"].as_table(all_info=True).sort_by("k")) + + assert tables[0].schema == tables[1].schema + assert tables[0].to_pylist() == tables[1].to_pylist() + assert [row["total"] for row in tables[0].to_pylist()] == [5, 9] From 4505e5aba4b7ffe6d10fc5c15bc827ef6011e527 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:42:03 -0800 Subject: [PATCH 3/6] feat(function_pod): calling a function pod drops the data columns its function does not take (ITL-807, ITL-193) pod(a, b) joins the inputs as before and, when the joined stream carries data columns that are not parameters, applies one SelectDataColumns before the pod. pod(..., strict=True) and pod.process(...) stay exact. Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/data_function.py | 15 ++++ src/orcapod/core/function_pod.py | 86 +++++++++++++++++-- .../protocols/core_protocols/data_function.py | 10 +++ .../test_function_pod_auto_select.py | 46 +++++++--- tests/test_core/test_newcomer_papercuts.py | 12 ++- 5 files changed, 145 insertions(+), 24 deletions(-) diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index abcd22861..f25cd6e0e 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -237,6 +237,17 @@ def input_data_schema(self) -> Schema: """Schema describing the input data this function accepts.""" ... + @property + def accepts_arbitrary_keywords(self) -> bool: + """Whether the function takes input columns beyond ``input_data_schema``. + + ``False`` for every data function kind today: the parameters are a fixed + set (``PythonDataFunction`` rejects ``**kwargs``). A kind that passes every + input column to its function overrides this to return ``True``; a function + pod then never drops a column for it (ITL-807). + """ + return False + @property @abstractmethod def output_data_schema(self) -> Schema: @@ -958,6 +969,10 @@ def canonical_function_name(self) -> str: def input_data_schema(self) -> Schema: return self._data_function.input_data_schema + @property + def accepts_arbitrary_keywords(self) -> bool: + return self._data_function.accepts_arbitrary_keywords + @property def output_data_schema(self) -> Schema: return self._data_function.output_data_schema diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 1580962fa..cdbac803e 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -655,12 +655,65 @@ def process( ... def __call__( - self, *streams: StreamProtocol, label: str | None = None + self, + *streams: StreamProtocol, + label: str | None = None, + strict: bool = False, ) -> StreamProtocol: - """Convenience alias for ``process``.""" + """Apply the pod, dropping the data columns its function does not take. + + This is the convenient form of ``process``. The inputs are joined as + ``process`` joins them. When the joined stream carries data columns that + are not parameters of the function, one ``SelectDataColumns`` keeping the + parameter columns is applied to it, and the result is handed to + ``process``. The select is an ordinary operator step: the pipeline is the + one in which the caller wrote that select by hand. When nothing would be + dropped no step is added, and the call is ``process``. Tag columns are + never dropped. + + Args: + *streams: Input streams to process. + label: Optional label for tracking. + strict: If ``True``, an extra data column is an error, as in + ``process``. It decides only that; it is part of no identity. + + Returns: + The resulting output stream. + + Raises: + ValueError: If a parameter without a default has no column, a column's + type is incompatible with its parameter, or (``strict=True``) the + input carries a data column that is not a parameter. + """ logger.debug(f"Invoking pod {self} on streams through __call__: {streams}") - # perform input stream validation - return self.process(*streams, label=label) + # A function that accepts arbitrary keywords takes every column, so + # nothing is unused and no select is added (developer, November 12, 2026, + # ITL-807). No data function kind accepts them today. + if strict or self.data_function.accepts_arbitrary_keywords: + return self.process(*streams, label=label) + + input_stream = self.handle_input_streams(*streams) + _, data_schema = input_stream.output_schema() + parameters = self.input_data_schema + dropped = [name for name in data_schema if name not in parameters] + kept = [name for name in data_schema if name in parameters] + # With no parameter column left there is nothing to select: a stream has + # at least one data column. ``process`` then raises its own error. + if dropped and kept: + # A missing column or a type mismatch is raised before a step is added. + self._validate_input_schema( + Schema({name: data_schema[name] for name in kept}) + ) + from orcapod.core.operators import SelectDataColumns + + logger.info( + "Function pod %r: dropped data columns %s, which are not " + "parameters of the function", + self.label, + dropped, + ) + input_stream = SelectDataColumns(columns=kept)(input_stream) + return self.process(input_stream, label=label) def argument_symmetry(self, streams: Collection[StreamProtocol]) -> ArgumentGroup: return self.multi_stream_handler().argument_symmetry(streams) @@ -781,12 +834,27 @@ def process( return output_stream def __call__( - self, *streams: StreamProtocol, label: str | None = None + self, + *streams: StreamProtocol, + label: str | None = None, + strict: bool = False, ) -> FunctionPodStream: - """Convenience alias for ``process``.""" - logger.debug(f"Invoking pod {self} on streams through __call__: {streams}") - # perform input stream validation - return self.process(*streams, label=label) + """Apply the pod, dropping the data columns its function does not take. + + See ``_FunctionPodBase.__call__``; this override only narrows the return + type to the ``FunctionPodStream`` that ``process`` returns. + + Args: + *streams: Input streams to process. + label: Optional label for tracking. + strict: If ``True``, an extra data column is an error, as in ``process``. + + Returns: + A ``FunctionPodStream`` wrapping the computation. + """ + return cast( + FunctionPodStream, super().__call__(*streams, label=label, strict=strict) + ) def to_config(self) -> dict[str, Any]: """Serialize this function pod to a JSON-compatible config dict. diff --git a/src/orcapod/protocols/core_protocols/data_function.py b/src/orcapod/protocols/core_protocols/data_function.py index c2c022c74..8ec7ba682 100644 --- a/src/orcapod/protocols/core_protocols/data_function.py +++ b/src/orcapod/protocols/core_protocols/data_function.py @@ -59,6 +59,16 @@ def input_data_schema(self) -> Schema: """Schema describing the input data this function accepts.""" ... + @property + def accepts_arbitrary_keywords(self) -> bool: + """Whether the function takes input columns beyond ``input_data_schema``. + + ``True`` means every data column of the input is passed to the function + (a ``**kwargs`` function), so no column is unused. A function pod never + drops a column for such a function. + """ + ... + @property def output_data_schema(self) -> Schema: """Schema describing the output data this function produces.""" diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py index 72c74721e..1a0c3c0d2 100644 --- a/tests/test_core/function_pod/test_function_pod_auto_select.py +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -128,6 +128,21 @@ def _rows(stream: StreamProtocol) -> list[dict[str, Any]]: return stream.as_table(all_info=True).sort_by("k").to_pylist() +def _without_source_info(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return rows without their source-info columns. + + Args: + rows: Rows as returned by ``_rows``. + + Returns: + The same rows, minus every column whose name starts with ``_source_``. + """ + return [ + {name: value for name, value in row.items() if not name.startswith("_source_")} + for row in rows + ] + + def _assert_same_stream(actual: StreamProtocol, expected: StreamProtocol) -> None: """Assert two streams have the same identity, schema and rows. @@ -141,7 +156,9 @@ def _assert_same_stream(actual: StreamProtocol, expected: StreamProtocol) -> Non assert actual.as_table(all_info=True).schema == ( expected.as_table(all_info=True).schema ) - assert _rows(actual) == _rows(expected) + # An output's source info names the computation that produced the row, which is + # new on every in-memory run; every other column must be equal. + assert _without_source_info(_rows(actual)) == _without_source_info(_rows(expected)) def _drop_messages(caplog: pytest.LogCaptureFixture) -> list[logging.LogRecord]: @@ -219,8 +236,12 @@ def test_stream_passed_for_its_tags_restricts_the_rows() -> None: # The hashes ``add.pod(_xs(), _ys())`` had on main before ITL-807 (commit 25650c42). -_JOINED_PIPELINE_HASH = "3202203f55b6da5b8a255f526d2a2e04792e88bde45b48588df7c31bd7c98b17" -_JOINED_CONTENT_HASH = "07a4093d77ff0f2d0267123f8403b860a4ce332857f84e02dbf8f35ed6492f46" +_JOINED_PIPELINE_HASH = ( + "3202203f55b6da5b8a255f526d2a2e04792e88bde45b48588df7c31bd7c98b17" +) +_JOINED_CONTENT_HASH = ( + "07a4093d77ff0f2d0267123f8403b860a4ce332857f84e02dbf8f35ed6492f46" +) def test_matching_inputs_build_the_pipeline_they_built_before() -> None: @@ -315,9 +336,7 @@ def test_missing_required_column_adds_no_step_to_a_job() -> None: with pytest.raises(ValueError, match="'y'"): add.pod(_xs(extra=1)) - assert sorted(type(node).__name__ for node in job.nodes.values()) == [ - "SourceJobNode" - ] + assert dict(job.nodes) == {} def test_input_with_no_parameter_column_is_rejected() -> None: @@ -433,7 +452,12 @@ def test_cached_function_pod_drops_unused_columns() -> None: automatic = cached(_wide()) by_hand = cached.process(SelectDataColumns(columns=["x", "y"])(_wide())) - _assert_same_stream(automatic, by_hand) + # Rows served from the pod cache carry other meta columns than computed rows, so + # the two tables are compared on identity and on their user columns. + assert automatic.pipeline_hash() == by_hand.pipeline_hash() + assert automatic.content_hash() == by_hand.content_hash() + assert automatic.output_schema() == by_hand.output_schema() + assert [(row["k"], row["total"]) for row in _rows(by_hand)] == [(1, 5), (2, 9)] assert [row["total"] for row in _rows(automatic)] == [5, 9] with pytest.raises(ValueError, match="column 'extra' is not a parameter"): cached(_wide(), strict=True) @@ -593,16 +617,12 @@ def test_job_equals_the_job_with_a_hand_written_select(tmp_path: Path) -> None: by_hand = PipelineJob(name="job", store=db) with by_hand: - counted.pod.process( - SelectDataColumns(columns=["x", "y"])(_wide()), label="add" - ) + counted.pod.process(SelectDataColumns(columns=["x", "y"])(_wide()), label="add") by_hand.run() assert len(calls) == 2 # every row came from the tables the automatic job wrote assert sorted(db.list_table_paths()) == automatic_paths - assert { - label: type(node).__name__ for label, node in automatic.nodes.items() - } == { + assert {label: type(node).__name__ for label, node in automatic.nodes.items()} == { "wide": "SourceJobNode", "SelectDataColumns": "OperatorJobNode", "add": "FunctionJobNode", diff --git a/tests/test_core/test_newcomer_papercuts.py b/tests/test_core/test_newcomer_papercuts.py index c7b70e5fd..ebbc5e654 100644 --- a/tests/test_core/test_newcomer_papercuts.py +++ b/tests/test_core/test_newcomer_papercuts.py @@ -92,16 +92,24 @@ def _wide_source() -> DictSource: ) +@pytest.mark.parametrize("call", ["process", "strict"]) def test_extra_column_error_names_the_column_and_logs_nothing( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, call: str ) -> None: """An extra column raises one error naming it, with no log line and no "data data". + The exact calls raise it: ``pod.process(stream)`` and ``pod(stream, strict=True)``. + A plain ``pod(stream)`` drops the extra column instead (ITL-807). + Args: caplog: Captures Orcapod's log records. + call: Which exact call is made. """ with caplog.at_level(logging.DEBUG), pytest.raises(ValueError) as excinfo: - compute_bmi.pod(_wide_source()) + if call == "process": + compute_bmi.pod.process(_wide_source()) + else: + compute_bmi.pod(_wide_source(), strict=True) message = str(excinfo.value) assert "data data" not in message From d624b439cf2491be62511b7e3e39d99a555ab8db Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:43:43 -0800 Subject: [PATCH 4/6] test(function_pod): type-clean the auto-select tests (ITL-807) Co-Authored-By: Claude Fable 5.2 --- .../test_function_pod_auto_select.py | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py index 1a0c3c0d2..80a819100 100644 --- a/tests/test_core/function_pod/test_function_pod_auto_select.py +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -19,7 +19,7 @@ from orcapod import FunctionPod, PipelineJob, function_pod from orcapod.core.cached_function_pod import CachedFunctionPod -from orcapod.core.data_function import DataFunctionWrapper, PythonDataFunction +from orcapod.core.data_function import DataFunctionWrapper from orcapod.core.function_pod import WrappedFunctionPod from orcapod.databases import InMemoryArrowDatabase from orcapod.errors import InputValidationError @@ -27,8 +27,11 @@ from orcapod.operators import Join, SelectDataColumns from orcapod.pipeline import AsyncPipelineOrchestrator from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator -from orcapod.protocols.core_protocols import StreamProtocol -from orcapod.side_effects import side_effect_pod +from orcapod.protocols.core_protocols import ( + PythonFunctionExecutorProtocol, + StreamProtocol, +) +from orcapod.side_effects import SideEffectPod from orcapod.sources import DictSource _POD_LOGGER = "orcapod.core.function_pod" @@ -62,6 +65,19 @@ def add_default(x: int, y: int = 10) -> int: return x + y +def _plain_add(x: int, y: int) -> int: + """Add two integers. + + Args: + x: First addend. + y: Second addend. + + Returns: + The sum. + """ + return x + y + + def _wide(extra: int = 9, source_id: str = "wide") -> DictSource: """Return a source with the columns of ``add`` plus one the function does not take. @@ -446,7 +462,8 @@ def add_with_ctx(x: int, y: int, ctx: InvocationContext) -> int: def test_cached_function_pod_drops_unused_columns() -> None: """A pod-cached function pod has the same convenient and strict calls.""" cached = CachedFunctionPod( - function_pod=add.pod, result_database=InMemoryArrowDatabase() + function_pod=FunctionPod.from_fn(_plain_add, output_keys="total"), + result_database=InMemoryArrowDatabase(), ) automatic = cached(_wide()) @@ -486,7 +503,7 @@ def cached_add(x: int, y: int) -> int: def test_wrapped_function_pod_drops_unused_columns() -> None: """A wrapper around a function pod has the same convenient and strict calls.""" - wrapped = WrappedFunctionPod(add.pod) + wrapped = WrappedFunctionPod(FunctionPod.from_fn(_plain_add, output_keys="total")) automatic = wrapped(_wide()) by_hand = wrapped.process(SelectDataColumns(columns=["x", "y"])(_wide())) @@ -504,7 +521,7 @@ def test_function_taking_arbitrary_keywords_gets_no_select() -> None: data function reporting ``True`` is applied exactly as ``process`` applies it. """ - class TakesAnything(DataFunctionWrapper): + class TakesAnything(DataFunctionWrapper[PythonFunctionExecutorProtocol]): """A data function that reports it accepts arbitrary keywords.""" @property @@ -512,21 +529,8 @@ def accepts_arbitrary_keywords(self) -> bool: """Report that any keyword is accepted.""" return True - def plain_add(x: int, y: int) -> int: - """Add two integers. - - Args: - x: First addend. - y: Second addend. - - Returns: - The sum. - """ - return x + y - - pod = FunctionPod( - data_function=TakesAnything(PythonDataFunction(plain_add, output_keys="total")) - ) + exact_pod = FunctionPod.from_fn(_plain_add, output_keys="total") + pod = FunctionPod(data_function=TakesAnything(exact_pod.data_function)) assert add.pod.data_function.accepts_arbitrary_keywords is False with pytest.raises(ValueError, match="column 'extra' is not a parameter"): @@ -536,7 +540,6 @@ def plain_add(x: int, y: int) -> int: def test_other_pods_take_no_strict_argument() -> None: """Operators and side-effect pods keep ``__call__`` as an alias of ``process``.""" - @side_effect_pod def record(x: int, y: int, extra: int, ctx: InvocationContext) -> None: """Do nothing with a row. @@ -547,11 +550,13 @@ def record(x: int, y: int, extra: int, ctx: InvocationContext) -> None: ctx: The invocation context. """ + pod = SideEffectPod(record) + with pytest.raises(TypeError): - record(_wide(), strict=False) # pyright: ignore[reportCallIssue] + pod(_wide(), strict=False) # pyright: ignore[reportCallIssue] with pytest.raises(TypeError): Join()(_xs(), _ys(), strict=False) - assert tuple(record(_wide()).keys()[1]) == ("x", "y", "extra") + assert tuple(pod(_wide()).keys()[1]) == ("x", "y", "extra") # --------------------------------------------------------------------------- From 8208f03f4e25d61181ba02d8c167f26bb23572cc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:51:06 -0800 Subject: [PATCH 5/6] docs(function_pod): document the convenient call, strict=True and process (ITL-807) Co-Authored-By: Claude Fable 5.2 --- .zed/rules | 19 ++++++ CLAUDE.md | 19 ++++++ changelog.d/ITL-807.md | 15 +++++ design_issues/ITL-807-F16.md | 23 +++++++ docs/concepts/function-pods.md | 117 +++++++++++++++++++++++++-------- docs/getting-started.md | 13 ++-- 6 files changed, 173 insertions(+), 33 deletions(-) create mode 100644 changelog.d/ITL-807.md create mode 100644 design_issues/ITL-807-F16.md diff --git a/.zed/rules b/.zed/rules index 7e13538e2..d6f7e6b7d 100644 --- a/.zed/rules +++ b/.zed/rules @@ -456,6 +456,25 @@ data. Never inspects tags. Two execution models: - FunctionPod → FunctionPodStream: lazy, in-memory - FunctionNode: DB-backed, two-phase (yield cached results first, then compute missing) +Calling a function pod (developer, November 12, 2026, ITL-807). pod(a, b) is the +convenient call and pod.process(a, b) the exact primitive: + +- pod(a, b) joins the inputs as process does and, when the joined stream carries data + columns that are not parameters of the function, places one automatic SelectDataColumns + after the automatic join and before the pod (_FunctionPodBase.__call__, the one place + that decides it). The select is an ordinary operator step: graph, hashes and stored tables + must equal those of a pipeline with that select written by hand. Nothing to drop means no + step. One info log names the dropped columns; it is not a warning. +- pod(a, b, strict=True) and pod.process(a, b) reject an extra column. strict is part of + no identity. process never takes a strict argument and never drops a column. +- A missing required column is an error in every form. Tags are never dropped. A data column + shared by two inputs fails at the join, used or not: no special handling. +- No select is added when the data function's accepts_arbitrary_keywords is true (no kind + today; PythonDataFunction rejects variadic keyword arguments). +- Function pods only. An operator's and a SideEffectPod's __call__ stay aliases of + process with no strict argument. There is no global setting. +- A test of the exact input contract uses process or strict=True. + Operator (core/operators/) — structural pod transforming streams without synthesizing new data values. All subclass StaticOutputPod: - UnaryOperator — 1 input (Batch, Select/Drop columns, Map, Filter) diff --git a/CLAUDE.md b/CLAUDE.md index 59048c0a1..7b1f0da05 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -471,6 +471,25 @@ data. Never inspects tags. Two execution models: - `FunctionPod` → `FunctionPodStream`: lazy, in-memory - `FunctionNode`: DB-backed, two-phase (yield cached results first, then compute missing) +**Calling a function pod** (developer, November 12, 2026, ITL-807). `pod(a, b)` is the +convenient call and `pod.process(a, b)` the exact primitive: + +- `pod(a, b)` joins the inputs as `process` does and, when the joined stream carries data + columns that are not parameters of the function, places one automatic `SelectDataColumns` + after the automatic join and before the pod (`_FunctionPodBase.__call__`, the one place + that decides it). The select is an ordinary operator step: graph, hashes and stored tables + must equal those of a pipeline with that select written by hand. Nothing to drop means no + step. One info log names the dropped columns; it is not a warning. +- `pod(a, b, strict=True)` and `pod.process(a, b)` reject an extra column. `strict` is part of + no identity. `process` never takes a `strict` argument and never drops a column. +- A missing required column is an error in every form. Tags are never dropped. A data column + shared by two inputs fails at the join, used or not: no special handling. +- No select is added when the data function's `accepts_arbitrary_keywords` is true (no kind + today; `PythonDataFunction` rejects `**kwargs`). +- Function pods only. An operator's and a `SideEffectPod`'s `__call__` stay aliases of + `process` with no `strict` argument. There is no global setting. +- A test of the exact input contract uses `process` or `strict=True`. + **Operator** (`core/operators/`) — structural pod transforming streams without synthesizing new data values. All subclass `StaticOutputPod`: - `UnaryOperator` — 1 input (Batch, Select/Drop columns, Map, Filter) diff --git a/changelog.d/ITL-807.md b/changelog.d/ITL-807.md new file mode 100644 index 000000000..7954445ab --- /dev/null +++ b/changelog.d/ITL-807.md @@ -0,0 +1,15 @@ +### Changed + +- Calling a function pod, `pod(a, b)`, now drops the data columns its function does not take + (ITL-807, ITL-193). The inputs are joined as before; when the joined stream carries data + columns that are not parameters of the function, one automatic `SelectDataColumns` is + placed after the automatic join and before the pod. It is an ordinary operator step: the + graph, the hashes and the stored tables are those of a pipeline with that select written by + hand, so a change in an unused column no longer causes a recomputation. When nothing would + be dropped no step is added and the pipeline is the one built before. One message at `INFO` + level names the dropped columns. `pod(a, b, strict=True)` rejects an extra column, and + `pod.process(a, b)` is the exact primitive and always rejects it. Code that relied on + `pod(wide)` raising for an extra column must use `strict=True` or `process`. A missing + column for a parameter without a default is an error in every form; tag columns are never + dropped; two inputs sharing a data column still fail at the join. Operators and + `SideEffectPod` are unchanged: their `__call__` stays an alias of `process`. diff --git a/design_issues/ITL-807-F16.md b/design_issues/ITL-807-F16.md new file mode 100644 index 000000000..96b2775e1 --- /dev/null +++ b/design_issues/ITL-807-F16.md @@ -0,0 +1,23 @@ +## `src/orcapod/core/function_pod.py` + +### F16 — Docs said a function pod extracts the columns it needs; the code rejected any extra column +**Status:** resolved +**Severity:** medium +**Issue:** ITL-807 + +`docs/concepts/function-pods.md` once said that a function pod picks the columns it needs out +of a wider stream, while `_FunctionPodBase._validate_input_schema` rejected every data column +that was not a parameter of the function. Users had to put a `SelectDataColumns` in front of +each pod by hand (ITL-193 asked for the automatic drop). `__call__` was a plain alias of +`process`. + +**Fix:** Developer decision of November 12, 2026. `_FunctionPodBase.__call__(*streams, +label=None, strict=False)` is the convenient call: it joins the inputs as `process` does and, +when the joined stream carries data columns that are not parameters, applies one +`SelectDataColumns(columns=)` after the automatic join and before the +pod, then calls `process`. The select is an ordinary operator step, so the graph, hashes and +stored tables equal those of a pipeline with the select written by hand; when nothing would +be dropped no step is added. One info message names the dropped columns. `strict=True` and +`process` reject an extra column as before; `strict` is in no identity. No select is added for +a data function whose `accepts_arbitrary_keywords` is true (none today). Operators and +`SideEffectPod` keep `__call__` as an alias of `process`. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 38bcf2e01..5fb96f465 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -24,9 +24,10 @@ Key points: - **`output_keys`** names the output data column(s). A single string means one output column; a list of strings means the function returns multiple values. -- **Function parameters** must match the input stream's data columns exactly: one data - column per parameter, with the same names. A parameter with a default value may be left out - of the stream, and the default is used. See [Input columns must match exactly](#input-columns-must-match-exactly). +- **Function parameters** are matched to the input stream's data columns by name: one data + column per parameter. A parameter with a default value may be left out of the stream, and + the default is used. Data columns the function does not take are dropped when the pod is + called. See [Input columns](#input-columns). - **Default values** are part of the function's signature hash, so a default must be a value Orcapod can hash: `None`, numbers, strings, `bytes`, and lists, tuples, sets and dicts of them. A default of another type, such as a `Path`, an `Enum` member, a `datetime` or a @@ -71,7 +72,7 @@ source = DictSource( ) # Apply the function pod to the source stream -result = compute_bmi.pod(source) # shorthand for compute_bmi.pod.process(source) +result = compute_bmi.pod(source) # Inspect the output schema -- tags pass through, data are replaced tag_schema, data_schema = result.output_schema() @@ -90,18 +91,16 @@ for tag, data in result.iter_data(): The function pod preserves tags and replaces data columns with the function's output. !!! tip - All standard pods support `__call__` as a shorthand for `.process()`, so - `compute_bmi.pod(source)` is equivalent to `compute_bmi.pod.process(source)`. + Calling a function pod, `compute_bmi.pod(source)`, is the convenient form of + `compute_bmi.pod.process(source)`: it first drops the data columns the function does not + take (see [Input columns](#input-columns)). When the stream has exactly the function's + columns, as here, the two are the same call. -## Input columns must match exactly +## Input columns -A function pod is strict about its input: the stream's data columns must be exactly the -function's parameters. Orcapod does **not** pick the columns it needs out of a wider stream. -A stream with an extra data column, a missing column for a parameter without a default, or a -column whose type is incompatible with its parameter is rejected with `ValueError` as soon as -the pod is applied. Types need to be compatible, not identical: a `str` column is accepted for -a `str | Path` parameter but rejected for an `int` one. (A parameter that has a default value -may be left out of the stream; the function then uses the default.) +A function takes one data column per parameter, matched by name. Calling the pod, +`compute_bmi.pod(stream)`, is the convenient form: data columns that are not parameters of +the function are dropped, and the function receives the rest. Tag columns are never dropped. ```python wide = DictSource( @@ -112,8 +111,52 @@ wide = DictSource( tag_columns=["subject_id"], ) +result = compute_bmi.pod(wide) # 'age' is not a parameter: it is dropped +for tag, data in result.iter_data(): + print(f"{tag.as_dict()} -> {data.as_dict()}") +# {'subject_id': 'mouse_01'} -> {'bmi': 1756.9444444444446} +# {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} +``` + +The drop is not hidden. Orcapod places one `SelectDataColumns` [operator](operators.md) +between the stream and the pod, exactly as if you had written it yourself, so the two +pipelines below are the same pipeline: same hashes, same graph, same stored tables. + +```python +from orcapod.operators import SelectDataColumns + +narrow = SelectDataColumns(columns=["weight", "height"])(wide) +by_hand = compute_bmi.pod(narrow) + +print(result.pipeline_hash() == by_hand.pipeline_hash()) +# True +print(result.content_hash() == by_hand.content_hash()) +# True +``` + +Because the unused columns are cut before the function's input is hashed, a change in an +unused column does not cause a recomputation. When the stream already has exactly the +function's columns, no step is added. + +Each call that drops columns logs one message at `INFO` level (logger +`orcapod.core.function_pod`) naming them. It is worth reading when a parameter has a default +value: a misspelt column (`Height` for a parameter `height=1.0`) is dropped like any other +unused column and the default is used, and that message is the only trace. + +### Exact calls: `strict=True` and `.process()` + +To have an extra column rejected instead of dropped, pass `strict=True`, or call +`.process()`, which is the exact primitive and never drops a column: + +```python try: - compute_bmi.pod(wide) + compute_bmi.pod(wide, strict=True) +except ValueError as err: + print(type(err).__name__) +# ValueError + +try: + compute_bmi.pod.process(wide) except ValueError as err: print(type(err).__name__) # ValueError @@ -124,19 +167,20 @@ The error message names both schemas and the first column that does not fit, for compatible with expected input schema Schema({'weight': float, 'height': float}): column 'age' is not a parameter of the function`. -Narrow the stream first with the `SelectDataColumns` [operator](operators.md) (or drop the -extras with `DropDataColumns`): +`strict` decides only whether an extra column is an error. It is not part of any hash: when +there is nothing to drop, `pod(stream)`, `pod(stream, strict=True)` and `pod.process(stream)` +build the same pipeline. -```python -from orcapod.operators import SelectDataColumns +### What is an error in every form -narrow = SelectDataColumns(columns=["weight", "height"])(wide) -result = compute_bmi.pod(narrow) -for tag, data in result.iter_data(): - print(f"{tag.as_dict()} -> {data.as_dict()}") -# {'subject_id': 'mouse_01'} -> {'bmi': 1756.9444444444446} -# {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} -``` +A missing column for a parameter without a default, or a column whose type is incompatible +with its parameter, is rejected with `ValueError` as soon as the pod is applied, however it +is applied. Types need to be compatible, not identical: a `str` column is accepted for a +`str | Path` parameter but rejected for an `int` one. (A parameter that has a default value +may be left out of the stream; the function then uses the default.) + +A stream must keep at least one data column, so a stream none of whose data columns is a +parameter is rejected as well. The same step is available as a stream method: `wide.select_data_columns(["weight", "height"])`. Tag columns are never affected by this rule -- a function pod passes all tags through @@ -615,7 +659,26 @@ for tag, data in result.iter_data(): ``` The join happens on shared tag columns, and the merged data columns are fed to the function. -The [exact-match rule](#input-columns-must-match-exactly) applies to the joined stream. +The [input column rules](#input-columns) apply to the joined stream: the join comes first, +then one automatic `SelectDataColumns` drops the joined columns the function does not take. +Two consequences: + +- Two inputs that carry the same data column are rejected by the join, whether or not the + function takes that column. Drop it from one input yourself. +- A stream can be passed to restrict the rows by its tags: it is joined, and its data + columns, which the function does not take, are then dropped. + +```python +chosen = DictSource( + data=[{"subject_id": "mouse_02", "cohort": "b"}], + tag_columns=["subject_id"], +) + +result = compute_bmi.pod(weights, heights, chosen) +for tag, data in result.iter_data(): + print(f"{tag.as_dict()} -> {data.as_dict()}") +# {'subject_id': 'mouse_02'} -> {'bmi': 2209.9999999999995} +``` ## DataFunction internals diff --git a/docs/getting-started.md b/docs/getting-started.md index 1bc272e87..289e1d1be 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -111,16 +111,16 @@ A few things to note about function pods: - **Parameter names match data column names.** The function's parameter names (`temperature`, `pressure`) must match the data column names from the input stream - exactly -- one parameter per data column. A stream with extra data columns is rejected; - narrow it first with `source.select_data_columns([...])` - (see [Function Pods](concepts/function-pods.md#input-columns-must-match-exactly)). + exactly -- one parameter per data column. Data columns the function does not take are + dropped when you call the pod; `pod(stream, strict=True)` and `pod.process(stream)` + reject them instead (see [Function Pods](concepts/function-pods.md#input-columns)). - **Type annotations define the schemas.** Orcapod uses them to build the output schema and to validate compatibility when the pod is applied. A parameter without an annotation needs its type in the decorator's `input_schema` mapping instead. - **`output_keys` names the output columns.** Since the function returns a tuple of two values, `output_keys` must be a list of two names. For a function returning a single value, pass a single string (e.g., `output_keys="result"`). -- **The decorator creates a `.pod` attribute.** You call `.pod.process()` to apply +- **The decorator creates a `.pod` attribute.** You call `.pod(stream)` to apply the function to a stream. Now apply it: @@ -130,8 +130,9 @@ result = analyze_conditions.pod(source) ``` !!! tip - All standard pods support `__call__` as a shorthand for `.process()`, so - `pod(stream)` is equivalent to `pod.process(stream)`. + For an operator, `op(stream)` is a shorthand for `op.process(stream)`. For a function + pod, `pod(stream)` also drops the data columns the function does not take, while + `pod.process(stream)` is exact and rejects them. The `result` is a new stream. Tags are preserved from the input; the data columns are replaced with the function's outputs. From 0ed0035a0393bf71c78d7ed8c35dc620fac59b99 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:20:10 -0800 Subject: [PATCH 6/6] docs(function_pod): show the stream's select_data_columns method in the hand-written equivalent (ITL-807) Pin by test that pod(wide) equals pod(wide.select_data_columns([...])), also with join. Co-Authored-By: Claude Sonnet 5.5 --- .zed/rules | 5 ++- CLAUDE.md | 5 ++- changelog.d/ITL-807.md | 2 +- design_issues/ITL-807-F16.md | 3 +- docs/concepts/function-pods.md | 10 ++--- .../test_function_pod_auto_select.py | 45 +++++++++++++++++++ 6 files changed, 59 insertions(+), 11 deletions(-) diff --git a/.zed/rules b/.zed/rules index d6f7e6b7d..4ce3620b9 100644 --- a/.zed/rules +++ b/.zed/rules @@ -463,8 +463,9 @@ convenient call and pod.process(a, b) the exact primitive: columns that are not parameters of the function, places one automatic SelectDataColumns after the automatic join and before the pod (_FunctionPodBase.__call__, the one place that decides it). The select is an ordinary operator step: graph, hashes and stored tables - must equal those of a pipeline with that select written by hand. Nothing to drop means no - step. One info log names the dropped columns; it is not a warning. + must equal those of a pipeline with that select written by hand (the stream's + `select_data_columns` method builds the same operator, and `a.join(b)` the same `Join`). + Nothing to drop means no step. One info log names the dropped columns; it is not a warning. - pod(a, b, strict=True) and pod.process(a, b) reject an extra column. strict is part of no identity. process never takes a strict argument and never drops a column. - A missing required column is an error in every form. Tags are never dropped. A data column diff --git a/CLAUDE.md b/CLAUDE.md index 7b1f0da05..ccca74a3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -478,8 +478,9 @@ convenient call and `pod.process(a, b)` the exact primitive: columns that are not parameters of the function, places one automatic `SelectDataColumns` after the automatic join and before the pod (`_FunctionPodBase.__call__`, the one place that decides it). The select is an ordinary operator step: graph, hashes and stored tables - must equal those of a pipeline with that select written by hand. Nothing to drop means no - step. One info log names the dropped columns; it is not a warning. + must equal those of a pipeline with that select written by hand (the stream's + `select_data_columns` method builds the same operator, and `a.join(b)` the same `Join`). + Nothing to drop means no step. One info log names the dropped columns; it is not a warning. - `pod(a, b, strict=True)` and `pod.process(a, b)` reject an extra column. `strict` is part of no identity. `process` never takes a `strict` argument and never drops a column. - A missing required column is an error in every form. Tags are never dropped. A data column diff --git a/changelog.d/ITL-807.md b/changelog.d/ITL-807.md index 7954445ab..d1eec85b3 100644 --- a/changelog.d/ITL-807.md +++ b/changelog.d/ITL-807.md @@ -5,7 +5,7 @@ columns that are not parameters of the function, one automatic `SelectDataColumns` is placed after the automatic join and before the pod. It is an ordinary operator step: the graph, the hashes and the stored tables are those of a pipeline with that select written by - hand, so a change in an unused column no longer causes a recomputation. When nothing would + hand (`pod(wide.select_data_columns([...]))`, or `pod(a.join(b).select_data_columns([...]))`), so a change in an unused column no longer causes a recomputation. When nothing would be dropped no step is added and the pipeline is the one built before. One message at `INFO` level names the dropped columns. `pod(a, b, strict=True)` rejects an extra column, and `pod.process(a, b)` is the exact primitive and always rejects it. Code that relied on diff --git a/design_issues/ITL-807-F16.md b/design_issues/ITL-807-F16.md index 96b2775e1..fc161fd42 100644 --- a/design_issues/ITL-807-F16.md +++ b/design_issues/ITL-807-F16.md @@ -16,7 +16,8 @@ label=None, strict=False)` is the convenient call: it joins the inputs as `proce when the joined stream carries data columns that are not parameters, applies one `SelectDataColumns(columns=)` after the automatic join and before the pod, then calls `process`. The select is an ordinary operator step, so the graph, hashes and -stored tables equal those of a pipeline with the select written by hand; when nothing would +stored tables equal those of a pipeline with the select written by hand (the stream methods +`select_data_columns` and `join` build the same operators; pinned by a test); when nothing would be dropped no step is added. One info message names the dropped columns. `strict=True` and `process` reject an extra column as before; `strict` is in no identity. No select is added for a data function whose `accepts_arbitrary_keywords` is true (none today). Operators and diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 5fb96f465..d9774df4e 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -119,13 +119,12 @@ for tag, data in result.iter_data(): ``` The drop is not hidden. Orcapod places one `SelectDataColumns` [operator](operators.md) -between the stream and the pod, exactly as if you had written it yourself, so the two -pipelines below are the same pipeline: same hashes, same graph, same stored tables. +between the stream and the pod, exactly as if you had written it yourself with the stream's +`select_data_columns` method, so the two pipelines below are the same pipeline: same hashes, +same graph, same stored tables. `SelectDataColumns` is the name that appears in the graph. ```python -from orcapod.operators import SelectDataColumns - -narrow = SelectDataColumns(columns=["weight", "height"])(wide) +narrow = wide.select_data_columns(["weight", "height"]) by_hand = compute_bmi.pod(narrow) print(result.pipeline_hash() == by_hand.pipeline_hash()) @@ -661,6 +660,7 @@ for tag, data in result.iter_data(): The join happens on shared tag columns, and the merged data columns are fed to the function. The [input column rules](#input-columns) apply to the joined stream: the join comes first, then one automatic `SelectDataColumns` drops the joined columns the function does not take. +Written by hand, that is `a.join(b).select_data_columns([...])` passed to the pod. Two consequences: - Two inputs that carry the same data column are rejected by the join, whether or not the diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py index 80a819100..edca9d26a 100644 --- a/tests/test_core/function_pod/test_function_pod_auto_select.py +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -708,3 +708,48 @@ def test_sync_and_async_runs_give_the_same_rows() -> None: assert tables[0].schema == tables[1].schema assert tables[0].to_pylist() == tables[1].to_pylist() assert [row["total"] for row in tables[0].to_pylist()] == [5, 9] + + +# --------------------------------------------------------------------------- +# The stream's convenience methods build the same operators (review of PR #327) +# --------------------------------------------------------------------------- + + +def test_call_equals_the_stream_convenience_methods() -> None: + """``pod(wide)`` is ``pod(wide.select_data_columns([...]))``; ``a.join(b)`` is ``Join()``.""" + automatic = add.pod(_wide()) + by_method = add.pod(_wide().select_data_columns(["x", "y"])) + _assert_same_stream(automatic, by_method) + assert automatic.pipeline_hash() == by_method.pipeline_hash() + assert automatic.content_hash() == by_method.content_hash() + + automatic_two = add.pod(_xs(a=1), _ys(b=2)) + by_method_two = add.pod(_xs(a=1).join(_ys(b=2)).select_data_columns(["x", "y"])) + _assert_same_stream(automatic_two, by_method_two) + assert automatic_two.pipeline_hash() == by_method_two.pipeline_hash() + assert automatic_two.content_hash() == by_method_two.content_hash() + + +def test_job_equals_the_job_built_with_convenience_methods() -> None: + """The job nodes and their hashes equal those of the stream-method pipeline.""" + calls: list[int] = [] + counted = _counting_pod(calls) + db = InMemoryArrowDatabase() + + automatic = PipelineJob(name="job", store=db) + with automatic: + counted.pod(_xs(a=1), _ys(b=2), label="add") + automatic.run() + + by_method = PipelineJob(name="job", store=db) + with by_method: + joined = _xs(a=1).join(_ys(b=2)) + counted.pod(joined.select_data_columns(["x", "y"]), label="add") + by_method.run() + + assert len(calls) == 2 + assert sorted(automatic.nodes) == sorted(by_method.nodes) + for label, node in by_method.nodes.items(): + assert type(automatic.nodes[label]) is type(node) + assert automatic.nodes[label].pipeline_hash() == node.pipeline_hash() + assert automatic.nodes[label].content_hash() == node.content_hash()