From 168ca65baae43b2c3a67ed21cb49632b5f66d4d8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:23:48 -0800 Subject: [PATCH 1/7] docs(plan): ITL-821 papercuts re-check and plan Co-Authored-By: Claude Opus 5.5 --- .../2026-11-11-itl-821-newcomer-papercuts.md | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md diff --git a/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md b/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md new file mode 100644 index 000000000..65ca1796d --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md @@ -0,0 +1,165 @@ +# ITL-821 Newcomer Papercuts Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the papercuts the docs audit found (ITL-821) that a v0.3 workshop attendee trips over, each with a +failing regression test first, and make the `examples/` directory run in the required test suite. + +**Architecture:** Small, contained fixes in the files that cause each papercut. Items 1, 2, 3 and 5 are +pre-approved. Items 4 and 6 wait for the developer's decision (routed through the coordinator). The examples +run as a pytest that executes each `examples/*.py` in a subprocess with a temporary working directory. That +test is part of `uv run pytest tests/`, so no CI workflow changes. + +**Tech Stack:** Python 3.11+, pytest, uv. + +**Branch:** `eywalker/itl-821-newcomer-papercuts-found-by-the-docs-audit` (based on origin/main `b197544e`). + +--- + +## Step 1 re-check against current main (`b197544e`, v0.2.0 + #291) + +| # | Item | Still reproduces? | Minimal reproduction | Proposed fix | Hash / stored identity | +|---|------|-------------------|----------------------|--------------|------------------------| +| 1 | `op.enable_file_hash_caching` not exported | **Yes** | `import orcapod as op; hasattr(op, "enable_file_hash_caching")` → `False` | Re-export from `orcapod/__init__.py` (+ `__all__`) | None | +| 2 | `DictSource(data=[])` misleading error | **Yes** | `DictSource(data=[], tag_columns=["a"])` → `ValueError: tag_columns not found in table: {'a'}. Available columns: []` | In `SourceStreamBuilder.build` raise `ValueError("Table is empty: ...")` when the table has no columns, before the tag check. Zero-row tables with columns (e.g. `DictSource(data=[], data_schema=...)`) keep working | None (error path only) | +| 3 | Stray line + "data data" | **Yes** | pod `(weight, height)` on stream with extra `age` → stderr `Key 'age' not found in parameter types.` (logging lastResort handler, `schema_utils.check_schema_compatibility` uses `logger.warning`) then `ValueError: Incoming data data type ...` | Add `schema_utils.schema_incompatibility(incoming, receiving) -> str \| None` returning the reason; `check_schema_compatibility` delegates to it and no longer logs. `FunctionPod._validate_input_schema` and `FunctionNode.__init__` raise `Incoming data schema ... is not compatible with expected input schema ...: ` | None (error path only) | +| 4a | `@function_pod` version `"v0.0"` vs `FunctionPod.from_fn` / `side_effect_function_pod` `"v1.0"` | **Yes** | `function_pod(output_keys="r")(f).pod.content_hash()` ≠ `FunctionPod.from_fn(f, output_keys="r").content_hash()` | **Decision needed** | Major version is in `DataFunctionBase.uri` → identity, pipeline hash and result DB path (`.../v0/...`). See decision section | +| 4b | `MapData`/`MapTags` keep unmapped vs `stream.map_data`/`map_tags` drop | **Yes** (documented as a warning in `docs/concepts/operators.md:420`) | `MapData(name_map=...)` default `drop_unmapped=False`; `StreamBase.map_data(..., drop_unmapped=True)` | **Decision needed** | `drop_unmapped` is in `MapData`/`MapTags.identity_structure` → operator content/pipeline hashes, plus output columns | +| 5 | `examples/save_and_load_pipelines.py` fails | **Yes** | `AttributeError: 'DeltaTableDatabase' object has no attribute 'register_tracker'` (the example passes a database where `Pipeline.__init__` now takes `tracker_manager`) | Rewrite the example around `PipelineJob(name, store=DeltaTableDatabase(...))`, `job.run()`, `job.save(path)`, `PipelineJob.load(path)` | None | +| 6a | protobuf gencode `UserWarning` on import | **No** (not reproducible) | `uv run python -W always -c "import orcapod"` and importing every `orcapod.*` module, with and without `--all-extras` (ray, pyspiral): no warnings, `google.protobuf` not loaded. protobuf 6.31.1 in `uv.lock` | Close with reason (likely an older lock/env) | — | +| 6b | deltalake `WARN deltalake_core::writer::stats Skipping column ... because it's a binary field` | **Yes** (11 lines for one tiny job run) | `PipelineJob(..., store=DeltaTableDatabase(dir)).run()` | **Decision needed** | Depends on option | + +`examples/` in CI: **not run anywhere** (no workflow or test references it). `async_vs_sync_pipeline.py` +runs in ~4.5 s and passes; `save_and_load_pipelines.py` fails. + +## Decisions pending (asked to coordinator) + +### Item 4a — `version` default +* **A (recommended): `from_fn` and `side_effect_function_pod` default to `"v0.0"`** (match the decorator, + the docs and the golden suite). Golden tests unchanged (golden pods use `@function_pod` with the default). + Hash move: only callers of `FunctionPod.from_fn` / `side_effect_function_pod` that omit `version`; their + function pod content/pipeline hash, every downstream node hash and the result-store path (`v1` → `v0`) + change, so their stored results would look missing (recompute). Both APIs are secondary: no docs/README use + them; `side_effect_function_pod` is documented as superseded. +* **B: decorator defaults to `"v1.0"`.** Moves the golden `pipeline/pod` and `pipeline/join_pod` entries and + every hash and stored result path of every decorator user who omits `version` (the documented path). +* **C: leave as is**, document the difference. + +### Item 4b — `drop_unmapped` default +* **A (recommended): `stream.map_data`/`map_tags` default to `False`** (match the classes; renaming keeps + the other columns, the non-destructive choice). Golden unchanged (golden pins `MapData(name_map=...)`, no + fluent call). Hash and output-column change for fluent callers that omit the arg. +* **B: classes default to `True`.** Moves golden `pipeline/map_data`; hash + output change for class users. +* **C: make `drop_unmapped` a required keyword on both.** No hash moves for anyone who passes it; callers that + omit it get a `TypeError` (API break; the golden `MapData` case and docs samples need the explicit `False`, + same hash). +* **D: leave as is** (already documented as a warning). + +### Item 6b — deltalake noise +* **A (recommended): in `orcapod.databases.delta_lake_databases`, before the first `deltalake` import, + `os.environ.setdefault("RUST_LOG", "warn,deltalake_core::writer::stats=error")`.** Verified: deltalake's + Rust `env_logger` reads `RUST_LOG` once when `deltalake` is first imported; setting it before that silences + the lines, setting it after has no effect. Respects a user's own `RUST_LOG`. Caveat: mutates process env, + and does nothing if the user imported `deltalake` before orcapod. +* **B: document** in troubleshooting. +* **C: table property** (`delta.dataSkippingStatsColumns`) — changes stored table metadata; not recommended. +* No dependency change in any option. + +### Examples in CI +`tests/test_examples/test_examples.py`: parametrized over `examples/*.py`, runs each with +`subprocess.run([sys.executable, path], cwd=tmp_path, timeout=120)` and asserts exit 0. No workflow edits. + +--- + +## File structure + +- Modify `src/orcapod/__init__.py` — re-export `enable_file_hash_caching` (item 1). Keep the hunk next to the + `.config` import and `"load_config"` in `__all__` to stay clear of PR #292's hunks. +- Modify `src/orcapod/core/sources/stream_builder.py` — zero-column check (item 2). +- Modify `src/orcapod/utils/schema_utils.py` — `schema_incompatibility` (item 3). PR #292 edits + `check_schema_compatibility`; resolve on merge. +- Modify `src/orcapod/core/function_pod.py`, `src/orcapod/core/nodes/function_node.py` — message (item 3). +- Modify `docs/concepts/function-pods.md` — prose quoting the message (item 3). +- Modify `docs/concepts/sources.md` — mention "Table is empty" rule (item 2). +- Rewrite `examples/save_and_load_pipelines.py` (item 5). +- Create `tests/test_examples/__init__.py`, `tests/test_examples/test_examples.py`. +- Create `tests/test_core/test_newcomer_papercuts.py` — regression tests for items 1–3. +- Modify `CHANGELOG.md` (`[0.3.0] — unreleased`), `DESIGN_ISSUES.md` (one entry appended). + +## Tasks + +### Task 1: export `enable_file_hash_caching` (item 1) +- [ ] Test (`tests/test_core/test_newcomer_papercuts.py`): + ```python + def test_enable_file_hash_caching_is_exported(): + import orcapod as op + from orcapod.contexts import enable_file_hash_caching + assert op.enable_file_hash_caching is enable_file_hash_caching + assert "enable_file_hash_caching" in op.__all__ + ``` +- [ ] Run, expect FAIL (`AttributeError`). +- [ ] Add `from .contexts import enable_file_hash_caching` after the `.config` import; add to `__all__`. +- [ ] Run, expect PASS. Commit `fix(api): export enable_file_hash_caching at top level (ITL-821)`. + +### Task 2: "Table is empty" for column-less input (item 2) +- [ ] Tests: + ```python + def test_dict_source_with_no_rows_and_no_schema_says_table_is_empty(): + with pytest.raises(ValueError, match="Table is empty"): + DictSource(data=[], tag_columns=["a"]) + + def test_dict_source_with_no_rows_and_a_schema_still_works(): + src = DictSource(data=[], tag_columns=["a"], data_schema={"a": int, "x": int}) + assert src.as_table().num_rows == 0 + ``` +- [ ] Run, expect first FAIL. +- [ ] In `SourceStreamBuilder.build`, after dropping system columns: + ```python + if not table.column_names: + raise ValueError( + "Table is empty: the input has no columns, so no tag or data columns can be found. " + "To build a source with zero rows, give it an explicit schema " + "(for example data_schema= on DictSource)." + ) + ``` +- [ ] Run, PASS. Update `docs/concepts/sources.md` construction rules. Commit. + +### Task 3: no stray log line, precise message (item 3) +- [ ] Tests (use `capsys` + `caplog` with a fresh logger state; the stray line comes from logging's + lastResort handler, so assert nothing on stderr): + ```python + def test_extra_column_error_names_the_column_and_prints_nothing(capfd): + @function_pod(output_keys="bmi") + def bmi(weight: float, height: float) -> float: ... + wide = DictSource(data=[{"id": 1, "weight": 70.0, "height": 1.8, "age": 30}], tag_columns=["id"]) + with pytest.raises(ValueError) as err: + bmi.pod(wide) + msg = str(err.value) + assert "data data" not in msg + assert "'age'" in msg + out, errout = capfd.readouterr() + assert out == "" and errout == "" + ``` + plus the same through a `PipelineJob` (FunctionNode path) and a type-mismatch and a missing-key case. +- [ ] Run, FAIL. +- [ ] `schema_utils.schema_incompatibility` returns: + `"column 'age' is not a parameter of the function"`, + `"column 'x' has type int, the function expects str"`, + `"required parameter 'height' has no matching column"`; `check_schema_compatibility` returns + `schema_incompatibility(...) is None`. Callers raise + `f"Incoming data schema {incoming} is not compatible with expected input schema {expected}: {reason}"`. +- [ ] Run, PASS; update `docs/concepts/function-pods.md` prose. Commit. + +### Task 4: fix and test examples (item 5 + CI) +- [ ] `tests/test_examples/test_examples.py` (subprocess per example, cwd=tmp_path). Run, expect + `save_and_load_pipelines.py` FAIL. +- [ ] Rewrite the example with `PipelineJob(name="my_pipeline", store=DeltaTableDatabase("./local_database"))`, + `job.run()`, `job.save("./pipeline.json")`, `PipelineJob.load("./pipeline.json")`, printing results. +- [ ] Run, PASS. Commit. + +### Task 5 / 6: items 4 and 6b per the developer's decision (pending). + +### Task 7: CHANGELOG, DESIGN_ISSUES, full suite, ruff, basedpyright, merge main, PR. + +## Progress notes +- 2026-11-11: Step 1 re-check done (table above). Asked coordinator about items 4 and 6. From d0219a753613ab61bcf32e820be26cab956ecfd5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:25:55 -0800 Subject: [PATCH 2/7] fix(api): export enable_file_hash_caching; clear errors for empty data and incompatible pod input (ITL-821) - orcapod.enable_file_hash_caching is a top-level export. - A source built from data with no columns raises 'Table is empty' instead of 'tag_columns not found in table'. - schema_utils.schema_incompatibility returns the reason a schema does not fit a pod's parameters; check_schema_compatibility no longer logs, so no stray 'Key ... not found' line reaches stderr. FunctionPod and FunctionNode name the offending column in the error and no longer say 'data data'. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 7 +- docs/concepts/sources.md | 4 + src/orcapod/__init__.py | 2 + src/orcapod/core/function_pod.py | 8 +- src/orcapod/core/nodes/function_node.py | 10 +- src/orcapod/core/sources/stream_builder.py | 13 ++- src/orcapod/utils/schema_utils.py | 54 +++++++-- tests/test_core/test_newcomer_papercuts.py | 126 +++++++++++++++++++++ 8 files changed, 203 insertions(+), 21 deletions(-) create mode 100644 tests/test_core/test_newcomer_papercuts.py diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8618052c..28a6712bf 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -113,9 +113,10 @@ except ValueError as err: # ValueError ``` -The error message names both schemas, for example `... Schema({'weight': float, 'height': -float, 'age': int}) is not compatible with expected input schema Schema({'weight': float, -'height': float})`. +The error message names both schemas and the first column that does not fit, for example +`Incoming data schema Schema({'weight': float, 'height': float, 'age': int}) is not +compatible with expected input schema Schema({'weight': float, 'height': float}): column +'age' is not a parameter of the function`. Narrow the stream first with the `SelectDataColumns` [operator](operators.md) (or drop the extras with `DropDataColumns`): diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 914c53e87..5a5e4ab5a 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -44,6 +44,10 @@ Two construction rules apply to every source and are the most common first error - At least one column must be left over as a data column, otherwise it raises `ValueError: No data columns found in the table`. +Data with no rows has no columns to read, so `DictSource(data=[], ...)` raises +`ValueError: Table is empty`. To build a source with no rows, pass `data_schema=` so the +columns are known. + ### `DerivedSource` A `DerivedSource` reads the stored results of a node of a diff --git a/src/orcapod/__init__.py b/src/orcapod/__init__.py index bc229f096..994888ec1 100644 --- a/src/orcapod/__init__.py +++ b/src/orcapod/__init__.py @@ -15,6 +15,7 @@ OrcapodConfig, load_config, ) +from .contexts import enable_file_hash_caching from .core.function_pod import ( FunctionPod, function_pod, @@ -78,6 +79,7 @@ "HashingConfig", "OrcapodConfig", "load_config", + "enable_file_hash_caching", "FunctionPod", "function_pod", "InvocationContext", diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index ba7e3b1d4..ed828edf1 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -238,12 +238,12 @@ def _validate_input_schema(self, input_schema: Schema) -> None: # check_schema_compatibility uses beartype.door.is_subhint which accepts # any concrete branch: is_subhint(str, str | Path) → True, # is_subhint(int, str | Path) → False. - if not schema_utils.check_schema_compatibility( - input_schema, expected_data_schema - ): + reason = schema_utils.schema_incompatibility(input_schema, expected_data_schema) + if reason is not None: # TODO: use custom exception type for better error handling raise ValueError( - f"Incoming data data type {input_schema} is not compatible with expected input schema {expected_data_schema}" + f"Incoming data schema {input_schema} is not compatible with " + f"expected input schema {expected_data_schema}: {reason}" ) def process_data( diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 7af980049..6db77bf92 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -184,12 +184,14 @@ def __init__( # Use the pod's exposed input schema (ctx excluded) rather than the # data function's full schema, because ctx is auto-injected per row. expected_data_schema = function_pod.input_data_schema - if not schema_utils.check_schema_compatibility( + reason = schema_utils.schema_incompatibility( incoming_data_types, expected_data_schema - ): + ) + if reason is not None: raise ValueError( - f"Incoming data data type {incoming_data_types} from {input_stream} " - f"is not compatible with expected input schema {expected_data_schema}" + f"Incoming data schema {incoming_data_types} from {input_stream} " + f"is not compatible with expected input schema " + f"{expected_data_schema}: {reason}" ) self._input_stream = input_stream diff --git a/src/orcapod/core/sources/stream_builder.py b/src/orcapod/core/sources/stream_builder.py index c060e3d5b..962a76303 100644 --- a/src/orcapod/core/sources/stream_builder.py +++ b/src/orcapod/core/sources/stream_builder.py @@ -146,7 +146,8 @@ def build( SourceStreamResult with enriched stream and metadata. Raises: - ValueError: If tag_columns or record_id_column are not in table. + ValueError: If the table has no columns ("Table is empty"), or if + tag_columns or record_id_column are not in the table. """ tag_columns_tuple = tuple(tag_columns) system_tag_columns_tuple = tuple(system_tag_columns) @@ -154,7 +155,15 @@ def build( # 1. Drop system columns from raw input. table = arrow_utils.drop_system_columns(table) - # 2. Validate tag_columns. + # 2. Validate tag_columns. A table with no columns at all (e.g. a + # DictSource built from no rows and no schema) gets its own error, since + # "tag column not found" would point at the wrong problem. + if not table.column_names: + raise ValueError( + "Table is empty: the input has no columns, so its tag and data " + "columns cannot be found. To build a source with no rows, give it " + "an explicit schema (for example data_schema= on DictSource)." + ) missing_tags = set(tag_columns_tuple) - set(table.column_names) if missing_tags: raise ValueError( diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index f6146a6b3..34e7cf9cd 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -39,25 +39,63 @@ def verify_data_schema(data: dict, schema: SchemaLike) -> bool: def check_schema_compatibility( incoming_types: SchemaLike, receiving_types: Schema ) -> bool: + """Return whether ``incoming_types`` can be passed to ``receiving_types``. + + See ``schema_incompatibility`` for the rules and for the reason when the + answer is no. + + Args: + incoming_types: Schema of the data offered (e.g. a stream's data schema). + receiving_types: Schema of the parameters that receive it. + + Returns: + True if every incoming column is a parameter of a compatible type and + every required parameter has a column. + """ + return schema_incompatibility(incoming_types, receiving_types) is None + + +def _type_label(type_info: Any) -> str: + """Return a short, readable label for a type in an error message.""" + if isinstance(type_info, type) and not get_args(type_info): + return type_info.__name__ + return str(type_info) + + +def schema_incompatibility( + incoming_types: SchemaLike, receiving_types: Schema +) -> str | None: + """Explain why ``incoming_types`` cannot be passed to ``receiving_types``. + + Every incoming column must be a parameter, its type must be a subhint of + the parameter's type, and every parameter without a default must have a + column. Nothing is logged: the caller decides how to report the reason. + + Args: + incoming_types: Schema of the data offered (e.g. a stream's data schema). + receiving_types: Schema of the parameters that receive it. + + Returns: + ``None`` when the schemas are compatible, otherwise a one-line reason + naming the first offending column or parameter. + """ from beartype.door import is_subhint for key, type_info in incoming_types.items(): if key not in receiving_types: - logger.warning(f"Key '{key}' not found in parameter types.") - return False + return f"column '{key}' is not a parameter of the function" if not is_subhint(type_info, receiving_types[key]): - logger.warning( - f"Type mismatch for key '{key}': expected {receiving_types[key]}, got {type_info}." + return ( + f"column '{key}' has type {_type_label(type_info)}, " + f"the function expects {_type_label(receiving_types[key])}" ) - return False # Every receiving key must be present in incoming OR be optional (has a default) for key in receiving_types: if key not in incoming_types and key not in receiving_types.optional_fields: - logger.warning(f"Required key '{key}' missing from incoming types.") - return False + return f"required parameter '{key}' has no matching column" - return True + return None def extract_function_schemas( diff --git a/tests/test_core/test_newcomer_papercuts.py b/tests/test_core/test_newcomer_papercuts.py new file mode 100644 index 000000000..7f6773917 --- /dev/null +++ b/tests/test_core/test_newcomer_papercuts.py @@ -0,0 +1,126 @@ +"""Regression tests for the newcomer papercuts found by the docs audit (ITL-821). + +Each test drives the public interface a workshop attendee would use. +""" + +from __future__ import annotations + +import logging + +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.nodes import FunctionNode +from orcapod.sources import DictSource + + +@function_pod(output_keys="bmi") +def compute_bmi(weight: float, height: float) -> float: + return weight / (height**2) + + +def _orcapod_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.name.startswith("orcapod") and record.levelno >= logging.WARNING + ] + + +# --------------------------------------------------------------------------- +# Item 1: enable_file_hash_caching is a top-level export +# --------------------------------------------------------------------------- + + +def test_enable_file_hash_caching_is_exported_at_top_level() -> None: + from orcapod.contexts import enable_file_hash_caching + + assert op.enable_file_hash_caching is enable_file_hash_caching + assert "enable_file_hash_caching" in op.__all__ + + +# --------------------------------------------------------------------------- +# Item 2: a source built from no rows and no schema says the table is empty +# --------------------------------------------------------------------------- + + +def test_dict_source_with_no_rows_and_no_schema_says_table_is_empty() -> None: + with pytest.raises(ValueError, match="Table is empty"): + DictSource(data=[], tag_columns=["a"]) + + +def test_dict_source_with_no_rows_and_a_schema_still_builds() -> None: + source = DictSource(data=[], tag_columns=["a"], data_schema={"a": int, "x": int}) + assert source.as_table().num_rows == 0 + + +# --------------------------------------------------------------------------- +# Item 3: an incompatible input schema raises one clear error and logs nothing +# --------------------------------------------------------------------------- + + +def _wide_source() -> DictSource: + return DictSource( + data=[{"subject": "s1", "weight": 70.0, "height": 1.8, "age": 30}], + tag_columns=["subject"], + ) + + +def test_extra_column_error_names_the_column_and_logs_nothing( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG): + with pytest.raises(ValueError) as excinfo: + compute_bmi.pod(_wide_source()) + + message = str(excinfo.value) + assert "data data" not in message + assert message.startswith("Incoming data schema ") + assert "column 'age' is not a parameter of the function" in message + assert _orcapod_warnings(caplog) == [] + + +def test_type_mismatch_error_names_the_column_and_both_types( + caplog: pytest.LogCaptureFixture, +) -> None: + source = DictSource( + data=[{"subject": "s1", "weight": "heavy", "height": 1.8}], + tag_columns=["subject"], + ) + with caplog.at_level(logging.DEBUG): + with pytest.raises(ValueError) as excinfo: + compute_bmi.pod(source) + + assert "column 'weight' has type str, the function expects float" in str( + excinfo.value + ) + assert _orcapod_warnings(caplog) == [] + + +def test_missing_parameter_error_names_the_parameter( + caplog: pytest.LogCaptureFixture, +) -> None: + source = DictSource( + data=[{"subject": "s1", "weight": 70.0}], tag_columns=["subject"] + ) + with caplog.at_level(logging.DEBUG): + with pytest.raises(ValueError) as excinfo: + compute_bmi.pod(source) + + assert "required parameter 'height' has no matching column" in str(excinfo.value) + assert _orcapod_warnings(caplog) == [] + + +def test_function_node_extra_column_error_is_the_same( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG): + with pytest.raises(ValueError) as excinfo: + FunctionNode(compute_bmi.pod, _wide_source()) + + message = str(excinfo.value) + assert "data data" not in message + assert message.startswith("Incoming data schema ") + assert "column 'age' is not a parameter of the function" in message + assert _orcapod_warnings(caplog) == [] From 85fbb734288717b7762655fa5e5f93e102080df1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:27:12 -0800 Subject: [PATCH 3/7] fix(examples): rewrite save_and_load_pipelines around PipelineJob; run every example in the test suite (ITL-821) The example passed a database to Pipeline(), whose second argument is now a tracker manager, and failed with AttributeError. It now records a PipelineJob with a DeltaTableDatabase store, saves it and loads it back to read the stored results. tests/test_examples runs each examples/*.py in a subprocess with a temporary working directory, so an example cannot break silently again. Co-Authored-By: Claude Opus 5.5 --- examples/save_and_load_pipelines.py | 51 +++++++++++++++++----------- tests/test_examples/__init__.py | 0 tests/test_examples/test_examples.py | 36 ++++++++++++++++++++ 3 files changed, 68 insertions(+), 19 deletions(-) create mode 100644 tests/test_examples/__init__.py create mode 100644 tests/test_examples/test_examples.py diff --git a/examples/save_and_load_pipelines.py b/examples/save_and_load_pipelines.py index 66051d27c..46bfffeae 100644 --- a/examples/save_and_load_pipelines.py +++ b/examples/save_and_load_pipelines.py @@ -1,34 +1,47 @@ -from orcapod import Pipeline, function_pod -from orcapod import sources, databases +"""Save a pipeline job to JSON, then load it back to inspect its stored results. -database = databases.DeltaTableDatabase("./local_database") -source1 = sources.DictSource( +Run from any directory: the database and the saved job are written to the +current working directory. +""" + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import DictSource + +database = DeltaTableDatabase("./local_database") + +source1 = DictSource( [{"id": 0, "x": 5}, {"id": 1, "x": 10}, {"id": 2, "x": 15}], tag_columns=["id"], - label="source1", + source_id="source1", ) -source1 = source1.cached(database) -source2 = sources.DictSource( +source2 = DictSource( [{"id": 0, "y": 3}, {"id": 2, "y": 6}, {"id": 4, "y": 9}], tag_columns=["id"], - label="source2", + source_id="source2", ) -source2 = source2.cached(database) - -pipeline = Pipeline("my_pipeline", database) - -@function_pod("sum") +@function_pod(output_keys="sum") def take_sum(x: int, y: int) -> int: return x + y -with pipeline: - result = take_sum.pod(source1.join(source2)) - -path = "./pipeline.json" -pipeline.save(path) +# Record the pipeline: nothing is computed inside the ``with`` block. +job = PipelineJob(name="my_pipeline", store=database) +with job: + take_sum.pod(source1.join(source2), label="sum") +job.run() +print(job.nodes["sum"].as_table().to_pandas()) -pipeline2 = Pipeline.load(path) +# Save the job (pipeline topology and store configuration) to JSON ... +path = "./pipeline.json" +job.save(path) + +# ... and load it back, for example in a later session, to inspect the stored +# results. A loaded job is read-only: its nodes read the records in the store +# with get_all_records(), and the job cannot run again. In-memory sources such +# as DictSource are not saved, so loading logs that they were skipped. +job2 = PipelineJob.load(path) +print(job2.nodes["sum"].get_all_records().to_pandas()) diff --git a/tests/test_examples/__init__.py b/tests/test_examples/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_examples/test_examples.py b/tests/test_examples/test_examples.py new file mode 100644 index 000000000..a57005cf7 --- /dev/null +++ b/tests/test_examples/test_examples.py @@ -0,0 +1,36 @@ +"""Run every script in ``examples/`` so an example cannot break silently (ITL-821). + +Each example runs in its own subprocess with a fresh temporary working directory, +because examples write files (databases, saved pipelines) relative to the +current directory. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +EXAMPLES_DIR = Path(__file__).resolve().parents[2] / "examples" +EXAMPLE_SCRIPTS = sorted(EXAMPLES_DIR.glob("*.py")) + + +def test_examples_directory_has_scripts() -> None: + assert EXAMPLE_SCRIPTS, f"no example scripts found in {EXAMPLES_DIR}" + + +@pytest.mark.parametrize("script", EXAMPLE_SCRIPTS, ids=lambda path: path.name) +def test_example_runs(script: Path, tmp_path: Path) -> None: + result = subprocess.run( + [sys.executable, str(script)], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=300, + ) + assert result.returncode == 0, ( + f"{script.name} exited with {result.returncode}\n" + f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) From d04cae95bb80c69350ade36cee3ab9cfbe78e452 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:27:38 -0800 Subject: [PATCH 4/7] docs: changelog and design-issues entry for ITL-821 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 +++++++++++++++ DESIGN_ISSUES.md | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8053fbb77..f209b6225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [0.3.0] — unreleased + +### Fixed + +- `orcapod.enable_file_hash_caching` is exported at the top level (ITL-821). +- A source built from data with no rows and no schema (e.g. `DictSource(data=[])`) raises + `ValueError: Table is empty` instead of `tag_columns not found in table` (ITL-821). +- Applying a function pod to a stream whose data schema does not fit no longer prints a stray + `Key ... not found in parameter types.` line, and the error names the offending column, + e.g. `Incoming data schema ... is not compatible with expected input schema ...: column + 'age' is not a parameter of the function` (it used to say `Incoming data data type`) + (ITL-821). +- `examples/save_and_load_pipelines.py` runs again, rewritten around `PipelineJob`; the test + suite now runs every example (ITL-821). + ## [0.2.0] — 2026-11-10 **Distribution:** Orcapod is MIT licensed starting with `0.2.0` (it was proprietary on diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 4be0e7e34..726b8bc28 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2245,3 +2245,38 @@ xfail. The async path checks key types on the first row of each input, because So when either input is empty (left or right), a key type mismatch that the sync path reports yields an empty result on the async path instead of an error. `Join`'s async key check (`_check_shared_key_types`) has the same limit. + +--- + +## Newcomer papercuts found by the docs audit + +### NP1 — Missing export, misleading errors, inconsistent defaults, broken example, write noise +**Status:** in progress +**Severity:** medium +**Issue:** ITL-821 + +The docs audit (PR #271) ran every sample against `main` and found behavior that trips a +newcomer. Re-checked on `b197544e` (v0.2.0 + #291): + +1. `orcapod.enable_file_hash_caching` was not exported. **Fix:** re-exported from `orcapod`. +2. `DictSource(data=[], tag_columns=[...])` raised `tag_columns not found in table: {...}. + Available columns: []`. **Fix:** `SourceStreamBuilder.build` raises `Table is empty` for + a table with no columns; a zero-row source with `data_schema=` still builds. +3. A pod applied to a stream with an extra column printed `Key 'age' not found in parameter + types.` (a `logger.warning` reaching logging's last-resort stderr handler) before raising + `Incoming data data type ...`. **Fix:** `schema_utils.schema_incompatibility` returns the + reason instead of logging; `FunctionPod` and `FunctionNode` raise `Incoming data schema + ... : column 'age' is not a parameter of the function`. +4. Defaults that disagree: `@function_pod` `version="v0.0"` vs `FunctionPod.from_fn` / + `side_effect_function_pod` `"v1.0"` (the major version is in the data function's `uri`, so + it moves identity hashes and the result-store path); `MapData`/`MapTags` + `drop_unmapped=False` vs `stream.map_data`/`map_tags` `True` (in the operator identity). + **Open:** awaiting the developer's choice of default. +5. `examples/save_and_load_pipelines.py` passed a database to `Pipeline()` and failed. + **Fix:** rewritten around `PipelineJob`; `tests/test_examples/test_examples.py` runs every + example in a subprocess so an example cannot break silently again. +6. Noise: the protobuf gencode `UserWarning`s on import no longer reproduce (every + `orcapod` module imported with and without all extras, `-W always`; protobuf is never + loaded), so that half is closed. deltalake's `WARN deltalake_core::writer::stats Skipping + column ... because it's a binary field` lines on every Delta write still appear. + **Open:** awaiting the developer's choice of how to silence them. From e17770d46b90c847a6eb66ca11245b1638325ed9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:28:51 -0800 Subject: [PATCH 5/7] test(itl-821): build the FunctionNode test pod with FunctionPod.from_fn for typing Co-Authored-By: Claude Opus 5.5 --- tests/test_core/test_newcomer_papercuts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_core/test_newcomer_papercuts.py b/tests/test_core/test_newcomer_papercuts.py index 7f6773917..2c8504b73 100644 --- a/tests/test_core/test_newcomer_papercuts.py +++ b/tests/test_core/test_newcomer_papercuts.py @@ -10,7 +10,7 @@ import pytest import orcapod as op -from orcapod import function_pod +from orcapod import FunctionPod, function_pod from orcapod.nodes import FunctionNode from orcapod.sources import DictSource @@ -115,9 +115,13 @@ def test_missing_parameter_error_names_the_parameter( def test_function_node_extra_column_error_is_the_same( caplog: pytest.LogCaptureFixture, ) -> None: + def bmi(weight: float, height: float) -> float: + return weight / (height**2) + + pod = FunctionPod.from_fn(bmi, output_keys="bmi") with caplog.at_level(logging.DEBUG): with pytest.raises(ValueError) as excinfo: - FunctionNode(compute_bmi.pod, _wide_source()) + _ = FunctionNode(pod, _wide_source()) message = str(excinfo.value) assert "data data" not in message From 23ca5b01fd478f0a1fb7e0e3bac2c3ea78e8b32a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:30:54 -0800 Subject: [PATCH 6/7] docs(plan): ITL-821 progress notes Co-Authored-By: Claude Opus 5.5 --- superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md b/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md index 65ca1796d..5f928ce9a 100644 --- a/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md +++ b/superpowers/plans/2026-11-11-itl-821-newcomer-papercuts.md @@ -163,3 +163,10 @@ runs in ~4.5 s and passes; `save_and_load_pipelines.py` fails. ## Progress notes - 2026-11-11: Step 1 re-check done (table above). Asked coordinator about items 4 and 6. +- 2026-11-11: Coordinator: 1, 2, 3, 5 + examples test approved; protobuf half of 6 closed with reason; + 4a, 4b and the deltalake RUST_LOG change HELD for the developer — do not implement until the + coordinator replies. Tasks 1–4 and 7 done; PR #296 opened ("Refs ITL-821"). Example note: a loaded + `PipelineJob` is read-only; its nodes read stored rows only via `get_all_records()`. +- Out-of-scope findings (report in worker_done, not filed): `PipelineJob.load` logs + "Could not reconstruct source ... skipping" for in-memory sources; a loaded node's `as_table()` raises + "no function pod and no database attached" although the store is attached. From 5820f705d3f71c8d69de11ad80291de3659e89d9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:46:56 -0800 Subject: [PATCH 7/7] test(itl-821): docstrings, single with-statements and explicit check=False (CodeRabbit) Co-Authored-By: Claude Opus 5.5 --- tests/test_core/test_newcomer_papercuts.py | 75 ++++++++++++++++++---- tests/test_examples/test_examples.py | 9 +++ 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/tests/test_core/test_newcomer_papercuts.py b/tests/test_core/test_newcomer_papercuts.py index 2c8504b73..c7b70e5fd 100644 --- a/tests/test_core/test_newcomer_papercuts.py +++ b/tests/test_core/test_newcomer_papercuts.py @@ -17,10 +17,27 @@ @function_pod(output_keys="bmi") def compute_bmi(weight: float, height: float) -> float: + """Compute a body-mass index; the pod under test takes ``weight`` and ``height``. + + Args: + weight: Body weight. + height: Body height. + + Returns: + ``weight`` divided by the square of ``height``. + """ return weight / (height**2) def _orcapod_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + """Return the messages Orcapod loggers recorded at WARNING or above. + + Args: + caplog: The pytest log-capture fixture of the running test. + + Returns: + The formatted messages of the matching records, in emission order. + """ return [ record.getMessage() for record in caplog.records @@ -34,6 +51,7 @@ def _orcapod_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: def test_enable_file_hash_caching_is_exported_at_top_level() -> None: + """``op.enable_file_hash_caching`` is the ``orcapod.contexts`` function and is in ``__all__``.""" from orcapod.contexts import enable_file_hash_caching assert op.enable_file_hash_caching is enable_file_hash_caching @@ -46,11 +64,13 @@ def test_enable_file_hash_caching_is_exported_at_top_level() -> None: def test_dict_source_with_no_rows_and_no_schema_says_table_is_empty() -> None: + """A ``DictSource`` over no rows and no schema raises "Table is empty".""" with pytest.raises(ValueError, match="Table is empty"): DictSource(data=[], tag_columns=["a"]) def test_dict_source_with_no_rows_and_a_schema_still_builds() -> None: + """A zero-row ``DictSource`` with ``data_schema`` still builds an empty source.""" source = DictSource(data=[], tag_columns=["a"], data_schema={"a": int, "x": int}) assert source.as_table().num_rows == 0 @@ -61,6 +81,11 @@ def test_dict_source_with_no_rows_and_a_schema_still_builds() -> None: def _wide_source() -> DictSource: + """Return a one-row source with an ``age`` column that ``compute_bmi`` does not take. + + Returns: + A ``DictSource`` tagged by ``subject`` with ``weight``, ``height`` and ``age``. + """ return DictSource( data=[{"subject": "s1", "weight": 70.0, "height": 1.8, "age": 30}], tag_columns=["subject"], @@ -70,9 +95,13 @@ def _wide_source() -> DictSource: def test_extra_column_error_names_the_column_and_logs_nothing( caplog: pytest.LogCaptureFixture, ) -> None: - with caplog.at_level(logging.DEBUG): - with pytest.raises(ValueError) as excinfo: - compute_bmi.pod(_wide_source()) + """An extra column raises one error naming it, with no log line and no "data data". + + Args: + caplog: Captures Orcapod's log records. + """ + with caplog.at_level(logging.DEBUG), pytest.raises(ValueError) as excinfo: + compute_bmi.pod(_wide_source()) message = str(excinfo.value) assert "data data" not in message @@ -84,13 +113,17 @@ def test_extra_column_error_names_the_column_and_logs_nothing( def test_type_mismatch_error_names_the_column_and_both_types( caplog: pytest.LogCaptureFixture, ) -> None: + """A column of the wrong type raises an error naming the column and both types. + + Args: + caplog: Captures Orcapod's log records. + """ source = DictSource( data=[{"subject": "s1", "weight": "heavy", "height": 1.8}], tag_columns=["subject"], ) - with caplog.at_level(logging.DEBUG): - with pytest.raises(ValueError) as excinfo: - compute_bmi.pod(source) + with caplog.at_level(logging.DEBUG), pytest.raises(ValueError) as excinfo: + compute_bmi.pod(source) assert "column 'weight' has type str, the function expects float" in str( excinfo.value @@ -101,12 +134,16 @@ def test_type_mismatch_error_names_the_column_and_both_types( def test_missing_parameter_error_names_the_parameter( caplog: pytest.LogCaptureFixture, ) -> None: + """A required parameter with no column raises an error naming the parameter. + + Args: + caplog: Captures Orcapod's log records. + """ source = DictSource( data=[{"subject": "s1", "weight": 70.0}], tag_columns=["subject"] ) - with caplog.at_level(logging.DEBUG): - with pytest.raises(ValueError) as excinfo: - compute_bmi.pod(source) + with caplog.at_level(logging.DEBUG), pytest.raises(ValueError) as excinfo: + compute_bmi.pod(source) assert "required parameter 'height' has no matching column" in str(excinfo.value) assert _orcapod_warnings(caplog) == [] @@ -115,13 +152,27 @@ def test_missing_parameter_error_names_the_parameter( def test_function_node_extra_column_error_is_the_same( caplog: pytest.LogCaptureFixture, ) -> None: + """``FunctionNode`` reports an extra column with the same message as ``FunctionPod``. + + Args: + caplog: Captures Orcapod's log records. + """ + def bmi(weight: float, height: float) -> float: + """Compute a body-mass index. + + Args: + weight: Body weight. + height: Body height. + + Returns: + ``weight`` divided by the square of ``height``. + """ return weight / (height**2) pod = FunctionPod.from_fn(bmi, output_keys="bmi") - with caplog.at_level(logging.DEBUG): - with pytest.raises(ValueError) as excinfo: - _ = FunctionNode(pod, _wide_source()) + with caplog.at_level(logging.DEBUG), pytest.raises(ValueError) as excinfo: + _ = FunctionNode(pod, _wide_source()) message = str(excinfo.value) assert "data data" not in message diff --git a/tests/test_examples/test_examples.py b/tests/test_examples/test_examples.py index a57005cf7..4a22f9a72 100644 --- a/tests/test_examples/test_examples.py +++ b/tests/test_examples/test_examples.py @@ -18,17 +18,26 @@ def test_examples_directory_has_scripts() -> None: + """The discovery glob finds at least one example, so the parametrized test is not vacuous.""" assert EXAMPLE_SCRIPTS, f"no example scripts found in {EXAMPLES_DIR}" @pytest.mark.parametrize("script", EXAMPLE_SCRIPTS, ids=lambda path: path.name) def test_example_runs(script: Path, tmp_path: Path) -> None: + """An example script exits 0 when run alone in an empty working directory. + + Args: + script: Path of the example script under ``examples/``. + tmp_path: Fresh directory used as the working directory, so files the + example writes do not leak into the repository or other tests. + """ result = subprocess.run( [sys.executable, str(script)], cwd=tmp_path, capture_output=True, text=True, timeout=300, + check=False, ) assert result.returncode == 0, ( f"{script.name} exited with {result.returncode}\n"