From 5b07349dfc523477d3fbd472d4a77e0eb7e53896 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:11:52 -0800 Subject: [PATCH 01/16] docs(itl-1043): investigation scripts and findings for loaded operator node reads Co-Authored-By: Claude Opus 5.5 --- superpowers/specs/itl-1043/FINDINGS.md | 120 ++++++++++++ superpowers/specs/itl-1043/q1_repro.py | 89 +++++++++ .../specs/itl-1043/q2_where_rows_go.py | 62 +++++++ superpowers/specs/itl-1043/q3_schemas.py | 95 ++++++++++ superpowers/specs/itl-1043/q4_blast_radius.py | 173 ++++++++++++++++++ .../specs/itl-1043/q4b_from_pipeline.py | 85 +++++++++ .../specs/itl-1043/q5_nullability_hash.py | 52 ++++++ 7 files changed, 676 insertions(+) create mode 100644 superpowers/specs/itl-1043/FINDINGS.md create mode 100644 superpowers/specs/itl-1043/q1_repro.py create mode 100644 superpowers/specs/itl-1043/q2_where_rows_go.py create mode 100644 superpowers/specs/itl-1043/q3_schemas.py create mode 100644 superpowers/specs/itl-1043/q4_blast_radius.py create mode 100644 superpowers/specs/itl-1043/q4b_from_pipeline.py create mode 100644 superpowers/specs/itl-1043/q5_nullability_hash.py diff --git a/superpowers/specs/itl-1043/FINDINGS.md b/superpowers/specs/itl-1043/FINDINGS.md new file mode 100644 index 000000000..465af00a4 --- /dev/null +++ b/superpowers/specs/itl-1043/FINDINGS.md @@ -0,0 +1,120 @@ +# ITL-1043 investigation findings (origin/main 4127e6bb, November 12, 2026) + +Scripts in this directory, each run with `uv run python superpowers/specs/itl-1043/`: +`q1_repro.py`, `q2_where_rows_go.py`, `q3_schemas.py`, `q4_blast_radius.py` (output in +`q4_output.txt`), `q4b_from_pipeline.py`, `q5_nullability_hash.py` (output in +`q5_output.txt`). + +## Q1: reproduction and supported usage + +Both behaviours reproduce on the in-memory and Delta stores (`q1_repro.py`): + +| tags | (a) no re-attach | (b) re-attached, REPLAY | +|---|---|---| +| non-null | `RuntimeError: ... no stored results available for this read` | 0 rows (2 stored) | +| nullable | same | `RuntimeError: ... the cache table is absent` at `schema:4c4a...` (run wrote `schema:659c...`) | + +Supported? Partly documented as not yet supported: + +- `docs/concepts/function-pods.md:688-693`: "A loaded operator without a readable cache raises + ... Saved descriptors do not yet preserve all runtime identity and cache configuration + needed for operator readback without reconstruction (follow-up ITL-977)." +- ITL-977 (Backlog, after v0.3) records that the saved operator descriptor has no + `pipeline_path`, no run-time `content_hash` and no `cache_mode`, and that the developer chose + not to change the saved file format for v0.3. `tests/test_pipeline/test_hash_step_combined.py` + injects the missing fields into the file to read an operator node back. +- `tests/test_pipeline/loaded_job/test_loaded_job_readback.py:758` + (`test_loaded_operator_node_raises_instead_of_returning_empty`) pins behaviour (a). +- The intended way to read cached operator output today: rebuild the job from code and attach + the operator node in `CacheMode.REPLAY` + (`tests/test_databases/test_delta_arrow_type_roundtrip.py:276-302`). That works in every case + tested here (`q4_blast_radius.py`, "rebuilt"). + +## Cause of (a) + +The operator's `cache_mode` is not saved: `Pipeline.save` writes no `cache_mode` +(`src/orcapod/pipeline/graph.py:205-212`), `OperatorJobNode.from_descriptor` defaults to +`"off"` (`src/orcapod/core/nodes/operator_node.py:783`), and `_distribute_databases` keeps OFF +(`src/orcapod/pipeline/job.py:500`). Documented (ITL-977). + +## Q2: the 0-row case + +The table is found at the right path and its rows are filtered out by content hash +(`q2_where_rows_go.py`): + +- Rows are written with the run's node content hash in `__node_content_hash` + (`operator_node.py:963`); `0fd8...` here. +- The loaded node is rebuilt with its operator (full mode, `operator_node.py:789`) over + `SourceJobNode` stubs whose in-memory sources were not saved (unbound), so its content hash is + the schema-only one (`source_node.py:85-87`): `1cb6...`. +- `_filter_by_content_hash` (`operator_node.py:922`) keeps rows equal to the loaded hash: none. +- Function nodes avoid this because `PipelineJob.save` writes their run content hash to + `run.node_job_content_hashes` (`job.py:1339-1351`) and `load` passes it to the stub + (`job.py:1564`). Operator nodes get nothing (`job.py:1585`). Known in ITL-977. + +## Q3: the hash question (nullable tags) + +`q3_schemas.py`. Running node and blueprint: source `a` tag `{id: int | None}`, pipeline hash +`9d1f...`; `joined` pipeline hash `659c...`. Saved descriptor of `a`: +`output_schema {'tag': {'id': 'int64'}, ...}` and `source_config.tag_schema {'id': 'int64'}`. +Loaded: source `a` tag `{id: int}`, pipeline hash `42d7...`; `joined` pipeline hash `4c4a...`, +equal to a run with non-null tags. `optional_fields` is empty in every case (nullability is +carried by the `T | None` type, not by `optional_fields`). + +Where it diverges: on SAVE. `serialize_schema` writes +`type_converter.python_type_to_arrow_type(T | None)`, an Arrow `DataType`, which has no +field nullability (`src/orcapod/pipeline/serialization.py:651-656`): `int | None` and `int` +both become `"int64"`. On load `Pipeline.load` rebuilds the `SourceNode` with `int` +(`graph.py:327-350`), `SourceNodeBase.pipeline_identity_structure` hashes that schema +(`source_node.py:95`), and the operator node, rebuilt with its operator, recomputes its path +from it (`operator_node.py:306`). Nested nullability is kept in the type string; only the +top-level flag is lost. + +Which hash is intended: the RUNNING node's (`659c...` here; `ea6e...` in the report). + +- `docs/reference/hashing.md:1204-1206`: nullability at every depth is part of the hash. +- Golden: `tests/test_hashing/golden/data/datagram.json:18` vs `:33` (ArrowTableSource, data + column `int64` non-null vs nullable: pipeline hash `979b...` vs `fcfb...`); + `schema.json:54` vs `:56` (`str | None` vs `str`). +- CLAUDE.md "Arrow types": nullability is never changed. + +No golden entry pins a loaded node or a save/load round trip (no such case in +`tests/test_hashing/golden/cases.py`). The loaded value is an artifact of information lost in +the saved file. + +## Q4: blast radius + +`q4_blast_radius.py` (tag and data nullability varied; function node `inc`, operator `joined`): + +- `PipelineJob.load`, loaded function node: pipeline hash, content hash and path EQUAL the run's + (the stub takes them from the descriptor, `function_node.py:1366-1394`); rows read back. Its + `output_schema()` reports `int` for `int | None`. +- Loaded source nodes: pipeline hash wrong whenever a tag or data column is nullable; content + hash always differs (in-memory sources unbound: expected); `output_schema()` loses `| None`. +- Loaded operator node: content hash always differs (Q2); pipeline hash and path wrong whenever + any input column (tag or data) is nullable. Data column order of a commutative `Join` can + differ too, because inputs are ordered by the (wrong) pipeline hash. +- Re-run from code with the same definitions: every node equal, function node + `computed=0 reused=2`, operator REPLAY reads its rows. No miss. +- A loaded `PipelineJob` cannot run (by design). `Pipeline.load` + `PipelineJob.from_pipeline` + fails for any pipeline with a function node (`to_invocations() cannot serialise FunctionNode + ... _function_pod is None`). +- `q4b_from_pipeline.py`: an operator-only pipeline through `Pipeline.load` + + `from_pipeline(sources=)`: content hash is right (bound sources), but with a + nullable tag the path is wrong (`605d...` instead of `68a6...`): REPLAY reads 0 rows, and a + LOG run would write a second table under the wrong path (a silent cache miss). Non-null: works. +- A bare `pa.table({...})` or `pa.Table.from_pandas(...)` makes every field nullable, so the + nullable case is the common one for `ArrowTableSource` and `DataFrameSource`. + +## Q5: is nullability part of the schema hash? + +Yes, as the declared type, not the presence of nulls (`q5_nullability_hash.py`): Arrow +non-nullable `5a41...`; nullable without nulls and nullable with a null both `3a86...`; +`DictSource` declared `int | None` `3a86...`; `DictSource` inferring from values gives `5a41...` +without a `None` and `3a86...` with one (inference only when no type is declared). + +## Related issues + +Not the same as ITL-1029, ITL-1039, ITL-1040, ITL-940 or ITL-1042. Causes (a) and the 0-row +case are ITL-977's deferred defect. The lost nullability in the saved schema is not recorded +anywhere found (Linear ITL-977, `design_issues/`). diff --git a/superpowers/specs/itl-1043/q1_repro.py b/superpowers/specs/itl-1043/q1_repro.py new file mode 100644 index 000000000..1e4c68e26 --- /dev/null +++ b/superpowers/specs/itl-1043/q1_repro.py @@ -0,0 +1,89 @@ +"""ITL-1043 Q1: reproduce reading a cached operator node from a loaded job. + +Run: uv run python superpowers/specs/itl-1043/q1_repro.py +""" + +import sys +import tempfile +from pathlib import Path + +import pyarrow as pa + +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import PipelineJob +from orcapod.types import CacheMode + +NAME = "j" + + +def tables(nullable: bool) -> tuple[pa.Table, pa.Table]: + """Two 2-row tables joined on tag ``id``; ``id`` nullable or not (no nulls).""" + + def one(col: str, vals: list[int]) -> pa.Table: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=nullable), + pa.field(col, pa.int64(), nullable=False), + ] + ) + return pa.table({"id": [0, 1], col: vals}, schema=schema) + + return one("a", [1, 2]), one("b", [3, 4]) + + +def run(store_kind: str, nullable: bool, tmp: Path) -> None: + store = ( + InMemoryArrowDatabase() + if store_kind == "memory" + else DeltaTableDatabase(str(tmp / "store")) + ) + ta, tb = tables(nullable) + job = PipelineJob(name=NAME, store=store) + with job: + left = ArrowTableSource(ta, tag_columns=["id"], source_id="a") + right = ArrowTableSource(tb, tag_columns=["id"], source_id="b") + Join()(left, right, label="joined") + node = job.nodes["joined"] + node.attach_databases(pipeline_database=store.at(NAME), cache_mode=CacheMode.LOG) + job.run() + print(f"--- store={store_kind} nullable_tag={nullable}") + print(" run: path", "/".join(node._stored_table_path)) + print(" run: content_hash", node.content_hash().to_string()) + print(" run: rows before save", node.as_table().num_rows) + saved = tmp / "job.json" + job.save(saved) + + loaded = PipelineJob.load(str(saved), store=store) + lnode = loaded.nodes["joined"] + print( + " load: node type", + type(lnode).__name__, + "operator present:", + lnode._operator is not None, + ) + try: + print(" (a) no re-attach rows:", lnode.as_table().num_rows) + except Exception as exc: # noqa: BLE001 - report every outcome + print(" (a) no re-attach:", type(exc).__name__, str(exc)[:110]) + lnode.attach_databases( + pipeline_database=store.at(NAME), cache_mode=CacheMode.REPLAY + ) + print(" load: path", "/".join(lnode._stored_table_path)) + print(" load: content_hash", lnode.content_hash().to_string()) + try: + print(" (b) REPLAY rows:", lnode.as_table().num_rows) + except Exception as exc: # noqa: BLE001 + print(" (b) REPLAY:", type(exc).__name__, str(exc)[:200]) + raw = store.at(NAME).get_all_records( + node._stored_table_path, record_id_column="__record_id" + ) + print(" stored rows at run path:", None if raw is None else raw.num_rows) + + +for kind in ("memory", "delta"): + for nullable in (False, True): + with tempfile.TemporaryDirectory() as d: + run(kind, nullable, Path(d)) +print(sys.version.split()[0]) diff --git a/superpowers/specs/itl-1043/q2_where_rows_go.py b/superpowers/specs/itl-1043/q2_where_rows_go.py new file mode 100644 index 000000000..aadca2fa4 --- /dev/null +++ b/superpowers/specs/itl-1043/q2_where_rows_go.py @@ -0,0 +1,62 @@ +"""ITL-1043 Q2: where do the stored rows go when a loaded REPLAY operator node reads 0? + +Non-null tags, in-memory store. Reads the raw operator-cache table at the loaded node's +path and compares its ``__node_content_hash`` values with the loaded node's content hash. + +Run: uv run python superpowers/specs/itl-1043/q2_where_rows_go.py +""" + +import tempfile +from pathlib import Path + +import pyarrow as pa +import pyarrow.compute as pc + +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import PipelineJob +from orcapod.system_constants import constants +from orcapod.types import CacheMode + +NAME = "j" + + +def src(col: str, sid: str) -> ArrowTableSource: + # Declared non-null: a bare pa.table({...}) makes every field nullable. + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field(col, pa.int64(), nullable=False), + ] + ) + return ArrowTableSource( + pa.table({"id": [0, 1], col: [1, 2]}, schema=schema), + tag_columns=["id"], + source_id=sid, + ) + + +store = InMemoryArrowDatabase() +job = PipelineJob(name=NAME, store=store) +with job: + Join()(src("a", "a"), src("b", "b"), label="joined") +node = job.nodes["joined"] +node.attach_databases(pipeline_database=store.at(NAME), cache_mode=CacheMode.LOG) +job.run() +with tempfile.TemporaryDirectory() as d: + job.save(Path(d) / "job.json") + lnode = PipelineJob.load(str(Path(d) / "job.json"), store=store).nodes["joined"] +lnode.attach_databases(pipeline_database=store.at(NAME), cache_mode=CacheMode.REPLAY) + +print("same table path:", lnode._stored_table_path == node._stored_table_path) +raw = store.at(NAME).get_all_records( + lnode._stored_table_path, record_id_column=node.HASH_COLUMN_NAME +) +stored = set(pc.unique(raw.column(constants.NODE_CONTENT_HASH_COL)).to_pylist()) +print("stored rows:", raw.num_rows) +print("stored __node_content_hash:", stored) +print("run node content hash: ", node.content_hash().to_prefixed_digest()) +print("loaded node content hash: ", lnode.content_hash().to_prefixed_digest()) +print("loaded inputs:", [(type(s).__name__, s._bound_source) for s in lnode.upstreams]) +print("loaded REPLAY rows:", lnode.as_table().num_rows) diff --git a/superpowers/specs/itl-1043/q3_schemas.py b/superpowers/specs/itl-1043/q3_schemas.py new file mode 100644 index 000000000..209b4ec39 --- /dev/null +++ b/superpowers/specs/itl-1043/q3_schemas.py @@ -0,0 +1,95 @@ +"""ITL-1043 Q3: the schemas hashed by the running and the loaded operator node. + +Prints, for every node of a Join job with a nullable tag, the hashed schema of each +input source node (types, ``optional_fields``) and the identity structures, then the +saved ``output_schema`` / ``source_config`` blocks, before save and after load. + +Run: uv run python superpowers/specs/itl-1043/q3_schemas.py +""" + +import json +import tempfile +from pathlib import Path + +import pyarrow as pa + +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import PipelineJob +from orcapod.types import CacheMode + +NAME = "j" + + +def table(col: str, vals: list[int], nullable: bool) -> pa.Table: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=nullable), + pa.field(col, pa.int64(), nullable=False), + ] + ) + return pa.table({"id": [0, 1], col: vals}, schema=schema) + + +def describe(schema) -> str: + fields = ", ".join(f"{k}: {v!r}" for k, v in schema.items()) + return ( + f"{{{fields}}} optional_fields={sorted(getattr(schema, 'optional_fields', ()))}" + ) + + +def show(label: str, node) -> None: + tag, data = node.output_schema() + print(f" {label}: {type(node).__name__}") + print(f" tag {describe(tag)}") + print(f" data {describe(data)}") + print(f" pipeline_hash {node.pipeline_hash().to_string()}") + print(f" content_hash {node.content_hash().to_string()}") + + +store = InMemoryArrowDatabase() +job = PipelineJob(name=NAME, store=store) +with job: + left = ArrowTableSource(table("a", [1, 2], True), tag_columns=["id"], source_id="a") + right = ArrowTableSource( + table("b", [3, 4], True), tag_columns=["id"], source_id="b" + ) + Join()(left, right, label="joined") +job.nodes["joined"].attach_databases( + pipeline_database=store.at(NAME), cache_mode=CacheMode.LOG +) +job.run() + +print("== source table schema (Arrow) ==") +print(" ", left.as_table().schema.field("id")) +print("== running job ==") +for label, node in job.nodes.items(): + show(label, node) +print("== compiled (blueprint) pipeline, which save() writes ==") +for label, node in job.compiled_pipeline.nodes.items(): + show(label, node) + +with tempfile.TemporaryDirectory() as d: + saved = Path(d) / "job.json" + job.save(saved) + data = json.loads(saved.read_text()) + print("== saved descriptors ==") + for desc in data["pipeline"]["nodes"].values(): + print(" ", desc["label"], "output_schema", desc["output_schema"]) + if "source_config" in desc: + sc = desc["source_config"] + print( + " source_config", + {"tag_schema": sc["tag_schema"], "data_schema": sc["data_schema"]}, + ) + print( + " content_hash", + desc["content_hash"], + "pipeline_hash", + desc["pipeline_hash"], + ) + loaded = PipelineJob.load(str(saved), store=store) + print("== loaded job ==") + for label, node in loaded.nodes.items(): + show(label, node) diff --git a/superpowers/specs/itl-1043/q4_blast_radius.py b/superpowers/specs/itl-1043/q4_blast_radius.py new file mode 100644 index 000000000..d1d018ed4 --- /dev/null +++ b/superpowers/specs/itl-1043/q4_blast_radius.py @@ -0,0 +1,173 @@ +"""ITL-1043 Q4: every node's hashes and paths before save and after load, and re-runs. + +Job: sources ``a`` (tag ``id``, data ``x``) and ``b`` (tag ``id``, data ``y``); function +node ``inc`` on ``a``; operator node ``joined`` = Join(inc, b) in LOG mode. Tag and data +nullability vary. For each case it prints, per node, pipeline_hash / content_hash / stored +path for: the run; ``PipelineJob.load``; the same job rebuilt from code; and +``Pipeline.load`` + ``PipelineJob.from_pipeline(sources=...)``. Then it re-runs the rebuilt +job and reports what was computed vs read from the store. + +Run: uv run python superpowers/specs/itl-1043/q4_blast_radius.py +""" + +import json +import tempfile +from pathlib import Path + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import Pipeline, PipelineJob +from orcapod.types import CacheMode + +NAME = "j" + + +@function_pod(output_keys="x1") +def inc(x: int) -> int: + return x + 1 + + +@function_pod(output_keys="x1") +def inc_opt(x: int | None) -> int | None: + return None if x is None else x + 1 + + +def sources( + tag_null: bool, data_null: bool +) -> tuple[ArrowTableSource, ArrowTableSource]: + def one(col: str, vals: list[int], sid: str) -> ArrowTableSource: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=tag_null), + pa.field(col, pa.int64(), nullable=data_null), + ] + ) + return ArrowTableSource( + pa.table({"id": [0, 1], col: vals}, schema=schema), + tag_columns=["id"], + source_id=sid, + ) + + return one("x", [1, 2], "a"), one("y", [3, 4], "b") + + +def build(store, tag_null: bool, data_null: bool, mode: CacheMode) -> PipelineJob: + a, b = sources(tag_null, data_null) + job = PipelineJob(name=NAME, store=store) + with job: + out = (inc_opt if data_null else inc).pod(a, label="inc") + Join()(out, b, label="joined") + job.nodes["joined"].attach_databases( + pipeline_database=store.at(NAME), cache_mode=mode + ) + return job + + +def path(node) -> str: + if hasattr(node, "_stored_table_path"): + return "/".join(node._stored_table_path) + if hasattr(node, "_versioned_pipeline_path"): + return "/".join(node._versioned_pipeline_path) + return "-" + + +def snapshot(job) -> dict[str, tuple[str, str, str, str]]: + out = {} + for label, node in job.nodes.items(): + tag, data = node.output_schema() + out[label] = ( + node.pipeline_hash().to_hex()[:8], + node.content_hash().to_hex()[:8], + path(node).split("/")[-2][:15] if path(node) != "-" else "-", + f"tag={dict(tag)} data={dict(data)}", + ) + return out + + +def compare(name: str, ref: dict, other: dict) -> None: + for label in sorted(ref): + r, o = ref[label], other.get(label) + flags = ( + [] + if o is None + else [ + k + for k, i in ( + ("pipeline", 0), + ("content", 1), + ("path", 2), + ("schema", 3), + ) + if r[i] != o[i] + ] + ) + print( + f" {name:14s} {label:7s} {o[:3] if o else None} differs: {flags or 'none'}" + ) + if o and "schema" in flags: + print(f" run {r[3]}\n {name:6s} {o[3]}") + + +for tag_null in (False, True): + for data_null in (False, True): + print(f"=== tag nullable={tag_null} data nullable={data_null}") + store = InMemoryArrowDatabase() + job = build(store, tag_null, data_null, CacheMode.LOG) + job.run() + ref = snapshot(job) + for label, v in ref.items(): + print(f" run {label:7s} {v[:3]}") + with tempfile.TemporaryDirectory() as d: + saved = Path(d) / "job.json" + job.save(saved) + loaded = PipelineJob.load(str(saved), store=store) + compare("load", ref, snapshot(loaded)) + print( + " load: inc rows", + loaded.nodes["inc"].as_table().num_rows, + end="; ", + ) + lj = loaded.nodes["joined"] + lj.attach_databases( + pipeline_database=store.at(NAME), cache_mode=CacheMode.REPLAY + ) + try: + print("joined REPLAY rows", lj.as_table().num_rows) + except RuntimeError as exc: + print("joined REPLAY:", str(exc)[:60]) + + # Rebuilt from code (definitions in hand), same store. + rebuilt = build(store, tag_null, data_null, CacheMode.REPLAY) + compare("rebuilt", ref, snapshot(rebuilt)) + summary = rebuilt.run() + print( + f" rebuilt re-run: inc computed={summary.nodes['inc'].computed} " + f"read={summary.nodes['inc'].reused}; joined REPLAY rows={rebuilt.nodes['joined'].as_table().num_rows}" + ) + + # Pipeline.load of the saved blueprint + from_pipeline with live sources. + pfile = Path(d) / "pipeline.json" + pfile.write_text(json.dumps(json.loads(saved.read_text())["pipeline"])) + pl = Pipeline.load(pfile) + a, b = sources(tag_null, data_null) + try: + fp = PipelineJob.from_pipeline( + pl, store=store, sources={"a": a, "b": b} + ) + compare("from_pipeline", ref, snapshot(fp)) + try: + s = fp.run() + print( + " from_pipeline run:", + {k: (v.computed, v.reused) for k, v in s.nodes.items()}, + ) + except Exception as exc: # noqa: BLE001 + print( + " from_pipeline run:", type(exc).__name__, str(exc)[:100] + ) + except Exception as exc: # noqa: BLE001 + print(" from_pipeline:", type(exc).__name__, str(exc)[:160]) diff --git a/superpowers/specs/itl-1043/q4b_from_pipeline.py b/superpowers/specs/itl-1043/q4b_from_pipeline.py new file mode 100644 index 000000000..078366e21 --- /dev/null +++ b/superpowers/specs/itl-1043/q4b_from_pipeline.py @@ -0,0 +1,85 @@ +"""ITL-1043 Q4b: an operator-only pipeline loaded with ``Pipeline.load`` and run again. + +``Pipeline.load`` rebuilds operators (not function pods), so ``from_pipeline`` with live +sources gives a runnable job. Does its operator node find the cache the first run wrote? + +Run: uv run python superpowers/specs/itl-1043/q4b_from_pipeline.py +""" + +import json +import tempfile +from pathlib import Path + +import pyarrow as pa + +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import Pipeline, PipelineJob +from orcapod.types import CacheMode + +NAME = "j" + + +def sources(tag_null: bool) -> tuple[ArrowTableSource, ArrowTableSource]: + def one(col: str, vals: list[int], sid: str) -> ArrowTableSource: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=tag_null), + pa.field(col, pa.int64(), nullable=False), + ] + ) + return ArrowTableSource( + pa.table({"id": [0, 1], col: vals}, schema=schema), + tag_columns=["id"], + source_id=sid, + ) + + return one("x", [1, 2], "a"), one("y", [3, 4], "b") + + +for tag_null in (False, True): + store = InMemoryArrowDatabase() + a, b = sources(tag_null) + job = PipelineJob(name=NAME, store=store) + with job: + Join()(a, b, label="joined") + node = job.nodes["joined"] + node.attach_databases(pipeline_database=store.at(NAME), cache_mode=CacheMode.LOG) + job.run() + with tempfile.TemporaryDirectory() as d: + saved = Path(d) / "job.json" + job.save(saved) + pfile = Path(d) / "pipeline.json" + pfile.write_text(json.dumps(json.loads(saved.read_text())["pipeline"])) + a2, b2 = sources(tag_null) + fp = PipelineJob.from_pipeline( + Pipeline.load(pfile), store=store, sources={"a": a2, "b": b2} + ) + fnode = fp.nodes["joined"] + fnode.attach_databases( + pipeline_database=store.at(NAME), cache_mode=CacheMode.REPLAY + ) + print(f"tag nullable={tag_null}") + print( + " run path", + node._stored_table_path[-2][:20], + "content", + node.content_hash().to_hex()[:8], + ) + print( + " load path", + fnode._stored_table_path[-2][:20], + "content", + fnode.content_hash().to_hex()[:8], + ) + print( + " slot schema", + dict(fp.nodes["a"].output_schema()[0]), + "bound source", + dict(a2.output_schema()[0]), + ) + try: + print(" REPLAY rows:", fnode.as_table().num_rows) + except Exception as exc: # noqa: BLE001 + print(" REPLAY:", type(exc).__name__, str(exc)[:90]) diff --git a/superpowers/specs/itl-1043/q5_nullability_hash.py b/superpowers/specs/itl-1043/q5_nullability_hash.py new file mode 100644 index 000000000..1d71f361f --- /dev/null +++ b/superpowers/specs/itl-1043/q5_nullability_hash.py @@ -0,0 +1,52 @@ +"""ITL-1043 Q5: is a tag's nullability part of the schema hash today? + +Pipeline hash of sources whose tag column is declared non-nullable, declared nullable +without nulls, declared nullable with a null; and DictSource inferring from values with +and without a None. + +Run: uv run python superpowers/specs/itl-1043/q5_nullability_hash.py +""" + +import pyarrow as pa + +from orcapod.core.sources import ArrowTableSource, DictSource + + +def arrow(nullable: bool, ids: list[int | None]) -> ArrowTableSource: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=nullable), + pa.field("v", pa.int64(), nullable=False), + ] + ) + return ArrowTableSource( + pa.table({"id": ids, "v": [1, 2]}, schema=schema), + tag_columns=["id"], + source_id="s", + ) + + +cases = { + "arrow non-nullable, no nulls": arrow(False, [0, 1]), + "arrow nullable, no nulls": arrow(True, [0, 1]), + "arrow nullable, one null": arrow(True, [0, None]), + "dict inferred, no None": DictSource( + data=[{"id": 0, "v": 1}, {"id": 1, "v": 2}], tag_columns=["id"], source_id="s" + ), + "dict inferred, one None": DictSource( + data=[{"id": 0, "v": 1}, {"id": None, "v": 2}], + tag_columns=["id"], + source_id="s", + ), + "dict declared int | None, no None": DictSource( + data=[{"id": 0, "v": 1}, {"id": 1, "v": 2}], + tag_columns=["id"], + source_id="s", + data_schema={"id": int | None, "v": int}, + ), +} +for name, src in cases.items(): + tag, _ = src.output_schema() + print( + f"{name:36s} tag={dict(tag)!s:28s} pipeline_hash={src.pipeline_hash().to_hex()[:12]}" + ) From 85667e1d2a33da9593e7034e3be0b9d5a0a55615 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:12:16 -0800 Subject: [PATCH 02/16] chore(itl-1043): type-clean the investigation scripts Co-Authored-By: Claude Opus 5.5 --- superpowers/specs/itl-1043/q2_where_rows_go.py | 1 + superpowers/specs/itl-1043/q4_blast_radius.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/superpowers/specs/itl-1043/q2_where_rows_go.py b/superpowers/specs/itl-1043/q2_where_rows_go.py index aadca2fa4..3bb08abbd 100644 --- a/superpowers/specs/itl-1043/q2_where_rows_go.py +++ b/superpowers/specs/itl-1043/q2_where_rows_go.py @@ -53,6 +53,7 @@ def src(col: str, sid: str) -> ArrowTableSource: raw = store.at(NAME).get_all_records( lnode._stored_table_path, record_id_column=node.HASH_COLUMN_NAME ) +assert raw is not None, "the run stored no table at the loaded path" stored = set(pc.unique(raw.column(constants.NODE_CONTENT_HASH_COL)).to_pylist()) print("stored rows:", raw.num_rows) print("stored __node_content_hash:", stored) diff --git a/superpowers/specs/itl-1043/q4_blast_radius.py b/superpowers/specs/itl-1043/q4_blast_radius.py index d1d018ed4..0e396bed1 100644 --- a/superpowers/specs/itl-1043/q4_blast_radius.py +++ b/superpowers/specs/itl-1043/q4_blast_radius.py @@ -88,7 +88,11 @@ def snapshot(job) -> dict[str, tuple[str, str, str, str]]: return out -def compare(name: str, ref: dict, other: dict) -> None: +def compare( + name: str, + ref: dict[str, tuple[str, str, str, str]], + other: dict[str, tuple[str, str, str, str]], +) -> None: for label in sorted(ref): r, o = ref[label], other.get(label) flags = ( From 51f3ab575aea5909b2af4b0fa8c0acfbf28c87da Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:15:29 -0800 Subject: [PATCH 03/16] test(operator_node): loaded operator node reads under its run's path, never shows stored rows as none (failing, ITL-1043) Co-Authored-By: Claude Opus 5.5 --- ...026-11-12-itl-1043-loaded-operator-node.md | 157 ++++++++++++++++++ .../loaded_job/test_loaded_operator_cache.py | 105 ++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md create mode 100644 tests/test_pipeline/loaded_job/test_loaded_operator_cache.py diff --git a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md new file mode 100644 index 000000000..3b692ebf0 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md @@ -0,0 +1,157 @@ +# ITL-1043 Loaded Operator Node Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A loaded operator node looks for its stored rows under the path its run wrote, and +never presents stored rows it cannot identify as zero rows. + +**Architecture:** Investigation in `superpowers/specs/itl-1043/FINDINGS.md`. Option 1 (approved +by the coordinator, November 12, 2026; no saved-format change, no hash, path or golden change): +an operator node rebuilt from a descriptor WITH its operator takes its pipeline hash from the +descriptor's saved `pipeline_hash` (the run's), as read-only stubs and function stubs already +do; a loaded node in REPLAY whose content-hash filter keeps 0 of N>0 stored rows raises. +Options 2 and 3 (saved operator run content hash; saved top-level nullability, riding the +`0.2.0` format of ITL-1026) are HELD until the coordinator confirms the developer's decision. + +**Tech Stack:** Python, PyArrow, pytest, `uv run`. + +## Status / notes for a successor + +- 2026-11-12: investigation done and reported (commit 5b07349d); option 1 approved; options + 2-3 to be prepared, not pushed until confirmed. ITL-1026 merges first; option 3 then adds + fields to the same `0.2.0` format. +- Pre-existing contradiction, not changed here: `OperatorJobNode.head` says "a loaded node + never raises", but `_head_from_store` raises when a loaded REPLAY node's table is absent. The + new 0-of-N case behaves like the absent table (raises from `head` too). Report it. + +## Files + +- Modify: `src/orcapod/core/nodes/operator_node.py` + - `OperatorNodeBase.pipeline_hash` (line ~192): use `_stored_pipeline_hash` whenever set. + - `OperatorNode.from_descriptor` full mode (~475) and `OperatorJobNode.from_descriptor` full + mode (~789): set `_stored_pipeline_hash` from the descriptor. + - `OperatorJobNode._read_stored_records` (~1020): split into + `_read_current_context_records` (no content-hash filter) + filter. + - `OperatorJobNode._load_cached_stream_from_db` (~1066): raise for a loaded node when the + filter keeps 0 of N>0 rows. +- Create: `tests/test_pipeline/loaded_job/test_loaded_operator_cache.py` +- Create: `changelog.d/ITL-1043.md` +- Create: `design_issues/ITL-1043-LJ1.md` (lost nullability in the saved schema; open until + option 3 lands or is declined) + +### Task 1: failing tests (public interface) + +**Files:** Create `tests/test_pipeline/loaded_job/test_loaded_operator_cache.py` + +- [ ] **Step 1: Write the tests** + +Job: `Join` of two `ArrowTableSource`s (tag `id`, data `a` / `b`), declared fields with the +tag nullable or not and the data nullable or not; operator node attached in `CacheMode.LOG` +(`node.attach_databases`, the public way); `job.run()`; `job.save`; `PipelineJob.load(path, +store=store)`; `loaded.nodes["joined"].attach_databases(store.at(NAME), CacheMode.REPLAY)`. +Stores: `InMemoryArrowDatabase`, `DeltaTableDatabase(tmp_path / "store")`. + +```python +def test_loaded_operator_node_keeps_the_pipeline_hash_and_path_of_its_run(...): + assert loaded_node.pipeline_hash() == run_node.pipeline_hash() + assert loaded_node.node_identity_path == run_node.node_identity_path + +@pytest.mark.parametrize("read", ["as_table", "iter_data", "head"]) +def test_loaded_replay_node_raises_rather_than_show_no_rows(...): + # In-memory sources are not saved, so the loaded node cannot reproduce the content + # hash its run stored the rows under: the 2 stored rows must not read as 0 rows. + with pytest.raises(RuntimeError, match=r"'joined'.*2 stored rows.*none"): + ... +``` + +- [ ] **Step 2: Run, verify they fail** + +`uv run pytest tests/test_pipeline/loaded_job/test_loaded_operator_cache.py -q` +Expected: nullable cases fail the hash test; every raise test fails (0 rows returned, or +"cache table is absent" for nullable cases). + +- [ ] **Step 3: Commit (WIP, failing tests)** + +### Task 2: loaded operator node takes the saved pipeline hash + +- [ ] **Step 1: Implement** + +```python +# OperatorNodeBase.pipeline_hash +if self._stored_pipeline_hash is not None: + return ContentHash.from_string(self._stored_pipeline_hash) +return super().pipeline_hash(hasher) +``` + +and in both full-mode branches of `from_descriptor`: + +```python +node._stored_pipeline_hash = descriptor.get("pipeline_hash") +``` + +(`_stored_pipeline_hash` is `None` for every node built by `__init__`, so a live node is +unchanged.) + +- [ ] **Step 2: Run the hash test; expect PASS.** + +### Task 3: 0 of N stored rows raises for a loaded node + +- [ ] **Step 1: Implement** + +```python +def _read_current_context_records(self) -> "pa.Table | None": + # body of today's _read_stored_records, returning before the content-hash filter + +def _read_stored_records(self) -> "pa.Table | None": + records = self._read_current_context_records() + return None if records is None else self._filter_by_content_hash(records) +``` + +In `_load_cached_stream_from_db`, read `_read_current_context_records()`, keep the absent-table +raise, filter, and: + +```python +if self._loaded and records.num_rows == 0 and stored.num_rows > 0: + raise RuntimeError( + f"OperatorJobNode {self.label!r}: its cache table ({path}) holds " + f"{stored.num_rows} stored rows, none of them under this node's content hash " + f"({self.content_hash().to_string()}). They were stored by other runs, or by " + "this job's run under a content hash the loaded job cannot reproduce (an input " + "source whose data was not saved with the job). Build the job from code and " + "attach the node with CacheMode.REPLAY to read them." + ) +``` + +- [ ] **Step 2: Run the new tests; expect PASS.** +- [ ] **Step 3: Run `tests/test_pipeline` and `tests/test_hashing/golden`; expect no change.** +- [ ] **Step 4: ruff format, ruff check, basedpyright (exit code 0). Commit, push.** + +### Task 4: changelog fragment, design issue entry, docs + +- [ ] `changelog.d/ITL-1043.md` under `### Fixed`. +- [ ] `design_issues/ITL-1043-LJ1.md`: the saved schema loses top-level nullability (status + `open`; option 3 pending the developer). +- [ ] `docs/concepts/function-pods.md:688-693`: state that a loaded operator node raises when + it cannot tell its run's stored rows. +- [ ] `uv run python scripts/assemble_changelog.py --check`, + `uv run python scripts/design_issues.py --check`. + +### Task 5 (HELD until the coordinator confirms): options 2 and 3 + +- Option 2: `PipelineJob.save` writes the run content hash of every `OperatorJobNode` into + `run.node_job_content_hashes`; `load` passes it to `OperatorJobNode.from_descriptor`, which + sets `_stored_content_hash`; `content_hash()` uses it whenever set (as `pipeline_hash`). +- Option 3: `serialize_schema` callers also write the names of top-level nullable columns as a + sibling list (`output_schema.nullable`, `source_config.nullable_tag` / `nullable_data`); + `deserialize_output_schema` / `Pipeline.load` restore `T | None` for them. `0.1.0` files load + as today (lossy); operator rows of such files raise per Task 3. +- Round-trip test: every node of a job with nullable and non-null tag and data columns has the + same pipeline hash, content hash, `node_identity_path` and `output_schema()` before save and + after load; and `Pipeline.load` + `PipelineJob.from_pipeline(sources=)` reads the + operator node's rows in REPLAY. + +### Task 6: finish + +- [ ] Merge origin/main (no rebase), full `uv run pytest tests/` (at most 2 test processes), + push, open PR against `main` ("Fixes ITL-1043" only if options 2-3 land; otherwise "Part of + ITL-1043"), link with `orca linear attach` and `orca worktree set --pr`. diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py new file mode 100644 index 000000000..58dc86d9b --- /dev/null +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -0,0 +1,105 @@ +"""Reading the stored cache of an operator node of a job loaded with ``PipelineJob.load``. + +ITL-1043. The job joins two in-memory ``ArrowTableSource``s and stores the join's output +(``CacheMode.LOG``). In-memory sources are not saved with a job, so the loaded node cannot +reproduce the content hash its run stored the rows under. It must still look for them +under the path its run wrote, also when a column is nullable, and it must not present the +stored rows as zero rows. +""" + +from __future__ import annotations + +import pathlib +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob +from orcapod.core.operators import Join +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.sources import ArrowTableSource +from orcapod.types import CacheMode + +JOB_NAME = "loaded_operator_cache" +NULLABILITY = { + "non_null": (False, False), + "nullable_tag": (True, False), + "nullable_data": (False, True), + "nullable_tag_and_data": (True, True), +} + + +def _source(column: str, tag_nullable: bool, data_nullable: bool) -> ArrowTableSource: + """A two-row source with tag ``id`` and data ``column``, nulls allowed as declared.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=tag_nullable), + pa.field(column, pa.int64(), nullable=data_nullable), + ] + ) + table = pa.table({"id": [0, 1], column: [1, 2]}, schema=schema) + return ArrowTableSource(table, tag_columns=["id"], source_id=f"src_{column}") + + +def _store(kind: str, tmp_path: pathlib.Path) -> ArrowDatabaseProtocol: + """An empty store of the given kind.""" + if kind == "delta": + return DeltaTableDatabase(base_path=tmp_path / "store") + return InMemoryArrowDatabase() + + +def _run_and_load( + store: ArrowDatabaseProtocol, nullability: str, tmp_path: pathlib.Path +) -> tuple[Any, Any]: + """Run the job with the join stored, save and load it; the run and loaded joins.""" + tag_nullable, data_nullable = NULLABILITY[nullability] + job = PipelineJob(name=JOB_NAME, store=store) + with job: + left = _source("a", tag_nullable, data_nullable) + right = _source("b", tag_nullable, data_nullable) + _ = Join()(left, right, label="joined") + run_node = job.nodes["joined"] + # OperatorNode.attach_databases is the public way to turn on LOG / REPLAY mode. + run_node.attach_databases( + pipeline_database=store.at(JOB_NAME), cache_mode=CacheMode.LOG + ) + _ = job.run() + saved = tmp_path / "job.json" + job.save(saved) + loaded_node = PipelineJob.load(str(saved), store=store).nodes["joined"] + loaded_node.attach_databases( + pipeline_database=store.at(JOB_NAME), cache_mode=CacheMode.REPLAY + ) + return run_node, loaded_node + + +@pytest.mark.parametrize("store_kind", ["memory", "delta"]) +@pytest.mark.parametrize("nullability", list(NULLABILITY)) +def test_loaded_operator_node_keeps_the_pipeline_hash_and_path_of_its_run( + store_kind: str, nullability: str, tmp_path: pathlib.Path +) -> None: + """The loaded node's pipeline hash and table path are its run's, nullable or not.""" + run_node, loaded_node = _run_and_load( + _store(store_kind, tmp_path), nullability, tmp_path + ) + assert loaded_node.pipeline_hash() == run_node.pipeline_hash() + assert loaded_node.node_identity_path == run_node.node_identity_path + + +@pytest.mark.parametrize("read", ["as_table", "iter_data", "head"]) +@pytest.mark.parametrize("store_kind", ["memory", "delta"]) +@pytest.mark.parametrize("nullability", list(NULLABILITY)) +def test_loaded_replay_node_raises_rather_than_show_no_rows( + store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path +) -> None: + """Two stored rows the loaded node cannot identify as its run's never read as none.""" + _, loaded_node = _run_and_load(_store(store_kind, tmp_path), nullability, tmp_path) + with pytest.raises( + RuntimeError, + match=r"'joined': its cache table \(.*\) holds 2 stored rows, none of them", + ): + result = getattr(loaded_node, read)() + if read == "iter_data": + _ = list(result) From c7e3e4d66f0f46b5de239991619552185eddc7e2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:20:05 -0800 Subject: [PATCH 04/16] fix(operator_node): a loaded operator node reads under its run's path and raises instead of showing stored rows as none (ITL-1043) A node rebuilt from a saved descriptor with its operator took a pipeline hash recomputed from inputs rebuilt from the saved schemas, which lose a column's nullability, so with a nullable column it looked for its cache under another table path. It now returns the descriptor's saved pipeline hash, the run's. A loaded REPLAY node whose stored rows all carry another content hash raised nothing and showed zero rows; it now raises. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 56 ++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 9f22703ec..388372f56 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -190,8 +190,14 @@ def content_hash(self, hasher=None) -> ContentHash: return super().content_hash(hasher) def pipeline_hash(self, hasher=None) -> ContentHash: - """Return the pipeline hash, using stored value in read-only mode.""" - if self._operator is None and self._stored_pipeline_hash is not None: + """Return the pipeline hash; a node rebuilt from a descriptor returns the saved one. + + A node rebuilt with its operator still returns the saved value: its inputs are + rebuilt from the saved schemas, which do not keep every property that the run's + inputs hashed (ITL-1043: a column's nullability), so recomputing could give + another value and another table path than the run's. + """ + if self._stored_pipeline_hash is not None: return ContentHash.from_string(self._stored_pipeline_hash) return super().pipeline_hash(hasher) @@ -481,6 +487,8 @@ def from_descriptor( table_scope=table_scope, ) node._descriptor = descriptor + # The run's pipeline hash, not one recomputed from the rebuilt inputs. + node._stored_pipeline_hash = descriptor.get("pipeline_hash") node._load_status = LoadStatus.FULL return node @@ -800,6 +808,9 @@ def from_descriptor( cache_mode=cache_mode, ) node._descriptor = descriptor + # The run's pipeline hash, not one recomputed from the rebuilt inputs: it + # names the table the run's rows are stored in. + node._stored_pipeline_hash = descriptor.get("pipeline_hash") node._load_status = LoadStatus.FULL node._loaded = True return node @@ -1030,6 +1041,21 @@ def _read_stored_records(self) -> "pa.Table | None": still present and without the ``__data_context`` column, or ``None`` when the table does not exist. """ + records = self._read_current_context_records() + if records is None: + return None + return self._filter_by_content_hash(records) + + def _read_current_context_records(self) -> "pa.Table | None": + """Read every row of this node's stored table written under the current data context. + + The rows of every run that shares the table, before ``_filter_by_content_hash`` + keeps this node's. Rows of another data context are left out and reported once. + + Returns: + The rows, with ``HASH_COLUMN_NAME`` and ``NODE_CONTENT_HASH_COL`` and without + the ``__data_context`` column, or ``None`` when the table does not exist. + """ assert self._pipeline_database is not None self._account_for_stale_tables() records = self._pipeline_database.get_all_records( @@ -1045,7 +1071,7 @@ def _read_stored_records(self) -> "pa.Table | None": self._pipeline_database.base_path + self._stored_table_path, stored_rows - records.num_rows, ) - return self._filter_by_content_hash(records.drop([constants.DATA_CONTEXT_COL])) + return records.drop([constants.DATA_CONTEXT_COL]) def _make_empty_table(self) -> "pa.Table": """Build a zero-row PyArrow table matching this node's full output schema. @@ -1081,13 +1107,33 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": (LOG/OFF modes may have stale historical records in the DB). Raises: - RuntimeError: If a loaded REPLAY node's cache table is absent. + RuntimeError: If a loaded REPLAY node's cache table is absent, or holds rows + none of which carries the node's content hash: a loaded node cannot + compute, so showing no rows would make stored results look dropped. """ if self._pipeline_database is None: return None if self._cache_mode != CacheMode.REPLAY: return None - records = self._read_stored_records() + stored = self._read_current_context_records() + records = None if stored is None else self._filter_by_content_hash(stored) + if ( + self._loaded + and stored is not None + and records is not None + and records.num_rows == 0 + and stored.num_rows > 0 + ): + raise RuntimeError( + f"OperatorJobNode {self.label!r}: its cache table " + f"({'/'.join(self._stored_table_path)}) holds {stored.num_rows} stored " + "rows, none of them under this node's content hash " + f"({self.content_hash().to_string()}). They were stored by other runs, " + "or by this job's run under a content hash the loaded job cannot " + "reproduce (an input source whose data was not saved with the job). " + "Build the job from code and attach the node with CacheMode.REPLAY to " + "read them." + ) if records is None: if self._loaded: raise RuntimeError( From 19f424eb168ce8d775551d3e189df28108d1750d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:20:55 -0800 Subject: [PATCH 05/16] docs(itl-1043): changelog fragment, design issue LJ1, loaded operator readback docs Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1043.md | 14 ++++++++++++++ design_issues/ITL-1043-LJ1.md | 33 +++++++++++++++++++++++++++++++++ docs/concepts/function-pods.md | 6 +++++- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ITL-1043.md create mode 100644 design_issues/ITL-1043-LJ1.md diff --git a/changelog.d/ITL-1043.md b/changelog.d/ITL-1043.md new file mode 100644 index 000000000..9d960d923 --- /dev/null +++ b/changelog.d/ITL-1043.md @@ -0,0 +1,14 @@ +### Fixed + +- **A loaded operator node looks for its stored rows under the path its run wrote.** An + operator node of a job loaded with `PipelineJob.load` recomputed its pipeline hash from + inputs rebuilt from the saved schemas, which do not keep a column's nullability. With a + nullable tag or data column (the default for a table built with `pa.table` or from + pandas) it looked under another table path and reported "the cache table is absent". It + now takes the pipeline hash saved with the job, the run's. No hash or stored path of a + running job changes. (ITL-1043) +- **A loaded operator node in `CacheMode.REPLAY` no longer shows stored rows as none.** When + its cache table holds rows but none carries the node's content hash (a job whose + in-memory input sources were not saved cannot reproduce the content hash its run stored + them under), `as_table()`, `iter_data()` and `head()` raise a `RuntimeError` saying how + many rows are stored and how to read them, instead of returning zero rows. (ITL-1043) diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md new file mode 100644 index 000000000..5bd24d313 --- /dev/null +++ b/design_issues/ITL-1043-LJ1.md @@ -0,0 +1,33 @@ +## `src/orcapod/pipeline/job.py` — reading a loaded `PipelineJob` + +### LJ1 — a saved job loses column nullability, and does not record the run content hash of an operator node +**Status:** in progress +**Severity:** high +**Issue:** ITL-1043 + +Found November 12, 2026. Evidence and scripts: `superpowers/specs/itl-1043/FINDINGS.md`. + +- `serialize_schema` (`src/orcapod/pipeline/serialization.py`) writes each column as the Arrow + type of its Python type. An Arrow `DataType` has no field nullability, so `int | None` and + `int` are both saved as `"int64"` (nested nullability survives in the type text; only the + top level is lost). A loaded job's source nodes therefore have non-null schemas, a + different pipeline hash from the run's, and every loaded node's `output_schema()` drops + `| None`. An operator node rebuilt with its operator recomputed its pipeline hash from them + and looked for its cache under another table path; `Pipeline.load` followed by + `PipelineJob.from_pipeline(sources=...)` still does, so REPLAY reads no rows there and a + LOG run writes a second table under the wrong path. +- `PipelineJob.save` records the run content hash of function nodes only + (`run.node_job_content_hashes`). A loaded operator node over in-memory sources (not saved) + cannot reproduce the content hash its rows are stored under, so it cannot select them. +- The operator's `cache_mode` is not saved (ITL-977). + +The running node's hashes are the intended ones: nullability is part of the schema hash +(`docs/reference/hashing.md`; golden `datagram.json` `arrow_source.int64` against +`arrow_source.int64_nullable`, `schema.json` `multi_field.nullable_type`). No live hash, +stored path or golden value is wrong; only what a loaded job reconstructs. + +**Partial fix (ITL-1043):** a loaded operator node takes the pipeline hash saved in its +descriptor, the run's, so it reads the table its run wrote; and in REPLAY it raises, instead +of returning zero rows, when that table holds rows none of which carries its content hash. +Saving nullability and the operator run content hash in the job file is a saved-format +decision pending with the developer. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index ec8e69073..9444c002e 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -688,7 +688,11 @@ as before. returns them; sort them if order matters. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. A loaded operator without a readable cache raises from `as_table()` and `iter_data()`. - Read its input nodes instead (an in-memory input source must be bound first, see below). + So does a loaded operator attached in `CacheMode.REPLAY` whose stored rows it cannot tell + apart as its run's, which happens when an input source's data was not saved with the job: + it never shows them as zero rows. Read its input nodes instead (an in-memory input source + must be bound first, see below), or build the job from your code and attach the operator + node in `CacheMode.REPLAY`. Saved descriptors do not yet preserve all runtime identity and cache configuration needed for operator readback without reconstruction (follow-up ITL-977). - Sources that read files, such as `CSVSource` and `DeltaTableSource`, are rebuilt from their From 128e8b7a2f648454022416c5b12f11e53fc0d3c1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:23:30 -0800 Subject: [PATCH 06/16] test(pipeline): save/load round-trips every node's identity and schema; loaded operator reads its rows (failing, ITL-1043) Co-Authored-By: Claude Opus 5.5 --- .../loaded_job/test_loaded_operator_cache.py | 234 +++++++++++++++--- 1 file changed, 202 insertions(+), 32 deletions(-) diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 58dc86d9b..07da9aee0 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -1,23 +1,29 @@ -"""Reading the stored cache of an operator node of a job loaded with ``PipelineJob.load``. +"""Saving and loading a job keeps every node's identity; a loaded operator reads its cache. -ITL-1043. The job joins two in-memory ``ArrowTableSource``s and stores the join's output -(``CacheMode.LOG``). In-memory sources are not saved with a job, so the loaded node cannot -reproduce the content hash its run stored the rows under. It must still look for them -under the path its run wrote, also when a column is nullable, and it must not present the -stored rows as zero rows. +ITL-1043. The job joins a function node over one in-memory ``ArrowTableSource`` with a +second one and stores the join's output (``CacheMode.LOG``). Columns are declared nullable +or not. A job file records each column's nullability and the run content hash of every +operator node, so the loaded job reproduces each node's pipeline hash, content hash, table +path and output schema, and a loaded operator node attached in ``CacheMode.REPLAY`` reads +the rows its run stored. A file without those fields (``0.1.0``, or a ``0.2.0`` file written +before them) loads as before, lossy: the operator node still looks under its run's path, +and raises rather than show the stored rows as none. """ from __future__ import annotations +import json import pathlib +from collections.abc import Callable from typing import Any import pyarrow as pa import pytest -from orcapod import PipelineJob +from orcapod import PipelineJob, function_pod from orcapod.core.operators import Join from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import Pipeline from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.sources import ArrowTableSource from orcapod.types import CacheMode @@ -29,6 +35,19 @@ "nullable_data": (False, True), "nullable_tag_and_data": (True, True), } +STORES = ["memory", "delta"] + + +@function_pod(output_keys="a1") +def inc(a: int) -> int: + """Add one.""" + return a + 1 + + +@function_pod(output_keys="a1") +def inc_nullable(a: int | None) -> int | None: + """Add one to a value that may be missing.""" + return None if a is None else a + 1 def _source(column: str, tag_nullable: bool, data_nullable: bool) -> ArrowTableSource: @@ -43,6 +62,15 @@ def _source(column: str, tag_nullable: bool, data_nullable: bool) -> ArrowTableS return ArrowTableSource(table, tag_columns=["id"], source_id=f"src_{column}") +def _sources(nullability: str) -> dict[str, ArrowTableSource]: + """The job's two sources, keyed by source node name.""" + tag_nullable, data_nullable = NULLABILITY[nullability] + return { + "src_a": _source("a", tag_nullable, data_nullable), + "src_b": _source("b", tag_nullable, data_nullable), + } + + def _store(kind: str, tmp_path: pathlib.Path) -> ArrowDatabaseProtocol: """An empty store of the given kind.""" if kind == "delta": @@ -50,56 +78,198 @@ def _store(kind: str, tmp_path: pathlib.Path) -> ArrowDatabaseProtocol: return InMemoryArrowDatabase() -def _run_and_load( - store: ArrowDatabaseProtocol, nullability: str, tmp_path: pathlib.Path -) -> tuple[Any, Any]: - """Run the job with the join stored, save and load it; the run and loaded joins.""" - tag_nullable, data_nullable = NULLABILITY[nullability] +def _run( + store: ArrowDatabaseProtocol, nullability: str, with_function: bool = True +) -> PipelineJob: + """Run the job with the join stored (``LOG``).""" + _, data_nullable = NULLABILITY[nullability] + sources = _sources(nullability) job = PipelineJob(name=JOB_NAME, store=store) with job: - left = _source("a", tag_nullable, data_nullable) - right = _source("b", tag_nullable, data_nullable) - _ = Join()(left, right, label="joined") - run_node = job.nodes["joined"] + left: Any = sources["src_a"] + if with_function: + pod = inc_nullable if data_nullable else inc + left = pod.pod(left, label="inc") + _ = Join()(left, sources["src_b"], label="joined") # OperatorNode.attach_databases is the public way to turn on LOG / REPLAY mode. - run_node.attach_databases( + job.nodes["joined"].attach_databases( pipeline_database=store.at(JOB_NAME), cache_mode=CacheMode.LOG ) _ = job.run() + return job + + +def _save( + job: PipelineJob, + tmp_path: pathlib.Path, + edit: Callable[[dict[str, Any]], None] | None = None, +) -> pathlib.Path: + """Save ``job``, optionally rewriting the saved file with ``edit``.""" saved = tmp_path / "job.json" job.save(saved) - loaded_node = PipelineJob.load(str(saved), store=store).nodes["joined"] - loaded_node.attach_databases( + if edit is not None: + data = json.loads(saved.read_text()) + edit(data) + saved.write_text(json.dumps(data)) + return saved + + +def _load_replaying(saved: pathlib.Path, store: ArrowDatabaseProtocol) -> PipelineJob: + """Load the job and attach its operator node in ``REPLAY`` mode.""" + loaded = PipelineJob.load(str(saved), store=store) + loaded.nodes["joined"].attach_databases( pipeline_database=store.at(JOB_NAME), cache_mode=CacheMode.REPLAY ) - return run_node, loaded_node + return loaded -@pytest.mark.parametrize("store_kind", ["memory", "delta"]) +def _sorted(table: pa.Table) -> pa.Table: + """``table`` with its columns in name order and its rows in ``id`` order.""" + return table.select(sorted(table.column_names)).sort_by("id") + + +def _drop_saved_nullability(data: dict[str, Any]) -> None: + """Remove the recorded nullability, as in a file written before it existed.""" + for descriptor in data["pipeline"]["nodes"].values(): + del descriptor["output_schema"]["nullable"] + if "source_config" in descriptor: + del descriptor["source_config"]["nullable"] + + +def _drop_saved_operator_content_hash(data: dict[str, Any]) -> None: + """Remove the join's run content hash, as in a file written before it existed.""" + del data["run"]["node_job_content_hashes"]["joined"] + + +# --------------------------------------------------------------------------- +# A file with the fields: every node round-trips, the operator reads its rows +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("store_kind", STORES) @pytest.mark.parametrize("nullability", list(NULLABILITY)) -def test_loaded_operator_node_keeps_the_pipeline_hash_and_path_of_its_run( +def test_every_node_keeps_its_identity_and_schema_through_save_and_load( store_kind: str, nullability: str, tmp_path: pathlib.Path ) -> None: - """The loaded node's pipeline hash and table path are its run's, nullable or not.""" - run_node, loaded_node = _run_and_load( - _store(store_kind, tmp_path), nullability, tmp_path + """Pipeline hash, content hash, table path and output schema are the run's.""" + store = _store(store_kind, tmp_path) + job = _run(store, nullability) + loaded = PipelineJob.load(str(_save(job, tmp_path)), store=store) + # In-memory sources are not saved: a source node's content hash is its bound + # source's, so bind the same sources before comparing. + loaded.bind(sources=_sources(nullability)) + assert set(loaded.nodes) == set(job.nodes) + for label, run_node in job.nodes.items(): + loaded_node = loaded.nodes[label] + assert loaded_node.pipeline_hash() == run_node.pipeline_hash(), label + assert loaded_node.content_hash() == run_node.content_hash(), label + assert loaded_node.output_schema() == run_node.output_schema(), label + if label in ("inc", "joined"): + assert loaded_node.node_identity_path == run_node.node_identity_path + + +@pytest.mark.parametrize("read", ["as_table", "iter_data", "head"]) +@pytest.mark.parametrize("store_kind", STORES) +@pytest.mark.parametrize("nullability", list(NULLABILITY)) +def test_loaded_replay_operator_reads_the_rows_its_run_stored( + store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path +) -> None: + """A loaded operator node in REPLAY reads exactly the rows read before saving.""" + store = _store(store_kind, tmp_path) + job = _run(store, nullability) + expected = _sorted(job.nodes["joined"].as_table()) + node = _load_replaying(_save(job, tmp_path), store).nodes["joined"] + if read == "as_table": + assert _sorted(node.as_table()).equals(expected) + elif read == "iter_data": + assert len(list(node.iter_data())) == expected.num_rows + else: + assert _sorted(node.head().table).equals(expected) + + +@pytest.mark.parametrize("nullability", list(NULLABILITY)) +def test_a_pipeline_loaded_and_bound_to_its_sources_reuses_the_operator_cache( + nullability: str, tmp_path: pathlib.Path +) -> None: + """``Pipeline.load`` + ``from_pipeline(sources=...)`` reads and writes the run's table.""" + store = InMemoryArrowDatabase() + # Pipeline.load rebuilds operators but not function pods: an operator-only pipeline. + job = _run(store, nullability, with_function=False) + expected = _sorted(job.nodes["joined"].as_table()) + saved = tmp_path / "pipeline.json" + job.pipeline.save(saved) + tables_after_run = set(store.at(JOB_NAME).list_table_paths()) + + def rebuilt(cache_mode: CacheMode) -> PipelineJob: + """The saved pipeline bound to the same sources, its join in ``cache_mode``.""" + again = PipelineJob.from_pipeline( + Pipeline.load(saved), store=store, sources=_sources(nullability) + ) + again.nodes["joined"].attach_databases( + pipeline_database=store.at(JOB_NAME), cache_mode=cache_mode + ) + return again + + assert _sorted(rebuilt(CacheMode.REPLAY).nodes["joined"].as_table()).equals( + expected ) - assert loaded_node.pipeline_hash() == run_node.pipeline_hash() - assert loaded_node.node_identity_path == run_node.node_identity_path + _ = rebuilt(CacheMode.LOG).run() + assert set(store.at(JOB_NAME).list_table_paths()) == tables_after_run + + +# --------------------------------------------------------------------------- +# A file without the fields: lossy, but never stored rows shown as none +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("store_kind", STORES) +@pytest.mark.parametrize("nullability", list(NULLABILITY)) +def test_without_saved_nullability_the_operator_still_reads_its_runs_table( + store_kind: str, nullability: str, tmp_path: pathlib.Path +) -> None: + """The saved pipeline hash, not the lossy schemas, names the operator's table.""" + store = _store(store_kind, tmp_path) + job = _run(store, nullability) + saved = _save(job, tmp_path, _drop_saved_nullability) + node = _load_replaying(saved, store).nodes["joined"] + assert node.pipeline_hash() == job.nodes["joined"].pipeline_hash() + assert node.node_identity_path == job.nodes["joined"].node_identity_path + assert _sorted(node.as_table()).equals(_sorted(job.nodes["joined"].as_table())) @pytest.mark.parametrize("read", ["as_table", "iter_data", "head"]) -@pytest.mark.parametrize("store_kind", ["memory", "delta"]) +@pytest.mark.parametrize("store_kind", STORES) @pytest.mark.parametrize("nullability", list(NULLABILITY)) -def test_loaded_replay_node_raises_rather_than_show_no_rows( +def test_without_the_run_content_hash_the_loaded_operator_raises( store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path ) -> None: - """Two stored rows the loaded node cannot identify as its run's never read as none.""" - _, loaded_node = _run_and_load(_store(store_kind, tmp_path), nullability, tmp_path) + """Stored rows the loaded node cannot identify as its run's never read as none.""" + store = _store(store_kind, tmp_path) + job = _run(store, nullability) + + def drop_both(data: dict[str, Any]) -> None: + """Remove every field this fix adds.""" + _drop_saved_nullability(data) + _drop_saved_operator_content_hash(data) + + node = _load_replaying(_save(job, tmp_path, drop_both), store).nodes["joined"] with pytest.raises( RuntimeError, match=r"'joined': its cache table \(.*\) holds 2 stored rows, none of them", ): - result = getattr(loaded_node, read)() + result = getattr(node, read)() if read == "iter_data": _ = list(result) + + +def test_an_empty_nullable_list_means_every_column_is_non_null( + tmp_path: pathlib.Path, +) -> None: + """A recorded empty list is a fact; only an absent field is unknown.""" + store = InMemoryArrowDatabase() + job = _run(store, "non_null") + data = json.loads(_save(job, tmp_path).read_text()) + joined = next( + d for d in data["pipeline"]["nodes"].values() if d["label"] == "joined" + ) + assert joined["output_schema"]["nullable"] == {"tag": [], "data": []} From 8546d7434001874236f422b53c254f08569fce86 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:31:51 -0800 Subject: [PATCH 07/16] feat(pipeline): saved jobs record column nullability and operator run content hashes (ITL-1043) A saved schema wrote each column's Arrow type, which carries no field nullability, so int | None loaded as int and a loaded node's pipeline hash and schema differed from its run's. The node descriptor's output_schema, a source node's source_config and a source's identity config now record the nullable tag and data columns, restored as T | None through the type converter. The job file records the run content hash of operator nodes as it did for function nodes, so a loaded operator node in REPLAY reads its run's rows. A file without these records loads lossy as before, and re-saving it records nothing it did not have. No hash, stored path or golden value of a running job changes. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1043.md | 39 ++-- design_issues/ITL-1043-LJ1.md | 18 +- docs/concepts/function-pods.md | 23 ++- src/orcapod/core/nodes/operator_node.py | 32 +++- src/orcapod/core/sources/base.py | 6 +- src/orcapod/pipeline/graph.py | 73 +++++--- src/orcapod/pipeline/job.py | 15 +- src/orcapod/pipeline/serialization.py | 169 ++++++++++++++++-- ...026-11-12-itl-1043-loaded-operator-node.md | 11 +- .../loaded_job/test_loaded_operator_cache.py | 65 ++++++- 10 files changed, 381 insertions(+), 70 deletions(-) diff --git a/changelog.d/ITL-1043.md b/changelog.d/ITL-1043.md index 9d960d923..458dfd8d4 100644 --- a/changelog.d/ITL-1043.md +++ b/changelog.d/ITL-1043.md @@ -1,14 +1,29 @@ ### Fixed -- **A loaded operator node looks for its stored rows under the path its run wrote.** An - operator node of a job loaded with `PipelineJob.load` recomputed its pipeline hash from - inputs rebuilt from the saved schemas, which do not keep a column's nullability. With a - nullable tag or data column (the default for a table built with `pa.table` or from - pandas) it looked under another table path and reported "the cache table is absent". It - now takes the pipeline hash saved with the job, the run's. No hash or stored path of a - running job changes. (ITL-1043) -- **A loaded operator node in `CacheMode.REPLAY` no longer shows stored rows as none.** When - its cache table holds rows but none carries the node's content hash (a job whose - in-memory input sources were not saved cannot reproduce the content hash its run stored - them under), `as_table()`, `iter_data()` and `head()` raise a `RuntimeError` saying how - many rows are stored and how to read them, instead of returning zero rows. (ITL-1043) +#### A loaded job keeps every node's identity, and a loaded operator reads its stored rows (ITL-1043) + +A job saved and loaded with `PipelineJob.load` could not read the stored output of an operator +node. With a nullable tag or data column (the default for a table built with `pa.table` or +from pandas) the loaded node looked for its cache under another table path and reported "the +cache table is absent"; otherwise it found the table and returned zero rows, although rows +were stored. + +- The saved file now records which columns of each schema may be null (`T | None`): the Arrow + type text it saves cannot say it, so `int | None` was saved as `int64` and loaded as `int`. + A node descriptor's `output_schema` gains `"nullable": {"tag": [...], "data": [...]}`, and a + source node's `source_config` and a source's identity config the same record. Loaded source, + function and operator nodes report the run's output schema, and `Pipeline.load` followed by + `PipelineJob.from_pipeline(sources=...)` reads and writes the run's operator cache table. +- The saved file records the run content hash of each operator node in + `run.node_job_content_hashes`, as it did for function nodes. A loaded operator node + attached in `CacheMode.REPLAY` reads exactly the rows its run stored. +- A loaded operator node takes the pipeline hash saved with the job, its run's, instead of + recomputing it from inputs rebuilt from the file. +- A loaded operator node in `CacheMode.REPLAY` whose cache table holds rows none of which + carries its content hash raises a `RuntimeError` saying how many rows are stored, instead + of returning zero rows. This is what a file written before these records (`0.1.0`) gives; + such a file still loads, with every column read as non-null, and saving it again does not + record nullability or run hashes it did not have. + +No hash, stored path or golden value of a running job changes: only what a loaded job +reconstructs. diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md index 5bd24d313..3506bd20a 100644 --- a/design_issues/ITL-1043-LJ1.md +++ b/design_issues/ITL-1043-LJ1.md @@ -1,7 +1,7 @@ ## `src/orcapod/pipeline/job.py` — reading a loaded `PipelineJob` ### LJ1 — a saved job loses column nullability, and does not record the run content hash of an operator node -**Status:** in progress +**Status:** resolved (cache mode not saved remains ITL-977) **Severity:** high **Issue:** ITL-1043 @@ -26,8 +26,14 @@ The running node's hashes are the intended ones: nullability is part of the sche `arrow_source.int64_nullable`, `schema.json` `multi_field.nullable_type`). No live hash, stored path or golden value is wrong; only what a loaded job reconstructs. -**Partial fix (ITL-1043):** a loaded operator node takes the pipeline hash saved in its -descriptor, the run's, so it reads the table its run wrote; and in REPLAY it raises, instead -of returning zero rows, when that table holds rows none of which carries its content hash. -Saving nullability and the operator run content hash in the job file is a saved-format -decision pending with the developer. +**Fix:** (ITL-1043; developer decision, November 12, 2026: record the information in the +saved file.) A node descriptor's `output_schema`, a source node's `source_config` and a +source's identity config record `"nullable": {"tag": [...], "data": [...]}`, restored as +`T | None` through the type converter's own rule (`serialization.serialize_nullability`, +`deserialize_output_schema`, `deserialize_schema(nullable=...)`). `PipelineJob.save` records +the run content hash of operator nodes in `run.node_job_content_hashes`; a loaded operator +node returns it, and the pipeline hash saved in its descriptor. A loaded REPLAY node whose +table holds rows none of which carries its content hash raises instead of returning zero +rows. A file without the records (`0.1.0`, or a `0.2.0` file written before them) loads as +before, lossy, and saving it again records nothing it did not have. No hash, stored path or +golden value of a running job changes. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 9444c002e..8094fa5f7 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -687,14 +687,21 @@ as before. saved run stored, with the types they were stored with. Rows come in the order the store returns them; sort them if order matters. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. - A loaded operator without a readable cache raises from `as_table()` and `iter_data()`. - So does a loaded operator attached in `CacheMode.REPLAY` whose stored rows it cannot tell - apart as its run's, which happens when an input source's data was not saved with the job: - it never shows them as zero rows. Read its input nodes instead (an in-memory input source - must be bound first, see below), or build the job from your code and attach the operator - node in `CacheMode.REPLAY`. - Saved descriptors do not yet preserve all runtime identity and cache configuration needed - for operator readback without reconstruction (follow-up ITL-977). + The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a + loaded operator node in `CacheMode.REPLAY` to read the rows its run stored: + `loaded.nodes["joined"].attach_databases(pipeline_database=store.at(""), + cache_mode=CacheMode.REPLAY)`. A loaded operator without a readable cache raises from + `as_table()` and `iter_data()`; it never shows stored rows as zero rows. Read its input + nodes instead (an in-memory input source must be bound first, see below), or build the job + from your code and attach the operator node in `CacheMode.REPLAY`. +- A loaded node has the pipeline hash, content hash, table path and output schema its run + had (a source node whose data was not saved gets its run's content hash once its source is + bound). The file records which columns may be null (`T | None`), because the Arrow type text + of a schema cannot say it, and the content hash of each node's run, which the stored rows + carry. A file written before these were recorded (format `0.1.0`) still loads, but lossy: + its schemas read every column as non-null, and a loaded operator node cannot tell its run's + stored rows apart, so reading them raises. Saving such a loaded job again does not record + what its file did not. - Sources that read files, such as `CSVSource` and `DeltaTableSource`, are rebuilt from their paths. In-memory sources (`DictSource`, `ListSource`, `ArrowTableSource`, `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 388372f56..9e5d14054 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -92,6 +92,10 @@ class OperatorNodeBase(StreamBase): node_type = "operator" HASH_COLUMN_NAME = "_record_hash" + # The content hash a saved job recorded for this node's run (ITL-1043); set only + # on a node rebuilt by ``OperatorJobNode.from_descriptor``. A class default, so + # nodes built without ``__init__`` have it too. + _saved_run_content_hash: str | None = None def __init__( self, @@ -184,7 +188,15 @@ def pipeline_identity_structure(self) -> Any: # ------------------------------------------------------------------ def content_hash(self, hasher=None) -> ContentHash: - """Return the content hash, using stored value in read-only mode.""" + """Return the content hash: the run's when a loaded job recorded it. + + A node of a job loaded from a file that records its run's content hash returns + that: the rows the run stored carry it, and inputs rebuilt from the file (an + in-memory source is not saved) cannot reproduce it (ITL-1043). Otherwise a + read-only node returns its descriptor's value, and any other node computes it. + """ + if self._saved_run_content_hash is not None: + return ContentHash.from_string(self._saved_run_content_hash) if self._operator is None and self._stored_content_hash is not None: return ContentHash.from_string(self._stored_content_hash) return super().content_hash(hasher) @@ -750,6 +762,7 @@ def from_descriptor( operator: OperatorPodProtocol | None, input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], databases: dict[str, Any], + job_content_hash: str | None = None, ) -> OperatorJobNode: """Construct an OperatorJobNode from a serialized descriptor. @@ -767,6 +780,8 @@ def from_descriptor( for read-only mode. databases: Mapping of database role names (``"pipeline"``) to database instances. + job_content_hash: The content hash of the node in the saved job's run, + which its stored rows carry; ``None`` when the file does not record it. Returns: A new ``OperatorJobNode`` instance. @@ -811,6 +826,7 @@ def from_descriptor( # The run's pipeline hash, not one recomputed from the rebuilt inputs: it # names the table the run's rows are stored in. node._stored_pipeline_hash = descriptor.get("pipeline_hash") + node._saved_run_content_hash = job_content_hash node._load_status = LoadStatus.FULL node._loaded = True return node @@ -862,6 +878,7 @@ def from_descriptor( node._descriptor = descriptor node._stored_schema = descriptor.get("output_schema", {}) node._stored_content_hash = descriptor.get("content_hash") + node._saved_run_content_hash = job_content_hash node._stored_pipeline_hash = descriptor.get("pipeline_hash") node._stored_pipeline_path = tuple(descriptor.get("pipeline_path", ())) node._stored_node_uri = tuple(descriptor.get("node_uri") or []) @@ -883,6 +900,19 @@ def from_descriptor( # Internal helpers # ------------------------------------------------------------------ + def _run_content_hash(self) -> ContentHash | None: + """Return the content hash this node's run stores its rows under, if known. + + A node that runs computes it. A loaded node knows it only when its saved job + recorded it (ITL-1043); otherwise ``None``, because the hash it recomputes from + inputs rebuilt from the file is not the run's. + """ + if not self._loaded: + return self.content_hash() + if self._saved_run_content_hash is None: + return None + return ContentHash.from_string(self._saved_run_content_hash) + def _no_stored_results_error(self) -> RuntimeError: """Return the error for reading a loaded operator node with nothing to serve. diff --git a/src/orcapod/core/sources/base.py b/src/orcapod/core/sources/base.py index 195473241..180c182f7 100644 --- a/src/orcapod/core/sources/base.py +++ b/src/orcapod/core/sources/base.py @@ -118,7 +118,10 @@ def _identity_config(self) -> dict[str, Any]: cannot be reconstructed from config, preserving identity hashes and schemas for downstream consumers. """ - from orcapod.pipeline.serialization import serialize_schema + from orcapod.pipeline.serialization import ( + serialize_nullability, + serialize_schema, + ) tag_schema, data_schema = self.output_schema() type_converter = self.data_context.type_converter @@ -127,6 +130,7 @@ def _identity_config(self) -> dict[str, Any]: "pipeline_hash": self.pipeline_hash().to_string(), "tag_schema": serialize_schema(tag_schema, type_converter), "data_schema": serialize_schema(data_schema, type_converter), + "nullable": serialize_nullability(tag_schema, data_schema, type_converter), } def pipeline_identity_structure(self) -> Any: diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 3bc2f386d..55108de42 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -67,6 +67,11 @@ class Pipeline(AbstractPipelineBase[GraphNode]): operator_node_class = OperatorNode side_effect_node_class = SideEffectNode + # False for a pipeline loaded from a file that does not record which columns may + # be null (ITL-1043): its schemas are lossy, so saving it again must not record + # their nullability as known. + _nullability_recorded: bool = True + def __init__( self, name: str | tuple[str, ...], @@ -141,6 +146,8 @@ def save(self, path: str | Path) -> None: import json as _json from orcapod.pipeline.serialization import ( PIPELINE_FORMAT_VERSION, + serialize_nullability, + serialize_output_schema, serialize_schema, ) from orcapod.core.nodes import OperatorNode, FunctionNode @@ -180,10 +187,9 @@ def save(self, path: str | Path) -> None: "label": node.label, "content_hash": node.content_hash().to_string(), "pipeline_hash": node.pipeline_hash().to_string(), - "output_schema": { - "tag": serialize_schema(tag_schema, type_converter), - "data": serialize_schema(data_schema, type_converter), - }, + "output_schema": serialize_output_schema( + tag_schema, data_schema, type_converter + ), "node_uri": list(node.node_uri), "data_context_key": data_context_key, "config": config_val, @@ -198,6 +204,9 @@ def save(self, path: str | Path) -> None: "data_schema": serialize_schema( node.data_schema, type_converter ), + "nullable": serialize_nullability( + node.tag_schema, node.data_schema, type_converter + ), } descriptor["reconstructable"] = True @@ -211,6 +220,10 @@ def save(self, path: str | Path) -> None: descriptor["operator_config"] = node._operator.to_config() descriptor["table_scope"] = node._table_scope + if not self._nullability_recorded: + # Loaded from a file without the record: what is known is unknown. + del descriptor["output_schema"]["nullable"] + descriptor.get("source_config", {}).pop("nullable", None) nodes[content_hash_str] = descriptor output: dict[str, Any] = { @@ -281,6 +294,12 @@ def load(cls, path: str | Path) -> "Pipeline": upstream_map.setdefault(down_hash, []).append(up_hash) reconstructed: dict[str, GraphNode] = {} + # A file written before nullability was recorded (ITL-1043) restores every + # column as non-null: lossy. Saving the loaded pipeline keeps it unrecorded. + nullability_recorded = all( + "nullable" in (descriptor.get("output_schema") or {}) + for descriptor in nodes_data.values() + ) for node_hash in topo_order: descriptor = nodes_data.get(node_hash) @@ -323,30 +342,34 @@ def load(cls, path: str | Path) -> "Pipeline": ) if not node_name: node_name = descriptor.get("label") or "unknown" + # Which columns may be null; None when the file does not say. + saved_nullable: dict[str, list[str]] | None if "tag_schema" in source_config and "data_schema" in source_config: - tag_schema = Schema( - deserialize_schema( - source_config["tag_schema"], context.type_converter - ) - ) - data_schema = Schema( - deserialize_schema( - source_config["data_schema"], context.type_converter - ) - ) + saved_tag = source_config["tag_schema"] + saved_data = source_config["data_schema"] + saved_nullable = source_config.get("nullable") else: - tag_schema = Schema( - deserialize_schema( - descriptor["output_schema"]["tag"], - context.type_converter, - ) + saved_tag = descriptor["output_schema"]["tag"] + saved_data = descriptor["output_schema"]["data"] + saved_nullable = descriptor["output_schema"].get("nullable") + tag_schema = Schema( + deserialize_schema( + saved_tag, + context.type_converter, + nullable=None + if saved_nullable is None + else saved_nullable["tag"], ) - data_schema = Schema( - deserialize_schema( - descriptor["output_schema"]["data"], - context.type_converter, - ) + ) + data_schema = Schema( + deserialize_schema( + saved_data, + context.type_converter, + nullable=None + if saved_nullable is None + else saved_nullable["data"], ) + ) node = SourceNodeClass( name=node_name, tag_schema=tag_schema, @@ -429,6 +452,7 @@ def load(cls, path: str | Path) -> "Pipeline": # Build Pipeline instance pipeline = cls(name=name, auto_compile=False) + pipeline._nullability_recorded = nullability_recorded pipeline._persistent_node_map = dict(reconstructed) nodes_by_label: dict[str, GraphNode] = {} @@ -512,6 +536,7 @@ def _clone_for_execution(self) -> "Pipeline": clone._node_graph = self._node_graph clone._auto_compile = self._auto_compile clone._compiled = self._compiled + clone._nullability_recorded = self._nullability_recorded # Mutable per-execution state — own copy so runs don't interfere clone._nodes = dict(self._nodes) return clone diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index afe0830b3..5c2c5bd53 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1330,7 +1330,8 @@ def save(self, path: "str | Path") -> None: else: status = "complete" - # Collect the live (data-inclusive) content hashes for FunctionJobNodes. + # Collect the live (data-inclusive) content hashes of function and operator + # job nodes. # The pipeline blueprint stores schema-only blueprint hashes; but the DB # records are written with the live hash (which includes the upstream data # content). Saving these ensures that PipelineJob.load() can reconstruct @@ -1351,6 +1352,13 @@ def save(self, path: "str | Path") -> None: node_has_stored_records[node.label] = ( node._expects_stored_records or node._has_stored_records() ) + elif isinstance(node, OperatorJobNode): + # An operator node's stored rows carry its run's content hash too + # (ITL-1043). A node loaded from a file without it cannot know it: its + # recomputed hash is not the run's, so it is left out, not guessed. + run_content_hash = node._run_content_hash() + if run_content_hash is not None: + node_job_content_hashes[node.label] = run_content_hash.to_string() output: dict[str, Any] = { "orcapod_pipeline_job_version": PIPELINE_JOB_FORMAT_VERSION, @@ -1525,7 +1533,7 @@ def load( # Extract the live (data-inclusive) content hashes saved at run time. # These are keyed by node label and used below to override the blueprint - # hash stored in each FunctionNode descriptor so that loaded stubs' + # hash stored in each function and operator node descriptor so that loaded stubs' # content_hash() matches the _node_content_hash column in the DB. run_block = data.get("run", {}) node_job_content_hashes: dict[str, str] = run_block.get( @@ -1582,11 +1590,14 @@ def load( ) # Use the reconstructed operator from the blueprint if available. operator = getattr(bp_node, "_operator", None) + # The run's content hash, when the file records it (ITL-1043): the + # rows the run stored carry it, and the rebuilt inputs cannot give it. ojn = OperatorJobNode.from_descriptor( descriptor, operator=operator, input_streams=input_streams, databases={}, + job_content_hash=node_job_content_hashes.get(bp_node.label), ) job._persistent_node_map[node_hash] = ojn diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index cb4653b72..46c7b7543 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -5,6 +5,7 @@ import hashlib import json as _json import logging +from collections.abc import Collection from enum import Enum from typing import TYPE_CHECKING, Any @@ -501,12 +502,14 @@ def _source_proxy_from_config( output_schema = node_descriptor.get("output_schema", {}) tag_schema_dict = output_schema.get("tag", {}) data_schema_dict = output_schema.get("data", {}) + saved_nullable = output_schema.get("nullable") else: # Inner sources (e.g. inside CachedSource) embed identity via _identity_config() content_hash = config.get("content_hash") pipeline_hash_val = config.get("pipeline_hash") tag_schema_dict = config.get("tag_schema", {}) data_schema_dict = config.get("data_schema", {}) + saved_nullable = config.get("nullable") if not content_hash or not pipeline_hash_val: raise ValueError( @@ -522,8 +525,19 @@ def _source_proxy_from_config( if source_type and source_type in SOURCE_REGISTRY: expected_class_name = SOURCE_REGISTRY[source_type].__name__ - tag_schema = Schema(deserialize_schema(tag_schema_dict)) - data_schema = Schema(deserialize_schema(data_schema_dict)) + # A config without the nullability record (written before ITL-1043) is lossy. + tag_schema = Schema( + deserialize_schema( + tag_schema_dict, + nullable=None if saved_nullable is None else saved_nullable["tag"], + ) + ) + data_schema = Schema( + deserialize_schema( + data_schema_dict, + nullable=None if saved_nullable is None else saved_nullable["data"], + ) + ) return SourceProxy( source_id=config.get("source_id", "unknown"), @@ -665,9 +679,87 @@ def serialize_schema(schema: Any, type_converter: Any | None = None) -> dict[str } +def nullable_column_names(schema: Any, type_converter: Any) -> list[str]: + """Return the names of the columns of ``schema`` whose values may be null. + + A column may be null when its type is ``T | None``, by the rule + ``type_converter.python_schema_to_arrow_schema`` uses to set a field's + nullability. ``serialize_schema`` writes an Arrow type, which carries no field + nullability, so a saved schema records these names beside its type strings + (ITL-1043). A column whose type has no Arrow type is left out: its saved type text + cannot be restored either. + + Args: + schema: A Schema-like mapping from column name to Python type. + type_converter: The type converter that writes the schema's type strings. + + Returns: + The names, in the schema's column order. + """ + names: list[str] = [] + for name, python_type in schema.items(): + try: + arrow_schema = type_converter.python_schema_to_arrow_schema( + {name: python_type} + ) + except Exception: + continue + if arrow_schema.field(name).nullable: + names.append(name) + return names + + +def serialize_nullability( + tag_schema: Any, data_schema: Any, type_converter: Any +) -> dict[str, list[str]]: + """Return the saved record of which tag and data columns may be null. + + Written next to the type strings of a saved ``(tag, data)`` schema pair, as + ``{"tag": [...], "data": [...]}``. An empty list records that no column of that + group may be null; a schema saved without the record has unknown nullability. + + Args: + tag_schema: The tag schema, a mapping from column name to Python type. + data_schema: The data schema, a mapping from column name to Python type. + type_converter: The type converter that writes the schemas' type strings. + + Returns: + The nullable tag and data column names. + """ + return { + "tag": nullable_column_names(tag_schema, type_converter), + "data": nullable_column_names(data_schema, type_converter), + } + + +def _python_type_of_saved_column( + name: str, arrow_type: Any, nullable: bool, type_converter: Any +) -> Any: + """Return the Python type of a saved column: its Arrow type, ``T | None`` if nullable. + + Goes through ``arrow_schema_to_python_schema``, the converter's own rule for a + field's nullability, so a restored type hashes as the running node's does. + """ + import pyarrow as pa + + field = pa.field(name, arrow_type, nullable=nullable) + return type_converter.arrow_schema_to_python_schema(pa.schema([field]))[name] + + +def _check_nullable_names(columns: Any, nullable: Any) -> None: + """Raise ``ValueError`` if a saved nullable column name is not a saved column.""" + unknown = sorted(set(nullable) - set(columns)) + if unknown: + raise ValueError( + f"The saved nullable columns {unknown} are not among the saved columns " + f"{list(columns)}." + ) + + def deserialize_schema( schema_dict: dict[str, str], type_converter: Any | None = None, + nullable: Collection[str] | None = None, ) -> dict[str, Any]: """Reconstruct a Python-type schema from Arrow type string values. @@ -681,21 +773,34 @@ def deserialize_schema( produced by `serialize_schema`. type_converter: Optional type converter for Arrow→Python conversion. When ``None``, the default data context's converter is used. + nullable: The columns saved as nullable (one group of + ``serialize_nullability``); each becomes ``T | None``. ``None`` when the + file records no nullability: every column is then restored as ``T``. Returns: A dict mapping field names to Python types (or raw strings if parsing fails). + + Raises: + ValueError: If ``nullable`` names a column that is not in ``schema_dict``. """ if type_converter is None: from orcapod.contexts import resolve_context type_converter = resolve_context(None).type_converter + if nullable is not None: + _check_nullable_names(schema_dict, nullable) result: dict[str, Any] = {} for name, type_str in schema_dict.items(): try: arrow_type = parse_arrow_type_string(type_str, type_converter) - result[name] = type_converter.arrow_type_to_python_type(arrow_type) + result[name] = _python_type_of_saved_column( + name, + arrow_type, + nullable is not None and name in nullable, + type_converter, + ) except LogicalTypeIdentifierError as exc: raise exc.for_column(name) from exc except SavedSchemaVersionError: @@ -705,16 +810,39 @@ def deserialize_schema( return result +def serialize_output_schema( + tag_schema: Any, data_schema: Any, type_converter: Any +) -> dict[str, Any]: + """Return a node's saved ``output_schema`` entry. + + ``{"tag": {...}, "data": {...}, "nullable": {"tag": [...], "data": [...]}}``: the + type strings of `serialize_schema` and the record of `serialize_nullability`. + + Args: + tag_schema: The node's tag schema. + data_schema: The node's data schema. + type_converter: The node's type converter. + + Returns: + The entry, ready for JSON. + """ + return { + "tag": serialize_schema(tag_schema, type_converter), + "data": serialize_schema(data_schema, type_converter), + "nullable": serialize_nullability(tag_schema, data_schema, type_converter), + } + + def deserialize_output_schema( - output_schema: dict[str, dict[str, str]], + output_schema: dict[str, Any], type_converter: Any, ) -> tuple[Schema, Schema]: """Reconstruct a node's ``(tag, data)`` output schema from its saved descriptor. Args: - output_schema: The descriptor's ``output_schema`` entry: ``{"tag": {...}, - "data": {...}}``, each mapping a column name to an Arrow type string as - written by `serialize_schema`. + output_schema: The descriptor's ``output_schema`` entry, as written by + `serialize_output_schema`. An entry without ``"nullable"`` (a file written + before it was recorded) restores every column as ``T``: lossy. type_converter: The node's type converter, used to turn each Arrow type back into the Python type that ``output_schema()`` reports. @@ -723,16 +851,24 @@ def deserialize_output_schema( Raises: ValueError: If a column's saved type cannot be restored to a Python type, - such as an unregistered logical type. The message names - the column and its saved type string. + such as an unregistered logical type, or a saved nullable column name is + not a saved column. The message names the column and its saved type string. """ + saved_nullable: dict[str, list[str]] | None = output_schema.get("nullable") - def restore(columns: dict[str, str]) -> Schema: + def restore(columns: dict[str, str], nullable: list[str] | None) -> Schema: + if nullable is not None: + _check_nullable_names(columns, nullable) restored: dict[str, Any] = {} for name, type_str in columns.items(): try: arrow_type = parse_arrow_type_string(type_str, type_converter) - restored[name] = type_converter.arrow_type_to_python_type(arrow_type) + restored[name] = _python_type_of_saved_column( + name, + arrow_type, + nullable is not None and name in nullable, + type_converter, + ) except LogicalTypeIdentifierError as exc: raise exc.for_column(name) from exc except SavedSchemaVersionError as exc: @@ -744,7 +880,16 @@ def restore(columns: dict[str, str]) -> Schema: ) from exc return Schema(restored) - return restore(output_schema.get("tag", {})), restore(output_schema.get("data", {})) + return ( + restore( + output_schema.get("tag", {}), + None if saved_nullable is None else saved_nullable["tag"], + ), + restore( + output_schema.get("data", {}), + None if saved_nullable is None else saved_nullable["data"], + ), + ) def parse_arrow_type_string(type_str: str, type_converter: Any | None = None) -> Any: diff --git a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md index 3b692ebf0..3c90f9781 100644 --- a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md +++ b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md @@ -17,9 +17,14 @@ Options 2 and 3 (saved operator run content hash; saved top-level nullability, r ## Status / notes for a successor -- 2026-11-12: investigation done and reported (commit 5b07349d); option 1 approved; options - 2-3 to be prepared, not pushed until confirmed. ITL-1026 merges first; option 3 then adds - fields to the same `0.2.0` format. +- 2026-11-12: investigation done and reported (commit 5b07349d); option 1 approved and + done (c7e3e4d6). Developer then chose option 3 (all three). Tasks 1-5 implemented locally; + presence of the `nullable` record / the operator's entry in `node_job_content_hashes` + decides lossless vs lossy load (never the version number). Pipeline._nullability_recorded + keeps a lossy file lossy on re-save; OperatorJobNode._run_content_hash() is None for a + loaded node without a saved run hash, so re-save leaves it out. +- PUSH RULE (coordinator): do not push the format fields until PR #345 (ITL-1026, format + 0.2.0) has merged; then merge origin/main and push once, open the PR (Fixes ITL-1043). - Pre-existing contradiction, not changed here: `OperatorJobNode.head` says "a loaded node never raises", but `_head_from_store` raises when a loaded REPLAY node's table is absent. The new 0-of-N case behaves like the absent table (raises from `head` too). Report it. diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 07da9aee0..68e6d0dfe 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -24,6 +24,7 @@ from orcapod.core.operators import Join from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.pipeline import Pipeline +from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.sources import ArrowTableSource from orcapod.types import CacheMode @@ -62,7 +63,7 @@ def _source(column: str, tag_nullable: bool, data_nullable: bool) -> ArrowTableS return ArrowTableSource(table, tag_columns=["id"], source_id=f"src_{column}") -def _sources(nullability: str) -> dict[str, ArrowTableSource]: +def _sources(nullability: str) -> dict[str, StreamProtocol]: """The job's two sources, keyed by source node name.""" tag_nullable, data_nullable = NULLABILITY[nullability] return { @@ -273,3 +274,65 @@ def test_an_empty_nullable_list_means_every_column_is_non_null( d for d in data["pipeline"]["nodes"].values() if d["label"] == "joined" ) assert joined["output_schema"]["nullable"] == {"tag": [], "data": []} + + +# --------------------------------------------------------------------------- +# Saving a loaded job again: known stays known, unknown stays unknown +# --------------------------------------------------------------------------- + + +def _recorded_fields(saved: pathlib.Path) -> dict[str, Any]: + """The nullability records and operator run hash a saved file holds.""" + data = json.loads(saved.read_text()) + records: dict[str, Any] = {} + for descriptor in data["pipeline"]["nodes"].values(): + label = descriptor["label"] + records[label] = descriptor["output_schema"].get("nullable") + if "source_config" in descriptor: + records[f"{label}.source_config"] = descriptor["source_config"].get( + "nullable" + ) + records["joined.run_content_hash"] = data["run"]["node_job_content_hashes"].get( + "joined" + ) + return records + + +@pytest.mark.parametrize("nullability", ["non_null", "nullable_tag_and_data"]) +def test_saving_a_loaded_job_again_keeps_what_its_file_recorded( + nullability: str, tmp_path: pathlib.Path +) -> None: + """A re-saved file records the same nullability and operator run hash.""" + store = InMemoryArrowDatabase() + saved = _save(_run(store, nullability), tmp_path) + resaved = tmp_path / "resaved.json" + PipelineJob.load(str(saved), store=store).save(resaved) + assert _recorded_fields(resaved) == _recorded_fields(saved) + + +def test_saving_a_lossy_loaded_job_again_records_nothing_it_does_not_know( + tmp_path: pathlib.Path, +) -> None: + """Unknown nullability and run hash are not written as facts when re-saving.""" + store = InMemoryArrowDatabase() + + def drop_both(data: dict[str, Any]) -> None: + """Remove every field this fix adds.""" + _drop_saved_nullability(data) + _drop_saved_operator_content_hash(data) + + saved = _save(_run(store, "nullable_tag"), tmp_path, drop_both) + resaved = tmp_path / "resaved.json" + PipelineJob.load(str(saved), store=store).save(resaved) + assert all(value is None for value in _recorded_fields(resaved).values()) + + +def test_a_source_proxy_keeps_the_nullability_of_the_source_it_stands_for() -> None: + """A source rebuilt as a ``SourceProxy`` from its config has the source's schema.""" + from orcapod.pipeline.serialization import resolve_source_from_config + + source = _source("a", tag_nullable=True, data_nullable=False) + config = {**source.to_config(), "source_type": "no_such_source_type"} + proxy = resolve_source_from_config(config, fallback_to_proxy=True) + assert proxy.output_schema() == source.output_schema() + assert proxy.pipeline_hash() == source.pipeline_hash() From 4f3f37a9733976984554cc2a55129a665c27d711 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:33:24 -0800 Subject: [PATCH 08/16] docs(itl-1043): old saved files load lossy; upgrading them is ITL-1048 Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1043.md | 14 +++++++++++--- docs/concepts/function-pods.md | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/changelog.d/ITL-1043.md b/changelog.d/ITL-1043.md index 458dfd8d4..8f3dc2133 100644 --- a/changelog.d/ITL-1043.md +++ b/changelog.d/ITL-1043.md @@ -21,9 +21,17 @@ were stored. recomputing it from inputs rebuilt from the file. - A loaded operator node in `CacheMode.REPLAY` whose cache table holds rows none of which carries its content hash raises a `RuntimeError` saying how many rows are stored, instead - of returning zero rows. This is what a file written before these records (`0.1.0`) gives; - such a file still loads, with every column read as non-null, and saving it again does not - record nullability or run hashes it did not have. + of returning zero rows. No hash, stored path or golden value of a running job changes: only what a loaded job reconstructs. + +### Known limitations + +- A job or pipeline file written before these records (format `0.1.0`, or a `0.2.0` file + written before this change) lacks the `"nullable"` record in every `output_schema`, + `source_config` and source identity config, and lacks the operator nodes' entries in + `run.node_job_content_hashes`. It still loads, lossy: every column reads as non-null, and + reading a loaded operator node's stored rows raises the error above. Saving such a loaded + job again does not record what its file lacked. This change does not rewrite or upgrade + old files; that is the job of the migration tool after v0.3 (ITL-1048). (ITL-1043) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 8094fa5f7..f82003175 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -701,7 +701,7 @@ as before. carry. A file written before these were recorded (format `0.1.0`) still loads, but lossy: its schemas read every column as non-null, and a loaded operator node cannot tell its run's stored rows apart, so reading them raises. Saving such a loaded job again does not record - what its file did not. + what its file did not. Upgrading such files is left to a migration tool after v0.3. - Sources that read files, such as `CSVSource` and `DeltaTableSource`, are rebuilt from their paths. In-memory sources (`DictSource`, `ListSource`, `ArrowTableSource`, `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs From a4f78bba6113a84ed4570d53fd4a94aaf5e1638d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:48:15 -0800 Subject: [PATCH 09/16] fix(operator_node): a loaded node's head() shows why its stored rows cannot be read instead of raising (ITL-1043) as_table() and iter_data() raise, naming the cause: a job file that does not record the run's content hash (ITL-1048 upgrades such files) or a store that holds only other runs' rows. head() keeps its contract never to raise on a loaded node, for this case and for an absent cache table. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1043.md | 9 +- design_issues/ITL-1043-LJ1.md | 3 +- docs/concepts/function-pods.md | 14 +-- src/orcapod/core/nodes/operator_node.py | 66 +++++++++++--- .../loaded_job/test_loaded_operator_cache.py | 87 +++++++++++++++++-- 5 files changed, 148 insertions(+), 31 deletions(-) diff --git a/changelog.d/ITL-1043.md b/changelog.d/ITL-1043.md index 8f3dc2133..26e6d54e0 100644 --- a/changelog.d/ITL-1043.md +++ b/changelog.d/ITL-1043.md @@ -20,8 +20,13 @@ were stored. - A loaded operator node takes the pipeline hash saved with the job, its run's, instead of recomputing it from inputs rebuilt from the file. - A loaded operator node in `CacheMode.REPLAY` whose cache table holds rows none of which - carries its content hash raises a `RuntimeError` saying how many rows are stored, instead - of returning zero rows. + carries its content hash raises a `RuntimeError` from `as_table()` and `iter_data()`, + saying how many rows are stored and why (a job file that does not record the run's content + hash, or a store holding only other runs' rows), instead of returning zero rows. `head()` + keeps its contract never to raise on a loaded node: it shows no rows with that reason, and + likewise for an absent cache table, which used to raise from `head()`. +- Whether a saved file has the nullability and run-hash records is decided by the records, + not by the format version. No hash, stored path or golden value of a running job changes: only what a loaded job reconstructs. diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md index 3506bd20a..066ce069b 100644 --- a/design_issues/ITL-1043-LJ1.md +++ b/design_issues/ITL-1043-LJ1.md @@ -34,6 +34,7 @@ source's identity config record `"nullable": {"tag": [...], "data": [...]}`, res the run content hash of operator nodes in `run.node_job_content_hashes`; a loaded operator node returns it, and the pipeline hash saved in its descriptor. A loaded REPLAY node whose table holds rows none of which carries its content hash raises instead of returning zero -rows. A file without the records (`0.1.0`, or a `0.2.0` file written before them) loads as +rows. `head()` shows no rows with the cause instead of raising, for that case and for +an absent cache table. A file without the records (`0.1.0`, or a `0.2.0` file written before them) loads as before, lossy, and saving it again records nothing it did not have. No hash, stored path or golden value of a running job changes. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index f82003175..46bb2604a 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -691,17 +691,21 @@ as before. loaded operator node in `CacheMode.REPLAY` to read the rows its run stored: `loaded.nodes["joined"].attach_databases(pipeline_database=store.at(""), cache_mode=CacheMode.REPLAY)`. A loaded operator without a readable cache raises from - `as_table()` and `iter_data()`; it never shows stored rows as zero rows. Read its input + `as_table()` and `iter_data()`, naming the cause; it never shows stored rows as zero rows. + `head()` does not raise: it shows no rows and says why. Read its input nodes instead (an in-memory input source must be bound first, see below), or build the job from your code and attach the operator node in `CacheMode.REPLAY`. - A loaded node has the pipeline hash, content hash, table path and output schema its run had (a source node whose data was not saved gets its run's content hash once its source is bound). The file records which columns may be null (`T | None`), because the Arrow type text of a schema cannot say it, and the content hash of each node's run, which the stored rows - carry. A file written before these were recorded (format `0.1.0`) still loads, but lossy: - its schemas read every column as non-null, and a loaded operator node cannot tell its run's - stored rows apart, so reading them raises. Saving such a loaded job again does not record - what its file did not. Upgrading such files is left to a migration tool after v0.3. + carry. Whether a file has these records is decided by the records themselves, not by the + file's format version: a file without them (format `0.1.0`, or a `0.2.0` file written + before they existed) still loads, but lossy: its schemas read every column as non-null, + and a loaded operator node cannot tell its run's stored rows apart, so reading them + raises. An empty record is not a missing one: it says that no column may be null. Saving + such a loaded job again does not record what its file did not. Upgrading such files is + left to a migration tool after v0.3. - Sources that read files, such as `CSVSource` and `DeltaTableSource`, are rebuilt from their paths. In-memory sources (`DictSource`, `ListSource`, `ArrowTableSource`, `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 9e5d14054..9f6e69289 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -83,6 +83,21 @@ # --------------------------------------------------------------------------- +class _LoadedCacheUnreadableError(RuntimeError): + """A loaded operator node in REPLAY cannot read its run's stored rows. + + ``as_table()`` and ``iter_data()`` raise it; ``head()``, which never raises on a + loaded node, shows no rows with ``head_note`` as the reason. + + Attributes: + head_note: The short reason a head shows. + """ + + def __init__(self, message: str, head_note: str) -> None: + super().__init__(message) + self.head_note = head_note + + class OperatorNodeBase(StreamBase): """Shared base for ``OperatorNode`` and ``OperatorJobNode``. @@ -1154,22 +1169,42 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": and records.num_rows == 0 and stored.num_rows > 0 ): - raise RuntimeError( + if self._saved_run_content_hash is None: + cause = ( + "the job file does not record the content hash of this node's run " + "(a file saved before Orcapod recorded it), and the loaded job " + "cannot reproduce it. Upgrading such files is left to the migration " + "tool after v0.3 (ITL-1048); until then build the job from code and " + "attach the node with CacheMode.REPLAY to read them." + ) + head_cause = ( + "the job file does not record its run's content hash " + "(an older file; ITL-1048)" + ) + else: + cause = ( + "they were stored by other runs: this store does not hold the rows " + "of the saved run. Load with the store the run wrote." + ) + head_cause = "they are other runs' rows (another store)" + raise _LoadedCacheUnreadableError( f"OperatorJobNode {self.label!r}: its cache table " f"({'/'.join(self._stored_table_path)}) holds {stored.num_rows} stored " "rows, none of them under this node's content hash " - f"({self.content_hash().to_string()}). They were stored by other runs, " - "or by this job's run under a content hash the loaded job cannot " - "reproduce (an input source whose data was not saved with the job). " - "Build the job from code and attach the node with CacheMode.REPLAY to " - "read them." + f"({self.content_hash().to_string()}): {cause}", + head_note=( + f"loaded node: {stored.num_rows} stored rows, none under its run's " + f"content hash; {head_cause}" + ), ) if records is None: if self._loaded: - raise RuntimeError( + raise _LoadedCacheUnreadableError( f"OperatorJobNode {self.label!r}: the cache table is absent " f"({'/'.join(self._stored_table_path)}). Load with the store " - "that holds the saved operator results." + "that holds the saved operator results.", + head_note="loaded node whose cache table is absent from the " + "attached store", ) if self._operator is None: # Read-only (deserialized) node with no operator: cannot derive @@ -1442,13 +1477,16 @@ def _head_from_store(self, n: int, config: ColumnConfig) -> StreamHead: """Return the first ``n`` of this node's stored rows. Reads the node's whole stored slice (no bounded read exists yet) through the - same path as ``as_table`` and keeps nothing on the node. - - Raises: - RuntimeError: If the node was loaded and its cache table is absent from - the attached store (the wrong store). + same path as ``as_table`` and keeps nothing on the node. A loaded node whose + run's rows cannot be read (its cache table is absent, or none of its rows is the + run's) shows no rows and says why, where ``as_table`` raises. """ - stream = self._load_cached_stream_from_db() + try: + stream = self._load_cached_stream_from_db() + except _LoadedCacheUnreadableError as exc: + return self._head_without_rows( + config, HeadStatus.UNAVAILABLE, note=exc.head_note + ) if stream is None or stream.as_table().num_rows == 0: return self._head_without_rows(config, HeadStatus.NOTHING_STORED) return stream.head(n, columns=config).for_stream(self, origin="stored") diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 68e6d0dfe..2de724827 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -22,6 +22,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.operators import Join +from orcapod.core.streams import HeadStatus from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.pipeline import Pipeline from orcapod.protocols.core_protocols import StreamProtocol @@ -238,7 +239,13 @@ def test_without_saved_nullability_the_operator_still_reads_its_runs_table( assert _sorted(node.as_table()).equals(_sorted(job.nodes["joined"].as_table())) -@pytest.mark.parametrize("read", ["as_table", "iter_data", "head"]) +def _drop_both(data: dict[str, Any]) -> None: + """Remove every field this fix adds, as in a file written before them.""" + _drop_saved_nullability(data) + _drop_saved_operator_content_hash(data) + + +@pytest.mark.parametrize("read", ["as_table", "iter_data"]) @pytest.mark.parametrize("store_kind", STORES) @pytest.mark.parametrize("nullability", list(NULLABILITY)) def test_without_the_run_content_hash_the_loaded_operator_raises( @@ -247,22 +254,84 @@ def test_without_the_run_content_hash_the_loaded_operator_raises( """Stored rows the loaded node cannot identify as its run's never read as none.""" store = _store(store_kind, tmp_path) job = _run(store, nullability) - - def drop_both(data: dict[str, Any]) -> None: - """Remove every field this fix adds.""" - _drop_saved_nullability(data) - _drop_saved_operator_content_hash(data) - - node = _load_replaying(_save(job, tmp_path, drop_both), store).nodes["joined"] + node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] with pytest.raises( RuntimeError, - match=r"'joined': its cache table \(.*\) holds 2 stored rows, none of them", + match=r"'joined': its cache table \(.*\) holds 2 stored rows, none of them " + r"under this node's content hash .* the job file does not record .*ITL-1048", ): result = getattr(node, read)() if read == "iter_data": _ = list(result) +@pytest.mark.parametrize("store_kind", STORES) +def test_without_the_run_content_hash_head_says_why_it_shows_no_rows( + store_kind: str, tmp_path: pathlib.Path +) -> None: + """``head()`` never raises on a loaded node: it shows no rows and names the cause.""" + store = _store(store_kind, tmp_path) + job = _run(store, "nullable_tag") + node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] + head = node.head() + assert head.status is HeadStatus.UNAVAILABLE + assert head.table.num_rows == 0 + assert "2 stored rows, none under its run's content hash" in head.header() + assert "ITL-1048" in head.header() + + +def test_a_store_holding_only_other_runs_rows_is_named_as_the_cause( + tmp_path: pathlib.Path, +) -> None: + """With the run's content hash recorded, rows of other runs only mean another store.""" + saved_store = InMemoryArrowDatabase() + saved = _save(_run(saved_store, "non_null"), tmp_path) + # Another store whose table at the same path holds another run's rows. + other_store = InMemoryArrowDatabase() + other = PipelineJob(name=JOB_NAME, store=other_store) + with other: + other_a = ArrowTableSource( + pa.table( + {"id": [0], "a": [9]}, + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("a", pa.int64(), nullable=False), + ] + ), + ), + tag_columns=["id"], + source_id="other_a", + ) + _ = Join()( + inc.pod(other_a, label="inc"), + _sources("non_null")["src_b"], + label="joined", + ) + other.nodes["joined"].attach_databases( + pipeline_database=other_store.at(JOB_NAME), cache_mode=CacheMode.LOG + ) + _ = other.run() + node = _load_replaying(saved, other_store).nodes["joined"] + with pytest.raises(RuntimeError, match=r"stored by other runs.*store"): + _ = node.as_table() + assert node.head().status is HeadStatus.UNAVAILABLE + + +def test_an_absent_cache_table_shows_no_rows_in_head_and_raises_elsewhere( + tmp_path: pathlib.Path, +) -> None: + """A store without the run's table: ``head()`` names it; ``as_table()`` raises.""" + store = InMemoryArrowDatabase() + saved = _save(_run(store, "non_null"), tmp_path) + node = _load_replaying(saved, InMemoryArrowDatabase()).nodes["joined"] + head = node.head() + assert head.status is HeadStatus.UNAVAILABLE + assert "cache table is absent" in head.header() + with pytest.raises(RuntimeError, match="'joined': the cache table is absent"): + _ = node.as_table() + + def test_an_empty_nullable_list_means_every_column_is_non_null( tmp_path: pathlib.Path, ) -> None: From 895420718099415a359a8f9039ff48cde95730e0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:52:46 -0800 Subject: [PATCH 10/16] test(pipeline): read the ITL-1026 format 0.1.0 fixture's join through its run's path (ITL-1043) Co-Authored-By: Claude Opus 5.5 --- .../loaded_job/test_loaded_operator_cache.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 2de724827..807bc1aec 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -405,3 +405,79 @@ def test_a_source_proxy_keeps_the_nullability_of_the_source_it_stands_for() -> N proxy = resolve_source_from_config(config, fallback_to_proxy=True) assert proxy.output_schema() == source.output_schema() assert proxy.pipeline_hash() == source.pipeline_hash() + + +# --------------------------------------------------------------------------- +# A real format 0.1.0 file (tests/test_pipeline/fixtures, written by ITL-1026) +# --------------------------------------------------------------------------- + +FORMAT_0_1_0 = pathlib.Path(__file__).parents[1] / "fixtures" / "job_format_0_1_0.json" + + +def _add_values(value: int, score: int) -> int: + """The fixture job's function.""" + return value + score + + +def _store_the_fixture_join(store: ArrowDatabaseProtocol) -> Any: + """Run the fixture's job with today's code, its join stored; the run's join node. + + The fixture was saved before running, with in-memory sources, so no store holds + its rows: this run stores them at the path the file's join node names. + """ + from orcapod.core.data_function import PythonDataFunction + from orcapod.core.function_pod import FunctionPod + + def source(column: str, values: list[int]) -> ArrowTableSource: + """One of the fixture's two sources.""" + table = pa.table( + { + "key": pa.array(["a", "b"], type=pa.large_string()), + column: pa.array(values, type=pa.int64()), + } + ) + return ArrowTableSource( + table, + tag_columns=["key"], + infer_nullable=True, + source_id=f"fixture_{column}", + ) + + name = json.loads(FORMAT_0_1_0.read_text())["pipeline"]["pipeline"]["name"] + job = PipelineJob(name=tuple(name), store=store) + with job: + joined = Join()(source("value", [10, 20]), source("score", [100, 200])) + pod = FunctionPod( + data_function=PythonDataFunction(_add_values, output_keys="total") + ) # pyright: ignore[reportArgumentType] -- as in the fixture's generator + _ = pod(joined) + run_join = job.nodes["Join"] + # OperatorNode.attach_databases is the public way to turn on LOG / REPLAY mode. + run_join.attach_databases( + pipeline_database=store.at(*name), cache_mode=CacheMode.LOG + ) + _ = job.run() + return run_join + + +def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_none() -> ( + None +): + """The real 0.1.0 file: the run's path, ``head()`` says why, ``as_table()`` raises.""" + store = InMemoryArrowDatabase() + run_join = _store_the_fixture_join(store) + loaded = PipelineJob.load(str(FORMAT_0_1_0), store=store) + node = loaded.nodes["Join"] + name = json.loads(FORMAT_0_1_0.read_text())["pipeline"]["pipeline"]["name"] + node.attach_databases( + pipeline_database=store.at(*name), cache_mode=CacheMode.REPLAY + ) + assert node.pipeline_hash() == run_join.pipeline_hash() + assert node.node_identity_path == run_join.node_identity_path + head = node.head() + assert head.status is HeadStatus.UNAVAILABLE + assert "ITL-1048" in head.header() + with pytest.raises( + RuntimeError, match=r"'Join': its cache table .* the job file does not record" + ): + _ = node.as_table() From 8c58bcf12fd44d40f9a9dafec80ab98329645451 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:53:59 -0800 Subject: [PATCH 11/16] test(pipeline): keep the targeted pyright ignore on its line Co-Authored-By: Claude Opus 5.5 --- .../test_pipeline/loaded_job/test_loaded_operator_cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 807bc1aec..6e606c701 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -447,9 +447,9 @@ def source(column: str, values: list[int]) -> ArrowTableSource: job = PipelineJob(name=tuple(name), store=store) with job: joined = Join()(source("value", [10, 20]), source("score", [100, 200])) - pod = FunctionPod( - data_function=PythonDataFunction(_add_values, output_keys="total") - ) # pyright: ignore[reportArgumentType] -- as in the fixture's generator + function = PythonDataFunction(_add_values, output_keys="total") + # As in the fixture's generator: a protocol mismatch shared with other tests. + pod = FunctionPod(data_function=function) # pyright: ignore[reportArgumentType] _ = pod(joined) run_join = job.nodes["Join"] # OperatorNode.attach_databases is the public way to turn on LOG / REPLAY mode. From 0a3f77698408a1792e5a50ff6e22146fe48013db Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:31:13 -0800 Subject: [PATCH 12/16] fix(pipeline): loaded replay never caches unreadable rows; re-save keeps nullability records per node (ITL-1043) CodeRabbit round 1 on #352: get_cached_output() of a loaded REPLAY node read through _read_stored_records and could keep an unreadable cache as an empty stream; it now goes through the validated read. Re-saving a loaded pipeline dropped every node's nullability record when one descriptor lacked it; it now drops only the records its file lacked. The other-runs message also names a run that produced no rows (an empty result stores nothing). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 18 ++++-- src/orcapod/pipeline/graph.py | 38 ++++++++----- ...026-11-12-itl-1043-loaded-operator-node.md | 2 +- .../loaded_job/test_loaded_operator_cache.py | 56 +++++++++++++++++++ 4 files changed, 96 insertions(+), 18 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 9f6e69289..2cd1bf7d9 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1183,10 +1183,11 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": ) else: cause = ( - "they were stored by other runs: this store does not hold the rows " - "of the saved run. Load with the store the run wrote." + "they were stored by other runs. Either this store does not hold " + "the rows of the saved run (load with the store the run wrote), or " + "the run produced no rows: an empty result stores nothing." ) - head_cause = "they are other runs' rows (another store)" + head_cause = "they are other runs' rows (another store, or its run produced none)" raise _LoadedCacheUnreadableError( f"OperatorJobNode {self.label!r}: its cache table " f"({'/'.join(self._stored_table_path)}) holds {stored.num_rows} stored " @@ -1320,8 +1321,17 @@ def _replay_from_cache(self) -> None: """Load cached results from DB, skip computation. If no cached records exist yet, produces an empty stream with - the correct schema (zero rows, correct columns). + the correct schema (zero rows, correct columns). A loaded node reads through + ``_load_cached_stream_from_db``, which raises when its run's rows cannot be + read, so an unreadable cache is never kept as an empty stream. """ + if self._loaded: + stream = self._load_cached_stream_from_db() + if stream is None: + raise self._no_stored_results_error() + self._cached_output_stream = stream + self._update_modified_time() + return records = self._read_stored_records() if records is None: records = self._make_empty_table() diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 12f93bf37..2d769cd80 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -68,10 +68,11 @@ class Pipeline(AbstractPipelineBase[GraphNode]): operator_node_class = OperatorNode side_effect_node_class = SideEffectNode - # False for a pipeline loaded from a file that does not record which columns may - # be null (ITL-1043): its schemas are lossy, so saving it again must not record - # their nullability as known. - _nullability_recorded: bool = True + # The nodes (by hash key) of a pipeline loaded from a file whose descriptor did not + # record which columns may be null (ITL-1043), in the output schema and in the + # source config. Their schemas are lossy, so saving again must not record them. + _unrecorded_output_nullability: frozenset[str] = frozenset() + _unrecorded_source_nullability: frozenset[str] = frozenset() def __init__( self, @@ -221,9 +222,10 @@ def save(self, path: str | Path) -> None: descriptor["operator_config"] = node._operator.to_config() descriptor["table_scope"] = node._table_scope - if not self._nullability_recorded: - # Loaded from a file without the record: what is known is unknown. + # A node loaded from a file without the record: its nullability is unknown. + if content_hash_str in self._unrecorded_output_nullability: del descriptor["output_schema"]["nullable"] + if content_hash_str in self._unrecorded_source_nullability: descriptor.get("source_config", {}).pop("nullable", None) nodes[content_hash_str] = descriptor @@ -296,11 +298,19 @@ def load(cls, path: str | Path) -> "Pipeline": upstream_map.setdefault(down_hash, []).append(up_hash) reconstructed: dict[str, GraphNode] = {} - # A file written before nullability was recorded (ITL-1043) restores every - # column as non-null: lossy. Saving the loaded pipeline keeps it unrecorded. - nullability_recorded = all( - "nullable" in (descriptor.get("output_schema") or {}) - for descriptor in nodes_data.values() + # A descriptor written before nullability was recorded (ITL-1043) restores + # every column as non-null: lossy. Saving the loaded pipeline keeps it + # unrecorded, node by node. + unrecorded_output_nullability = frozenset( + node_hash + for node_hash, descriptor in nodes_data.items() + if "nullable" not in (descriptor.get("output_schema") or {}) + ) + unrecorded_source_nullability = frozenset( + node_hash + for node_hash, descriptor in nodes_data.items() + if "source_config" in descriptor + and "nullable" not in descriptor["source_config"] ) for node_hash in topo_order: @@ -454,7 +464,8 @@ def load(cls, path: str | Path) -> "Pipeline": # Build Pipeline instance pipeline = cls(name=name, auto_compile=False) - pipeline._nullability_recorded = nullability_recorded + pipeline._unrecorded_output_nullability = unrecorded_output_nullability + pipeline._unrecorded_source_nullability = unrecorded_source_nullability pipeline._persistent_node_map = dict(reconstructed) nodes_by_label: dict[str, GraphNode] = {} @@ -538,7 +549,8 @@ def _clone_for_execution(self) -> "Pipeline": clone._node_graph = self._node_graph clone._auto_compile = self._auto_compile clone._compiled = self._compiled - clone._nullability_recorded = self._nullability_recorded + clone._unrecorded_output_nullability = self._unrecorded_output_nullability + clone._unrecorded_source_nullability = self._unrecorded_source_nullability # Mutable per-execution state — own copy so runs don't interfere clone._nodes = dict(self._nodes) return clone diff --git a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md index 3c90f9781..993d91365 100644 --- a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md +++ b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md @@ -20,7 +20,7 @@ Options 2 and 3 (saved operator run content hash; saved top-level nullability, r - 2026-11-12: investigation done and reported (commit 5b07349d); option 1 approved and done (c7e3e4d6). Developer then chose option 3 (all three). Tasks 1-5 implemented locally; presence of the `nullable` record / the operator's entry in `node_job_content_hashes` - decides lossless vs lossy load (never the version number). Pipeline._nullability_recorded + decides lossless vs lossy load (never the version number). Pipeline._unrecorded_*_nullability (per node) keeps a lossy file lossy on re-save; OperatorJobNode._run_content_hash() is None for a loaded node without a saved run hash, so re-save leaves it out. - PUSH RULE (coordinator): do not push the format fields until PR #345 (ITL-1026, format diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 6e606c701..6bced9513 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -481,3 +481,59 @@ def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_no RuntimeError, match=r"'Join': its cache table .* the job file does not record" ): _ = node.as_table() + + +# --------------------------------------------------------------------------- +# get_cached_output on a loaded node (CodeRabbit, PR #352) +# --------------------------------------------------------------------------- + + +def test_get_cached_output_of_a_loaded_node_reads_its_runs_rows( + tmp_path: pathlib.Path, +) -> None: + """``get_cached_output()`` on a loaded REPLAY node returns its run's rows.""" + store = InMemoryArrowDatabase() + job = _run(store, "nullable_tag") + node = _load_replaying(_save(job, tmp_path), store).nodes["joined"] + cached = node.get_cached_output() + assert cached is not None + assert _sorted(cached.as_table()).equals(_sorted(job.nodes["joined"].as_table())) + + +def test_get_cached_output_of_a_loaded_node_never_caches_unreadable_rows_as_none( + tmp_path: pathlib.Path, +) -> None: + """``get_cached_output()`` raises as ``as_table()`` does; later reads raise too.""" + store = InMemoryArrowDatabase() + job = _run(store, "nullable_tag") + node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] + with pytest.raises(RuntimeError, match="the job file does not record"): + _ = node.get_cached_output() + with pytest.raises(RuntimeError, match="the job file does not record"): + _ = node.as_table() + + +# --------------------------------------------------------------------------- +# A file whose nodes record nullability unevenly (CodeRabbit, PR #352) +# --------------------------------------------------------------------------- + + +def test_saving_again_drops_only_the_records_its_file_lacked( + tmp_path: pathlib.Path, +) -> None: + """A node without a record stays unrecorded; the other nodes keep theirs.""" + store = InMemoryArrowDatabase() + + def drop_src_a(data: dict[str, Any]) -> None: + """Remove the records of source node ``src_a`` only.""" + for descriptor in data["pipeline"]["nodes"].values(): + if descriptor["label"] == "src_a": + del descriptor["output_schema"]["nullable"] + del descriptor["source_config"]["nullable"] + + saved = _save(_run(store, "nullable_tag_and_data"), tmp_path, drop_src_a) + resaved = tmp_path / "resaved.json" + PipelineJob.load(str(saved), store=store).save(resaved) + assert _recorded_fields(resaved) == _recorded_fields(saved) + assert _recorded_fields(resaved)["src_a"] is None + assert _recorded_fields(resaved)["src_b"] == {"tag": ["id"], "data": ["b"]} From f5e1967bb6e137ad7714f9554bf25076635316b8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:39:55 -0800 Subject: [PATCH 13/16] fix(operator_node): the store is the source of truth for a loaded operator node (ITL-1043) Developer ruling (Nov 12): a loaded REPLAY node that knows its key reads as the node built from code does, so an absent table or a table with no rows under its run's content hash is an empty result (head: NOTHING_STORED), never an error. Only a node whose job file lacks the run content hash, which cannot name its run's rows, raises (head: UNAVAILABLE) and points at ITL-1048. No row count is saved. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1043.md | 19 ++- design_issues/ITL-1043-LJ1.md | 18 +-- docs/concepts/function-pods.md | 22 +-- src/orcapod/core/nodes/operator_node.py | 73 ++++------ superpowers/specs/itl-1043/q6_live_replay.py | 68 +++++++++ .../loaded_job/test_loaded_job_readback.py | 18 ++- .../loaded_job/test_loaded_operator_cache.py | 135 ++++++++++++++---- 7 files changed, 257 insertions(+), 96 deletions(-) create mode 100644 superpowers/specs/itl-1043/q6_live_replay.py diff --git a/changelog.d/ITL-1043.md b/changelog.d/ITL-1043.md index 26e6d54e0..eda54f269 100644 --- a/changelog.d/ITL-1043.md +++ b/changelog.d/ITL-1043.md @@ -19,12 +19,16 @@ were stored. attached in `CacheMode.REPLAY` reads exactly the rows its run stored. - A loaded operator node takes the pipeline hash saved with the job, its run's, instead of recomputing it from inputs rebuilt from the file. -- A loaded operator node in `CacheMode.REPLAY` whose cache table holds rows none of which - carries its content hash raises a `RuntimeError` from `as_table()` and `iter_data()`, - saying how many rows are stored and why (a job file that does not record the run's content - hash, or a store holding only other runs' rows), instead of returning zero rows. `head()` - keeps its contract never to raise on a loaded node: it shows no rows with that reason, and - likewise for an absent cache table, which used to raise from `head()`. +- The store is the source of truth: a loaded operator node attached in `CacheMode.REPLAY` + reads exactly as the same node built from code does. When the store holds no rows under + its key (no table, or none of the table's rows), the result is empty, with the node's + columns, and `head()` reports `NOTHING_STORED`. A run whose result was empty stores + nothing and reads back the same way. +- A loaded operator node whose job file does not record its run's content hash cannot name + its run's rows. When the store holds none under the hash the loaded job computes, + `as_table()` and `iter_data()` raise a `RuntimeError` saying so, instead of returning zero + rows, and `head()` shows no rows with that reason (`UNAVAILABLE`) instead of raising. + `get_cached_output()` follows the same rule. - Whether a saved file has the nullability and run-hash records is decided by the records, not by the format version. @@ -37,6 +41,7 @@ reconstructs. written before this change) lacks the `"nullable"` record in every `output_schema`, `source_config` and source identity config, and lacks the operator nodes' entries in `run.node_job_content_hashes`. It still loads, lossy: every column reads as non-null, and - reading a loaded operator node's stored rows raises the error above. Saving such a loaded + a loaded operator node that finds no rows under the content hash it computes raises the + error above. Saving such a loaded job again does not record what its file lacked. This change does not rewrite or upgrade old files; that is the job of the migration tool after v0.3 (ITL-1048). (ITL-1043) diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md index 066ce069b..0ff7e01fe 100644 --- a/design_issues/ITL-1043-LJ1.md +++ b/design_issues/ITL-1043-LJ1.md @@ -30,11 +30,13 @@ stored path or golden value is wrong; only what a loaded job reconstructs. saved file.) A node descriptor's `output_schema`, a source node's `source_config` and a source's identity config record `"nullable": {"tag": [...], "data": [...]}`, restored as `T | None` through the type converter's own rule (`serialization.serialize_nullability`, -`deserialize_output_schema`, `deserialize_schema(nullable=...)`). `PipelineJob.save` records -the run content hash of operator nodes in `run.node_job_content_hashes`; a loaded operator -node returns it, and the pipeline hash saved in its descriptor. A loaded REPLAY node whose -table holds rows none of which carries its content hash raises instead of returning zero -rows. `head()` shows no rows with the cause instead of raising, for that case and for -an absent cache table. A file without the records (`0.1.0`, or a `0.2.0` file written before them) loads as -before, lossy, and saving it again records nothing it did not have. No hash, stored path or -golden value of a running job changes. +`deserialize_output_schema`, `deserialize_schema(nullable=...)`). `PipelineJob.save` +records the run content hash of operator nodes in `run.node_job_content_hashes`; a loaded +operator node returns it, and the pipeline hash saved in its descriptor. The store is the +source of truth (developer, November 12, 2026): a loaded REPLAY node that knows its key +reads as the node built from code does, so no rows under its key (no table, or none of its +rows) is an empty result. A node whose file lacks the run content hash cannot name its +rows: with none under the hash it computes, `as_table()`/`iter_data()` raise and `head()` +shows no rows with the reason. A file without the records (`0.1.0`, or a `0.2.0` file +written before them) loads as before, lossy, and saving it again records nothing it did +not have. No hash, stored path or golden value of a running job changes. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b54044081..a9473a6ff 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -698,11 +698,13 @@ as before. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a loaded operator node in `CacheMode.REPLAY` to read the rows its run stored: `loaded.nodes["joined"].attach_databases(pipeline_database=store.at(""), - cache_mode=CacheMode.REPLAY)`. A loaded operator without a readable cache raises from - `as_table()` and `iter_data()`, naming the cause; it never shows stored rows as zero rows. - `head()` does not raise: it shows no rows and says why. Read its input - nodes instead (an in-memory input source must be bound first, see below), or build the job - from your code and attach the operator node in `CacheMode.REPLAY`. + cache_mode=CacheMode.REPLAY)`. The store is the source of truth: a loaded job shows what + the store holds for it. A loaded operator node reads exactly as the same node built from + your code and attached in `CacheMode.REPLAY`; when the store holds no rows under the + node's key (no table, or none of its rows), the answer is an empty result, as for that + node. A loaded operator not attached in `REPLAY` has no readable cache and raises from + `as_table()` and `iter_data()`; read its input nodes instead (an in-memory input source + must be bound first, see below), or build the job from your code. - A loaded node has the pipeline hash, content hash, table path and output schema its run had (a source node whose data was not saved gets its run's content hash once its source is bound). The file records which columns may be null (`T | None`), because the Arrow type text @@ -710,10 +712,12 @@ as before. carry. Whether a file has these records is decided by the records themselves, not by the file's format version: a file without them (format `0.1.0`, or a `0.2.0` file written before they existed) still loads, but lossy: its schemas read every column as non-null, - and a loaded operator node cannot tell its run's stored rows apart, so reading them - raises. An empty record is not a missing one: it says that no column may be null. Saving - such a loaded job again does not record what its file did not. Upgrading such files is - left to a migration tool after v0.3. + and a loaded operator node cannot name its run's rows. When the store holds none under the + content hash the loaded job computes, `as_table()` and `iter_data()` raise, and `head()` + shows no rows and says why, rather than presenting stored rows as none. An empty record is + not a missing one: it says that no column may be null. Saving such a loaded job again does + not record what its file did not. Upgrading such files is left to a migration tool after + v0.3. - Sources that read files, such as `CSVSource` and `DeltaTableSource`, are rebuilt from their paths. In-memory sources (`DictSource`, `ListSource`, `ArrowTableSource`, `DataFrameSource`) keep their data only in memory, so they are not saved; loading logs diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 2cd1bf7d9..d246b058d 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1151,10 +1151,15 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": - Returns ``None`` if ``_cache_mode != CacheMode.REPLAY`` (LOG/OFF modes may have stale historical records in the DB). + The store is the source of truth: a loaded node whose job file records its + run's content hash reads exactly as a node that runs does, so an absent table or + a table with no rows under that hash is an empty result. + Raises: - RuntimeError: If a loaded REPLAY node's cache table is absent, or holds rows - none of which carries the node's content hash: a loaded node cannot - compute, so showing no rows would make stored results look dropped. + RuntimeError: If a loaded REPLAY node's job file does not record its run's + content hash and the store holds no rows under the hash the loaded job + computes: the node cannot name its run's rows, so an empty result would + make stored results look dropped. """ if self._pipeline_database is None: return None @@ -1164,50 +1169,29 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": records = None if stored is None else self._filter_by_content_hash(stored) if ( self._loaded - and stored is not None - and records is not None - and records.num_rows == 0 - and stored.num_rows > 0 + and self._saved_run_content_hash is None + and (records is None or records.num_rows == 0) ): - if self._saved_run_content_hash is None: - cause = ( - "the job file does not record the content hash of this node's run " - "(a file saved before Orcapod recorded it), and the loaded job " - "cannot reproduce it. Upgrading such files is left to the migration " - "tool after v0.3 (ITL-1048); until then build the job from code and " - "attach the node with CacheMode.REPLAY to read them." - ) - head_cause = ( - "the job file does not record its run's content hash " - "(an older file; ITL-1048)" - ) - else: - cause = ( - "they were stored by other runs. Either this store does not hold " - "the rows of the saved run (load with the store the run wrote), or " - "the run produced no rows: an empty result stores nothing." - ) - head_cause = "they are other runs' rows (another store, or its run produced none)" + found = ( + "is absent from the attached store" + if stored is None + else f"holds {stored.num_rows} stored rows, none of them under the " + f"content hash the loaded job computes ({self.content_hash().to_string()})" + ) raise _LoadedCacheUnreadableError( - f"OperatorJobNode {self.label!r}: its cache table " - f"({'/'.join(self._stored_table_path)}) holds {stored.num_rows} stored " - "rows, none of them under this node's content hash " - f"({self.content_hash().to_string()}): {cause}", + f"OperatorJobNode {self.label!r}: the job file does not record the " + "content hash of this node's run (a file saved before Orcapod recorded " + "it), so the loaded job cannot name the run's rows, and its cache table " + f"({'/'.join(self._stored_table_path)}) {found}. Upgrading such files is " + "left to the migration tool after v0.3 (ITL-1048); until then build the " + "job from code and attach the node with CacheMode.REPLAY to read them.", head_note=( - f"loaded node: {stored.num_rows} stored rows, none under its run's " - f"content hash; {head_cause}" + "loaded node: the job file does not record its run's content hash " + "(an older file; ITL-1048)" ), ) if records is None: - if self._loaded: - raise _LoadedCacheUnreadableError( - f"OperatorJobNode {self.label!r}: the cache table is absent " - f"({'/'.join(self._stored_table_path)}). Load with the store " - "that holds the saved operator results.", - head_note="loaded node whose cache table is absent from the " - "attached store", - ) - if self._operator is None: + if self._operator is None and not self._loaded: # Read-only (deserialized) node with no operator: cannot derive # schema without a live operator. Return None so callers fall # back to iter([]) / _make_empty_table() guarded paths. @@ -1498,7 +1482,12 @@ def _head_from_store(self, n: int, config: ColumnConfig) -> StreamHead: config, HeadStatus.UNAVAILABLE, note=exc.head_note ) if stream is None or stream.as_table().num_rows == 0: - return self._head_without_rows(config, HeadStatus.NOTHING_STORED) + return self._head_without_rows( + config, + HeadStatus.NOTHING_STORED, + # A loaded job cannot run: say what the store holds instead. + note="the store holds no rows for its run" if self._loaded else None, + ) return stream.head(n, columns=config).for_stream(self, origin="stored") # ------------------------------------------------------------------ diff --git a/superpowers/specs/itl-1043/q6_live_replay.py b/superpowers/specs/itl-1043/q6_live_replay.py new file mode 100644 index 000000000..bce92c82a --- /dev/null +++ b/superpowers/specs/itl-1043/q6_live_replay.py @@ -0,0 +1,68 @@ +"""ITL-1043 Q6: what a LIVE operator node in REPLAY does with no rows of its own. + +(1) table absent from the store; (2) table present, holding only another run's rows. + +Run: uv run python superpowers/specs/itl-1043/q6_live_replay.py +""" + +import pyarrow as pa + +from orcapod import PipelineJob +from orcapod.core.operators import Join +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import ArrowTableSource +from orcapod.types import CacheMode + + +def src(col: str, ids: list[int], sid: str) -> ArrowTableSource: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field(col, pa.int64(), nullable=False), + ] + ) + return ArrowTableSource( + pa.table({"id": ids, col: [1] * len(ids)}, schema=schema), + tag_columns=["id"], + source_id=sid, + ) + + +def job(store, a_ids: list[int], mode: CacheMode) -> PipelineJob: + j = PipelineJob(name="q6", store=store) + with j: + Join()(src("a", a_ids, f"a{a_ids}"), src("b", [0, 1], "b"), label="joined") + j.nodes["joined"].attach_databases( + pipeline_database=store.at("q6"), cache_mode=mode + ) + return j + + +def show(case: str, node) -> None: + try: + t = node.as_table() + print(f"{case}: as_table rows={t.num_rows} cols={t.column_names}") + except Exception as exc: # noqa: BLE001 + print(f"{case}: as_table raised {type(exc).__name__}: {exc}") + try: + print(f"{case}: iter_data rows={len(list(node.iter_data()))}") + except Exception as exc: # noqa: BLE001 + print(f"{case}: iter_data raised {type(exc).__name__}: {exc}") + h = node.head() + print( + f"{case}: head status={h.status} rows={h.table.num_rows} header={h.header()!r}" + ) + + +# (1) absent table +show( + "live REPLAY, table absent", + job(InMemoryArrowDatabase(), [0, 1], CacheMode.REPLAY).nodes["joined"], +) +# (2) table with another run's rows only +store = InMemoryArrowDatabase() +job(store, [0, 1], CacheMode.LOG).run() +show( + "live REPLAY, only other runs' rows", + job(store, [1], CacheMode.REPLAY).nodes["joined"], +) diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index 7c85693c6..add0c153e 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -867,18 +867,24 @@ def test_in_memory_source_node_is_readable_after_bind(saved: pathlib.Path) -> No @pytest.mark.parametrize("method", ["as_table", "iter_data"]) -def test_loaded_replay_operator_reports_absent_cache_table( +def test_loaded_replay_operator_reads_an_absent_cache_table_as_no_rows( tmp_path: pathlib.Path, method: str ) -> None: - """A missing REPLAY table must not look like a successful zero-row result.""" + """The store is the source of truth: no table under the node's key is no rows. + + The job file records the operator's run content hash (ITL-1043), so the loaded node + knows its key and reads as a running node does. A file without that hash raises + instead; see ``test_loaded_operator_cache.py``. + """ from orcapod.types import CacheMode node = _loaded_operator_only_job(tmp_path).nodes["joined"] node.attach_databases(InMemoryArrowDatabase(), cache_mode=CacheMode.REPLAY) - with pytest.raises(RuntimeError, match="joined.*cache table is absent"): - result = getattr(node, method)() - if method == "iter_data": - list(result) + result = getattr(node, method)() + if method == "iter_data": + assert list(result) == [] + else: + assert result.num_rows == 0 def test_loaded_job_is_never_runnable(tmp_path: pathlib.Path) -> None: diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 6bced9513..256b7c589 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -245,24 +245,40 @@ def _drop_both(data: dict[str, Any]) -> None: _drop_saved_operator_content_hash(data) +UNKNOWN_KEY = ( + r"'joined': the job file does not record the content hash of this node's run " + r".*ITL-1048" +) + + @pytest.mark.parametrize("read", ["as_table", "iter_data"]) @pytest.mark.parametrize("store_kind", STORES) @pytest.mark.parametrize("nullability", list(NULLABILITY)) def test_without_the_run_content_hash_the_loaded_operator_raises( store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path ) -> None: - """Stored rows the loaded node cannot identify as its run's never read as none.""" + """A node that cannot name its run's rows never shows the stored rows as none.""" store = _store(store_kind, tmp_path) job = _run(store, nullability) node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] - with pytest.raises( - RuntimeError, - match=r"'joined': its cache table \(.*\) holds 2 stored rows, none of them " - r"under this node's content hash .* the job file does not record .*ITL-1048", - ): + with pytest.raises(RuntimeError, match=UNKNOWN_KEY) as excinfo: result = getattr(node, read)() if read == "iter_data": _ = list(result) + assert "holds 2 stored rows, none of them under the content hash" in str( + excinfo.value + ) + + +def test_without_the_run_content_hash_an_absent_table_raises_too( + tmp_path: pathlib.Path, +) -> None: + """Without its run's content hash, an absent table is no answer either.""" + saved = _save(_run(InMemoryArrowDatabase(), "non_null"), tmp_path, _drop_both) + node = _load_replaying(saved, InMemoryArrowDatabase()).nodes["joined"] + with pytest.raises(RuntimeError, match=UNKNOWN_KEY) as excinfo: + _ = node.as_table() + assert "is absent from the attached store" in str(excinfo.value) @pytest.mark.parametrize("store_kind", STORES) @@ -276,16 +292,48 @@ def test_without_the_run_content_hash_head_says_why_it_shows_no_rows( head = node.head() assert head.status is HeadStatus.UNAVAILABLE assert head.table.num_rows == 0 - assert "2 stored rows, none under its run's content hash" in head.header() + assert "does not record its run's content hash" in head.header() assert "ITL-1048" in head.header() -def test_a_store_holding_only_other_runs_rows_is_named_as_the_cause( +# --------------------------------------------------------------------------- +# The store is the source of truth: a known key with no rows is an empty result +# --------------------------------------------------------------------------- + + +def _live_replaying(store: ArrowDatabaseProtocol, job: PipelineJob) -> Any: + """The join of ``job``'s definition rebuilt from code, in REPLAY over ``store``.""" + rebuilt = PipelineJob.from_pipeline(job.pipeline, store=store) + rebuilt.bind(sources=job.sources) + node = rebuilt.nodes["joined"] + node.attach_databases( + pipeline_database=store.at(JOB_NAME), cache_mode=CacheMode.REPLAY + ) + return node + + +def _assert_reads_as_live_with_no_rows(loaded_node: Any, live_node: Any) -> None: + """The loaded node reads what a running node reads from the same store: no rows.""" + for config in ({}, {"all_info": True}): + table = loaded_node.as_table(**config) + live = live_node.as_table(**config) + assert table.num_rows == 0 + assert table.schema.equals(live.schema), (table.schema, live.schema) + assert list(loaded_node.iter_data()) == [] + head = loaded_node.head() + assert head.status is HeadStatus.NOTHING_STORED + assert head.table.num_rows == 0 + assert "the store holds no rows for its run" in head.header() + assert live_node.head().status is HeadStatus.NOTHING_STORED + + +def test_a_store_holding_only_other_runs_rows_gives_an_empty_result( tmp_path: pathlib.Path, ) -> None: - """With the run's content hash recorded, rows of other runs only mean another store.""" + """With its key known, a store without the run's rows answers: no rows.""" saved_store = InMemoryArrowDatabase() - saved = _save(_run(saved_store, "non_null"), tmp_path) + job = _run(saved_store, "non_null") + saved = _save(job, tmp_path) # Another store whose table at the same path holds another run's rows. other_store = InMemoryArrowDatabase() other = PipelineJob(name=JOB_NAME, store=other_store) @@ -313,23 +361,62 @@ def test_a_store_holding_only_other_runs_rows_is_named_as_the_cause( ) _ = other.run() node = _load_replaying(saved, other_store).nodes["joined"] - with pytest.raises(RuntimeError, match=r"stored by other runs.*store"): - _ = node.as_table() - assert node.head().status is HeadStatus.UNAVAILABLE + _assert_reads_as_live_with_no_rows(node, _live_replaying(other_store, job)) -def test_an_absent_cache_table_shows_no_rows_in_head_and_raises_elsewhere( - tmp_path: pathlib.Path, +@pytest.mark.parametrize("nullability", list(NULLABILITY)) +def test_an_absent_cache_table_gives_an_empty_result( + nullability: str, tmp_path: pathlib.Path ) -> None: - """A store without the run's table: ``head()`` names it; ``as_table()`` raises.""" + """With its key known, a store without the table answers: no rows, typed.""" + job = _run(InMemoryArrowDatabase(), nullability) + empty_store = InMemoryArrowDatabase() + node = _load_replaying(_save(job, tmp_path), empty_store).nodes["joined"] + _assert_reads_as_live_with_no_rows(node, _live_replaying(empty_store, job)) + + +def _run_empty_join(store: ArrowDatabaseProtocol) -> PipelineJob: + """Run the operator-only job over inputs with no common ``id``: an empty join.""" + + def source(column: str, ids: list[int], source_id: str) -> ArrowTableSource: + """A non-null source with tag ``id`` and data ``column``.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field(column, pa.int64(), nullable=False), + ] + ) + table = pa.table({"id": ids, column: [1] * len(ids)}, schema=schema) + return ArrowTableSource(table, tag_columns=["id"], source_id=source_id) + + job = PipelineJob(name=JOB_NAME, store=store) + with job: + _ = Join()( + source("a", [0, 1], "src_a"), + source("b", [7, 8], "src_b_disjoint"), + label="joined", + ) + # OperatorNode.attach_databases is the public way to turn on LOG / REPLAY mode. + job.nodes["joined"].attach_databases( + pipeline_database=store.at(JOB_NAME), cache_mode=CacheMode.LOG + ) + _ = job.run() + return job + + +@pytest.mark.parametrize("other_runs_stored", [False, True]) +def test_a_loaded_run_with_an_empty_result_reads_as_empty( + other_runs_stored: bool, tmp_path: pathlib.Path +) -> None: + """A run whose result was empty (it stores nothing) reads back as empty.""" store = InMemoryArrowDatabase() - saved = _save(_run(store, "non_null"), tmp_path) - node = _load_replaying(saved, InMemoryArrowDatabase()).nodes["joined"] - head = node.head() - assert head.status is HeadStatus.UNAVAILABLE - assert "cache table is absent" in head.header() - with pytest.raises(RuntimeError, match="'joined': the cache table is absent"): - _ = node.as_table() + if other_runs_stored: + # Another run of the same pipeline stores rows in the shared table. + _ = _run(store, "non_null", with_function=False) + job = _run_empty_join(store) + assert job.nodes["joined"].as_table().num_rows == 0 + node = _load_replaying(_save(job, tmp_path), store).nodes["joined"] + _assert_reads_as_live_with_no_rows(node, _live_replaying(store, job)) def test_an_empty_nullable_list_means_every_column_is_non_null( @@ -478,7 +565,7 @@ def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_no assert head.status is HeadStatus.UNAVAILABLE assert "ITL-1048" in head.header() with pytest.raises( - RuntimeError, match=r"'Join': its cache table .* the job file does not record" + RuntimeError, match=r"'Join': the job file does not record .*ITL-1048" ): _ = node.as_table() From 87196d0f3c5231bfbb2e0485c91726b4e8fc1212 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:59:27 -0800 Subject: [PATCH 14/16] feat(pipeline): optional advisory run.expected_rows; a loaded node warns, never raises, on another store's content (ITL-1043) Developer decision (Nov 12): the store is the source of truth. A saved job may record, in its own optional run.expected_rows section, how many rows the store held under each function and operator node's run key, written only after a run that finished and committed, and kept (never invented) when a loaded job is saved again. A loaded node returns what its store holds and issues one StoreContentWarning per node when the number differs. A loaded function node no longer raises on a store without its run's results: the ITL-904 node_has_stored_records flag is neither written nor read. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1043.md | 21 +++ design_issues/ITL-1043-LJ1.md | 8 +- docs/concepts/function-pods.md | 21 ++- src/orcapod/contexts/stored.py | 66 +++++++- src/orcapod/core/nodes/function_node.py | 136 +++++++++------- src/orcapod/core/nodes/operator_node.py | 52 ++++++ src/orcapod/errors.py | 12 ++ src/orcapod/pipeline/job.py | 45 +++--- ...026-11-12-itl-1043-loaded-operator-node.md | 5 + .../loaded_job/test_loaded_job_readback.py | 150 +++++++++++------- .../loaded_job/test_loaded_operator_cache.py | 149 ++++++++++++++++- 11 files changed, 523 insertions(+), 142 deletions(-) diff --git a/changelog.d/ITL-1043.md b/changelog.d/ITL-1043.md index eda54f269..8863cda45 100644 --- a/changelog.d/ITL-1043.md +++ b/changelog.d/ITL-1043.md @@ -1,3 +1,24 @@ +### Added + +- **`run.expected_rows` and `StoreContentWarning`.** A saved job file holds an optional, + advisory section, `run.expected_rows`: for each function and operator node, how many rows + the store held under its run's key when the job was saved, written only when that is + certain (after a run that finished and committed; never invented when a loaded job is + saved again). A loaded node always returns what its store holds; when the store holds + another number of rows, the first read issues one `orcapod.errors.StoreContentWarning` + naming the node, both numbers and the store. Deleting the section from the file changes + nothing but this check. (ITL-1043) + +### Changed + +- **A loaded function node no longer raises when its store lacks the run's results** (it + returns what the store holds, with a `StoreContentWarning`). The store is the source of + truth: a loaded job shows what the store holds for it. This supersedes the + `run.node_has_stored_records` flag and its error described under ITL-904, both unreleased: + the flag is no longer written, and a file that still has it loads without a check. A + dangling reference, a pipeline record whose result is missing from the result store, is + logged and left out, for a loaded node as for a running one. (ITL-1043) + ### Fixed #### A loaded job keeps every node's identity, and a loaded operator reads its stored rows (ITL-1043) diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md index 0ff7e01fe..d5a5794ee 100644 --- a/design_issues/ITL-1043-LJ1.md +++ b/design_issues/ITL-1043-LJ1.md @@ -39,4 +39,10 @@ rows) is an empty result. A node whose file lacks the run content hash cannot na rows: with none under the hash it computes, `as_table()`/`iter_data()` raise and `head()` shows no rows with the reason. A file without the records (`0.1.0`, or a `0.2.0` file written before them) loads as before, lossy, and saving it again records nothing it did -not have. No hash, stored path or golden value of a running job changes. +not have. No hash, stored path or golden value of a running job changes. Facts about +stored data stay out of the definition: the job file holds only an optional, advisory +`run.expected_rows` section (rows under each node's run key at save time, written only +when certain); a loaded node returns what the store holds and issues one +`StoreContentWarning` when the number differs. A loaded function node no longer raises on +a store without its run's results (the ITL-904 `node_has_stored_records` flag is no longer +written or read). diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index a9473a6ff..16302f498 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -691,6 +691,11 @@ A loaded job is for reading stored results. It cannot be run again: function pod saved with the job, so `loaded.run()` raises. To compute again, build the job from your code as before. +The store is the source of truth: a loaded job shows what the store holds for it. The job +file names each run's rows by the content hash the run stored them under, so a loaded node +reads exactly what the same node built from your code would read from that store, and no +rows under that key is an empty result, not an error. + - A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows the saved run stored, with the types they were stored with. Rows come in the order the store returns them; sort them if order matters. @@ -698,9 +703,8 @@ as before. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a loaded operator node in `CacheMode.REPLAY` to read the rows its run stored: `loaded.nodes["joined"].attach_databases(pipeline_database=store.at(""), - cache_mode=CacheMode.REPLAY)`. The store is the source of truth: a loaded job shows what - the store holds for it. A loaded operator node reads exactly as the same node built from - your code and attached in `CacheMode.REPLAY`; when the store holds no rows under the + cache_mode=CacheMode.REPLAY)`. A loaded operator node reads exactly as the same node built + from your code and attached in `CacheMode.REPLAY`; when the store holds no rows under the node's key (no table, or none of its rows), the answer is an empty result, as for that node. A loaded operator not attached in `REPLAY` has no readable cache and raises from `as_table()` and `iter_data()`; read its input nodes instead (an in-memory input source @@ -724,9 +728,14 @@ as before. this at INFO level. This does not affect reading the other nodes. To read such a source node itself, bind a source to it: `loaded.bind(sources={"mice": source})`. - The results are read from the store recorded in the file. Pass - `PipelineJob.load(path, store=...)` if the store has moved. If the store you give lacks - results the saved run stored, reading the node raises an error naming the store, instead - of returning no rows. + `PipelineJob.load(path, store=...)` if the store has moved. +- The job file may also hold an optional, advisory section, `run.expected_rows`: for each + node, how many rows the store held under its run's key when the job was saved (written + only when that is certain: after a run that finished). Reading a loaded node always + returns what the store holds; when the store holds another number of rows, the read + issues one `StoreContentWarning` per node, naming the node, both numbers and the store: + the store may not be the one the run wrote, or it may have changed since. Deleting the + section from the file changes nothing but this check. Logical types in saved schema text carry their complete identifier, written as `extension`, including inside nested containers. The loader checks the diff --git a/src/orcapod/contexts/stored.py b/src/orcapod/contexts/stored.py index af5a4d6c4..adaa7d745 100644 --- a/src/orcapod/contexts/stored.py +++ b/src/orcapod/contexts/stored.py @@ -30,13 +30,15 @@ from __future__ import annotations import logging +import warnings from collections.abc import Collection from dataclasses import dataclass, field -from typing import TYPE_CHECKING, NamedTuple, Protocol, cast +from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, cast from orcapod.errors import ( InputValidationError, SideEffectLogDataContextError, + StoreContentWarning, TableListingNotSupportedError, ) from orcapod.system_constants import ( @@ -185,6 +187,68 @@ def report_rows_of_other_data_contexts( ) +class _DescribedStore(Protocol): + """The part of ``ArrowDatabaseProtocol`` that names a store (no import cycle).""" + + @property + def base_path(self) -> tuple[str, ...]: + """Path of this store's view from the root of the underlying store.""" + ... + + def to_config(self) -> dict[str, Any]: + """Return the store's configuration, with its location when it has one.""" + ... + + +def describe_store_table( + database: _DescribedStore, record_path: tuple[str, ...] +) -> str: + """Name a stored table for a message: the store's location and the table's path. + + Args: + database: The store, or a view of it. + record_path: Path of the table relative to ``database``. + + Returns: + For example ``"store at '/data/store' (table 'job/node/...')"``. + """ + store_config = database.to_config() + location = store_config.get("root_uri") + store = ( + f"store at {location!r}" + if location + else f"{store_config.get('type', 'attached')} store" + ) + table = "/".join(database.base_path + record_path) + return f"{store} (table {table!r})" + + +def report_unexpected_row_count( + node: str, location: str, expected: int, found: int +) -> None: + """Warn that a loaded node's store holds another number of rows than its run's. + + The store is the source of truth: the read returns what it holds. The warning says + that it differs from what the job file recorded (ITL-1043). + + Args: + node: The node, as messages name it (``"FunctionJobNode 'dbl'"``). + location: The store and table, from ``describe_store_table``. + expected: The rows the job file records for the node's run. + found: The rows the store holds under the node's run key. + """ + warnings.warn( + StoreContentWarning( + f"{node}: the job file records {expected} " + f"{'row' if expected == 1 else 'rows'} for its run, but the {location} " + f"holds {found} under the run's key. The read returns what the store " + "holds; the store may not be the one the run wrote, or it may have " + "changed since." + ), + stacklevel=3, + ) + + def report_old_layout_rows(record_path: tuple[str, ...], rows: int) -> None: """Warn that a table written at an older table layout was found and is not used. diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 5d3660d5e..011a7cde3 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -56,8 +56,10 @@ keep_rows_of_data_context, require_same_data_context, account_for_stale_tables, + describe_store_table, find_stale_tables, report_rows_of_other_data_contexts, + report_unexpected_row_count, ) from orcapod.utils.store_errors import commit_best_effort, keep_store_error from orcapod.protocols.core_protocols import ( @@ -999,6 +1001,10 @@ class FunctionJobNode(FunctionNodeBase): # Set once a read has reported the stored rows it skipped (rows of another data # context); a class default so nodes rebuilt from a saved descriptor start unset. _skipped_rows_reported: bool = False + # The job file's expected row count and whether it was checked (ITL-1043); class + # defaults so a node attached before ``from_descriptor`` sets them has them. + _expected_row_count: int | None = None + _row_count_checked: bool = False # Set once stored rows of other generations have been accounted for. A node # that delivers side effects unsets it at the start of every run. _stale_tables_checked: bool = False @@ -1048,9 +1054,11 @@ def __init__( # .node_config setter or PipelineJob.apply_node_config(). self._node_config: NodeConfig = NodeConfig() - # Set by from_descriptor() for a loaded node whose saved run had stored - # records; a live node computes its own results. - self._expects_stored_records: bool = False + # Set by from_descriptor() for a loaded node whose job file records how many + # pipeline records its run had in the store (ITL-1043); checked once per + # store, with a warning only: the store is the source of truth. + self._expected_row_count: int | None = None + self._row_count_checked: bool = False # Batched commits, set only while PipelineJob.run runs the node (ITL-919). self._commit_every_rows: int | None = None self._rows_since_commit = 0 @@ -1236,6 +1244,8 @@ def attach_databases( self._current_tables_checked = False self._unlisted_store_reported = False self._skipped_rows_reported = False + # ... and its own rows to compare with the job file's expected count. + self._row_count_checked = False if pipeline_database is not None: if result_database is None: # Default result database is pipeline_database scoped to "_result" @@ -1284,7 +1294,7 @@ def from_descriptor( input_stream: StreamProtocol | None = None, databases: dict[str, Any] | None = None, job_content_hash: str | None = None, - expect_stored_records: bool = False, + expected_row_count: int | None = None, ) -> "FunctionJobNode": """Create a read-only ``FunctionJobNode`` stub from a serialized descriptor. @@ -1311,11 +1321,11 @@ def from_descriptor( column written to the DB at execution time. Required for ``get_all_records()`` to correctly filter DB rows when ``table_scope="pipeline_hash"`` (the default). - expect_stored_records: Whether the store held at least one pipeline - record of this node (one with this node's content hash) when the - job was saved. When ``True``, reading the node from a store with - none of them raises instead of returning zero rows, because the - store is not the one the job ran against. + expected_row_count: How many pipeline records of this node (with its + run's content hash) the store held when the job was saved, from the + job file's optional ``run.expected_rows``; ``None`` when not recorded. + Advisory: a read always returns what the store holds and only warns + (``StoreContentWarning``) when the store holds another number. Returns: A ``FunctionJobNode`` in read-only stub mode @@ -1414,7 +1424,8 @@ def from_descriptor( # FunctionJobNode — orchestrator config node._node_config = NodeConfig() - node._expects_stored_records = expect_stored_records + node._expected_row_count = expected_row_count + node._row_count_checked = False node._commit_every_rows = None node._rows_since_commit = 0 node._commit_failed = False @@ -1518,17 +1529,17 @@ def _cannot_compute_error(self) -> RuntimeError: "as_table() or iter_data()." ) - def _has_stored_records(self) -> bool: - """Return whether the attached store holds a pipeline record of this node. + def _stored_record_count(self) -> int: + """Return how many pipeline records of this node the attached store holds. A record belongs to this node when its ``_node_content_hash`` is this node's content hash: the rows ``iter_data()`` and ``get_all_records()`` read back. Records of other runs or data contexts do not count. """ if self._pipeline_database is None: - return False + return 0 if not self._pipeline_database.table_exists(self._versioned_pipeline_path): - return False + return 0 own_records = self._pipeline_database.get_records_with_column_value( self._versioned_pipeline_path, { @@ -1536,51 +1547,57 @@ def _has_stored_records(self) -> bool: constants.DATA_CONTEXT_COL: self._stored_data_context_key, }, ) - return own_records is not None and own_records.num_rows > 0 + return 0 if own_records is None else own_records.num_rows + + def _expected_row_count_to_save(self, run_finished: bool) -> int | None: + """Return the count ``PipelineJob.save`` records for this node, if certain. + + A loaded node keeps the count its file recorded, or none. A node of a job + whose last run finished (its writes committed) records the pipeline records + the store holds under its run's key; otherwise nothing is certain. + + Args: + run_finished: Whether the job's last run finished and committed. + """ + if self._function_pod is None: + return self._expected_row_count + if not run_finished or self._pipeline_database is None: + return None + return self._stored_record_count() def _require_readable_stub(self) -> None: """Raise unless this loaded node can read its stored results. Raises: - RuntimeError: If no store is attached, or the saved run stored records - that the attached store does not have. + RuntimeError: If no store is attached. """ from orcapod.pipeline.serialization import LoadStatus if self.load_status == LoadStatus.UNAVAILABLE: raise self._no_store_error() - self._require_expected_records() + self._check_expected_rows() - def _require_expected_records(self) -> None: - """Raise if the saved run stored this node's records but the store has none. + def _check_expected_rows(self) -> None: + """Warn, once per store, if the store holds another number of rows than recorded. - A loaded node reads only what its store holds. When the job was saved - with records of this node in the store and the attached store has none - of them, the store is not the one the job ran against; reading zero rows - would make the stored results look lost. - - Raises: - RuntimeError: If the saved run stored records for this node and the - attached store has none of them. + The store is the source of truth: the read goes on and returns what it holds + (ITL-1043). Only a job file that records the node's count is checked. """ - if not self._expects_stored_records or self._has_stored_records(): + if self._expected_row_count is None or self._row_count_checked: return - assert self._pipeline_database is not None # attached: status is READ_ONLY - store_config = self._pipeline_database.to_config() - location = store_config.get("root_uri") - store = ( - f"store at {location!r}" - if location - else f"{store_config.get('type', 'attached')} store" - ) - table = "/".join( - self._pipeline_database.base_path + self._versioned_pipeline_path - ) - raise RuntimeError( - f"FunctionJobNode {self.label!r}: the saved run stored results for this " - f"node, but the attached {store} has none of them (table {table!r}). Load " - "the job with the store it ran against: PipelineJob.load(path, store=...)." - ) + self._row_count_checked = True + if self._pipeline_database is None: + return + found = self._stored_record_count() + if found != self._expected_row_count: + report_unexpected_row_count( + f"FunctionJobNode {self.label!r}", + describe_store_table( + self._pipeline_database, self._versioned_pipeline_path + ), + self._expected_row_count, + found, + ) def _require_pipeline_database(self) -> ArrowDatabaseProtocol: """Return the attached pipeline database, raising if there is none. @@ -2503,8 +2520,12 @@ def get_all_records( Raises: RuntimeError: If the node was loaded by ``PipelineJob.load`` and no - store is attached, or its saved run stored records that the - attached store does not have. + store is attached. + + Warns: + StoreContentWarning: Once per store, if the node was loaded from a job file + that records another number of pipeline records for its run than the + store holds. The store is the source of truth. """ if self._function_pod is None: self._require_readable_stub() @@ -3019,9 +3040,8 @@ def head( rows: the node's whole stored slice is read and nothing is kept on the node. With nothing stored, there are no rows. See ``StreamBase.head``. - Raises: - RuntimeError: If the node was loaded and the attached store does not hold - the records its saved run stored (the wrong store). + Warns: + StoreContentWarning: As for ``as_table``. """ from orcapod.pipeline.serialization import LoadStatus @@ -3047,7 +3067,7 @@ def _head_from_store(self, n: int, config: ColumnConfig) -> StreamHead: """ if self._function_pod is None: # A loaded node: its table is built from the stored Arrow records. - self._require_expected_records() + self._check_expected_rows() fetched = self._fetch_joined_records() stored = None if fetched is None else self._stored_rows_stream(fetched) stored_ids = set(stored[0]) if stored is not None else set() @@ -3094,10 +3114,14 @@ def as_table( table has zero rows and the node's columns. Raises: - RuntimeError: If the node was loaded and no store is attached, its - saved run stored records the attached store does not have, or - some of its results were kept in an ephemeral store, which is - not saved with the job. + RuntimeError: If the node was loaded and no store is attached, or some of + its results were kept in an ephemeral store, which is not saved with + the job. + + Warns: + StoreContentWarning: Once per store, if the node was loaded from a job file + that records another number of pipeline records for its run than the + store holds. The read returns what the store holds. """ if self._function_pod is None: self._require_readable_stub() @@ -3158,7 +3182,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: if self._function_pod is None: # A loaded node: its rows can come only from the store. - self._require_expected_records() + self._check_expected_rows() if not self._cached_output_datas and self._cached_function_pod is not None: # Hot-load from DB on the first call when store is empty. loaded = self._load_cached_entries() diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index d246b058d..e65a0dac4 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -49,8 +49,10 @@ keep_rows_of_data_context, require_same_data_context, account_for_stale_tables, + describe_store_table, find_stale_tables, report_rows_of_other_data_contexts, + report_unexpected_row_count, with_data_context, ) from orcapod.protocols.core_protocols import ( @@ -641,6 +643,10 @@ class OperatorJobNode(OperatorNodeBase): _skipped_rows_reported: bool = False # Set once a table found at the older layout has been accounted for. _stale_tables_checked: bool = False + # The job file's expected row count for a loaded node, and whether it was checked + # against the attached store (ITL-1043). Advisory: a mismatch only warns. + _expected_row_count: int | None = None + _row_count_checked: bool = False @property def _stored_table_path(self) -> tuple[str, ...]: @@ -757,6 +763,7 @@ def attach_databases( # Another store has its own tables of other generations to account for. self._stale_tables_checked = False self._skipped_rows_reported = False + self._row_count_checked = False self._pipeline_database = pipeline_database self._cache_mode = cache_mode @@ -778,6 +785,7 @@ def from_descriptor( input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], databases: dict[str, Any], job_content_hash: str | None = None, + expected_row_count: int | None = None, ) -> OperatorJobNode: """Construct an OperatorJobNode from a serialized descriptor. @@ -797,6 +805,10 @@ def from_descriptor( to database instances. job_content_hash: The content hash of the node in the saved job's run, which its stored rows carry; ``None`` when the file does not record it. + expected_row_count: How many rows the store held under the run's key when + the job was saved, from the job file's optional ``run.expected_rows``; + ``None`` when not recorded. Advisory: a read returns what the store + holds and only warns (``StoreContentWarning``) when it differs. Returns: A new ``OperatorJobNode`` instance. @@ -842,6 +854,7 @@ def from_descriptor( # names the table the run's rows are stored in. node._stored_pipeline_hash = descriptor.get("pipeline_hash") node._saved_run_content_hash = job_content_hash + node._expected_row_count = expected_row_count node._load_status = LoadStatus.FULL node._loaded = True return node @@ -894,6 +907,7 @@ def from_descriptor( node._stored_schema = descriptor.get("output_schema", {}) node._stored_content_hash = descriptor.get("content_hash") node._saved_run_content_hash = job_content_hash + node._expected_row_count = expected_row_count node._stored_pipeline_hash = descriptor.get("pipeline_hash") node._stored_pipeline_path = tuple(descriptor.get("pipeline_path", ())) node._stored_node_uri = tuple(descriptor.get("node_uri") or []) @@ -928,6 +942,42 @@ def _run_content_hash(self) -> ContentHash | None: return None return ContentHash.from_string(self._saved_run_content_hash) + def _expected_row_count_to_save(self, run_finished: bool) -> int | None: + """Return the count ``PipelineJob.save`` records for this node, if certain. + + A loaded node keeps the count its file recorded, or none. A node of a job whose + last run finished (its writes committed) records the rows the store holds under + its run's content hash; otherwise nothing is certain. + + Args: + run_finished: Whether the job's last run finished and committed. + """ + if self._loaded: + return self._expected_row_count + if not run_finished or self._pipeline_database is None: + return None + records = self._read_stored_records() + return 0 if records is None else records.num_rows + + def _check_expected_rows(self, found: int) -> None: + """Warn, once per store, if a loaded node finds another number of rows than recorded. + + The store is the source of truth: the read returns what it holds (ITL-1043). + + Args: + found: The rows the store holds under the node's run key. + """ + if self._expected_row_count is None or self._row_count_checked: + return + self._row_count_checked = True + if found != self._expected_row_count and self._pipeline_database is not None: + report_unexpected_row_count( + f"OperatorJobNode {self.label!r}", + describe_store_table(self._pipeline_database, self._stored_table_path), + self._expected_row_count, + found, + ) + def _no_stored_results_error(self) -> RuntimeError: """Return the error for reading a loaded operator node with nothing to serve. @@ -1190,6 +1240,8 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": "(an older file; ITL-1048)" ), ) + if self._loaded: + self._check_expected_rows(0 if records is None else records.num_rows) if records is None: if self._operator is None and not self._loaded: # Read-only (deserialized) node with no operator: cannot derive diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index f7b604d6d..b5379e262 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -367,6 +367,18 @@ class RowFailuresWarning(UserWarning): """ +class StoreContentWarning(UserWarning): + """A loaded node's store holds another number of rows than the saved run recorded. + + A job file may record, in its optional ``run.expected_rows`` section, how many rows + the store held under each node's run key when the job was saved. Reading a loaded + node always returns what the attached store holds, which is the source of truth; + when that differs from the recorded number, this warning is issued once per node. + The store may not be the one the run wrote, or it may have changed since. A file + without the section, or without a node's entry, is not checked. + """ + + class EmptyDataAccessError(Exception): """Raised when a payload-access method is called on an ``EmptyData`` instance. diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 310b37878..7f9cd1b7f 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -181,6 +181,9 @@ def __init__( # the same class name (e.g. two ArrowTableSource objects). self._sources_by_hash: dict[str, cp.StreamProtocol] = {} self._has_run: bool = False + # True only while the last run started has finished and committed its writes: + # what the store then holds for each node is what that run left (ITL-1043). + self._last_run_finished: bool = False self._run_id: str | None = None # The summary of the last run this object completed (ITL-997). self._last_run: RunSummary | None = None @@ -1057,6 +1060,8 @@ def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": "is for reading stored results only. To run a new computation, " "create a fresh PipelineJob from the original pipeline definition." ) + # Until this run finishes, what the store holds for a node is not certain. + self._last_run_finished = False table_committing = as_table_committing(self._store) if ( @@ -1248,6 +1253,7 @@ def _finish_run( # Mutate self in place. self._has_run = True + self._last_run_finished = True self._run_id = prepared.run_id node_summaries: list[NodeRunSummary] = [] for node in prepared.exec_dag.topological_sort(): @@ -1339,20 +1345,18 @@ def save(self, path: "str | Path") -> None: # stubs whose content_hash() matches the _node_content_hash column in the # DB — required for _filter_by_content_hash() in get_all_records(). node_job_content_hashes: dict[str, str] = {} - # Whether the store holds at least one pipeline record of each function - # node (a record with the node's content hash), so that load() can tell a - # store that holds this run's results from one that does not (a - # different store passed to load(), or a moved one). - node_has_stored_records: dict[str, bool] = {} + # Advisory only (ITL-1043): how many rows the store holds under each node's + # run key, written only when certain (the last run finished and committed, or + # a loaded node's own record). A loaded node compares it with its store and + # warns on a difference; the store stays the source of truth. + expected_rows: dict[str, int] = {} for node in (self._persistent_node_map or {}).values(): + if isinstance(node, (FunctionJobNode, OperatorJobNode)): + count = node._expected_row_count_to_save(self._last_run_finished) + if count is not None: + expected_rows[node.label] = count if isinstance(node, FunctionJobNode): node_job_content_hashes[node.label] = node.content_hash().to_string() - # A loaded node keeps the expectation it was saved with: a store - # attached since then that lacks the table does not mean the - # original run stored nothing. - node_has_stored_records[node.label] = ( - node._expects_stored_records or node._has_stored_records() - ) elif isinstance(node, OperatorJobNode): # An operator node's stored rows carry its run's content hash too # (ITL-1043). A node loaded from a file without it cannot know it: its @@ -1369,7 +1373,9 @@ def save(self, path: "str | Path") -> None: "status": status, "unbound_sources": list(self.unbound_sources), "node_job_content_hashes": node_job_content_hashes, - "node_has_stored_records": node_has_stored_records, + # A separate, optional section: deleting it changes nothing but the + # check (ITL-1043). + **({"expected_rows": expected_rows} if expected_rows else {}), }, "pipeline": pipeline_data, "bindings": { @@ -1541,11 +1547,11 @@ def load( node_job_content_hashes: dict[str, str] = run_block.get( "node_job_content_hashes", {} ) - # Absent in files saved before ITL-904: such nodes read zero rows from a - # store that has none, as before. - node_has_stored_records: dict[str, bool] = run_block.get( - "node_has_stored_records", {} - ) + # The optional, advisory section (ITL-1043): a node with an entry warns when + # its store holds another number of rows. Without it nothing is checked. The + # ITL-904 flag (node_has_stored_records) of older files is not used: the store + # is the source of truth. + expected_rows: dict[str, int] = run_block.get("expected_rows", {}) for node_hash in _nx.topological_sort(job._hash_graph): bp_node = pipeline._persistent_node_map.get(node_hash) @@ -1576,9 +1582,7 @@ def load( descriptor, input_stream=input_stream, job_content_hash=live_content_hash, - expect_stored_records=node_has_stored_records.get( - bp_node.label, False - ), + expected_row_count=expected_rows.get(bp_node.label), ) job._persistent_node_map[node_hash] = fjn @@ -1600,6 +1604,7 @@ def load( input_streams=input_streams, databases={}, job_content_hash=node_job_content_hashes.get(bp_node.label), + expected_row_count=expected_rows.get(bp_node.label), ) job._persistent_node_map[node_hash] = ojn diff --git a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md index 993d91365..863775094 100644 --- a/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md +++ b/superpowers/plans/2026-11-12-itl-1043-loaded-operator-node.md @@ -23,6 +23,11 @@ Options 2 and 3 (saved operator run content hash; saved top-level nullability, r decides lossless vs lossy load (never the version number). Pipeline._unrecorded_*_nullability (per node) keeps a lossy file lossy on re-save; OperatorJobNode._run_content_hash() is None for a loaded node without a saved run hash, so re-save leaves it out. +- Final shape (developer, Nov 12): store is the source of truth. Known key (run content + hash in file) -> loaded node reads like live REPLAY (empty, NOTHING_STORED). Unknown key + -> raise / head UNAVAILABLE (ITL-1048). No row count as truth; optional advisory + run.expected_rows -> StoreContentWarning once per node. node_has_stored_records dropped + (function nodes warn instead of raise). Dangling result refs: unchanged (logged, matches live). - PUSH RULE (coordinator): do not push the format fields until PR #345 (ITL-1026, format 0.2.0) has merged; then merge origin/main and push once, open the PR (Fixes ITL-1043). - Pre-existing contradiction, not changed here: `OperatorJobNode.head` says "a loaded node diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index add0c153e..5c9be80d7 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -16,6 +16,7 @@ import subprocess import sys import uuid +import warnings from collections.abc import Callable from typing import Any @@ -27,6 +28,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.operators import Join from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import StoreContentWarning from orcapod.pipeline.serialization import LoadStatus from orcapod.sources import DictSource from orcapod.types import NodeConfig @@ -157,7 +159,8 @@ def record() -> None: path = _save_small_job(tmp_path, record) data = json.loads(pathlib.Path(path).read_text()) - data["run"]["node_has_stored_records"]["stamp"] = False + # Without the advisory count, an empty store is read without any check. + data["run"].pop("expected_rows", None) _ = pathlib.Path(path).write_text(json.dumps(data)) table = ( PipelineJob.load(path, store=InMemoryArrowDatabase()).nodes["stamp"].as_table() @@ -346,7 +349,7 @@ def double(x: int) -> int: def test_other_context_records_do_not_satisfy_saved_expectation( tmp_path: pathlib.Path, read_method: str ) -> None: - """Matching node hashes from another context cannot hide missing saved results.""" + """Records of another data context are not the run's: none is read, with a warning.""" path = _save_small_job(tmp_path, _record_double) written = DeltaTableDatabase(str(tmp_path / "store")) other = InMemoryArrowDatabase() @@ -364,10 +367,12 @@ def test_other_context_records_do_not_satisfy_saved_expectation( other.flush() node = PipelineJob.load(path, store=other).nodes["dbl"] - with pytest.raises(RuntimeError, match="saved run stored results.*has none"): + # The store is the source of truth (ITL-1043): it holds none of the run's + # records, so none is read; the job file's count makes it a warning. + with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): result = getattr(node, read_method)() - if read_method == "iter_data": - list(result) + rows = list(result) if read_method == "iter_data" else result + assert rows is None or len(rows) == 0 def test_recomputed_entry_is_read_once_with_its_latest_result( @@ -484,43 +489,53 @@ def test_newer_unavailable_result_does_not_expose_old_persistent_result( # --------------------------------------------------------------------------- -def test_loaded_function_node_saved_with_records_raises_on_a_store_without_them( +def test_loaded_function_node_warns_once_on_a_store_without_its_runs_records( saved: pathlib.Path, ) -> None: - """A store lacking the saved run's records raises, on every read method.""" - # The saved run stored this node's records, so a store without them is not the - # store the job ran against: zero rows would make the results look lost. + """A store lacking the run's records is read as it is, with one warning. + + The store is the source of truth (ITL-1043): every read returns what it holds. + The job file's advisory count makes the difference a ``StoreContentWarning``, + issued once per node. + """ node = _load(saved, store=InMemoryArrowDatabase()).nodes["dbl"] - reads = (node.as_table, lambda: list(node.iter_data()), node.get_all_records) - for read in reads: - with pytest.raises(RuntimeError) as excinfo: - _ = read() - message = str(excinfo.value) - assert message.startswith( - "FunctionJobNode 'dbl': the saved run stored results for this node, but " - "the attached in_memory store has none of them (table 'loaded_demo/double/" - ) - assert message.endswith( - "Load the job with the store it ran against: " - "PipelineJob.load(path, store=...)." - ) + with pytest.warns(StoreContentWarning) as record: + table = node.as_table() + rows = list(node.iter_data()) + records = node.get_all_records() + head = node.head() + assert table.num_rows == 0 + assert table.column_names == ["id", "dbl"] + assert rows == [] + assert records is None + assert head.table.num_rows == 0 + messages = [str(w.message) for w in record if w.category is StoreContentWarning] + assert len(messages) == 1 + assert messages[0].startswith( + "FunctionJobNode 'dbl': the job file records 2 rows for its run, but the " + "in_memory store (table 'loaded_demo/double/" + ) + assert messages[0].endswith( + "holds 0 under the run's key. The read returns what the store holds; the " + "store may not be the one the run wrote, or it may have changed since." + ) -def test_mismatched_delta_store_error_names_the_store_location( +def test_store_content_warning_names_a_delta_stores_location( saved: pathlib.Path, tmp_path: pathlib.Path ) -> None: - """The mismatched-store error names a Delta store's location.""" + """The warning names a Delta store's location.""" other = tmp_path / "other_store" node = _load(saved, store=DeltaTableDatabase(str(other))).nodes["dbl"] - with pytest.raises(RuntimeError, match="the attached store at '") as excinfo: + with pytest.warns(StoreContentWarning, match="the store at '") as record: _ = node.as_table() - assert str(other) in str(excinfo.value) + assert str(other) in str(record[0].message) -def test_store_with_only_another_runs_records_is_a_mismatched_store( +def test_store_with_only_another_runs_records_reads_none_with_a_warning( tmp_path: pathlib.Path, ) -> None: - """A shared table holding another run's records, not this node's, raises.""" + """A shared table holding another run's records, not this node's, reads none.""" def record_other_source() -> None: """Record ``double`` over another source: same table, other records.""" @@ -545,37 +560,41 @@ def double(x: int) -> int: other_store = DeltaTableDatabase(str(tmp_path / "other_run" / "store")) node = PipelineJob.load(path, store=other_store).nodes["dbl"] # Non-public on purpose: confirm the shared table exists, so this test checks - # the per-record test, not the missing-table case. + # the per-record count, not the missing-table case. assert node._pipeline_database is not None assert node._pipeline_database.table_exists(node._versioned_pipeline_path) - with pytest.raises(RuntimeError, match="the saved run stored results"): - _ = node.as_table() + with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): + assert node.as_table().num_rows == 0 -def test_saved_file_records_which_nodes_had_stored_records( +def test_saved_file_records_the_rows_each_node_had_in_the_store( saved: pathlib.Path, ) -> None: - """``save()`` records, per function node, that its records were stored.""" + """``save()`` records, per job node, the rows its run left in the store.""" run = json.loads((saved / "job.json").read_text())["run"] - assert run["node_has_stored_records"] == { - "dbl": True, - "inc": True, - "half": True, - "sq": True, + # The join is not cached (cache mode OFF): its run stored nothing. + assert run["expected_rows"] == { + "dbl": 2, + "inc": 2, + "half": 2, + "sq": 2, + "joined": 0, } + assert "node_has_stored_records" not in run -def test_saving_a_loaded_job_keeps_the_stored_record_flags( +def test_saving_a_loaded_job_keeps_the_expected_rows( saved: pathlib.Path, tmp_path: pathlib.Path ) -> None: - """Re-saving a job loaded with the wrong store keeps the saved expectations.""" + """Re-saving a job loaded with another store keeps the saved counts.""" resaved = tmp_path / "resaved.json" _load(saved, store=InMemoryArrowDatabase()).save(str(resaved)) + original = json.loads((saved / "job.json").read_text())["run"]["expected_rows"] run = json.loads(resaved.read_text())["run"] - assert run["node_has_stored_records"]["dbl"] is True + assert run["expected_rows"] == original node = PipelineJob.load(str(resaved), store=InMemoryArrowDatabase()).nodes["dbl"] - with pytest.raises(RuntimeError, match="the saved run stored results"): - _ = node.as_table() + with pytest.warns(StoreContentWarning, match="records 2 rows"): + assert node.as_table().num_rows == 0 def test_loaded_function_node_with_nothing_stored_returns_zero_rows( @@ -585,30 +604,51 @@ def test_loaded_function_node_with_nothing_stored_returns_zero_rows( def nothing_stored(data: dict[str, Any]) -> None: """Mark the node as saved with nothing stored.""" - data["run"]["node_has_stored_records"]["dbl"] = False + data["run"]["expected_rows"]["dbl"] = 0 path = _rewrite_job_file(saved, tmp_path, nothing_stored) node = PipelineJob.load(path, store=InMemoryArrowDatabase()).nodes["dbl"] - table = node.as_table() + with warnings.catch_warnings(): + # Zero expected and zero found: nothing to warn about. + warnings.simplefilter("error", StoreContentWarning) + table = node.as_table() assert table.num_rows == 0 assert table.column_names == ["id", "dbl"] assert list(node.iter_data()) == [] -def test_file_saved_without_the_stored_records_field_reads_zero_rows( +def test_file_without_the_expected_rows_section_reads_without_a_check( saved: pathlib.Path, tmp_path: pathlib.Path ) -> None: - """A file saved before the field existed loads as before.""" + """The section is optional: deleting it changes nothing but the check.""" + + def drop_section(data: dict[str, Any]) -> None: + """Remove the advisory section, by hand.""" + del data["run"]["expected_rows"] + + path = _rewrite_job_file(saved, tmp_path, drop_section) + with warnings.catch_warnings(): + warnings.simplefilter("error", StoreContentWarning) + job = PipelineJob.load(path, store=InMemoryArrowDatabase()) + assert job.nodes["dbl"].as_table().num_rows == 0 + assert PipelineJob.load(path).nodes["dbl"].as_table().num_rows == 2 - # A store without the node's records gives zero rows. - def drop_field(data: dict[str, Any]) -> None: - """Remove the stored-records field, as in an older file.""" - del data["run"]["node_has_stored_records"] - path = _rewrite_job_file(saved, tmp_path, drop_field) - job = PipelineJob.load(path, store=InMemoryArrowDatabase()) - assert job.nodes["dbl"].as_table().num_rows == 0 - assert PipelineJob.load(path).nodes["dbl"].as_table().num_rows == 2 +def test_a_file_with_the_old_stored_records_flag_loads_without_a_check( + saved: pathlib.Path, tmp_path: pathlib.Path +) -> None: + """The ITL-904 flag of an earlier file is not used: it never raises or warns.""" + + def old_flag(data: dict[str, Any]) -> None: + """Write the file as before ITL-1043: the flag, no section.""" + del data["run"]["expected_rows"] + data["run"]["node_has_stored_records"] = {"dbl": True} + + path = _rewrite_job_file(saved, tmp_path, old_flag) + with warnings.catch_warnings(): + warnings.simplefilter("error", StoreContentWarning) + node = PipelineJob.load(path, store=InMemoryArrowDatabase()).nodes["dbl"] + assert node.as_table().num_rows == 0 def test_loaded_function_node_without_store_says_how_to_attach_one( diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index 256b7c589..d71d652c7 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import warnings import pathlib from collections.abc import Callable from typing import Any @@ -24,7 +25,8 @@ from orcapod.core.operators import Join from orcapod.core.streams import HeadStatus from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase -from orcapod.pipeline import Pipeline +from orcapod.errors import StoreContentWarning +from orcapod.pipeline import Pipeline, SyncPipelineOrchestrator from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.sources import ArrowTableSource @@ -361,7 +363,9 @@ def test_a_store_holding_only_other_runs_rows_gives_an_empty_result( ) _ = other.run() node = _load_replaying(saved, other_store).nodes["joined"] - _assert_reads_as_live_with_no_rows(node, _live_replaying(other_store, job)) + # The job file records the run's 2 rows: an advisory warning, not an error. + with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): + _assert_reads_as_live_with_no_rows(node, _live_replaying(other_store, job)) @pytest.mark.parametrize("nullability", list(NULLABILITY)) @@ -372,7 +376,9 @@ def test_an_absent_cache_table_gives_an_empty_result( job = _run(InMemoryArrowDatabase(), nullability) empty_store = InMemoryArrowDatabase() node = _load_replaying(_save(job, tmp_path), empty_store).nodes["joined"] - _assert_reads_as_live_with_no_rows(node, _live_replaying(empty_store, job)) + # The job file records the run's 2 rows: an advisory warning, not an error. + with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): + _assert_reads_as_live_with_no_rows(node, _live_replaying(empty_store, job)) def _run_empty_join(store: ArrowDatabaseProtocol) -> PipelineJob: @@ -624,3 +630,140 @@ def drop_src_a(data: dict[str, Any]) -> None: assert _recorded_fields(resaved) == _recorded_fields(saved) assert _recorded_fields(resaved)["src_a"] is None assert _recorded_fields(resaved)["src_b"] == {"tag": ["id"], "data": ["b"]} + + +# --------------------------------------------------------------------------- +# run.expected_rows: an optional, advisory count; the store stays the truth +# --------------------------------------------------------------------------- + +READ_NODES = ["inc", "joined"] + + +def _read_all(job: PipelineJob, label: str) -> pa.Table: + """Read a loaded node three ways and return its table.""" + node = job.nodes[label] + _ = list(node.iter_data()) + _ = node.head() + return node.as_table() + + +def _loaded(saved: pathlib.Path, store: ArrowDatabaseProtocol) -> PipelineJob: + """Load the job; its join replays (the file does not record its cache mode).""" + return _load_replaying(saved, store) + + +def test_the_job_file_records_the_rows_each_node_has_in_the_store( + tmp_path: pathlib.Path, +) -> None: + """``run.expected_rows``: the run's rows under each node's key, in its own section.""" + data = json.loads( + _save(_run(InMemoryArrowDatabase(), "non_null"), tmp_path).read_text() + ) + assert data["run"]["expected_rows"] == {"inc": 2, "joined": 2} + + +@pytest.mark.parametrize("label", READ_NODES) +@pytest.mark.parametrize("store_kind", STORES) +def test_fewer_rows_than_expected_warn_once_and_read_what_the_store_holds( + store_kind: str, label: str, tmp_path: pathlib.Path +) -> None: + """Another store with none of the run's rows: no rows, one warning per node.""" + saved = _save(_run(InMemoryArrowDatabase(), "non_null"), tmp_path) + loaded = _loaded(saved, _store(store_kind, tmp_path)) + with pytest.warns(StoreContentWarning) as record: + table = _read_all(loaded, label) + _ = _read_all(loaded, label) + assert table.num_rows == 0 + messages = [str(w.message) for w in record if w.category is StoreContentWarning] + assert len(messages) == 1 + assert f"Node {label!r}: the job file records 2 rows for its run" in messages[0] + assert "holds 0 under the run's key" in messages[0] + + +@pytest.mark.parametrize("label", READ_NODES) +@pytest.mark.parametrize("store_kind", STORES) +def test_more_rows_than_expected_warn_and_read_them_all( + store_kind: str, label: str, tmp_path: pathlib.Path +) -> None: + """The store holds more than the file records: all of them, one warning.""" + store = _store(store_kind, tmp_path) + job = _run(store, "non_null") + + def expect_one(data: dict[str, Any]) -> None: + """Record one row where the run left two.""" + data["run"]["expected_rows"][label] = 1 + + loaded = _loaded(_save(job, tmp_path, expect_one), store) + with pytest.warns(StoreContentWarning, match="records 1 row for its run.*holds 2"): + table = _read_all(loaded, label) + assert _sorted(table).equals(_sorted(job.nodes[label].as_table())) + + +@pytest.mark.parametrize("store_kind", STORES) +def test_matching_counts_and_zero_expected_zero_found_do_not_warn( + store_kind: str, tmp_path: pathlib.Path +) -> None: + """No warning when the store holds what the file records, zero included.""" + store = _store(store_kind, tmp_path) + job = _run(store, "non_null") + empty = _run_empty_join(_store("memory", tmp_path)) + with warnings.catch_warnings(): + warnings.simplefilter("error", StoreContentWarning) + loaded = _loaded(_save(job, tmp_path), store) + for label in READ_NODES: + assert _read_all(loaded, label).num_rows == 2 + empty_saved = _save(empty, tmp_path / "empty") + assert json.loads(empty_saved.read_text())["run"]["expected_rows"] == { + "joined": 0 + } + node = _load_replaying(empty_saved, InMemoryArrowDatabase()).nodes["joined"] + assert node.as_table().num_rows == 0 + + +def test_deleting_the_section_by_hand_loads_identically_without_a_check( + tmp_path: pathlib.Path, +) -> None: + """Without ``run.expected_rows`` every read gives the same rows and no warning.""" + store = InMemoryArrowDatabase() + job = _run(store, "nullable_tag_and_data") + + def delete_section(data: dict[str, Any]) -> None: + """Delete the advisory section.""" + del data["run"]["expected_rows"] + + with_section = _loaded(_save(job, tmp_path), store) + without = _loaded(_save(job, tmp_path / "bare", delete_section), store) + other = _loaded( + _save(job, tmp_path / "bare2", delete_section), InMemoryArrowDatabase() + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", StoreContentWarning) + for label in READ_NODES: + assert _sorted(_read_all(without, label)).equals( + _sorted(_read_all(with_section, label)) + ) + assert _read_all(other, label).num_rows == 0 + + +def test_a_run_that_did_not_finish_records_no_counts(tmp_path: pathlib.Path) -> None: + """Counts are written only when certain: not before a run, nor after a failed one.""" + store = InMemoryArrowDatabase() + sources = _sources("non_null") + job = PipelineJob(name=JOB_NAME, store=store) + with job: + _ = Join()(sources["src_a"], sources["src_b"], label="joined") + pending = _save(job, tmp_path / "pending") + assert "expected_rows" not in json.loads(pending.read_text())["run"] + + @function_pod(output_keys="a1") + def fails(a: int) -> int: + """Fail on every row.""" + raise ValueError(f"no {a}") + + failing = PipelineJob(name=JOB_NAME, store=store) + with failing: + _ = fails.pod(_sources("non_null")["src_a"], label="fails") + with pytest.raises(ValueError, match="no 0|no 1"): + _ = failing.run(orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast")) + failed = _save(failing, tmp_path / "failed") + assert "expected_rows" not in json.loads(failed.read_text())["run"] From cb98a39b7155087411341fdb04fd0c0c77c558c4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 00:00:04 -0800 Subject: [PATCH 15/16] docs(design_issues): note on ITL-904 PJLOAD1: stored-records flag superseded by ITL-1043 Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-904-PJLOAD1.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/design_issues/ITL-904-PJLOAD1.md b/design_issues/ITL-904-PJLOAD1.md index 699505c63..919a4351c 100644 --- a/design_issues/ITL-904-PJLOAD1.md +++ b/design_issues/ITL-904-PJLOAD1.md @@ -89,3 +89,12 @@ latest-index selection (persistent wins equal indices). Loaded REPLAY operators when their cache table is absent; loaded function `run()` rejects computation, and loaded jobs report `is_runnable()` as false even after binding sources. Source-schema restoration uses the saved context converter in both normal and fallback branches. + +**Superseded in part (ITL-1043, developer decision, November 12, 2026):** the store is the +source of truth, with one rule for function and operator nodes. `run.node_has_stored_records` +is no longer written, and is ignored when an older file has it: a loaded function node no +longer raises on a store without its run's records; it returns what the store holds. The job +file's optional, advisory `run.expected_rows` makes a difference one `StoreContentWarning` +per node. Likewise, a loaded REPLAY operator whose job file records its run's content hash +reads an absent cache table as no rows instead of raising. See +`design_issues/ITL-1043-LJ1.md`. From cc1038d72e187114a36bcfde581ad9932a51167c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 00:38:36 -0800 Subject: [PATCH 16/16] fix(pipeline): advisory check on a loaded operator's get_all_records; source node falls back to its output schema's nullability record (ITL-1043) CodeRabbit round 2 on #352. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 3 ++ src/orcapod/pipeline/graph.py | 6 ++- .../loaded_job/test_loaded_operator_cache.py | 40 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index e65a0dac4..624be3b4f 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1562,6 +1562,9 @@ def get_all_records( return None results = self._read_stored_records() + if self._loaded: + # The advisory check of every read of a loaded node (ITL-1043). + self._check_expected_rows(0 if results is None else results.num_rows) if results is None: return None diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 2d769cd80..409b8622d 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -359,7 +359,11 @@ def load(cls, path: str | Path) -> "Pipeline": if "tag_schema" in source_config and "data_schema" in source_config: saved_tag = source_config["tag_schema"] saved_data = source_config["data_schema"] - saved_nullable = source_config.get("nullable") + # A source node's output schema has the same columns: its + # record stands in when the source config has none. + saved_nullable = source_config.get( + "nullable", descriptor["output_schema"].get("nullable") + ) else: saved_tag = descriptor["output_schema"]["tag"] saved_data = descriptor["output_schema"]["data"] diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index d71d652c7..f1d4ce575 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -767,3 +767,43 @@ def fails(a: int) -> int: _ = failing.run(orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast")) failed = _save(failing, tmp_path / "failed") assert "expected_rows" not in json.loads(failed.read_text())["run"] + + +# --------------------------------------------------------------------------- +# CodeRabbit round 2 (PR #352) +# --------------------------------------------------------------------------- + + +def test_get_all_records_of_a_loaded_operator_runs_the_advisory_check( + tmp_path: pathlib.Path, +) -> None: + """``get_all_records()`` warns once, like the other reads, and returns no rows.""" + saved = _save(_run(InMemoryArrowDatabase(), "non_null"), tmp_path) + node = _loaded(saved, InMemoryArrowDatabase()).nodes["joined"] + with pytest.warns(StoreContentWarning) as record: + assert node.get_all_records() is None + assert node.get_all_records() is None + assert len([w for w in record if w.category is StoreContentWarning]) == 1 + + +def test_a_source_node_without_its_own_record_takes_its_output_schemas( + tmp_path: pathlib.Path, +) -> None: + """A source config without the record restores nullability from the output schema.""" + store = InMemoryArrowDatabase() + job = _run(store, "nullable_tag_and_data") + + def drop_source_config_records(data: dict[str, Any]) -> None: + """Remove only the source configs' records, by hand.""" + for descriptor in data["pipeline"]["nodes"].values(): + descriptor.get("source_config", {}).pop("nullable", None) + + saved = _save(job, tmp_path, drop_source_config_records) + loaded = PipelineJob.load(str(saved), store=store) + for label in ("src_a", "src_b"): + assert loaded.nodes[label].output_schema() == job.nodes[label].output_schema() + resaved = tmp_path / "resaved.json" + loaded.save(resaved) + fields = _recorded_fields(resaved) + assert fields["src_a"] == {"tag": ["id"], "data": ["a"]} + assert fields["src_a.source_config"] is None