diff --git a/changelog.d/ITL-988.md b/changelog.d/ITL-988.md new file mode 100644 index 000000000..2fda9d792 --- /dev/null +++ b/changelog.d/ITL-988.md @@ -0,0 +1,4 @@ +### Fixed + +- A function pod created with `pod_cache_database=` (a `CachedFunctionPod`) is now recorded when it is applied inside a `PipelineJob`: the job has the source and the function node, and runs, saves and loads like an uncached pod. Its inputs are joined once. The pod cache is honoured in a job (a row already in it is not recomputed). The pod cache database is not written into the saved job, like the node's own databases, so a loaded job does not use it. +- The table of a `CachedFunctionPod` stream has the same columns, types and order whether a row was computed or served from the cache, and no meta columns unless asked for (`columns={"meta": True}` or `all_info=True`); then both states carry only `__computed`. Before, a served row also carried `__pf_var_*`, `__pf_exec_*` and `__pod_ts`, and the plain table showed `__computed`. diff --git a/design_issues/ITL-988-CP1.md b/design_issues/ITL-988-CP1.md new file mode 100644 index 000000000..44a57910b --- /dev/null +++ b/design_issues/ITL-988-CP1.md @@ -0,0 +1,10 @@ +## `src/orcapod/core/cached_function_pod.py` — pod-level result cache (ITL-988) + +### CP1 — a pod-cached function pod was not recorded in a job, and its table showed cache state +**Status:** resolved +**Severity:** high +**Issue:** ITL-988 + +`CachedFunctionPod.process` never recorded the invocation with the tracker, so a pod created with `pod_cache_database=` and applied in a `PipelineJob` left the job with no nodes; it also joined its inputs twice. Its stream's `as_table()` carried `__computed` for computed rows and `__pf_var_*` / `__pf_exec_*` / `__pod_ts` for rows served from the cache, so meta columns leaked into the plain table and the columns depended on cache state. + +**Fix:** `process` mirrors `FunctionPod.process` (join once, validate, record, return), `CachedFunctionPod.to_config` serializes the inner pod so a job saves as it does for an uncached pod, and the stream keeps only `__computed` and drops meta columns unless asked for. The pod cache database is not saved in a job, so a loaded job does not use it. `CachedFunctionPod` still has no `from_config` (it cannot be rebuilt without its cache database). diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index bb36c512a..f40a67cf5 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -4,10 +4,13 @@ import logging from datetime import datetime, timezone -from typing import TYPE_CHECKING +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from orcapod.core.datagrams import Datagram -from orcapod.core.function_pod import WrappedFunctionPod +from orcapod.core.function_pod import FunctionPodStream, WrappedFunctionPod +from orcapod.system_constants import constants +from orcapod.types import ColumnConfig from orcapod.contexts.stored import StaleTables from orcapod.core.result_cache import ResultCache from orcapod.hooks import InvocationStatus @@ -399,22 +402,72 @@ def get_all_cached_outputs( include_system_columns=include_system_columns, ) + def to_config(self) -> dict[str, Any]: + """Serialize this pod as its inner function pod. + + A saved job describes the function, not where its results are cached, so a + pod-cached pod is saved exactly as the uncached pod would be. The pod cache + database is not part of the saved job, and a loaded job does not use it. + + Returns: + The inner function pod's config. + """ + return self._function_pod.to_config() + def process( self, *streams: StreamProtocol, label: str | None = None ) -> StreamProtocol: """Invoke the inner pod but with pod-level caching on process_data. The stream returned uses *this* pod's ``process_data`` (which - includes caching) rather than the inner pod's. + includes caching) rather than the inner pod's. Like ``FunctionPod.process`` + it joins the inputs once, validates them and records the invocation with the + active trackers (so a ``PipelineJob`` records this pod). """ - from orcapod.core.function_pod import FunctionPodStream - - # Validate and prepare the input stream - input_stream = self._function_pod.handle_input_streams(*streams) - self._function_pod.validate_inputs(*streams) - - return FunctionPodStream( + input_stream = self.handle_input_streams(*streams) + self._validate_input_schema(input_stream.output_schema()[1]) + # The pod has no ``from_config`` (it cannot be rebuilt without its cache + # database), the one member of FunctionPodProtocol it lacks; the tracker uses + # none of it. + self.tracker_manager.record_function_pod_invocation( + self, # pyright: ignore[reportArgumentType] + input_stream, + label=label, + ) + return _CachedFunctionPodStream( function_pod=self, input_stream=input_stream, label=label, ) + + +class _CachedFunctionPodStream(FunctionPodStream): + """The stream of a ``CachedFunctionPod``, with the same columns in every cache state. + + A row computed in this process carries only the ``__computed`` meta column; a row + served from the cache also carries the function variation and execution columns it + was stored with. This stream keeps ``__computed`` and drops the others, and its + table has no meta columns unless asked for, as an uncached pod's table. + """ + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + keep = CachedFunctionPod.RESULT_COMPUTED_FLAG + for tag, data in super().iter_data(): + stored = [name for name in data.meta_columns if name != keep] + if stored: + data = data.drop_meta_columns(*stored) + yield tag, data + + def as_table( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "pa.Table": + table = super().as_table(columns=columns, all_info=all_info) + column_config = ColumnConfig.handle_config(columns, all_info=all_info) + if column_config.meta or column_config.all_info: + return table + return table.drop_columns( + [n for n in table.column_names if n.startswith(constants.META_PREFIX)] + ) diff --git a/superpowers/plans/2026-11-12-itl-988-cached-pod-job.md b/superpowers/plans/2026-11-12-itl-988-cached-pod-job.md new file mode 100644 index 000000000..e68b023ff --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-988-cached-pod-job.md @@ -0,0 +1,20 @@ +# ITL-988 — pod-cached function pod in a PipelineJob; plain table independent of cache state + +Branch `eywalker/itl-988-a-pod-cached-function-pod-is-not-recorded`. Only `src/orcapod/core/cached_function_pod.py` changes (plus tests, changelog fragment, design_issues file, docs sentence). + +## Decisions (coordinator, 2026-11-12) +- Defect 1: `CachedFunctionPod.process` mirrors `FunctionPod.process`: join once, `_validate_input_schema`, `tracker_manager.record_function_pod_invocation(self, input_stream, label=label)`, return the stream. Record SELF (the pod cache is honoured: a row in the pod cache is not recomputed in the job). Job save needs `CachedFunctionPod.to_config` delegating to the inner pod's config (identical saved file to the uncached pod, apart from `run_id`). The pod cache database is not saved, so a loaded job does not use it. +- Defect 2: `process` returns a `FunctionPodStream` subclass in this file: it strips the stored `__pf_var_*`/`__pf_exec_*`/`__pod_ts` meta from served rows (keeps `__computed`), and `as_table` drops meta columns unless `meta` is requested. `process_data` is untouched (FunctionNode and hooks read `__computed` from it). +- #327 only changes `_FunctionPodBase.__call__`; process receives the already-narrowed stream. Re-check after it merges. + +## Facts measured on main 25650c42 +- Uncached node: ph a1b47334ae8c, ch d4bc20dfcee4; record SELF gives the same, same store paths; save of SELF fails (no `to_config`). +- Table columns: computed `[k,total,__computed]`; served `[k,total,__pf_*...,__pod_ts,__computed]`; uncached `[k,total]`. + +## Conditions to test +(a) saved file equals uncached save except `run/run_id`; (b) save+load in memory and Delta, sync and async, loaded job reads stored results without pod cache; (c) docs/changelog state pod cache DB is not saved; (d) report from_config/proxy paths that cannot round-trip; (e) a row is computed once, appears in pod cache and job result table; second run computes nothing. +Fallback if any fails: record INNER plus info log; send facts to coordinator. + +## Steps +1. Tests `tests/test_core/function_pod/test_cached_function_pod_job.py` (fail first). +2. Implement. 3. Full suite, ruff, basedpyright, golden. 4. changelog.d/ITL-988.md, design_issues/ITL-988-CP1.md, docs. 5. Merge origin/main, push, PR. diff --git a/tests/test_core/function_pod/test_cached_function_pod_job.py b/tests/test_core/function_pod/test_cached_function_pod_job.py new file mode 100644 index 000000000..eaf30a2c2 --- /dev/null +++ b/tests/test_core/function_pod/test_cached_function_pod_job.py @@ -0,0 +1,241 @@ +"""A pod-cached function pod in a ``PipelineJob``, and its table (ITL-988). + +- The job records the pod's source and function node, like an uncached pod, and runs, + saves and loads it (in memory and on Delta, with the sync and the async orchestrator). +- The pod cache is honoured in a job: a row is computed once. +- The plain table of a ``CachedFunctionPod`` stream has the columns of an uncached pod's + table whether a row was computed or served from the cache; meta columns come only on + request, and then the same ones in both states. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from orcapod import PipelineJob +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.function_pod import FunctionPod +from orcapod.core.operators import Join +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.sources import DictSource +from orcapod.types import ColumnConfig + +ORCHESTRATORS: dict[str, Callable[[], Any]] = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} +ROWS = [{"k": 1, "x": 2, "y": 3}, {"k": 2, "x": 5, "y": 5}] + + +def make_source(rows: list[dict[str, int]] | None = None) -> DictSource: + return DictSource(rows or ROWS, tag_columns=["k"], source_id="s") + + +class Counter: + """An ``add`` function that records the ``x`` of every call.""" + + def __init__(self) -> None: + self.calls: list[int] = [] + + def fn(self) -> Callable[[int, int], int]: + def add(x: int, y: int) -> int: + self.calls.append(x) + return x + y + + return add + + def pod(self) -> FunctionPod: + return FunctionPod.from_fn(self.fn(), output_keys="total") + + +def make_db(kind: str, tmp_path: Path, name: str): + if kind == "memory": + return InMemoryArrowDatabase() + return DeltaTableDatabase(tmp_path / name) + + +def columns(table) -> list[tuple[str, str]]: + return [(f.name, str(f.type)) for f in table.schema] + + +# -- the table --------------------------------------------------------------------- + + +@pytest.mark.parametrize("all_info", [False, True]) +def test_table_columns_do_not_depend_on_cache_state(all_info: bool) -> None: + counter = Counter() + cached = CachedFunctionPod( + function_pod=counter.pod(), result_database=InMemoryArrowDatabase() + ) + computed = cached.process(make_source()).as_table(all_info=all_info) + served = cached.process(make_source()).as_table(all_info=all_info) + assert counter.calls == [2, 5] # the second table came from the cache + assert columns(computed) == columns(served) + + +def test_plain_table_equals_the_uncached_pods_table() -> None: + counter = Counter() + uncached = counter.pod().process(make_source()).as_table() + cached = CachedFunctionPod( + function_pod=counter.pod(), result_database=InMemoryArrowDatabase() + ) + for _ in range(2): # computed, then served + table = cached.process(make_source()).as_table() + assert columns(table) == columns(uncached) + assert table.to_pylist() == uncached.to_pylist() + + +def test_meta_columns_only_when_asked_for_and_the_same_in_both_states() -> None: + cached = CachedFunctionPod( + function_pod=Counter().pod(), result_database=InMemoryArrowDatabase() + ) + computed = cached.process(make_source()).as_table(columns={"meta": True}) + served = cached.process(make_source()).as_table(columns={"meta": True}) + assert columns(computed) == columns(served) + assert [n for n, _ in columns(computed) if n.startswith("__")] == ["__computed"] + assert computed.column("__computed").to_pylist() == [True, True] + assert served.column("__computed").to_pylist() == [False, False] + + +@pytest.mark.parametrize("columns", [ColumnConfig(all_info=True), {"all_info": True}]) +def test_all_info_in_the_column_config_keeps_the_meta_columns(columns: Any) -> None: + cached = CachedFunctionPod( + function_pod=Counter().pod(), result_database=InMemoryArrowDatabase() + ) + table = cached.process(make_source()).as_table(columns=columns) + assert "__computed" in table.column_names + + +# -- the job ----------------------------------------------------------------------- + + +def run_job(pod, store, orchestrator: str) -> PipelineJob: + job = PipelineJob(name="j", store=store) + with job: + pod.process(make_source(), label="add") + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + return job + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize("kind", ["memory", "delta"]) +def test_cached_pod_is_recorded_and_run_like_an_uncached_pod( + kind: str, orchestrator: str, tmp_path: Path +) -> None: + counter = Counter() + plain = run_job(counter.pod(), make_db(kind, tmp_path, "plain"), orchestrator) + cached_pod = CachedFunctionPod( + function_pod=counter.pod(), result_database=make_db(kind, tmp_path, "pod") + ) + job = run_job(cached_pod, make_db(kind, tmp_path, "cached"), orchestrator) + + assert list(job.nodes) == list(plain.nodes) == ["s", "add"] + assert job.nodes["add"].pipeline_hash() == plain.nodes["add"].pipeline_hash() + assert job.nodes["add"].content_hash() == plain.nodes["add"].content_hash() + assert columns(job.nodes["add"].as_table()) == columns( + plain.nodes["add"].as_table() + ) + assert ( + job.nodes["add"].as_table().to_pylist() + == plain.nodes["add"].as_table().to_pylist() + ) + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +@pytest.mark.parametrize("kind", ["memory", "delta"]) +def test_row_is_computed_once_and_stored_in_both_caches( + kind: str, orchestrator: str, tmp_path: Path +) -> None: + counter = Counter() + pod_db = make_db(kind, tmp_path, "pod") + cached_pod = CachedFunctionPod(function_pod=counter.pod(), result_database=pod_db) + store = make_db(kind, tmp_path, "cached") + + run_job(cached_pod, store, orchestrator) + assert sorted(counter.calls) == [2, 5] + pod_rows = cached_pod.get_all_cached_outputs() + assert pod_rows is not None and pod_rows.num_rows == 2 + assert sorted(counter.calls) == [2, 5] + + # A second run over the same store computes nothing. + run_job(cached_pod, store, orchestrator) + assert sorted(counter.calls) == [2, 5] + + +def test_row_in_the_pod_cache_is_not_recomputed_by_a_job() -> None: + counter = Counter() + cached_pod = CachedFunctionPod( + function_pod=counter.pod(), result_database=InMemoryArrowDatabase() + ) + cached_pod.process(make_source(ROWS[:1])).as_table() # fills the pod cache + counter.calls.clear() + run_job(cached_pod, InMemoryArrowDatabase(), "sync") + assert counter.calls == [5] + + +def test_inputs_are_joined_once() -> None: + counter = Counter() + cached_pod = CachedFunctionPod( + function_pod=counter.pod(), result_database=InMemoryArrowDatabase() + ) + left = DictSource([{"k": 1, "x": 2}], tag_columns=["k"], source_id="l") + right = DictSource([{"k": 1, "y": 3}], tag_columns=["k"], source_id="r") + joins: list[int] = [] + original = Join.process + + def counting(self, *streams, **kwargs): + joins.append(len(streams)) + return original(self, *streams, **kwargs) + + # Counting the calls of the join is the test: the public result is the same. + Join.process = counting # type: ignore[method-assign] + try: + stream = cached_pod.process(left, right) + finally: + Join.process = original # type: ignore[method-assign] + assert joins == [2] + assert stream.as_table().to_pylist() == [{"k": 1, "total": 5}] + + +# -- save and load ----------------------------------------------------------------- + + +def saved(job: PipelineJob, path: Path) -> dict[str, Any]: + job.save(path) + return json.loads(path.read_text()) + + +@pytest.mark.parametrize("kind", ["memory", "delta"]) +def test_saved_job_equals_the_uncached_pods_and_loads( + kind: str, tmp_path: Path +) -> None: + counter = Counter() + plain_store = make_db(kind, tmp_path, "plain") + plain = run_job(counter.pod(), plain_store, "sync") + store = make_db(kind, tmp_path, "cached") + cached_pod = CachedFunctionPod( + function_pod=counter.pod(), result_database=make_db(kind, tmp_path, "pod") + ) + job = run_job(cached_pod, store, "sync") + + plain_saved = saved(plain, tmp_path / "plain.json") + cached_saved = saved(job, tmp_path / "cached.json") + # Fields that differ between any two saves: the run id and the store's location. + for doc in (plain_saved, cached_saved): + doc["run"].pop("run_id") + doc["bindings"].pop("store") + assert cached_saved == plain_saved + + # A loaded job reads its stored results; the pod cache is not part of it. + loaded = PipelineJob.load(tmp_path / "cached.json", store=store) + assert loaded.nodes["add"].as_table().to_pylist() == [ + {"k": 1, "total": 5}, + {"k": 2, "total": 10}, + ] diff --git a/tests/test_core/operators/test_join_meta_columns.py b/tests/test_core/operators/test_join_meta_columns.py index 7eb102705..8f1ad5903 100644 --- a/tests/test_core/operators/test_join_meta_columns.py +++ b/tests/test_core/operators/test_join_meta_columns.py @@ -40,23 +40,6 @@ META = constants.META_PREFIX -# The meta columns a cached function pod's output carries on a cache hit: the -# result cache's function-variation, execution and timestamp columns plus the -# computed flag. -CACHE_HIT_META = { - f"{META}pf_var_function_name", - f"{META}pf_var_function_version", - f"{META}pf_var_function_signature_hash", - f"{META}pf_var_function_content_hash", - f"{META}pf_var_git_hash", - f"{META}pf_exec_executor_type", - f"{META}pf_exec_executor_info", - f"{META}pf_exec_python_version", - f"{META}pf_exec_extra_info", - f"{META}pod_ts", - f"{META}computed", -} - def add_one_a(a: int) -> int: return a + 1 @@ -120,7 +103,8 @@ def _function_node_streams() -> tuple[StreamProtocol, StreamProtocol]: {f"{META}computed"}, id="cached_pods_compute", ), - pytest.param(_cache_hit_streams, CACHE_HIT_META, id="cached_pods_cache_hit"), + # A stream served from the cache carries the same meta as a computed one (ITL-988). + pytest.param(_cache_hit_streams, {f"{META}computed"}, id="cached_pods_cache_hit"), pytest.param(_function_node_streams, {f"{META}computed"}, id="function_nodes"), pytest.param( lambda: (