Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelog.d/ITL-988.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Fixed

- A function pod created with `pod_cache_database=` (a `CachedFunctionPod`) is now recorded when it is applied inside a `PipelineJob`: the job has the source and the function node, and runs, saves and loads like an uncached pod. Its inputs are joined once. The pod cache is honoured in a job (a row already in it is not recomputed). The pod cache database is not written into the saved job, like the node's own databases, so a loaded job does not use it.
- The table of a `CachedFunctionPod` stream has the same columns, types and order whether a row was computed or served from the cache, and no meta columns unless asked for (`columns={"meta": True}` or `all_info=True`); then both states carry only `__computed`. Before, a served row also carried `__pf_var_*`, `__pf_exec_*` and `__pod_ts`, and the plain table showed `__computed`.
10 changes: 10 additions & 0 deletions design_issues/ITL-988-CP1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
## `src/orcapod/core/cached_function_pod.py` — pod-level result cache (ITL-988)

### CP1 — a pod-cached function pod was not recorded in a job, and its table showed cache state
**Status:** resolved
**Severity:** high
**Issue:** ITL-988

`CachedFunctionPod.process` never recorded the invocation with the tracker, so a pod created with `pod_cache_database=` and applied in a `PipelineJob` left the job with no nodes; it also joined its inputs twice. Its stream's `as_table()` carried `__computed` for computed rows and `__pf_var_*` / `__pf_exec_*` / `__pod_ts` for rows served from the cache, so meta columns leaked into the plain table and the columns depended on cache state.

**Fix:** `process` mirrors `FunctionPod.process` (join once, validate, record, return), `CachedFunctionPod.to_config` serializes the inner pod so a job saves as it does for an uncached pod, and the stream keeps only `__computed` and drops meta columns unless asked for. The pod cache database is not saved in a job, so a loaded job does not use it. `CachedFunctionPod` still has no `from_config` (it cannot be rebuilt without its cache database).
73 changes: 63 additions & 10 deletions src/orcapod/core/cached_function_pod.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,13 @@

import logging
from datetime import datetime, timezone
from typing import TYPE_CHECKING
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any

from orcapod.core.datagrams import Datagram
from orcapod.core.function_pod import WrappedFunctionPod
from orcapod.core.function_pod import FunctionPodStream, WrappedFunctionPod
from orcapod.system_constants import constants
from orcapod.types import ColumnConfig
from orcapod.contexts.stored import StaleTables
from orcapod.core.result_cache import ResultCache
from orcapod.hooks import InvocationStatus
Expand Down Expand Up @@ -399,22 +402,72 @@ def get_all_cached_outputs(
include_system_columns=include_system_columns,
)

def to_config(self) -> dict[str, Any]:
"""Serialize this pod as its inner function pod.

A saved job describes the function, not where its results are cached, so a
pod-cached pod is saved exactly as the uncached pod would be. The pod cache
database is not part of the saved job, and a loaded job does not use it.

Returns:
The inner function pod's config.
"""
return self._function_pod.to_config()

def process(
self, *streams: StreamProtocol, label: str | None = None
) -> StreamProtocol:
"""Invoke the inner pod but with pod-level caching on process_data.

The stream returned uses *this* pod's ``process_data`` (which
includes caching) rather than the inner pod's.
includes caching) rather than the inner pod's. Like ``FunctionPod.process``
it joins the inputs once, validates them and records the invocation with the
active trackers (so a ``PipelineJob`` records this pod).
"""
from orcapod.core.function_pod import FunctionPodStream

# Validate and prepare the input stream
input_stream = self._function_pod.handle_input_streams(*streams)
self._function_pod.validate_inputs(*streams)

return FunctionPodStream(
input_stream = self.handle_input_streams(*streams)
self._validate_input_schema(input_stream.output_schema()[1])
# The pod has no ``from_config`` (it cannot be rebuilt without its cache
# database), the one member of FunctionPodProtocol it lacks; the tracker uses
# none of it.
self.tracker_manager.record_function_pod_invocation(
self, # pyright: ignore[reportArgumentType]
input_stream,
label=label,
)
return _CachedFunctionPodStream(
function_pod=self,
input_stream=input_stream,
label=label,
)


class _CachedFunctionPodStream(FunctionPodStream):
"""The stream of a ``CachedFunctionPod``, with the same columns in every cache state.

A row computed in this process carries only the ``__computed`` meta column; a row
served from the cache also carries the function variation and execution columns it
was stored with. This stream keeps ``__computed`` and drops the others, and its
table has no meta columns unless asked for, as an uncached pod's table.
"""

def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]:
keep = CachedFunctionPod.RESULT_COMPUTED_FLAG
for tag, data in super().iter_data():
stored = [name for name in data.meta_columns if name != keep]
if stored:
data = data.drop_meta_columns(*stored)
yield tag, data

def as_table(
self,
*,
columns: ColumnConfig | dict[str, Any] | None = None,
all_info: bool = False,
) -> "pa.Table":
table = super().as_table(columns=columns, all_info=all_info)
column_config = ColumnConfig.handle_config(columns, all_info=all_info)
if column_config.meta or column_config.all_info:
return table
return table.drop_columns(
[n for n in table.column_names if n.startswith(constants.META_PREFIX)]
)
20 changes: 20 additions & 0 deletions superpowers/plans/2026-11-12-itl-988-cached-pod-job.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# ITL-988 — pod-cached function pod in a PipelineJob; plain table independent of cache state

Branch `eywalker/itl-988-a-pod-cached-function-pod-is-not-recorded`. Only `src/orcapod/core/cached_function_pod.py` changes (plus tests, changelog fragment, design_issues file, docs sentence).

## Decisions (coordinator, 2026-11-12)
- Defect 1: `CachedFunctionPod.process` mirrors `FunctionPod.process`: join once, `_validate_input_schema`, `tracker_manager.record_function_pod_invocation(self, input_stream, label=label)`, return the stream. Record SELF (the pod cache is honoured: a row in the pod cache is not recomputed in the job). Job save needs `CachedFunctionPod.to_config` delegating to the inner pod's config (identical saved file to the uncached pod, apart from `run_id`). The pod cache database is not saved, so a loaded job does not use it.
- Defect 2: `process` returns a `FunctionPodStream` subclass in this file: it strips the stored `__pf_var_*`/`__pf_exec_*`/`__pod_ts` meta from served rows (keeps `__computed`), and `as_table` drops meta columns unless `meta` is requested. `process_data` is untouched (FunctionNode and hooks read `__computed` from it).
- #327 only changes `_FunctionPodBase.__call__`; process receives the already-narrowed stream. Re-check after it merges.

## Facts measured on main 25650c42
- Uncached node: ph a1b47334ae8c, ch d4bc20dfcee4; record SELF gives the same, same store paths; save of SELF fails (no `to_config`).
- Table columns: computed `[k,total,__computed]`; served `[k,total,__pf_*...,__pod_ts,__computed]`; uncached `[k,total]`.

## Conditions to test
(a) saved file equals uncached save except `run/run_id`; (b) save+load in memory and Delta, sync and async, loaded job reads stored results without pod cache; (c) docs/changelog state pod cache DB is not saved; (d) report from_config/proxy paths that cannot round-trip; (e) a row is computed once, appears in pod cache and job result table; second run computes nothing.
Fallback if any fails: record INNER plus info log; send facts to coordinator.

## Steps
1. Tests `tests/test_core/function_pod/test_cached_function_pod_job.py` (fail first).
2. Implement. 3. Full suite, ruff, basedpyright, golden. 4. changelog.d/ITL-988.md, design_issues/ITL-988-CP1.md, docs. 5. Merge origin/main, push, PR.
241 changes: 241 additions & 0 deletions tests/test_core/function_pod/test_cached_function_pod_job.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,241 @@
"""A pod-cached function pod in a ``PipelineJob``, and its table (ITL-988).

- The job records the pod's source and function node, like an uncached pod, and runs,
saves and loads it (in memory and on Delta, with the sync and the async orchestrator).
- The pod cache is honoured in a job: a row is computed once.
- The plain table of a ``CachedFunctionPod`` stream has the columns of an uncached pod's
table whether a row was computed or served from the cache; meta columns come only on
request, and then the same ones in both states.
"""

from __future__ import annotations

import json
from collections.abc import Callable
from pathlib import Path
from typing import Any

import pytest

from orcapod import PipelineJob
from orcapod.core.cached_function_pod import CachedFunctionPod
from orcapod.core.function_pod import FunctionPod
from orcapod.core.operators import Join
from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase
from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator
from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator
from orcapod.sources import DictSource
from orcapod.types import ColumnConfig

ORCHESTRATORS: dict[str, Callable[[], Any]] = {
"sync": SyncPipelineOrchestrator,
"async": AsyncPipelineOrchestrator,
}
ROWS = [{"k": 1, "x": 2, "y": 3}, {"k": 2, "x": 5, "y": 5}]


def make_source(rows: list[dict[str, int]] | None = None) -> DictSource:
return DictSource(rows or ROWS, tag_columns=["k"], source_id="s")


class Counter:
"""An ``add`` function that records the ``x`` of every call."""

def __init__(self) -> None:
self.calls: list[int] = []

def fn(self) -> Callable[[int, int], int]:
def add(x: int, y: int) -> int:
self.calls.append(x)
return x + y

return add

def pod(self) -> FunctionPod:
return FunctionPod.from_fn(self.fn(), output_keys="total")


def make_db(kind: str, tmp_path: Path, name: str):
if kind == "memory":
return InMemoryArrowDatabase()
return DeltaTableDatabase(tmp_path / name)


def columns(table) -> list[tuple[str, str]]:
return [(f.name, str(f.type)) for f in table.schema]


# -- the table ---------------------------------------------------------------------


@pytest.mark.parametrize("all_info", [False, True])
def test_table_columns_do_not_depend_on_cache_state(all_info: bool) -> None:
counter = Counter()
cached = CachedFunctionPod(
function_pod=counter.pod(), result_database=InMemoryArrowDatabase()
)
computed = cached.process(make_source()).as_table(all_info=all_info)
served = cached.process(make_source()).as_table(all_info=all_info)
assert counter.calls == [2, 5] # the second table came from the cache
assert columns(computed) == columns(served)


def test_plain_table_equals_the_uncached_pods_table() -> None:
counter = Counter()
uncached = counter.pod().process(make_source()).as_table()
cached = CachedFunctionPod(
function_pod=counter.pod(), result_database=InMemoryArrowDatabase()
)
for _ in range(2): # computed, then served
table = cached.process(make_source()).as_table()
assert columns(table) == columns(uncached)
assert table.to_pylist() == uncached.to_pylist()


def test_meta_columns_only_when_asked_for_and_the_same_in_both_states() -> None:
cached = CachedFunctionPod(
function_pod=Counter().pod(), result_database=InMemoryArrowDatabase()
)
computed = cached.process(make_source()).as_table(columns={"meta": True})
served = cached.process(make_source()).as_table(columns={"meta": True})
assert columns(computed) == columns(served)
assert [n for n, _ in columns(computed) if n.startswith("__")] == ["__computed"]
assert computed.column("__computed").to_pylist() == [True, True]
assert served.column("__computed").to_pylist() == [False, False]


@pytest.mark.parametrize("columns", [ColumnConfig(all_info=True), {"all_info": True}])
def test_all_info_in_the_column_config_keeps_the_meta_columns(columns: Any) -> None:
cached = CachedFunctionPod(
function_pod=Counter().pod(), result_database=InMemoryArrowDatabase()
)
table = cached.process(make_source()).as_table(columns=columns)
assert "__computed" in table.column_names


# -- the job -----------------------------------------------------------------------


def run_job(pod, store, orchestrator: str) -> PipelineJob:
job = PipelineJob(name="j", store=store)
with job:
pod.process(make_source(), label="add")
job.run(orchestrator=ORCHESTRATORS[orchestrator]())
return job


@pytest.mark.parametrize("orchestrator", ["sync", "async"])
@pytest.mark.parametrize("kind", ["memory", "delta"])
def test_cached_pod_is_recorded_and_run_like_an_uncached_pod(
kind: str, orchestrator: str, tmp_path: Path
) -> None:
counter = Counter()
plain = run_job(counter.pod(), make_db(kind, tmp_path, "plain"), orchestrator)
cached_pod = CachedFunctionPod(
function_pod=counter.pod(), result_database=make_db(kind, tmp_path, "pod")
)
job = run_job(cached_pod, make_db(kind, tmp_path, "cached"), orchestrator)

assert list(job.nodes) == list(plain.nodes) == ["s", "add"]
assert job.nodes["add"].pipeline_hash() == plain.nodes["add"].pipeline_hash()
assert job.nodes["add"].content_hash() == plain.nodes["add"].content_hash()
assert columns(job.nodes["add"].as_table()) == columns(
plain.nodes["add"].as_table()
)
assert (
job.nodes["add"].as_table().to_pylist()
== plain.nodes["add"].as_table().to_pylist()
)


@pytest.mark.parametrize("orchestrator", ["sync", "async"])
@pytest.mark.parametrize("kind", ["memory", "delta"])
def test_row_is_computed_once_and_stored_in_both_caches(
kind: str, orchestrator: str, tmp_path: Path
) -> None:
counter = Counter()
pod_db = make_db(kind, tmp_path, "pod")
cached_pod = CachedFunctionPod(function_pod=counter.pod(), result_database=pod_db)
store = make_db(kind, tmp_path, "cached")

run_job(cached_pod, store, orchestrator)
assert sorted(counter.calls) == [2, 5]
pod_rows = cached_pod.get_all_cached_outputs()
assert pod_rows is not None and pod_rows.num_rows == 2
assert sorted(counter.calls) == [2, 5]

# A second run over the same store computes nothing.
run_job(cached_pod, store, orchestrator)
assert sorted(counter.calls) == [2, 5]


def test_row_in_the_pod_cache_is_not_recomputed_by_a_job() -> None:
counter = Counter()
cached_pod = CachedFunctionPod(
function_pod=counter.pod(), result_database=InMemoryArrowDatabase()
)
cached_pod.process(make_source(ROWS[:1])).as_table() # fills the pod cache
counter.calls.clear()
run_job(cached_pod, InMemoryArrowDatabase(), "sync")
assert counter.calls == [5]


def test_inputs_are_joined_once() -> None:
counter = Counter()
cached_pod = CachedFunctionPod(
function_pod=counter.pod(), result_database=InMemoryArrowDatabase()
)
left = DictSource([{"k": 1, "x": 2}], tag_columns=["k"], source_id="l")
right = DictSource([{"k": 1, "y": 3}], tag_columns=["k"], source_id="r")
joins: list[int] = []
original = Join.process

def counting(self, *streams, **kwargs):
joins.append(len(streams))
return original(self, *streams, **kwargs)

# Counting the calls of the join is the test: the public result is the same.
Join.process = counting # type: ignore[method-assign]
try:
stream = cached_pod.process(left, right)
finally:
Join.process = original # type: ignore[method-assign]
assert joins == [2]
assert stream.as_table().to_pylist() == [{"k": 1, "total": 5}]


# -- save and load -----------------------------------------------------------------


def saved(job: PipelineJob, path: Path) -> dict[str, Any]:
job.save(path)
return json.loads(path.read_text())


@pytest.mark.parametrize("kind", ["memory", "delta"])
def test_saved_job_equals_the_uncached_pods_and_loads(
kind: str, tmp_path: Path
) -> None:
counter = Counter()
plain_store = make_db(kind, tmp_path, "plain")
plain = run_job(counter.pod(), plain_store, "sync")
store = make_db(kind, tmp_path, "cached")
cached_pod = CachedFunctionPod(
function_pod=counter.pod(), result_database=make_db(kind, tmp_path, "pod")
)
job = run_job(cached_pod, store, "sync")

plain_saved = saved(plain, tmp_path / "plain.json")
cached_saved = saved(job, tmp_path / "cached.json")
# Fields that differ between any two saves: the run id and the store's location.
for doc in (plain_saved, cached_saved):
doc["run"].pop("run_id")
doc["bindings"].pop("store")
assert cached_saved == plain_saved

# A loaded job reads its stored results; the pod cache is not part of it.
loaded = PipelineJob.load(tmp_path / "cached.json", store=store)
assert loaded.nodes["add"].as_table().to_pylist() == [
{"k": 1, "total": 5},
{"k": 2, "total": 10},
]
Loading
Loading