From 84d1312c277b5af2dea6a534913cd72649c6e80f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 01:06:33 -0800 Subject: [PATCH 01/19] feat(referential_tags): stored tables and saved jobs keep tag bindings (ITL-1019) A function node's pipeline table marks each referential tag column with Arrow field metadata (orcapod:tag_kind = referential, one constant in referential_tags); a plain tag carries no marker and the result table never records referential status. The node reads the binding back from its table and refuses a table that disagrees with its pod or, for a loaded job, the saved file. The marker moves no hash, path or layout. Jobs and pipelines holding referential tags and fall-through pods save and load with unchanged hashes. Format 0.3.0 writes tag_only_columns for every source that takes it and fall_through_references for every function node; the version decides meaning (0.1.0/0.2.0 are all plain), and a 0.3.0 file missing or contradicting these keys is refused. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1019.md | 27 ++ design_issues/ITL-1023-RT1.md | 2 +- docs/concepts/referential-tags.md | 45 ++- src/orcapod/core/function_pod.py | 43 ++- src/orcapod/core/nodes/function_node.py | 98 ++++- src/orcapod/core/referential_tags.py | 112 +++++- .../core/sources/arrow_table_source.py | 2 + src/orcapod/core/sources/base.py | 39 +- src/orcapod/core/sources/csv_source.py | 8 +- src/orcapod/core/sources/data_frame_source.py | 2 + src/orcapod/core/sources/db_table_source.py | 4 +- .../core/sources/delta_table_source.py | 8 +- src/orcapod/core/sources/dict_source.py | 2 + .../core/sources/postgresql_table_source.py | 3 +- .../core/sources/spiraldb_table_source.py | 8 +- .../core/sources/sqlite_table_source.py | 3 +- src/orcapod/pipeline/graph.py | 28 +- src/orcapod/pipeline/job.py | 7 + src/orcapod/pipeline/serialization.py | 161 +++++++- .../test_model_and_sources.py | 14 +- .../referential_tags/test_persistence.py | 354 ++++++++++++++++++ .../test_loaded_side_effect_nodes.py | 5 + .../test_loaded_slot_unregistered_class.py | 3 + .../test_saved_job_before_referential_tags.py | 6 +- .../test_saved_job_with_referential_tags.py | 265 +++++++++++++ 25 files changed, 1197 insertions(+), 52 deletions(-) create mode 100644 changelog.d/ITL-1019.md create mode 100644 tests/test_core/referential_tags/test_persistence.py create mode 100644 tests/test_pipeline/test_saved_job_with_referential_tags.py diff --git a/changelog.d/ITL-1019.md b/changelog.d/ITL-1019.md new file mode 100644 index 000000000..40ccb20a1 --- /dev/null +++ b/changelog.d/ITL-1019.md @@ -0,0 +1,27 @@ +### Added + +- A stored table states which of its tags are referential (ITL-1019): a function node's + pipeline table marks each referential tag column with Arrow field metadata + (`orcapod:tag_kind` = `referential`, `referential_tags.REFERENTIAL_TAG_METADATA_KEY`); a + plain tag carries no marker. The result table never records referential status. The + marker moves no hash and no stored path, and needs no new table layout. +- A function node reads its bindings back from its pipeline table and refuses + (`InputValidationError`, naming the table and both lists) a table whose marker disagrees + with what the node declares: its pod's fall-through tags, or a loaded job's saved record. +- A job or pipeline holding referential tags and fall-through pods can be saved and + loaded; it keeps its hashes and bindings. `ReferenceFallThroughPod.to_config` and + `from_config` work. + +### Changed + +- Saved files of format `0.3.0` state every binding: each source config that takes the + option writes `tag_only_columns` (`True` when every tag is plain), and each function + node writes `fall_through_references` (empty for a plain pod). The format version decides + what a file means, never a missing key: a file at `0.1.0` or `0.2.0` holds only plain + tags, and a `0.3.0` file without these keys, or whose keys disagree with its saved + schemas, is refused on load. + +### Known limitations + +- The SQLite and PostgreSQL databases keep no Arrow field metadata and refuse a pipeline + table with a referential tag, as they refuse logical types. diff --git a/design_issues/ITL-1023-RT1.md b/design_issues/ITL-1023-RT1.md index b791c065d..b2bd1f411 100644 --- a/design_issues/ITL-1023-RT1.md +++ b/design_issues/ITL-1023-RT1.md @@ -5,4 +5,4 @@ **Severity:** medium **Issue:** ITL-1023 -The workshop slice of referential tags (specification attached to ITL-1004) covers sources, function pods, `Join`, `GroupBy`, select and drop, renames, filters, `DropNulls`, `SemiJoin`, `Pick` and `Index`. Everything else raises "not supported with referential tags yet" when a referential tag reaches it, instead of giving a wrong answer: `Batch` (ITL-1017), `MergeJoin` (ITL-1018), storing an operator node's results in `LOG` or `REPLAY` mode, `DerivedSource` over a referential stream and saving a pipeline or job (ITL-1019), and any operator that does not declare `REFERENTIAL_TAGS_ISSUE = None` (ITL-1022). `PollingSource` keeps its tags plain. The identities of referential streams are not frozen yet. The project's issues (ITL-1009 onward) land the full design and close this entry. +The workshop slice of referential tags (specification attached to ITL-1004) covers sources, function pods, `Join`, `GroupBy`, select and drop, renames, filters, `DropNulls`, `SemiJoin`, `Pick` and `Index`. Everything else raises "not supported with referential tags yet" when a referential tag reaches it, instead of giving a wrong answer: `Batch` (ITL-1017), `MergeJoin` (ITL-1018), storing an operator node's results in `LOG` or `REPLAY` mode, and `DerivedSource` over a referential stream (ITL-1019; saving a pipeline or job and the function node's stored bindings are supported since ITL-1019), and any operator that does not declare `REFERENTIAL_TAGS_ISSUE = None` (ITL-1022). `PollingSource` keeps its tags plain. The identities of referential streams are not frozen yet. The project's issues (ITL-1009 onward) land the full design and close this entry. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 6fb99d58e..2208c914d 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -133,9 +133,52 @@ referential_tags.reference_materialization = "silent" referential_tags.reference_materialization = "warn" # the default ``` +## Stored tables and saved jobs + +A stored table states which of its tags are referential. A function node stores each +row's tags in its pipeline table, and a referential tag is stored there once, as a tag +column that carries Arrow field metadata: the key `orcapod:tag_kind` with the value +`referential` (`referential_tags.REFERENTIAL_TAG_METADATA_KEY`). A plain tag carries no +marker. The function's result table never records referential status, so plain and +referential runs of the same function share their results. The marker moves no hash, and +the stores keep it on every write (first write, append, merge, schema evolution; in +memory and on Delta). The SQLite and PostgreSQL databases keep no Arrow +metadata at all and refuse such a table, as they refuse logical types. + +```python +from orcapod.core.referential_tags import stored_references +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import PipelineJob + +database = InMemoryArrowDatabase() +job = PipelineJob(database=database) +with job: + scale.pod(spikes, label="scale") +job.run() +pipeline_table = next( + database.get_all_records(path) + for path in database.list_table_paths() + if "_result" not in path +) +print(stored_references(pipeline_table.schema)) +# ('unit', 'time') +``` + +A node reads the binding back from its table, and refuses a table whose marker disagrees +with what the node declares (its pod's fall-through tags, or for a loaded job the saved +file's). + +A saved job or pipeline (format `0.3.0`) states every binding: each source config writes +`tag_only_columns` (`True` when every tag is plain), and each function node writes +`fall_through_references` (empty for a plain pod). The format version, never a missing +key, decides what a file means: a file at `0.1.0` or `0.2.0` holds only plain tags, and a +`0.3.0` file without these keys is refused. A loaded job keeps its hashes and bindings, and +the file is checked against the stored tables when a node reads them. + ## Not supported yet These raise "not supported with referential tags yet" when a referential tag reaches them: `Batch`, `MergeJoin`, `SelectColumns`, `DropColumns`, storing an operator node's results (`LOG` and `REPLAY` cache modes), -`DerivedSource` over a stream with referential tags, and saving a pipeline or job. +and `DerivedSource` over a stream with referential tags (so `node.as_source()` of a node +with referential tags). `PollingSource` keeps its tags plain. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index a2e8e3498..adf888e8e 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -17,7 +17,6 @@ from orcapod.core.datagrams import Data from orcapod.core.referential_tags import ( materialization_diagnostic_off, - not_supported_yet, stream_references, ) from orcapod.core.streams.base import ( @@ -1925,21 +1924,47 @@ def plain_pod(self) -> FunctionPod: return self._plain_pod def to_config(self) -> dict[str, Any]: - """Raise: a pipeline holding referential tags cannot be saved yet (ITL-1019). + """Serialize this pod: the plain pod's config and the references (ITL-1019). - Raises: - NotImplementedError: Always. + Returns: + The plain pod's config with ``fall_through_references``, the referential + tags that fall through. """ - raise not_supported_yet("Saving a fall-through function pod", "ITL-1019") + from orcapod.pipeline.serialization import FALL_THROUGH_REFERENCES_KEY + + return { + **self._plain_pod.to_config(), + FALL_THROUGH_REFERENCES_KEY: list(self._references), + } @classmethod - def from_config(cls, config: dict[str, Any]) -> "ReferenceFallThroughPod": - """Raise: a fall-through pod is never saved, so never loaded (ITL-1019). + def from_config( + cls, config: dict[str, Any], *, fallback_to_proxy: bool = False + ) -> "ReferenceFallThroughPod": + """Rebuild the pod from ``to_config``'s output. + + Args: + config: A config written by ``to_config``. + fallback_to_proxy: Passed to ``FunctionPod.from_config``. + + Returns: + The fall-through pod over the rebuilt plain pod. Raises: - NotImplementedError: Always. + ValueError: If the config names no ``fall_through_references``. """ - raise not_supported_yet("Loading a fall-through function pod", "ITL-1019") + from orcapod.pipeline.serialization import FALL_THROUGH_REFERENCES_KEY + + if FALL_THROUGH_REFERENCES_KEY not in config: + raise ValueError( + f"a fall-through pod config names {FALL_THROUGH_REFERENCES_KEY}; " + "this one does not" + ) + plain = {k: v for k, v in config.items() if k != FALL_THROUGH_REFERENCES_KEY} + return cls( + FunctionPod.from_config(plain, fallback_to_proxy=fallback_to_proxy), + config[FALL_THROUGH_REFERENCES_KEY], + ) def identity_structure(self) -> Any: return (self._plain_pod, ("fall_through_references", self._references)) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index b644f58d3..880f47f11 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -36,6 +36,11 @@ result_pod, with_fall_through, ) +from orcapod.core.referential_tags import ( + mark_references, + stored_references, + without_reference_marks, +) from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig from orcapod.core.cached_function_pod import CachedFunctionPod @@ -431,6 +436,31 @@ def __init__( self._table_scope = table_scope self._node_identity_path_cache: tuple[str, ...] | None = None + def declared_references(self) -> tuple[str, ...]: + """Return the referential tags this node lets fall through, as it declares them. + + A node with a pod declares the pod's (``ReferenceFallThroughPod``); a node + loaded from a saved file declares the file's record (``fall_through_references`` + of its descriptor, ITL-1019). They are what the node's stored pipeline table + must mark. + + Returns: + The referential tags; empty for a plain pod. + + Raises: + ValueError: If the node has neither a pod nor a saved record. + """ + if self._function_pod is not None: + return fall_through_references(self._function_pod) + from orcapod.pipeline.serialization import FALL_THROUGH_REFERENCES_KEY + + if FALL_THROUGH_REFERENCES_KEY not in self._descriptor: + raise ValueError( + f"Function node {self.label!r} has no pod and no saved " + f"{FALL_THROUGH_REFERENCES_KEY}: its referential tags are unknown" + ) + return tuple(self._descriptor[FALL_THROUGH_REFERENCES_KEY]) + # ------------------------------------------------------------------ # load_status # ------------------------------------------------------------------ @@ -1142,12 +1172,15 @@ class _JoinedRecords(NamedTuple): empty_taginfo_rows: Mapping from ``base_entry_id`` bytes to the raw taginfo row dict for each entry in ``empty_data_tokens``. Used by ``_load_cached_entries`` to reconstruct the tag for each token. + references: The referential tags the pipeline table marks (ITL-1019), + checked against the node's pod when it has one. """ table: pa.Table taginfo_columns: tuple[str, ...] empty_data_tokens: dict[bytes, EmptyData] empty_taginfo_rows: dict[bytes, dict] + references: tuple[str, ...] class FunctionJobNode(RunHoldingNode, FunctionNodeBase): @@ -2911,11 +2944,16 @@ def _add_pipeline_record( } ) - # Combine: tag (with system tags) + input source columns + meta columns - combined_record = arrow_utils.hstack_tables( - tag.as_table(columns={"system_tags": True}), - input_source_table, - meta_table, + # Combine: tag (with system tags) + input source columns + meta columns. The + # table states which tags are referential: the ones that fall through this + # node's pod, marked on their stored column (ITL-1019). + combined_record = mark_references( + arrow_utils.hstack_tables( + tag.as_table(columns={"system_tags": True}), + input_source_table, + meta_table, + ), + fall_through_references(self._function_pod), ) try: @@ -3117,6 +3155,7 @@ def _fetch_joined_records( taginfo_columns = tuple(taginfo.column_names) taginfo_schema = taginfo.schema + references = self._stored_references(taginfo_schema) is_ephemeral_col = constants.IS_EPHEMERAL_COL taginfo_df = pl.DataFrame(taginfo) @@ -3275,6 +3314,7 @@ def _fetch_joined_records( taginfo_columns=taginfo_columns, empty_data_tokens=empty_data_tokens, empty_taginfo_rows=empty_taginfo_rows, + references=references, ) # Apply base_entry_id filter if requested @@ -3298,8 +3338,48 @@ def _fetch_joined_records( taginfo_columns=taginfo_columns, empty_data_tokens=empty_data_tokens, empty_taginfo_rows=empty_taginfo_rows, + references=references, ) + def _stored_references(self, taginfo_schema: "pa.Schema") -> tuple[str, ...]: + """Return the referential tags the pipeline table marks, checked against the pod. + + The stored table states its bindings (developer, November 16, 2026, ITL-1019): + a referential tag column carries the marker of ``referential_tags``. The node + reads the table only when it agrees with what the node declares + (``declared_references``: its pod's, or a loaded node's saved record, an + independent point of verification). + + Args: + taginfo_schema: The schema of the stored pipeline table. + + Returns: + The referential tags, in the order the node declares them. + + Raises: + InputValidationError: If the table's marker is invalid, or the table + and the node disagree on which tags are referential. + """ + table_path = self._require_pipeline_database().base_path + ( + self._versioned_pipeline_path + ) + try: + stored = stored_references(taginfo_schema) + except ValueError as exc: + raise InputValidationError( + f"Function node {self.label!r}, pipeline table {table_path}: {exc}" + ) from exc + expected = self.declared_references() + if set(stored) != set(expected): + source = "pod" if self._function_pod is not None else "saved job" + raise InputValidationError( + f"Function node {self.label!r}: the pipeline table {table_path} marks " + f"the referential tags {sorted(stored)}, but the node's {source} " + f"declares {sorted(expected)}. The table was not written by this " + "node's pipeline; nothing was read." + ) + return expected + def _stored_tag_keys(self, fetched: _JoinedRecords) -> tuple[str, ...]: """Return the user tag columns of the records in *fetched*. @@ -3376,11 +3456,13 @@ def _stored_rows_stream( uuid.UUID(bytes=bytes(cast(bytes, record_id))) for record_id in joined.column(constants.DATA_RECORD_ID).to_pylist() ] - # A fall-through column is the stored tag column in its data role (ITL-1023). + # A fall-through column is the stored tag column in its data role (ITL-1023): + # the table states which (ITL-1019). In the stream the binding is the shared + # name, so the marker stays with the stored table. stream = ArrowTableStream( - joined.drop(drop_cols), + without_reference_marks(joined.drop(drop_cols)), tag_columns=self._stored_tag_keys(fetched), - reference_tags=fall_through_references(self._function_pod), + reference_tags=fetched.references, ) return base_entry_ids, stream, data_ids diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index 61142366f..1dea111b9 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -25,12 +25,19 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from orcapod.errors import InputValidationError from orcapod.types import SourceInfoValue from orcapod.utils import arrow_utils +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import pyarrow as pa +else: + pa = LazyModule("pyarrow") + # Diagnostic emitted when a structural operator materializes a reference (section 7.4). # ``"warn"`` emits a ``ReferenceMaterializationWarning``; ``"silent"`` emits nothing. The # setting never changes a result, a schema or a hash. A module-level default for now. @@ -164,6 +171,109 @@ def warn_materialized(operation: object, names: Iterable[str]) -> None: ) +# The marker of a referential tag in a stored table (developer, November 16, 2026, +# ITL-1019): Arrow field metadata on the stored tag column, at the level of a logical +# type's extension metadata. Only a referential tag carries it; a plain tag carries no +# marker. It is not ``ARROW:extension:*`` metadata, so no schema or table hash reads it. +REFERENTIAL_TAG_METADATA_KEY = b"orcapod:tag_kind" +REFERENTIAL_TAG_METADATA_VALUE = b"referential" + + +def mark_references(table: "pa.Table", references: Iterable[str]) -> "pa.Table": + """Return ``table`` with exactly the columns ``references`` marked referential. + + A stored table states its own bindings: the writer marks the columns it knows to be + referential and removes a marker any other column carries (one that came in from + upstream metadata is not this writer's binding). Values and every other field + property (name, type, nullability, other metadata) are unchanged. + + Args: + table: The rows about to be stored. + references: The referential tags of the rows; each must be a column of + ``table``. + + Returns: + The table with the marker on exactly the ``references`` columns. + + Raises: + ValueError: If a reference is not a column of ``table``. + """ + refs = set(references) + if missing := refs - set(table.column_names): + raise ValueError( + f"referential tags {sorted(missing)} are not columns of the stored table" + ) + schema = table.schema + changed = False + for index, field in enumerate(schema): + metadata: dict[bytes | str, bytes | str] = { + key: value for key, value in (field.metadata or {}).items() + } + marked = metadata.get(REFERENTIAL_TAG_METADATA_KEY) is not None + if field.name in refs: + if metadata.get(REFERENTIAL_TAG_METADATA_KEY) == ( + REFERENTIAL_TAG_METADATA_VALUE + ): + continue + metadata[REFERENTIAL_TAG_METADATA_KEY] = REFERENTIAL_TAG_METADATA_VALUE + elif marked: + del metadata[REFERENTIAL_TAG_METADATA_KEY] + else: + continue + schema = schema.set( + index, + field.with_metadata(metadata) if metadata else field.remove_metadata(), + ) + changed = True + if not changed: + return table + # The same arrays under the new field metadata: no cast, no copy. + return pa.Table.from_arrays(table.columns, schema=schema) + + +def stored_references(schema: "pa.Schema") -> tuple[str, ...]: + """Return the columns a stored table marks as referential tags, in column order. + + Args: + schema: The Arrow schema of a stored table. + + Returns: + The names of the columns that carry the referential tag marker. + + Raises: + ValueError: If a column carries the marker key with another value. + """ + names: list[str] = [] + for field in schema: + value = (field.metadata or {}).get(REFERENTIAL_TAG_METADATA_KEY) + if value is None: + continue + if value != REFERENTIAL_TAG_METADATA_VALUE: + raise ValueError( + f"stored column {field.name!r} has {REFERENTIAL_TAG_METADATA_KEY!r} " + f"metadata {value!r}; the only valid value is " + f"{REFERENTIAL_TAG_METADATA_VALUE!r}" + ) + names.append(field.name) + return tuple(names) + + +def without_reference_marks(table: "pa.Table") -> "pa.Table": + """Return ``table`` with no referential tag marker on any column. + + A stored table read into a stream drops the marker: in a stream the binding is the + name intersection of the tag and data keys, and a marker carried further would be + written by a downstream store whose binding it is not. + + Args: + table: Rows read from a stored table. + + Returns: + The same rows; no field carries the marker. + """ + return mark_references(table, ()) + + def not_supported_yet(operation: str, issue: str) -> NotImplementedError: """Return the error for an operation that does not handle referential tags yet. diff --git a/src/orcapod/core/sources/arrow_table_source.py b/src/orcapod/core/sources/arrow_table_source.py index 890fbfd9f..801d57d2c 100644 --- a/src/orcapod/core/sources/arrow_table_source.py +++ b/src/orcapod/core/sources/arrow_table_source.py @@ -70,6 +70,8 @@ class ArrowTableSource(RootSource): and set ``record_id_column`` to a column that is unique per row. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # The config type key; also the kind prefix of the default source id. SOURCE_KIND: ClassVar[str] = "arrow_table" diff --git a/src/orcapod/core/sources/base.py b/src/orcapod/core/sources/base.py index e4fd05ed9..6ffa783eb 100644 --- a/src/orcapod/core/sources/base.py +++ b/src/orcapod/core/sources/base.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from orcapod import contexts from orcapod.config import OrcapodConfig @@ -16,6 +16,29 @@ from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +def saved_tag_only_columns(config: dict[str, Any]) -> bool | list[str]: + """Return the ``tag_only_columns`` of a saved source config (ITL-1019). + + A config of format 0.3.0 always holds it; the loader of an older file, whose tags + are all plain, puts ``True`` in first (``serialization.source_config_at_version``). + + Args: + config: A saved source config. + + Returns: + ``True`` (every tag plain) or the names of the plain tags. + + Raises: + ValueError: If the config does not hold ``tag_only_columns``. + """ + if "tag_only_columns" not in config: + raise ValueError( + f"source config {config.get('source_type')!r} has no tag_only_columns: " + "a saved source config states its plain tags (format 0.3.0)" + ) + return config["tag_only_columns"] + + class RootSource(StreamBase): """Abstract base class for all root sources in Orcapod. @@ -49,6 +72,10 @@ class RootSource(StreamBase): ``identity_structure()``. """ + # Whether the source takes ``tag_only_columns`` and saves it in its config + # (ITL-1019): from format 0.3.0 on such a config always holds it. + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = False + def __init__( self, source_id: str | None = None, @@ -111,16 +138,16 @@ def _identity_config(self) -> dict[str, Any]: } def _tag_only_config(self) -> dict[str, Any]: - """Return ``tag_only_columns`` for ``to_config``, when any tag is referential. + """Return ``tag_only_columns`` for ``to_config``: always written (ITL-1019). - A source whose tags are all plain writes nothing, so its config is the one - v0.3 wrote; a config without the key reads back with every tag plain - (``tag_only_columns=True``), as v0.3 configs mean. + The format version decides what a saved file means, never a missing key: a + config of format 0.3.0 states its plain tags, ``True`` when every tag is plain + and the names otherwise (``saved_tag_only_columns`` reads it back). """ tag_columns = tuple(getattr(self, "_tag_columns", ())) tag_only = tuple(getattr(self, "_tag_only_columns", tag_columns)) if tag_only == tag_columns: - return {} + return {"tag_only_columns": True} return {"tag_only_columns": list(tag_only)} def pipeline_identity_structure(self) -> Any: diff --git a/src/orcapod/core/sources/csv_source.py b/src/orcapod/core/sources/csv_source.py index f62761979..ee043a75f 100644 --- a/src/orcapod/core/sources/csv_source.py +++ b/src/orcapod/core/sources/csv_source.py @@ -2,11 +2,11 @@ import os from collections.abc import Collection -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override -from orcapod.core.sources.base import RootSource +from orcapod.core.sources.base import RootSource, saved_tag_only_columns from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.reload import keep_nullable_columns, same_rows from orcapod.core.sources.stream_builder import ( @@ -44,6 +44,8 @@ class CSVSource(RootSource): value, or the row's position. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + def __init__( self, file_path: str, @@ -171,7 +173,7 @@ def from_config(cls, config: dict[str, Any], db_registry=None) -> CSVSource: return cls( file_path=config["file_path"], tag_columns=config.get("tag_columns", ()), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/data_frame_source.py b/src/orcapod/core/sources/data_frame_source.py index c5b9af61e..3488e957b 100644 --- a/src/orcapod/core/sources/data_frame_source.py +++ b/src/orcapod/core/sources/data_frame_source.py @@ -29,6 +29,8 @@ class DataFrameSource(RootSource): ``SourceStreamBuilder`` (source-info, schema-hash, system tags). """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # The config type key; also the kind prefix of the default source id. SOURCE_KIND: ClassVar[str] = "data_frame" diff --git a/src/orcapod/core/sources/db_table_source.py b/src/orcapod/core/sources/db_table_source.py index 7572c13c4..18589f562 100644 --- a/src/orcapod/core/sources/db_table_source.py +++ b/src/orcapod/core/sources/db_table_source.py @@ -14,7 +14,7 @@ from collections.abc import Collection, Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override @@ -73,6 +73,8 @@ class DBTableSource(RootSource): provided, or is empty. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + def __init__( self, connector: DBConnectorProtocol, diff --git a/src/orcapod/core/sources/delta_table_source.py b/src/orcapod/core/sources/delta_table_source.py index 69143e4c1..159dcdacf 100644 --- a/src/orcapod/core/sources/delta_table_source.py +++ b/src/orcapod/core/sources/delta_table_source.py @@ -2,11 +2,11 @@ from collections.abc import Collection from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override -from orcapod.core.sources.base import RootSource +from orcapod.core.sources.base import RootSource, saved_tag_only_columns from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.reload import same_rows from orcapod.core.sources.stream_builder import ( @@ -41,6 +41,8 @@ class DeltaTableSource(RootSource): Record ids keep their rule: the ``record_id_column`` value, or the row's position. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + def __init__( self, delta_table_path: PathLike, @@ -168,7 +170,7 @@ def from_config( return cls( delta_table_path=config["delta_table_path"], tag_columns=config.get("tag_columns", ()), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/dict_source.py b/src/orcapod/core/sources/dict_source.py index 6c9b29a95..26dbeb971 100644 --- a/src/orcapod/core/sources/dict_source.py +++ b/src/orcapod/core/sources/dict_source.py @@ -26,6 +26,8 @@ class DictSource(RootSource): enriched by ``SourceStreamBuilder`` (source-info, schema-hash, system tags). """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # The config type key; also the kind prefix of the default source id. SOURCE_KIND: ClassVar[str] = "dict" diff --git a/src/orcapod/core/sources/postgresql_table_source.py b/src/orcapod/core/sources/postgresql_table_source.py index 2343f4262..a86fb13c3 100644 --- a/src/orcapod/core/sources/postgresql_table_source.py +++ b/src/orcapod/core/sources/postgresql_table_source.py @@ -20,6 +20,7 @@ from typing_extensions import override from orcapod.core.sources.db_table_source import DBTableSource +from orcapod.core.sources.base import saved_tag_only_columns from orcapod.databases.postgresql_connector import PostgreSQLConnector if TYPE_CHECKING: @@ -142,7 +143,7 @@ def from_config( dsn=config["dsn"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 686cd75e6..aa9cad100 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -38,11 +38,11 @@ import datetime from collections.abc import AsyncIterator, Collection, Generator, Iterator, Mapping from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple, ClassVar from typing_extensions import override -from orcapod.core.sources.base import RootSource +from orcapod.core.sources.base import RootSource, saved_tag_only_columns from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.derived_source import validate_tags_as_data from orcapod.core.sources.stream_builder import SourceStreamBuilder @@ -238,6 +238,8 @@ class SpiralDBTableSource(RootSource): given twice, or is reserved for system columns. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # Whether a notebook display reads the first rows. False shows the schema and the # table's recorded row count only, with no scan (kept for the ITL-991 ruling on what # a preview may read). @@ -862,7 +864,7 @@ def from_config( table_name=config["table_name"], dataset=config.get("dataset", "default"), tag_columns=config.get("tag_columns"), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), where=_locationfrom_config(config.get("where") or {}), limit=config.get("limit"), diff --git a/src/orcapod/core/sources/sqlite_table_source.py b/src/orcapod/core/sources/sqlite_table_source.py index 934daa9a7..41a387125 100644 --- a/src/orcapod/core/sources/sqlite_table_source.py +++ b/src/orcapod/core/sources/sqlite_table_source.py @@ -30,6 +30,7 @@ from typing_extensions import override from orcapod.core.sources.db_table_source import DBTableSource +from orcapod.core.sources.base import saved_tag_only_columns from orcapod.databases.sqlite_connector import SQLiteConnector from orcapod.utils.schema_utils import _normalize_column_list @@ -186,7 +187,7 @@ def from_config( db_path=config["db_path"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 70b068f9f..418d60a26 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -174,6 +174,7 @@ def save(self, path: str | Path) -> None: import json as _json from orcapod.pipeline.serialization import ( + FALL_THROUGH_REFERENCES_KEY, PIPELINE_FORMAT_VERSION, serialize_nullability, serialize_output_schema, @@ -182,15 +183,8 @@ def save(self, path: str | Path) -> None: from orcapod.core.nodes import OperatorNode, FunctionNode from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass - from orcapod.core.referential_tags import ( - not_supported_yet, - stream_references, - ) - nodes: dict[str, Any] = {} for content_hash_str, node in self._persistent_node_map.items(): - if stream_references(node): - raise not_supported_yet("Saving a pipeline", "ITL-1019") tag_schema, data_schema = node.output_schema() try: type_converter = node.data_context.type_converter @@ -264,6 +258,10 @@ def save(self, path: str | Path) -> None: if node._function_pod is not None: descriptor["function_config"] = node._function_pod.to_config() descriptor["table_scope"] = node._table_scope + # Written for every function node, also a plain one (ITL-1019). + descriptor[FALL_THROUGH_REFERENCES_KEY] = list( + node.declared_references() + ) case OperatorNode(): if node._operator is not None: @@ -309,8 +307,11 @@ def load(cls, path: str | Path) -> "Pipeline": """ import json as _json from orcapod.pipeline.serialization import ( + FALL_THROUGH_REFERENCES_KEY, SUPPORTED_FORMAT_VERSIONS, + check_saved_output_schema_is_plain, deserialize_schema, + saved_fall_through_references, ) from orcapod.core.nodes import FunctionNode, OperatorNode from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass @@ -372,6 +373,19 @@ def load(cls, path: str | Path) -> "Pipeline": deserialize_output_schema, ) + # The format version decides which tags may be referential (ITL-1019). + try: + check_saved_output_schema_is_plain(descriptor, version) + if descriptor.get("node_type") == "function": + # Stated explicitly from here on, whatever the version. + descriptor[FALL_THROUGH_REFERENCES_KEY] = list( + saved_fall_through_references(descriptor, version) + ) + except ValueError as exc: + raise ValueError( + f"{path}: saved node {descriptor.get('label')!r}: {exc}" + ) from exc + # Validate even a leaf node's saved identifiers before exposing the job. context = resolve_saved_context(descriptor.get("data_context_key")) try: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 605dbf415..7f830dc1f 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1586,6 +1586,7 @@ def load( from orcapod.pipeline.serialization import ( SUPPORTED_JOB_FORMAT_VERSIONS, resolve_source_from_config, + source_config_at_version, ) path = _Path(path) @@ -1619,6 +1620,12 @@ def load( data.get("bindings", {}).get("sources", {}).items() ): source_type = src_config.get("source_type") + # The file's version decides which tags are plain (ITL-1019): a refusal + # is the file's error, not a source that could not be rebuilt. + try: + src_config = source_config_at_version(src_config, version) + except ValueError as exc: + raise ValueError(f"{path}: source {spec_name!r}: {exc}") from exc try: source = resolve_source_from_config(src_config) if source is not None: diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index aae816224..64291d96c 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -34,8 +34,14 @@ # 0.3.0: a source's tags may be referential (ITL-1023). A file at 0.1.0 or 0.2.0 # holds v0.3 sources: their configs carry no ``tag_only_columns`` and read # back with every declared tag plain, so their saved hashes match. +# Plain tags are decided by this version, never by a missing key (ITL-1159, ITL-1019): +# everything in a file at a ``PLAIN_TAG_FORMAT_VERSIONS`` version is plain, and such a +# file never holds ``tag_only_columns`` or ``fall_through_references``. From 0.3.0 on, +# every source config that takes the option writes ``tag_only_columns`` and every +# function node writes ``fall_through_references``; a load refuses a file without them. PIPELINE_FORMAT_VERSION = "0.3.0" SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0", "0.3.0"}) +PLAIN_TAG_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0"}) # --------------------------------------------------------------------------- # LoadStatus @@ -1292,6 +1298,159 @@ def _get_arrow_primitive_types() -> dict[str, Any]: # --------------------------------------------------------------------------- # Same scheme as ``PIPELINE_FORMAT_VERSION``: the written version, and the set a -# load accepts (``PipelineJob.load``). +# load accepts (``PipelineJob.load``). The plain-tag rule stated there holds for a +# job file too (``PLAIN_TAG_FORMAT_VERSIONS``, ``source_config_at_version``). PIPELINE_JOB_FORMAT_VERSION = "0.3.0" SUPPORTED_JOB_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0", "0.3.0"}) + + +# --------------------------------------------------------------------------- +# Referential tags in saved files (ITL-1019) +# --------------------------------------------------------------------------- + +FALL_THROUGH_REFERENCES_KEY = "fall_through_references" +"""The function node descriptor key naming the referential tags its pod lets fall +through (a list; empty for a plain pod).""" + + +def saved_fall_through_references( + descriptor: dict[str, Any], version: str +) -> tuple[str, ...]: + """Return the referential tags a saved function node lets fall through. + + The file's format version decides what a file may hold (ITL-1019): at a + ``PLAIN_TAG_FORMAT_VERSIONS`` version nothing is referential and the key is never + written; from 0.3.0 on the key is always written. The saved output schema is + checked against it: its tag and data halves share exactly these names. When the + descriptor holds the pod's config, the config's list must agree too. + + Args: + descriptor: A function node descriptor of a saved pipeline. + version: The file's format version. + + Returns: + The referential tags, in the saved order. + + Raises: + ValueError: If the key is present in a file of a plain-tag version, missing + or not a list of names in a later one, or disagrees with the saved output + schema or the saved pod config. + """ + from orcapod.core.referential_tags import reference_names + + output_schema = descriptor.get("output_schema") or {} + shared = reference_names( + output_schema.get("tag", {}), output_schema.get("data", {}) + ) + if version in PLAIN_TAG_FORMAT_VERSIONS: + if FALL_THROUGH_REFERENCES_KEY in descriptor: + raise ValueError( + f"a file at format {version} holds only plain tags, but it names " + f"{FALL_THROUGH_REFERENCES_KEY} {descriptor[FALL_THROUGH_REFERENCES_KEY]!r}" + ) + references: tuple[str, ...] = () + else: + if FALL_THROUGH_REFERENCES_KEY not in descriptor: + raise ValueError( + f"a file at format {version} names the fall-through references of " + f"every function node, but this one has no {FALL_THROUGH_REFERENCES_KEY}" + ) + saved = descriptor[FALL_THROUGH_REFERENCES_KEY] + if not isinstance(saved, list) or not all(isinstance(n, str) for n in saved): + raise ValueError( + f"{FALL_THROUGH_REFERENCES_KEY} must be a list of names, got {saved!r}" + ) + references = tuple(saved) + if set(shared) != set(references): + raise ValueError( + f"the saved output schema has the referential tags {sorted(shared)}, but " + f"{FALL_THROUGH_REFERENCES_KEY} is {sorted(references)}" + ) + function_config = descriptor.get("function_config") + if function_config is not None and version not in PLAIN_TAG_FORMAT_VERSIONS: + configured = function_config.get(FALL_THROUGH_REFERENCES_KEY, []) + if set(configured) != set(references): + raise ValueError( + f"the saved pod lets {sorted(configured)} fall through, but " + f"{FALL_THROUGH_REFERENCES_KEY} is {sorted(references)}" + ) + return references + + +def check_saved_output_schema_is_plain( + descriptor: dict[str, Any], version: str +) -> None: + """Refuse a saved node with a referential tag in a file of a plain-tag version. + + Args: + descriptor: A node descriptor of a saved pipeline. + version: The file's format version. + + Raises: + ValueError: If the version holds only plain tags and the node's saved tag and + data halves share a name. + """ + from orcapod.core.referential_tags import reference_names + + if version not in PLAIN_TAG_FORMAT_VERSIONS: + return + output_schema = descriptor.get("output_schema") or {} + if shared := reference_names( + output_schema.get("tag", {}), output_schema.get("data", {}) + ): + raise ValueError( + f"a file at format {version} holds only plain tags, but the saved output " + f"schema has the referential tags {list(shared)}" + ) + + +def source_config_at_version(config: dict[str, Any], version: str) -> dict[str, Any]: + """Return a saved source config with its ``tag_only_columns`` as its version means. + + A source config of a ``PLAIN_TAG_FORMAT_VERSIONS`` file never holds + ``tag_only_columns``: every one of its tags is plain, which the returned config + states (``tag_only_columns=True``) for each source in it that takes the option, + nested sources (the inner source of a cached source, a derived source's origin) + included. A config of a later version holds the option for each such source and + is returned as it is. + + Args: + config: A saved source config. + version: The file's format version. + + Returns: + The config to rebuild the source from (a copy when anything was added). + + Raises: + ValueError: If a config of a plain-tag version holds ``tag_only_columns``, or + one of a later version lacks it. + """ + _ensure_registries() + plain_version = version in PLAIN_TAG_FORMAT_VERSIONS + + def upgrade(value: Any) -> Any: + if isinstance(value, list): + return [upgrade(item) for item in value] + if not isinstance(value, dict): + return value + upgraded = {key: upgrade(item) for key, item in value.items()} + cls = SOURCE_REGISTRY.get(upgraded.get("source_type")) # pyright: ignore[reportArgumentType] # any saved value + if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): + return upgraded + if plain_version and "tag_only_columns" in upgraded: + raise ValueError( + f"a file at format {version} holds only plain tags, but source " + f"config {upgraded.get('source_type')!r} names tag_only_columns " + f"{upgraded['tag_only_columns']!r}" + ) + if plain_version: + upgraded["tag_only_columns"] = True + elif "tag_only_columns" not in upgraded: + raise ValueError( + f"a file at format {version} states the plain tags of every source, " + f"but source config {upgraded.get('source_type')!r} has no " + "tag_only_columns" + ) + return upgraded + + return upgrade(config) diff --git a/tests/test_core/referential_tags/test_model_and_sources.py b/tests/test_core/referential_tags/test_model_and_sources.py index 81ff31369..f100accf4 100644 --- a/tests/test_core/referential_tags/test_model_and_sources.py +++ b/tests/test_core/referential_tags/test_model_and_sources.py @@ -104,23 +104,27 @@ def test_arrow_table_source_default_and_tag_only() -> None: assert data.arrow_schema().field("x").type == pa.int8() -def test_to_config_writes_tag_only_columns_only_for_referential_sources( - tmp_path: object, -) -> None: +def test_to_config_always_writes_tag_only_columns(tmp_path: object) -> None: + """Format 0.3.0 states every source's plain tags; the version, not a missing key, + decides an older file's meaning (ITL-1019).""" from pathlib import Path path = Path(str(tmp_path)) / "rows.csv" path.write_text("x,y\n1,0.5\n2,0.75\n") plain = op.sources.CSVSource(str(path), tag_columns=["x"], tag_only_columns=True) referential = op.sources.CSVSource(str(path), tag_columns=["x"]) - assert "tag_only_columns" not in plain.to_config() + assert plain.to_config()["tag_only_columns"] is True assert referential.to_config()["tag_only_columns"] == [] - # A v0.3 config (no key) reads back with every tag plain. assert op.sources.CSVSource.from_config(plain.to_config()).keys() == plain.keys() assert ( op.sources.CSVSource.from_config(referential.to_config()).keys() == referential.keys() ) + # A config without the key is refused: its meaning is not guessed. + config = plain.to_config() + del config["tag_only_columns"] + with pytest.raises(ValueError, match="has no tag_only_columns"): + op.sources.CSVSource.from_config(config) @pytest.mark.parametrize("tag_only", [(), ("k",)], ids=["referential", "plain"]) diff --git a/tests/test_core/referential_tags/test_persistence.py b/tests/test_core/referential_tags/test_persistence.py new file mode 100644 index 000000000..cd5ec6941 --- /dev/null +++ b/tests/test_core/referential_tags/test_persistence.py @@ -0,0 +1,354 @@ +"""A stored table states which of its tags are referential (ITL-1019). + +Developer decision, November 16, 2026: the stored table fully expresses what it stores. A +stored tag column that is a referential tag carries Arrow field metadata +(``referential_tags.REFERENTIAL_TAG_METADATA_KEY``); a plain tag carries none. A function +node marks the referential tags of its pipeline (tag) table, reads the binding back from +that table, and refuses a table that disagrees with its pod. The result table never records +referential status. The marker moves no hash. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.contexts import resolve_context +from orcapod.core.referential_tags import ( + REFERENTIAL_TAG_METADATA_KEY, + REFERENTIAL_TAG_METADATA_VALUE, + mark_references, + stored_references, + without_reference_marks, +) +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol + +CALLS: list[float] = [] + +ROWS = [ + {"unit": 1, "time": 10, "amp": 0.5}, + {"unit": 2, "time": 20, "amp": 1.5}, +] +MORE_ROWS = [*ROWS, {"unit": 3, "time": 30, "amp": 2.5}] + + +@op.function_pod(output_keys=["y"]) +def double(amp: float) -> float: + """Double the amplitude; reads no tag.""" + CALLS.append(amp) + return 2 * amp + + +def _source( + rows: list[dict[str, float]], tag_only: object = ("time",) +) -> op.sources.DictSource: + """``unit`` referential and ``time`` plain unless ``tag_only`` says otherwise.""" + return op.sources.DictSource( + rows, + tag_columns=["unit", "time"], + tag_only_columns=tag_only, # pyright: ignore[reportArgumentType] # the test varies it + source_id="spikes", + ) + + +def _in_memory(_: Path) -> ArrowDatabaseProtocol: + return InMemoryArrowDatabase() + + +def _delta(tmp_path: Path) -> ArrowDatabaseProtocol: + return DeltaTableDatabase(tmp_path / "store") + + +STORES = pytest.mark.parametrize( + "make_store", [_in_memory, _delta], ids=["memory", "delta"] +) +ORCHESTRATORS = pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) + + +def _run( + store: ArrowDatabaseProtocol, + rows: list[dict[str, float]], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], + tag_only: object = ("time",), + load_only: bool = False, +) -> PipelineJob: + job = PipelineJob(database=store) + with job: + double.pod(_source(rows, tag_only), label="double") + job.run(orchestrator=orchestrator(), load_only=load_only) + return job + + +def _tables(store: ArrowDatabaseProtocol) -> dict[str, pa.Table]: + """Return the stored tables by kind: ``pipeline`` (tag) and ``result``.""" + tables: dict[str, pa.Table] = {} + for path in store.list_table_paths(): + table = store.get_all_records(path) + assert table is not None + kind = "result" if "_result" in path else "pipeline" + assert kind not in tables, f"more than one {kind} table: {path}" + tables[kind] = table + return tables + + +def _marked(schema: pa.Schema) -> list[str]: + """The columns carrying the marker key, whatever its value.""" + return [ + f.name for f in schema if REFERENTIAL_TAG_METADATA_KEY in (f.metadata or {}) + ] + + +# --------------------------------------------------------------------------- +# The marker itself +# --------------------------------------------------------------------------- + + +def test_mark_references_marks_exactly_the_named_columns() -> None: + stray = pa.field("b", pa.int64(), metadata={REFERENTIAL_TAG_METADATA_KEY: b"x"}) + table = pa.Table.from_arrays( + [pa.array([1]), pa.array([2]), pa.array(["c"])], + schema=pa.schema( + [ + pa.field("a", pa.int64(), nullable=False), + stray, + pa.field("c", pa.string()), + ] + ), + ) + marked = mark_references(table, ["a"]) + # A marker that came in on another column is not this writer's binding. + assert _marked(marked.schema) == ["a"] + assert marked.schema.field("a").metadata == { + REFERENTIAL_TAG_METADATA_KEY: REFERENTIAL_TAG_METADATA_VALUE + } + assert marked.schema.field("b").metadata is None + # Values, types and nullability are untouched. + assert marked.schema.field("a").nullable is False + assert marked.equals(table) + assert stored_references(marked.schema) == ("a",) + # The read side drops every marker and gives back the unmarked schema. + assert without_reference_marks(marked).schema.equals( + mark_references(table, []).schema, check_metadata=True + ) + + +def test_mark_references_keeps_other_field_metadata() -> None: + field = pa.field("a", pa.int64(), metadata={b"other": b"kept"}) + table = pa.Table.from_arrays([pa.array([1])], schema=pa.schema([field])) + marked = mark_references(table, ["a"]) + assert marked.schema.field("a").metadata == { + b"other": b"kept", + REFERENTIAL_TAG_METADATA_KEY: REFERENTIAL_TAG_METADATA_VALUE, + } + assert without_reference_marks(marked).schema.field("a").metadata == { + b"other": b"kept" + } + + +def test_an_unmarked_table_is_returned_as_it_is() -> None: + table = pa.table({"a": [1]}) + assert mark_references(table, []) is table + + +def test_marking_a_column_the_table_lacks_raises() -> None: + with pytest.raises(ValueError, match="not columns of the stored table"): + mark_references(pa.table({"a": [1]}), ["b"]) + + +def test_a_marker_with_another_value_is_refused_on_read() -> None: + field = pa.field("a", pa.int64(), metadata={REFERENTIAL_TAG_METADATA_KEY: b"plain"}) + with pytest.raises(ValueError, match="only valid value"): + stored_references(pa.schema([field])) + + +def test_the_marker_moves_no_table_hash() -> None: + """Audit probe 6: the table hash ignores the marker (no golden value can move).""" + hasher = resolve_context(None).arrow_hasher + table = pa.table({"unit": [1, 2], "y": [0.5, 1.5]}) + marked = mark_references(table, ["unit"]) + assert marked.schema.field("unit").metadata + assert hasher.hash_table(marked) == hasher.hash_table(table) + + +# --------------------------------------------------------------------------- +# A function node's pipeline table +# --------------------------------------------------------------------------- + + +@STORES +@ORCHESTRATORS +def test_the_pipeline_table_marks_only_the_referential_tags( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + _run(store, ROWS, orchestrator) + tables = _tables(store) + pipeline = tables["pipeline"] + # ``unit`` is referential and is marked; ``time`` is plain and carries nothing. + assert _marked(pipeline.schema) == ["unit"] + assert REFERENTIAL_TAG_METADATA_KEY not in ( + pipeline.schema.field("time").metadata or {} + ) + assert stored_references(pipeline.schema) == ("unit",) + # The result table never records referential status: no marker, no tag column. + assert _marked(tables["result"].schema) == [] + assert "unit" not in tables["result"].column_names + + +@STORES +@ORCHESTRATORS +def test_a_plain_pipeline_table_carries_no_marker( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + _run(store, ROWS, orchestrator, tag_only=True) + for table in _tables(store).values(): + assert _marked(table.schema) == [] + + +@STORES +@ORCHESTRATORS +def test_the_marker_survives_an_append( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + _run(store, ROWS, orchestrator) + # More rows of the same pipeline: the same table, appended to. + CALLS.clear() + _run(store, MORE_ROWS, orchestrator) + assert CALLS == [2.5] + pipeline = _tables(store)["pipeline"] + assert pipeline.num_rows == 3 + assert stored_references(pipeline.schema) == ("unit",) + # A fresh handle on the same store reads the marker back from storage. + if isinstance(store, DeltaTableDatabase): + assert stored_references( + _tables(DeltaTableDatabase(tmp_path / "store"))["pipeline"].schema + ) == ("unit",) + + +@STORES +@ORCHESTRATORS +def test_the_binding_is_read_back_from_the_table( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + first = _run(store, ROWS, orchestrator).nodes["double"] + CALLS.clear() + # A load-only run reads every row from the store. + loaded = _run(store, ROWS, orchestrator, load_only=True).nodes["double"] + assert CALLS == [] + assert loaded.keys() == first.keys() == (("unit", "time"), ("unit", "y")) + table = loaded.as_table() + assert table.column_names.count("unit") == 1 + # The marker belongs to the stored table, not to the stream read from it. + assert _marked(table.schema) == [] + assert sorted(table.to_pylist(), key=lambda r: r["unit"]) == sorted( + first.as_table().to_pylist(), key=lambda r: r["unit"] + ) + for tag, data in loaded.iter_data(): + assert tag["unit"] == data["unit"] + + +@ORCHESTRATORS +def test_a_table_that_disagrees_with_the_pod_is_refused( + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + written = InMemoryArrowDatabase() + _run(written, ROWS, orchestrator) + # The same tables with the marker removed: as if another pipeline wrote them. + tampered = InMemoryArrowDatabase() + for path in written.list_table_paths(): + id_column = "__tampered_id" + table = written.get_all_records(path, record_id_column=id_column) + assert table is not None + tampered.add_records( + path, without_reference_marks(table), record_id_column=id_column, flush=True + ) + with pytest.raises(InputValidationError, match=r"marks the referential tags \[\]"): + _run(tampered, ROWS, orchestrator, load_only=True) + + +# --------------------------------------------------------------------------- +# Every write path of a store keeps the marker +# --------------------------------------------------------------------------- + + +def _rows(ids: list[int], extra: bool = False) -> pa.Table: + """Marked rows keyed by a binary record id; ``extra`` adds a column.""" + columns: dict[str, pa.Array[Any]] = { + "rid": pa.array([str(i).encode() for i in ids], pa.large_binary()), + "unit": pa.array(ids, pa.int64()), + "y": pa.array([float(i) for i in ids], pa.float64()), + } + if extra: + columns["z"] = pa.array(ids, pa.int64()) + return mark_references(pa.Table.from_pydict(columns), ["unit"]) + + +@pytest.mark.parametrize( + "make_store", + [ + _in_memory, + lambda tmp_path: DeltaTableDatabase( + tmp_path / "store", allow_schema_evolution=True + ), + ], + ids=["memory", "delta"], +) +def test_every_write_path_keeps_the_marker( + tmp_path: Path, make_store: Callable[[Path], ArrowDatabaseProtocol] +) -> None: + store = make_store(tmp_path) + path = ("node", "table") + + def references() -> tuple[str, ...]: + table = store.get_all_records(path) + assert table is not None + return stored_references(table.schema) + + store.add_records(path, _rows([1, 2]), record_id_column="rid", flush=True) + assert references() == ("unit",) # first write + store.add_records(path, _rows([3]), record_id_column="rid") + assert references() == ("unit",) # stored rows read together with pending ones + store.flush() + assert references() == ("unit",) # append + store.add_records( + path, _rows([3, 4]), record_id_column="rid", skip_duplicates=True, flush=True + ) + assert references() == ("unit",) # some ids stored already: a merge + unit = store.get_records_with_column_value(path, {"unit": 1}) + assert unit is not None and stored_references(unit.schema) == ("unit",) + if isinstance(store, DeltaTableDatabase): + # A new column: the stored schema evolves (in-memory tables do not evolve). + store.add_records( + path, _rows([5], extra=True), record_id_column="rid", flush=True + ) + assert references() == ("unit",) + reopened = DeltaTableDatabase(tmp_path / "store") + table = reopened.get_all_records(path) + assert table is not None and stored_references(table.schema) == ("unit",) diff --git a/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py b/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py index 93a1db6df..8710ff4cf 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py +++ b/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py @@ -227,6 +227,11 @@ def test_a_file_without_recorded_nullability_restores_side_effect_nodes( for descriptor in data["pipeline"]["nodes"].values(): del descriptor["output_schema"]["nullable"] descriptor.get("source_config", {}).pop("nullable", None) + # A 0.1.0 file holds only plain tags and never names them (ITL-1019). + descriptor.pop("fall_through_references", None) + descriptor.get("function_config", {}).pop("fall_through_references", None) + for config in data["bindings"]["sources"].values(): + config.pop("tag_only_columns", None) old = tmp_path / "format_0_1_0.json" old.write_text(json.dumps(data)) diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index c548f5c57..e04528aa6 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -147,6 +147,9 @@ def read_u(pp: cls) -> int: if descriptor["node_type"] == "source": descriptor.pop("pipeline_hash", None) descriptor.pop("content_hash", None) + # A 0.1.0 job file never names plain tags (ITL-1019). + for config in data["bindings"]["sources"].values(): + config.pop("tag_only_columns", None) with open(work + "/job.json", "w") as f: json.dump(data, f) # Nothing registers the class before the load. diff --git a/tests/test_pipeline/test_saved_job_before_referential_tags.py b/tests/test_pipeline/test_saved_job_before_referential_tags.py index c61997926..5249eff11 100644 --- a/tests/test_pipeline/test_saved_job_before_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_before_referential_tags.py @@ -23,6 +23,7 @@ SUPPORTED_FORMAT_VERSIONS, SUPPORTED_JOB_FORMAT_VERSIONS, resolve_database_from_config, + source_config_at_version, ) from orcapod.sources import CSVSource @@ -70,7 +71,10 @@ def test_a_v03_job_loads_with_unchanged_hashes(saved: dict[str, Any]) -> None: def test_a_v03_source_config_rebuilds_the_v03_source(saved: dict[str, Any]) -> None: config = saved["bindings"]["sources"]["subjects"] assert "tag_only_columns" not in config - source = CSVSource.from_config(config) + # The file's version (0.1.0) states that every tag is plain (ITL-1019). + upgraded = source_config_at_version(config, saved["orcapod_pipeline_job_version"]) + assert upgraded["tag_only_columns"] is True + source = CSVSource.from_config(upgraded) assert source.keys() == (("subject",), ("age",)) assert source.content_hash().to_string() == config["content_hash"] assert source.pipeline_hash().to_string() == config["pipeline_hash"] diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py new file mode 100644 index 000000000..f37c3af1a --- /dev/null +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -0,0 +1,265 @@ +"""Saving and loading a job or pipeline that holds referential tags (ITL-1019). + +A file of format 0.3.0 states its bindings: every source config that takes the option +writes ``tag_only_columns``, and every function node writes ``fall_through_references`` +(empty for a plain pod). Files at 0.1.0 and 0.2.0 hold only plain tags, decided by the +version, never by a missing key. The saved file is an independent point of verification: +a loaded node reads its stored table only when the table's marker agrees with the file. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import orcapod as op +from orcapod.core.function_pod import FunctionPod, ReferenceFallThroughPod +from orcapod.core.referential_tags import without_reference_marks +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + Pipeline, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.pipeline.serialization import ( + PLAIN_TAG_FORMAT_VERSIONS, + source_config_at_version, +) + +ORCHESTRATORS = pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) + + +@op.function_pod(output_keys=["y"]) +def double(amp: float) -> float: + """Double the amplitude; reads no tag, so ``unit`` falls through.""" + return 2 * amp + + +@op.function_pod(output_keys=["z"]) +def shift(y: float) -> float: + """A second step over the first one's output.""" + return y + 1 + + +def _csv(tmp_path: Path) -> str: + path = tmp_path / "spikes.csv" + path.write_text("unit,time,amp\n1,10,0.5\n2,20,1.5\n") + return str(path) + + +def _job(tmp_path: Path, tag_only: object = ("time",)) -> PipelineJob: + """A CSV source (``unit`` referential unless ``tag_only`` says otherwise), two pods.""" + source = op.sources.CSVSource( + _csv(tmp_path), + tag_columns=["unit", "time"], + tag_only_columns=tag_only, # pyright: ignore[reportArgumentType] # the test varies it + source_id="spikes", + ) + job = PipelineJob(database=DeltaTableDatabase(tmp_path / "store")) + with job: + doubled = double.pod(source, label="double") + shift.pod(doubled, label="shift") + return job + + +def _hashes(job: PipelineJob) -> dict[str, tuple[str, str, Any]]: + return { + label: ( + node.content_hash().to_string(), + node.pipeline_hash().to_string(), + node.keys(), + ) + for label, node in job.nodes.items() + } + + +def _saved(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def _function_descriptors(data: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + d["label"]: d + for d in data["pipeline"]["nodes"].values() + if d["node_type"] == "function" + } + + +@ORCHESTRATORS +def test_a_referential_job_saves_and_loads_with_its_hashes_and_bindings( + tmp_path: Path, + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + job = _job(tmp_path) + job.run(orchestrator=orchestrator()) + assert job.nodes["double"].keys() == (("unit", "time"), ("unit", "y")) + rows = sorted(job.nodes["shift"].as_table().to_pylist(), key=lambda r: r["unit"]) + job.save(tmp_path / "job.json") + + loaded = PipelineJob.load(tmp_path / "job.json") + assert _hashes(loaded) == _hashes(job) + # A loaded job reads its stored rows; the binding comes back with them. + loaded.run(load_only=True, orchestrator=orchestrator()) + table = loaded.nodes["shift"].as_table() + assert table.column_names.count("unit") == 1 + assert sorted(table.to_pylist(), key=lambda r: r["unit"]) == rows + records = loaded.nodes["double"].get_all_records() + assert records is not None and sorted(records.column("y").to_pylist()) == [1.0, 3.0] + + +def test_the_file_states_every_binding(tmp_path: Path) -> None: + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + assert data["orcapod_pipeline_job_version"] == "0.3.0" + assert data["bindings"]["sources"]["spikes"]["tag_only_columns"] == ["time"] + descriptors = _function_descriptors(data) + for label in ("double", "shift"): + descriptor = descriptors[label] + # Written for every function node, and the pod's config agrees. + assert descriptor["fall_through_references"] == ["unit"] + assert descriptor["function_config"]["fall_through_references"] == ["unit"] + + +def test_a_plain_job_states_its_plain_tags(tmp_path: Path) -> None: + job = _job(tmp_path, tag_only=True) + job.run() + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + assert data["bindings"]["sources"]["spikes"]["tag_only_columns"] is True + for descriptor in _function_descriptors(data).values(): + assert descriptor["fall_through_references"] == [] + assert "fall_through_references" not in descriptor["function_config"] + loaded = PipelineJob.load(tmp_path / "job.json") + assert _hashes(loaded) == _hashes(job) + + +def test_a_loaded_job_saved_again_keeps_its_bindings(tmp_path: Path) -> None: + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + PipelineJob.load(tmp_path / "job.json").save(tmp_path / "again.json") + for descriptor in _function_descriptors(_saved(tmp_path / "again.json")).values(): + assert descriptor["fall_through_references"] == ["unit"] + assert _hashes(PipelineJob.load(tmp_path / "again.json")) == _hashes(job) + + +def test_a_pipeline_saves_and_loads_its_bindings(tmp_path: Path) -> None: + job = _job(tmp_path) + job.compiled_pipeline.save(tmp_path / "pipeline.json") + loaded = Pipeline.load(tmp_path / "pipeline.json") + for label in ("spikes", "double", "shift"): + assert loaded.nodes[label].keys() == job.nodes[label].keys() + assert ( + loaded.nodes[label].pipeline_hash().to_string() + == job.nodes[label].pipeline_hash().to_string() + ) + + +def test_the_fall_through_pod_config_round_trips() -> None: + plain = double.pod + assert isinstance(plain, FunctionPod) + pod = ReferenceFallThroughPod(plain, ["unit"]) + config = pod.to_config() + assert config["fall_through_references"] == ["unit"] + rebuilt = ReferenceFallThroughPod.from_config(config) + assert rebuilt.references == ("unit",) + assert rebuilt.pipeline_hash() == pod.pipeline_hash() + del config["fall_through_references"] + with pytest.raises(ValueError, match="fall_through_references"): + ReferenceFallThroughPod.from_config(config) + + +# --------------------------------------------------------------------------- +# The format version decides; a missing or contradicting record is refused +# --------------------------------------------------------------------------- + + +def _edited(tmp_path: Path, edit: Any) -> Path: + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + edit(data) + path = tmp_path / "edited.json" + path.write_text(json.dumps(data)) + return path + + +def test_a_030_source_config_without_tag_only_columns_is_refused( + tmp_path: Path, +) -> None: + def edit(data: dict[str, Any]) -> None: + del data["bindings"]["sources"]["spikes"]["tag_only_columns"] + + # The file is malformed, not a source that moved: the load fails. + with pytest.raises(ValueError, match="source 'spikes'.*has no tag_only_columns"): + PipelineJob.load(_edited(tmp_path, edit)) + + +def test_a_030_function_node_without_its_references_is_refused(tmp_path: Path) -> None: + def edit(data: dict[str, Any]) -> None: + for descriptor in _function_descriptors(data).values(): + del descriptor["fall_through_references"] + + with pytest.raises(ValueError, match="has no fall_through_references"): + PipelineJob.load(_edited(tmp_path, edit)) + + +def test_references_that_disagree_with_the_saved_schema_are_refused( + tmp_path: Path, +) -> None: + def edit(data: dict[str, Any]) -> None: + _function_descriptors(data)["double"]["fall_through_references"] = [] + + with pytest.raises(ValueError, match="saved output schema has the referential"): + PipelineJob.load(_edited(tmp_path, edit)) + + +@pytest.mark.parametrize("version", sorted(PLAIN_TAG_FORMAT_VERSIONS)) +def test_an_older_file_holds_only_plain_tags(tmp_path: Path, version: str) -> None: + def edit(data: dict[str, Any]) -> None: + data["orcapod_pipeline_job_version"] = version + data["pipeline"]["orcapod_pipeline_version"] = version + + # A referential schema in a file of a plain-tag version is not that version. + with pytest.raises(ValueError, match=f"a file at format {version} holds only"): + PipelineJob.load(_edited(tmp_path, edit)) + with pytest.raises(ValueError, match="names tag_only_columns"): + source_config_at_version( + {"source_type": "csv", "tag_only_columns": []}, version + ) + assert source_config_at_version({"source_type": "csv"}, version) == { + "source_type": "csv", + "tag_only_columns": True, + } + + +def test_the_table_is_checked_against_the_saved_job(tmp_path: Path) -> None: + """The file and the store are independent: a loaded node refuses a disagreement.""" + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + store = DeltaTableDatabase(tmp_path / "store") + # The same rows without the marker, as if another pipeline had written them. + tampered = InMemoryArrowDatabase() + for path in store.list_table_paths(): + table = store.get_all_records(path, record_id_column="__copy_id") + assert table is not None + tampered.add_records( + path, without_reference_marks(table), record_id_column="__copy_id" + ) + tampered.flush() + loaded = PipelineJob.load(tmp_path / "job.json", database=tampered) + with pytest.raises(InputValidationError, match="saved job declares \\['unit'\\]"): + loaded.nodes["double"].get_all_records() From 199221ccbca7300f9da60b39a5164c9bce8a04aa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 01:21:02 -0800 Subject: [PATCH 02/19] feat(referential_tags): refuse a database that cannot keep the marker before writing (ITL-1019) A function node with referential tags asks its pipeline database, when the database is attached, whether it can store the marker (RecordValidatingDatabaseProtocol: ConnectorArrowDatabase.validate_records). The SQLite and PostgreSQL connector databases keep no field metadata, so such a node is refused with an InputValidationError naming the database and the two ways out (a database that keeps field metadata, or tag_only_columns), before any result or record is written. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1019.md | 6 ++- docs/concepts/referential-tags.md | 4 +- src/orcapod/core/nodes/function_node.py | 6 +++ src/orcapod/core/referential_tags.py | 38 +++++++++++++++++++ .../databases/connector_arrow_database.py | 10 +++++ .../databases/extension_aware_database.py | 13 +++++++ src/orcapod/protocols/database_protocols.py | 15 ++++++++ .../referential_tags/test_persistence.py | 35 +++++++++++++++++ 8 files changed, 124 insertions(+), 3 deletions(-) diff --git a/changelog.d/ITL-1019.md b/changelog.d/ITL-1019.md index 40ccb20a1..f534c7756 100644 --- a/changelog.d/ITL-1019.md +++ b/changelog.d/ITL-1019.md @@ -23,5 +23,7 @@ ### Known limitations -- The SQLite and PostgreSQL databases keep no Arrow field metadata and refuse a pipeline - table with a referential tag, as they refuse logical types. +- The SQLite and PostgreSQL databases keep no Arrow field metadata, so a function node + that would store a referential tag there is refused (`InputValidationError`, naming the + database and the two ways out: a database that keeps field metadata, or + `tag_only_columns`) before anything is written. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 2208c914d..fad4b0555 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -143,7 +143,9 @@ marker. The function's result table never records referential status, so plain a referential runs of the same function share their results. The marker moves no hash, and the stores keep it on every write (first write, append, merge, schema evolution; in memory and on Delta). The SQLite and PostgreSQL databases keep no Arrow -metadata at all and refuse such a table, as they refuse logical types. +metadata at all: a function node that would store a referential tag there is refused when +its database is attached, before anything is written. Use a database that keeps field +metadata, or keep those tags plain with `tag_only_columns`. ```python from orcapod.core.referential_tags import stored_references diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 880f47f11..62730b11a 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -38,6 +38,7 @@ ) from orcapod.core.referential_tags import ( mark_references, + require_database_keeps_marker, stored_references, without_reference_marks, ) @@ -1461,6 +1462,11 @@ def attach_databases( result_database = pipeline_database.at("_result") if self._function_pod is not None: + # A table must state its referential tags: refuse a database that + # cannot, before anything is written (ITL-1019). + require_database_keeps_marker( + pipeline_database, fall_through_references(self._function_pod) + ) # Normal path: wrap in CachedFunctionPod for compute + cache. # A fall-through pod's results are its plain pod's (ITL-1023). self._cached_function_pod = CachedFunctionPod( diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index 1dea111b9..614d586ca 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -258,6 +258,44 @@ def stored_references(schema: "pa.Schema") -> tuple[str, ...]: return tuple(names) +def require_database_keeps_marker(database: object, references: Iterable[str]) -> None: + """Refuse, before anything is written, a database that cannot store the marker. + + A stored table must state what it stores (developer, November 16, 2026, ITL-1019), + so a referential tag is never stored unmarked. A database that can check rows + before writing them (``RecordValidatingDatabaseProtocol``) is asked whether it + keeps the marker; one that cannot is refused. + + Args: + database: The database a node will store its referential tags in. + references: The referential tags it will store; nothing is checked when empty. + + Raises: + InputValidationError: If the database cannot keep the marker, naming the + database kind and the two ways out. + """ + from orcapod.protocols.database_protocols import RecordValidatingDatabaseProtocol + + references = tuple(references) + if not references or not isinstance(database, RecordValidatingDatabaseProtocol): + return + probe = mark_references( + pa.table({name: pa.array([], type=pa.null()) for name in references}), + references, + ) + try: + database.validate_records(probe) + except ValueError as exc: + raise InputValidationError( + f"{type(database).__name__} cannot store the referential tags " + f"{list(references)}: a stored referential tag carries Arrow field " + f"metadata, and this database keeps none ({exc}). Use a database that " + "keeps field metadata (DeltaTableDatabase, InMemoryArrowDatabase, the " + "SpiralDB database), or keep these tags plain with tag_only_columns on " + "the source. Nothing was written." + ) from exc + + def without_reference_marks(table: "pa.Table") -> "pa.Table": """Return ``table`` with no referential tag marker on any column. diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index af48cecd8..25bb2d8f3 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -269,6 +269,16 @@ def add_record( flush=flush, ) + def validate_records(self, records: pa.Table) -> None: + """Raise ``ValueError`` if the connector could not store ``records`` as given. + + The check ``add_records`` applies, without writing anything. + + Args: + records: The rows (any number, none included) to check. + """ + self._connector.validate_records(records) + def add_records( self, record_path: tuple[str, ...], diff --git a/src/orcapod/databases/extension_aware_database.py b/src/orcapod/databases/extension_aware_database.py index 01e4fa8af..74f719a1c 100644 --- a/src/orcapod/databases/extension_aware_database.py +++ b/src/orcapod/databases/extension_aware_database.py @@ -164,6 +164,19 @@ def add_records( flush=flush, ) + def validate_records(self, records: pa.Table) -> None: + """Raise ``ValueError`` if the wrapped database could not store ``records``. + + Delegates to the wrapped database when it can check rows before writing + (``RecordValidatingDatabaseProtocol``); otherwise accepts them. + """ + from orcapod.protocols.database_protocols import ( + RecordValidatingDatabaseProtocol, + ) + + if isinstance(self._db, RecordValidatingDatabaseProtocol): + self._db.validate_records(records) + def flush(self) -> None: self._db.flush() diff --git a/src/orcapod/protocols/database_protocols.py b/src/orcapod/protocols/database_protocols.py index fb585495c..7e533ae3c 100644 --- a/src/orcapod/protocols/database_protocols.py +++ b/src/orcapod/protocols/database_protocols.py @@ -227,6 +227,21 @@ def flush_batch(self, record_path: tuple[str, ...]) -> None: ... +@runtime_checkable +class RecordValidatingDatabaseProtocol(Protocol): + """A database that can say, before writing, whether it can store some rows. + + A database whose backend cannot keep every part of an Arrow table (the SQLite and + PostgreSQL connector databases keep no field metadata) refuses such rows on + write. ``validate_records`` applies the same check without writing, so a caller + can refuse up front (ITL-1019: a referential tag marker). + """ + + def validate_records(self, records: pa.Table) -> None: + """Raise ``ValueError`` if the database could not store ``records`` as given.""" + ... + + @runtime_checkable class DatabaseRegistryProtocol(Protocol): """Protocol for a database config registry used during pipeline save/load. diff --git a/tests/test_core/referential_tags/test_persistence.py b/tests/test_core/referential_tags/test_persistence.py index cd5ec6941..ebb103794 100644 --- a/tests/test_core/referential_tags/test_persistence.py +++ b/tests/test_core/referential_tags/test_persistence.py @@ -352,3 +352,38 @@ def references() -> tuple[str, ...]: reopened = DeltaTableDatabase(tmp_path / "store") table = reopened.get_all_records(path) assert table is not None and stored_references(table.schema) == ("unit",) + + +# --------------------------------------------------------------------------- +# A database that cannot keep the marker is refused before anything is written +# --------------------------------------------------------------------------- + + +def test_a_database_without_field_metadata_is_refused_before_any_write() -> None: + from orcapod.databases.connector_arrow_database import ConnectorArrowDatabase + from orcapod.databases.sqlite_connector import SQLiteConnector + + connector = SQLiteConnector(":memory:") + database = ConnectorArrowDatabase(connector) + CALLS.clear() + with pytest.raises(InputValidationError) as raised: + _run(database, ROWS, SyncPipelineOrchestrator) + message = str(raised.value) + # The database kind and both ways out are named. + assert "SQLiteConnector does not preserve Arrow metadata" in message + assert "keeps field metadata" in message and "tag_only_columns" in message + assert CALLS == [] + assert database.list_table_paths() == [] + + +def test_plain_tags_ask_nothing_of_the_database() -> None: + from orcapod.core.referential_tags import require_database_keeps_marker + from orcapod.databases.connector_arrow_database import ConnectorArrowDatabase + from orcapod.databases.sqlite_connector import SQLiteConnector + + database = ConnectorArrowDatabase(SQLiteConnector(":memory:")) + require_database_keeps_marker(database, ()) + with pytest.raises(InputValidationError, match="cannot store the referential"): + require_database_keeps_marker(database, ["unit"]) + # A database that keeps field metadata is accepted. + require_database_keeps_marker(InMemoryArrowDatabase(), ["unit"]) From fc1db0ab5d813896aa9230e65a34b0bec8a6a8db Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:06:23 -0800 Subject: [PATCH 03/19] fix(pipeline): a saved-node error of a loaded job names the job file (ITL-1019) PipelineJob.load reads its embedded pipeline from a temporary file; an error Pipeline.load raises for a saved node named that deleted file. It now names the job file, keeping the error's type. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 6 ++++++ .../test_pipeline/test_saved_job_with_referential_tags.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 7f830dc1f..8fe236939 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1606,6 +1606,12 @@ def load( _json.dump(data["pipeline"], tmp) try: pipeline = Pipeline.load(tmp_path) + except ValueError as exc: + # The embedded pipeline was read from a temporary file: name the job file. + message = str(exc) + if tmp_path in message: + exc.args = (message.replace(tmp_path, str(path)),) + raise finally: os.unlink(tmp_path) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index f37c3af1a..0e9894e1e 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -212,8 +212,11 @@ def edit(data: dict[str, Any]) -> None: for descriptor in _function_descriptors(data).values(): del descriptor["fall_through_references"] - with pytest.raises(ValueError, match="has no fall_through_references"): - PipelineJob.load(_edited(tmp_path, edit)) + path = _edited(tmp_path, edit) + with pytest.raises(ValueError, match="has no fall_through_references") as raised: + PipelineJob.load(path) + # The error names the job file, not the temporary file of its embedded pipeline. + assert str(raised.value).startswith(f"{path}: saved node") def test_references_that_disagree_with_the_saved_schema_are_refused( From 00818bd80a5c7d3739787c8eb5f8adcfe36b7c47 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:17:49 -0800 Subject: [PATCH 04/19] fix(pipeline): refuse a saved source config whose bindings contradict its slot (ITL-1019) A job file states a source's bindings twice: in the source config (tag_columns, tag_only_columns) and in its slot's saved schema. When they disagree the load now fails with an error naming the job file, the source and both lists, instead of logging a warning and leaving the source unbound (the path kept for a source whose data changed on disk). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 6 +++ src/orcapod/pipeline/serialization.py | 40 ++++++++++++++++++- .../test_saved_job_with_referential_tags.py | 13 ++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 8fe236939..1dbb845ea 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1585,6 +1585,7 @@ def load( from orcapod.pipeline.graph import Pipeline from orcapod.pipeline.serialization import ( SUPPORTED_JOB_FORMAT_VERSIONS, + check_source_config_binding, resolve_source_from_config, source_config_at_version, ) @@ -1630,6 +1631,11 @@ def load( # is the file's error, not a source that could not be rebuilt. try: src_config = source_config_at_version(src_config, version) + if spec_name in slots: + slot = slots[spec_name] + check_source_config_binding( + src_config, slot.tag_schema.keys(), slot.data_schema.keys() + ) except ValueError as exc: raise ValueError(f"{path}: source {spec_name!r}: {exc}") from exc try: diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 64291d96c..08a612823 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -5,7 +5,7 @@ import hashlib import json as _json import logging -from collections.abc import Collection +from collections.abc import Collection, Iterable from enum import Enum from typing import TYPE_CHECKING, Any, cast @@ -1454,3 +1454,41 @@ def upgrade(value: Any) -> Any: return upgraded return upgrade(config) + + +def check_source_config_binding( + config: dict[str, Any], tag_names: Iterable[str], data_names: Iterable[str] +) -> None: + """Refuse a saved source config whose referential tags contradict its slot. + + The source config (``tag_columns`` and ``tag_only_columns``) and the slot's saved + schema (a name in both halves is a referential tag) each state the source's + bindings, independently; a file in which they disagree is refused before the + source is rebuilt (ITL-1019). A config of a source that does not take the option, + or one written without ``tag_columns``, is not checked here. + + Args: + config: A saved source config, as its file version means it + (``source_config_at_version``). + tag_names: The slot's saved tag column names. + data_names: The slot's saved data column names. + + Raises: + ValueError: If the config and the slot name different referential tags. + """ + from orcapod.core.referential_tags import reference_names, source_reference_tags + + _ensure_registries() + cls = SOURCE_REGISTRY.get(config.get("source_type")) # pyright: ignore[reportArgumentType] # any saved value + if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): + return + tag_columns = config.get("tag_columns") + if not tag_columns or "tag_only_columns" not in config: + return + configured = source_reference_tags(tag_columns, config["tag_only_columns"]) + declared = reference_names(tag_names, data_names) + if set(configured) != set(declared): + raise ValueError( + f"its config makes {sorted(configured)} referential, but its slot's saved " + f"schema declares {sorted(declared)}" + ) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index 0e9894e1e..5a0d3137c 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -207,6 +207,19 @@ def edit(data: dict[str, Any]) -> None: PipelineJob.load(_edited(tmp_path, edit)) +def test_a_source_config_that_contradicts_its_slot_is_refused(tmp_path: Path) -> None: + def edit(data: dict[str, Any]) -> None: + # The slot declares ``unit`` referential; the config says every tag is plain. + data["bindings"]["sources"]["spikes"]["tag_only_columns"] = True + + path = _edited(tmp_path, edit) + with pytest.raises(ValueError, match="source 'spikes'") as raised: + PipelineJob.load(path) + message = str(raised.value) + assert str(path) in message + assert "['unit']" in message and "[]" in message + + def test_a_030_function_node_without_its_references_is_refused(tmp_path: Path) -> None: def edit(data: dict[str, Any]) -> None: for descriptor in _function_descriptors(data).values(): From ec5984a75ee285a03e6d3a457aa59c276269a0dd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:25:36 -0800 Subject: [PATCH 05/19] fix(pipeline): check an explicitly empty tag_columns against the slot too (ITL-1019) Only an absent or None tag_columns (a database source that takes its tags from the table's keys) skips the source config vs slot binding check. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/serialization.py | 6 ++++-- .../test_saved_job_with_referential_tags.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 08a612823..80c9f471d 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -1465,7 +1465,8 @@ def check_source_config_binding( schema (a name in both halves is a referential tag) each state the source's bindings, independently; a file in which they disagree is refused before the source is rebuilt (ITL-1019). A config of a source that does not take the option, - or one written without ``tag_columns``, is not checked here. + or one whose ``tag_columns`` is absent or ``None`` (a database source that takes + its tags from the table's keys), is not checked here. Args: config: A saved source config, as its file version means it @@ -1483,7 +1484,8 @@ def check_source_config_binding( if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): return tag_columns = config.get("tag_columns") - if not tag_columns or "tag_only_columns" not in config: + # ``None``: a database source takes its tags from the table's keys when read. + if tag_columns is None or "tag_only_columns" not in config: return configured = source_reference_tags(tag_columns, config["tag_only_columns"]) declared = reference_names(tag_names, data_names) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index 5a0d3137c..d155613b9 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -207,10 +207,17 @@ def edit(data: dict[str, Any]) -> None: PipelineJob.load(_edited(tmp_path, edit)) -def test_a_source_config_that_contradicts_its_slot_is_refused(tmp_path: Path) -> None: +@pytest.mark.parametrize( + "contradiction", + [{"tag_only_columns": True}, {"tag_columns": [], "tag_only_columns": True}], + ids=["all_plain", "no_tags"], +) +def test_a_source_config_that_contradicts_its_slot_is_refused( + tmp_path: Path, contradiction: dict[str, Any] +) -> None: def edit(data: dict[str, Any]) -> None: - # The slot declares ``unit`` referential; the config says every tag is plain. - data["bindings"]["sources"]["spikes"]["tag_only_columns"] = True + # The slot declares ``unit`` referential; the config makes nothing referential. + data["bindings"]["sources"]["spikes"].update(contradiction) path = _edited(tmp_path, edit) with pytest.raises(ValueError, match="source 'spikes'") as raised: From 26d981a87e275e6bc8ffe2c1656052982958eeae Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 03:40:22 -0800 Subject: [PATCH 06/19] feat(referential_tags): operator nodes store and derived sources present referential bindings (ITL-1019) An operator node in LOG mode marks the referential tags of its operator's output in its stored table; every read checks the marker against the node (its operator's prediction, or a loaded job's saved output schema) and drops it in the stream. A database that cannot keep field metadata is refused when attached in LOG or REPLAY mode. DerivedSource carries the bindings of what it reads: a job node's stored table (checked against its keys) or a source's own. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1019.md | 19 +- docs/concepts/referential-tags.md | 30 +- src/orcapod/core/nodes/operator_node.py | 155 +++++++- src/orcapod/core/sources/derived_source.py | 71 +++- .../referential_tags/test_derived_source.py | 270 +++++++++++++ .../test_operator_node_storage.py | 376 ++++++++++++++++++ .../referential_tags/test_operators.py | 5 - ...ved_job_with_referential_operator_nodes.py | 260 ++++++++++++ 8 files changed, 1147 insertions(+), 39 deletions(-) create mode 100644 tests/test_core/referential_tags/test_derived_source.py create mode 100644 tests/test_core/referential_tags/test_operator_node_storage.py create mode 100644 tests/test_pipeline/test_saved_job_with_referential_operator_nodes.py diff --git a/changelog.d/ITL-1019.md b/changelog.d/ITL-1019.md index f534c7756..763af1cf1 100644 --- a/changelog.d/ITL-1019.md +++ b/changelog.d/ITL-1019.md @@ -11,6 +11,22 @@ - A job or pipeline holding referential tags and fall-through pods can be saved and loaded; it keeps its hashes and bindings. `ReferenceFallThroughPod.to_config` and `from_config` work. +- An operator node stores a referential stream (`LOG`) and replays it (`REPLAY`): its + table marks the referential tags of the operator's own output with the same marker, + never one that came in from upstream. Every read of the table (a replay, the stored + view `get_all_records()`, a loaded job's read) checks the marker against the node (its + operator's prediction, or a loaded job's saved output schema) and refuses a + disagreement (`InputValidationError`, naming the table and both lists). Source info + stays part of each row's record hash; the marker is put on after the hash and never + enters it. +- `DerivedSource` over a referential stream: `node.as_source()` of a function or operator + node presents the bindings its stored table states (checked against the node's keys), + in the plain form and with `tags_as_data`; `DerivedSource(source, tags_as_data=...)` + over a referential source keeps the source's own. A `tags_as_data` copy of a + referential tag is a new plain data column. Nothing new enters the identity for the + binding: the plain form's content hash is the origin node's, and every pipeline hash + hashes the output schema, where a referential tag is in both halves. A saved derived + source writes no new key: its saved schema states the binding. ### Changed @@ -26,4 +42,5 @@ - The SQLite and PostgreSQL databases keep no Arrow field metadata, so a function node that would store a referential tag there is refused (`InputValidationError`, naming the database and the two ways out: a database that keeps field metadata, or - `tag_only_columns`) before anything is written. + `tag_only_columns`) before anything is written. The same holds for an operator node + attached to such a database in `LOG` or `REPLAY` mode. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index fad4b0555..740526058 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -170,6 +170,32 @@ A node reads the binding back from its table, and refuses a table whose marker d with what the node declares (its pod's fall-through tags, or for a loaded job the saved file's). +An operator node that stores its output (`LOG`) marks its table the same way: the +referential tags of the operator's own output carry the marker, written from the +operator's binding, never from a marker that came in from upstream. A `REPLAY` read, and +the stored view `get_all_records()`, check the marker against the node (its operator's +prediction, or for a loaded job the saved output schema, where a referential tag is in +both halves). A database that keeps no field metadata is refused when it is attached in +`LOG` or `REPLAY` mode. Source info stays part of each stored row's record hash; the +marker never enters it. + +`node.as_source()` gives the stored rows as a source with the bindings the stored table +states, in the plain form and with `tags_as_data` (a copy of a referential tag is a new, +plain data column). Nothing is added to the derived source's identity for the binding: its +plain form's content hash is the node's, which carries the binding already, and its +pipeline hash is its output schema, where a referential tag is in both halves. + +```python +derived = job.nodes["scale"].as_source() +print(derived.keys()) +# (('unit', 'time'), ('unit', 'time', 'scaled')) +print(derived.as_table().column_names) +# ['unit', 'time', 'scaled'] +``` + +In a stream the binding is the shared name, so the marker stays with the stored table: +the rows of a replay or of a derived source carry none. + A saved job or pipeline (format `0.3.0`) states every binding: each source config writes `tag_only_columns` (`True` when every tag is plain), and each function node writes `fall_through_references` (empty for a plain pod). The format version, never a missing @@ -180,7 +206,5 @@ the file is checked against the stored tables when a node reads them. ## Not supported yet These raise "not supported with referential tags yet" when a referential tag reaches them: -`Batch`, `MergeJoin`, `SelectColumns`, `DropColumns`, storing an operator node's results (`LOG` and `REPLAY` cache modes), -and `DerivedSource` over a stream with referential tags (so `node.as_source()` of a node -with referential tags). +`Batch`, `MergeJoin`, `SelectColumns` and `DropColumns`. `PollingSource` keeps its tags plain. diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index d4b9244d1..902d43115 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -25,7 +25,14 @@ from typing import TYPE_CHECKING, Any, Literal, cast from orcapod import contexts -from orcapod.core.referential_tags import not_supported_yet, stream_references +from orcapod.core.referential_tags import ( + mark_references, + reference_names, + require_database_keeps_marker, + stored_references, + stream_references, + without_reference_marks, +) from orcapod.channels import Channel, ChannelClosed, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod @@ -48,6 +55,7 @@ from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER from orcapod.core.nodes.staleness import RunHoldingNode from orcapod.errors import ( + InputValidationError, PipelineJobRequiredError, OrcapodDatabaseReadError, OrcapodDatabaseWriteError, @@ -843,7 +851,17 @@ def attach_databases( Args: pipeline_database: Database for pipeline records. cache_mode: Caching behaviour (OFF, LOG, or REPLAY). + + Raises: + InputValidationError: If the node's output has referential tags, the cache + mode stores or replays them (``LOG`` or ``REPLAY``), and the database + cannot keep the marker that states them (ITL-1019). Nothing is + attached or written. """ + if cache_mode != CacheMode.OFF: + # A stored table states its referential tags; refuse a database that + # cannot, before anything runs or is written (ITL-1019). + require_database_keeps_marker(pipeline_database, stream_references(self)) # Another database has its own tables of other generations to account for. self._stale_tables_checked = False self._skipped_rows_reported = False @@ -1009,6 +1027,88 @@ def from_descriptor( return node + # ------------------------------------------------------------------ + # Referential tags in the stored table (ITL-1019) + # ------------------------------------------------------------------ + + def declared_references(self) -> tuple[str, ...]: + """Return the referential tags of this node's output, as the node declares them. + + A node built in this process declares its operator's predicted output: the + names in both halves of ``keys()``. A node loaded by ``PipelineJob.load`` + declares its saved descriptor's output schema, the saved job being an + independent point of verification of the stored table (developer, November 16, + 2026). These are the tags the node's stored table must mark. + + Returns: + The referential tags, in tag order; empty for a plain output. + """ + saved = self._descriptor.get("output_schema") if self._loaded else None + if saved is not None: + return reference_names(saved.get("tag", {}), saved.get("data", {})) + return stream_references(self) + + def _stored_references(self, schema: "pa.Schema") -> tuple[str, ...]: + """Return the referential tags the stored table marks, checked against the node. + + The stored table states its bindings (developer, November 16, 2026, ITL-1019): + a referential tag column carries the marker of ``referential_tags``. The node + reads the table only when it agrees with what the node declares + (``declared_references``). + + Args: + schema: The schema of the node's stored table. + + Returns: + The referential tags, in the order the node declares them. + + Raises: + InputValidationError: If the table's marker is invalid, or the table and + the node disagree on which tags are referential. + """ + assert self._pipeline_database is not None + table_path = self._pipeline_database.base_path + self._stored_table_path + try: + stored = stored_references(schema) + except ValueError as exc: + raise InputValidationError( + f"Operator node {self.label!r}, stored table {table_path}: {exc}" + ) from exc + expected = self.declared_references() + if set(stored) != set(expected): + declarer = "saved job" if self._loaded else "operator" + raise InputValidationError( + f"Operator node {self.label!r}: the stored table {table_path} marks " + f"the referential tags {sorted(stored)}, but the node's {declarer} " + f"declares {sorted(expected)}. The table was not written by this " + "node's pipeline; nothing was read." + ) + return expected + + def _stream_over_stored( + self, records: "pa.Table", references: Sequence[str] + ) -> ArrowTableStream: + """Return a stream over rows read from the stored table, without the marker. + + In a stream the binding is the shared name of a tag and a data column, so the + marker stays with the stored table: carried on, it would be written by a + downstream store whose binding it is not. + + Args: + records: The stored rows, without the bookkeeping columns. + references: The referential tags the table marks (``_stored_references``). + + Returns: + The stream, in this node's data context. + """ + # Stored rows are this node's: give them its data context, not the default. + return ArrowTableStream( + without_reference_marks(records), + tag_columns=self.keys()[0], + reference_tags=references, + data_context=self.data_context, + ) + # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ @@ -1124,11 +1224,12 @@ def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: def _store_output_stream(self, stream: StreamProtocol) -> None: """Materialize stream and store in the pipeline database with per-row dedup. - Raises: - NotImplementedError: If the stream has a referential tag (ITL-1019). + The stored table states its bindings (developer, November 16, 2026, ITL-1019): + the columns of the node's referential tags carry the marker of + ``referential_tags``, written from the operator's own output binding and + never from metadata that came in from upstream. The marker is put on after the + per-row record hashes are computed, so it never enters them. """ - if stream_references(stream): - raise not_supported_yet("Operator node storage", "ITL-1019") pipeline_database = self._pipeline_database if pipeline_database is None: raise RuntimeError( @@ -1201,7 +1302,10 @@ def _store_output_stream(self, stream: StreamProtocol) -> None: try: pipeline_database.add_records( self._stored_table_path, - with_data_context(output_table, self.data_context_key), + with_data_context( + mark_references(output_table, stream_references(self)), + self.data_context_key, + ), record_id_column=self.HASH_COLUMN_NAME, skip_duplicates=True, ) @@ -1241,9 +1345,17 @@ def _read_current_context_records(self) -> "pa.Table | None": The rows of every run that shares the table, before ``_filter_by_content_hash`` keeps this node's. Rows of another data context are left out and reported once. + Every read of the stored table goes through here, so every read checks the + table's referential tag marker against the node (``_stored_references``). The + rows keep the marker; a read into a stream drops it (``_stream_over_stored``). + Returns: The rows, with ``HASH_COLUMN_NAME`` and ``NODE_CONTENT_HASH_COL`` and without the ``__data_context`` column, or ``None`` when the table does not exist. + + Raises: + InputValidationError: If the table and the node disagree on which tags are + referential (ITL-1019). """ assert self._pipeline_database is not None self._account_for_stale_tables() @@ -1252,6 +1364,7 @@ def _read_current_context_records(self) -> "pa.Table | None": ) if records is None: return None + _ = self._stored_references(records.schema) stored_rows = records.num_rows records = keep_rows_of_data_context(records, self.data_context_key) if records.num_rows < stored_rows and not self._skipped_rows_reported: @@ -1443,13 +1556,8 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": ] if cols_to_drop: records_table = records_table.drop(cols_to_drop) - tag_keys = self.keys()[0] - if records is not None and stream_references(self): - raise not_supported_yet("Operator node storage", "ITL-1019") - # Stored rows are this node's: give them its data context, not the default. - return ArrowTableStream( - records_table, tag_columns=tag_keys, data_context=self.data_context - ) + # The table's binding was checked against the node's when it was read. + return self._stream_over_stored(records_table, self.declared_references()) def get_cached_output(self) -> StreamProtocol | None: """Return cached output stream in REPLAY mode, else None. @@ -1589,11 +1697,9 @@ def _replay_from_cache(self) -> None: if cols_to_drop: records = records.drop(cols_to_drop) - tag_keys = self.keys()[0] - if stream_references(self): - raise not_supported_yet("Operator node storage", "ITL-1019") - self._cached_output_stream = ArrowTableStream( - records, tag_columns=tag_keys, data_context=self.data_context + # The table's binding was checked against the node's when it was read. + self._cached_output_stream = self._stream_over_stored( + records, self.declared_references() ) self._update_modified_time() @@ -1788,7 +1894,10 @@ def as_table( ), ) return ArrowTableStream( - empty, tag_columns=tuple(tag_schema), data_context=self.data_context + empty, + tag_columns=tuple(tag_schema), + reference_tags=self.declared_references(), + data_context=self.data_context, ).as_table(columns=columns, all_info=all_info) empty_stream = ArrowTableStream( self._make_empty_table(), @@ -1846,6 +1955,14 @@ def get_all_records( per ``ColumnConfig`` conventions. Returns ``None`` when no database is attached or no matching records exist. Switching to OFF does not remove records stored by an earlier LOG run. + + This is the raw records view: a referential tag column keeps the marker the + stored table states it with (ITL-1019), checked against the node on read. + ``as_source()`` reads the binding from it. + + Raises: + InputValidationError: If the stored table and the node disagree on which + tags are referential. """ if self._pipeline_database is None: return None diff --git a/src/orcapod/core/sources/derived_source.py b/src/orcapod/core/sources/derived_source.py index 318bf96a3..52aa3e300 100644 --- a/src/orcapod/core/sources/derived_source.py +++ b/src/orcapod/core/sources/derived_source.py @@ -5,7 +5,11 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, ClassVar -from orcapod.core.referential_tags import not_supported_yet, stream_references +from orcapod.core.referential_tags import ( + stored_references, + stream_references, + without_reference_marks, +) from typing_extensions import override from orcapod.core.sources.base import RootSource @@ -166,6 +170,16 @@ class DerivedSource(RootSource): rows with the correct columns. In the ``tags_as_data`` form the rows appear once the node has run. + Referential tags (ITL-1019): the source carries the bindings of what it reads, + in both forms. Over a job node they are the ones the node's stored table states + (the marker of ``referential_tags``), which must be the node's own (a name in + both halves of its keys); a table that disagrees is refused. Over a source they + are the source's own. A ``tags_as_data`` copy of a referential tag is a new, + plain data column. Nothing is added to the identity for the binding, because it + is already there: the plain form's ``content_hash`` is the origin's, which carries + it, and every ``pipeline_hash`` and the ``tags_as_data`` form's ``content_hash`` + hash the output schema, in which a referential tag is in both halves. + Args: origin: The source or DB-backed job node to derive from. source_id: Canonical source name. Defaults as described above. @@ -236,8 +250,6 @@ def __init__( record_id_column: str | None = None, **kwargs: Any, ) -> None: - if stream_references(origin): - raise not_supported_yet("DerivedSource", "ITL-1019") if tags_as_data is None: if isinstance(origin, RootSource): raise TypeError( @@ -392,11 +404,46 @@ def keys( return self._origin.keys(columns=columns, all_info=all_info) return self._get_stream().keys(columns=columns, all_info=all_info) + def _references(self) -> tuple[str, ...]: + """Return the referential tags this source presents: its origin's. + + A derived source carries the bindings of what it reads (gap audit Q7). Over a + job node they are the ones its stored table states, which ``_origin_records`` + checks against the node's keys on every read; over a source, the source's own. + """ + return stream_references(self._origin) + def _origin_records(self) -> "pa.Table | None": - """Return the origin's rows, or ``None`` when a job node has none stored.""" + """Return the origin's rows, or ``None`` when a job node has none stored. + + A job node's stored rows state their referential tags with a marker on the + column (ITL-1019). The derived source presents exactly those: the marker must + name the tags the node's keys share between tag and data, and is then + dropped, since in a stream the binding is the shared name. + + Raises: + InputValidationError: If a job node's stored rows mark other referential + tags than the node's keys declare, or carry an invalid marker. + """ if isinstance(self._origin, RootSource): return self._origin.as_table() - return self._origin.get_all_records() + records = self._origin.get_all_records() + if records is None: + return None + try: + stored = stored_references(records.schema) + except ValueError as exc: + raise InputValidationError( + f"DerivedSource over {self._origin.label!r}: {exc}" + ) from exc + declared = self._references() + if set(stored) != set(declared): + raise InputValidationError( + f"DerivedSource over {self._origin.label!r}: the stored records mark " + f"the referential tags {sorted(stored)}, but the node declares " + f"{sorted(declared)}; nothing was read." + ) + return without_reference_marks(records) def _empty_origin_table(self) -> "pa.Table": """Return a zero-row table with the origin's declared columns.""" @@ -445,6 +492,7 @@ def _derive_from( result = builder.build( table, tag_columns=self._origin.keys()[0], + reference_tags=self._references(), source_id=self.source_id, record_id_column=self._record_id_column, ) @@ -469,8 +517,7 @@ def _get_stream(self) -> ArrowTableStream: self._cached_table = self._empty_origin_table() else: self._cached_table = records - tag_keys = self._origin.keys()[0] - return ArrowTableStream(self._cached_table, tag_columns=tag_keys) + return self._stream_over(self._cached_table) def _held_rows(self) -> "pa.Table | None": """The rows this source holds from its origin, or ``None`` before a read.""" @@ -562,9 +609,7 @@ def head( self, origin=origin_kind ) if self._tags_as_data is None and self._cached_table is not None: - stream = ArrowTableStream( - self._cached_table, tag_columns=self._origin.keys()[0] - ) + stream = self._stream_over(self._cached_table) return stream.head(n, columns=config).for_stream(self, origin=origin_kind) if isinstance(self._origin, RootSource): origin_head = self._origin.head(n) @@ -630,7 +675,11 @@ def _stream_over(self, table: "pa.Table") -> ArrowTableStream: """Return this source's stream over the origin rows ``table``.""" if self._tags_as_data is not None: return self._derive_from(table, self._tags_as_data) - return ArrowTableStream(table, tag_columns=self._origin.keys()[0]) + return ArrowTableStream( + table, + tag_columns=self._origin.keys()[0], + reference_tags=self._references(), + ) def iter_data(self): return self._get_stream().iter_data() diff --git a/tests/test_core/referential_tags/test_derived_source.py b/tests/test_core/referential_tags/test_derived_source.py new file mode 100644 index 000000000..25ad00f61 --- /dev/null +++ b/tests/test_core/referential_tags/test_derived_source.py @@ -0,0 +1,270 @@ +"""A derived source carries the bindings of what it reads (ITL-1019, gap audit S1 and Q7). + +``node.as_source()`` over a function or operator node whose stored records hold referential +tags presents the bindings the stored table states (the marker of ``referential_tags``), +checked against the node's keys, in the plain form and the ``tags_as_data`` form. Nothing is +added to the identity for the binding: it is in the origin's content hash and in the output +schema, where a referential tag is in both halves. +""" + +from __future__ import annotations + +import warnings +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.core.operators import Join +from orcapod.core.referential_tags import ( + REFERENTIAL_TAG_METADATA_KEY, + ReferenceMaterializationWarning, + without_reference_marks, +) +from orcapod.core.sources import DerivedSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.types import CacheMode + +JOB = "derived" +REFERENTIAL_KEYS = (("unit", "time"), ("unit", "y")) + + +def _in_memory(_: Path) -> ArrowDatabaseProtocol: + return InMemoryArrowDatabase() + + +def _delta(tmp_path: Path) -> ArrowDatabaseProtocol: + return DeltaTableDatabase(tmp_path / "store") + + +STORES = pytest.mark.parametrize( + "make_store", [_in_memory, _delta], ids=["memory", "delta"] +) +ORCHESTRATORS = pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +Orchestrator = type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator] + + +@op.function_pod(output_keys=["y"]) +def double(amp: float) -> float: + """Double the amplitude; reads no tag, so ``unit`` falls through.""" + return 2 * amp + + +@op.function_pod(output_keys=["w"]) +def scale(unit: int, y: float) -> float: + """Reads the referential tag ``unit`` as data.""" + return unit * y + + +def _source(tag_only: object = ("time",), rows: int = 2) -> op.sources.DictSource: + """``unit`` referential and ``time`` plain unless ``tag_only`` says otherwise.""" + return op.sources.DictSource( + [{"unit": i, "time": 10 * i, "amp": i - 0.5} for i in range(1, rows + 1)], + tag_columns=["unit", "time"], + tag_only_columns=tag_only, # pyright: ignore[reportArgumentType] # the test varies it + source_id="spikes", + ) + + +def _function_job( + store: ArrowDatabaseProtocol, + orchestrator: Orchestrator = SyncPipelineOrchestrator, + tag_only: object = ("time",), + run: bool = True, +) -> PipelineJob: + job = PipelineJob(name=JOB, database=store) + with job: + double.pod(_source(tag_only), label="double") + if run: + job.run(orchestrator=orchestrator()) + return job + + +def _operator_job( + store: ArrowDatabaseProtocol, orchestrator: Orchestrator = SyncPipelineOrchestrator +) -> PipelineJob: + """A ``LOG`` join of the function node's output with itself filtered by ``unit``.""" + job = PipelineJob(name=JOB, database=store) + with job: + doubled = double.pod(_source(), label="double") + _ = Join()(doubled, _source().select_data_columns(["unit"]), label="joined") + job.nodes["joined"].attach_databases( + pipeline_database=store.at(JOB), cache_mode=CacheMode.LOG + ) + job.run(orchestrator=orchestrator()) + return job + + +def _marked(schema: pa.Schema) -> list[str]: + return [ + f.name for f in schema if REFERENTIAL_TAG_METADATA_KEY in (f.metadata or {}) + ] + + +def _by_unit(table: pa.Table) -> list[dict[str, object]]: + return sorted(table.to_pylist(), key=lambda row: str(row["unit"])) + + +# --------------------------------------------------------------------------- +# The plain form presents the bindings the stored table states +# --------------------------------------------------------------------------- + + +@STORES +@ORCHESTRATORS +def test_as_source_of_a_function_node_keeps_its_bindings( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: Orchestrator, +) -> None: + job = _function_job(make_store(tmp_path), orchestrator) + node = job.nodes["double"] + source = node.as_source() + assert source.keys() == node.keys() == REFERENTIAL_KEYS + assert source.output_schema() == node.output_schema() + table = source.as_table(all_info=True) + assert table.column_names.count("unit") == 1 + assert _marked(table.schema) == [] + assert _by_unit(source.as_table()) == _by_unit(node.as_table()) + for tag, data in source.iter_data(): + assert tag["unit"] == data["unit"] + head = source.head() + assert head.table.column_names.count("unit") == 1 + + +@STORES +@ORCHESTRATORS +def test_as_source_of_an_operator_node_keeps_its_bindings( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: Orchestrator, +) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ReferenceMaterializationWarning) + job = _operator_job(make_store(tmp_path), orchestrator) + node = job.nodes["joined"] + source = node.as_source() + assert source.keys() == node.keys() + assert "unit" in source.keys()[0] and "unit" in source.keys()[1] + assert _by_unit(source.as_table()) == _by_unit(node.as_table()) + assert _marked(source.as_table(all_info=True).schema) == [] + + +def test_a_derived_source_before_its_node_ran_has_the_binding_and_no_rows() -> None: + job = _function_job(InMemoryArrowDatabase(), run=False) + source = job.nodes["double"].as_source() + assert source.keys() == REFERENTIAL_KEYS + assert source.as_table().num_rows == 0 + assert source.as_table().column_names.count("unit") == 1 + + +def test_a_derived_source_refuses_records_that_disagree_with_the_node() -> None: + written = InMemoryArrowDatabase() + _function_job(written) + tampered = InMemoryArrowDatabase() + for path in written.list_table_paths(): + table = written.get_all_records(path, record_id_column="__copy_id") + assert table is not None + tampered.add_records( + path, + without_reference_marks(table), + record_id_column="__copy_id", + flush=True, + ) + source = _function_job(tampered, run=False).nodes["double"].as_source() + with pytest.raises(InputValidationError, match=r"marks the referential tags \[\]"): + _ = source.as_table() + + +# --------------------------------------------------------------------------- +# The tags_as_data form +# --------------------------------------------------------------------------- + + +@ORCHESTRATORS +def test_tags_as_data_over_a_referential_node(orchestrator: Orchestrator) -> None: + job = _function_job(InMemoryArrowDatabase(), orchestrator) + node = job.nodes["double"] + source = node.as_source(tags_as_data={"time": "start", "unit": "unit_copy"}) + # The referential tag keeps its binding; each copy is a new plain data column. + assert source.keys() == (("unit", "time"), ("unit", "y", "start", "unit_copy")) + table = source.as_table(all_info=True) + assert table.column_names.count("unit") == 1 + assert _marked(table.schema) == [] + # Fresh provenance: every data column, the referential one included, gets a token. + assert {"_source_unit", "_source_y", "_source_start", "_source_unit_copy"} <= set( + table.column_names + ) + for row in source.as_table().to_pylist(): + assert row["unit_copy"] == row["unit"] and row["start"] == 10 * row["unit"] + + +def test_tags_as_data_over_a_referential_source() -> None: + source = DerivedSource(_source(), tags_as_data={"time": "start"}) + assert source.keys() == (("unit", "time"), ("unit", "amp", "start")) + assert source.as_table().column_names.count("unit") == 1 + + +# --------------------------------------------------------------------------- +# Identity: the binding enters through what is already hashed +# --------------------------------------------------------------------------- + + +def test_the_binding_enters_the_identity_through_the_origin_and_the_schema() -> None: + referential = _function_job(InMemoryArrowDatabase()).nodes["double"].as_source() + plain = ( + _function_job(InMemoryArrowDatabase(), tag_only=True) + .nodes["double"] + .as_source() + ) + assert plain.keys() == (("unit", "time"), ("y",)) + # Plain form: the content hash is the origin node's, which carries the binding. + assert referential.identity_structure() == ( + referential._origin.content_hash(), # pyright: ignore[reportPrivateUsage] # the identity rule under test + ) + assert referential.content_hash() != plain.content_hash() + # Pipeline hash: schema only; a referential tag is in both halves of it. + assert referential.pipeline_identity_structure() == referential.output_schema() + assert referential.pipeline_hash() != plain.pipeline_hash() + # The default source id keeps its construction: derived: