From 84d1312c277b5af2dea6a534913cd72649c6e80f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 01:06:33 -0800 Subject: [PATCH 1/8] feat(referential_tags): stored tables and saved jobs keep tag bindings (ITL-1019) A function node's pipeline table marks each referential tag column with Arrow field metadata (orcapod:tag_kind = referential, one constant in referential_tags); a plain tag carries no marker and the result table never records referential status. The node reads the binding back from its table and refuses a table that disagrees with its pod or, for a loaded job, the saved file. The marker moves no hash, path or layout. Jobs and pipelines holding referential tags and fall-through pods save and load with unchanged hashes. Format 0.3.0 writes tag_only_columns for every source that takes it and fall_through_references for every function node; the version decides meaning (0.1.0/0.2.0 are all plain), and a 0.3.0 file missing or contradicting these keys is refused. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1019.md | 27 ++ design_issues/ITL-1023-RT1.md | 2 +- docs/concepts/referential-tags.md | 45 ++- src/orcapod/core/function_pod.py | 43 ++- src/orcapod/core/nodes/function_node.py | 98 ++++- src/orcapod/core/referential_tags.py | 112 +++++- .../core/sources/arrow_table_source.py | 2 + src/orcapod/core/sources/base.py | 39 +- src/orcapod/core/sources/csv_source.py | 8 +- src/orcapod/core/sources/data_frame_source.py | 2 + src/orcapod/core/sources/db_table_source.py | 4 +- .../core/sources/delta_table_source.py | 8 +- src/orcapod/core/sources/dict_source.py | 2 + .../core/sources/postgresql_table_source.py | 3 +- .../core/sources/spiraldb_table_source.py | 8 +- .../core/sources/sqlite_table_source.py | 3 +- src/orcapod/pipeline/graph.py | 28 +- src/orcapod/pipeline/job.py | 7 + src/orcapod/pipeline/serialization.py | 161 +++++++- .../test_model_and_sources.py | 14 +- .../referential_tags/test_persistence.py | 354 ++++++++++++++++++ .../test_loaded_side_effect_nodes.py | 5 + .../test_loaded_slot_unregistered_class.py | 3 + .../test_saved_job_before_referential_tags.py | 6 +- .../test_saved_job_with_referential_tags.py | 265 +++++++++++++ 25 files changed, 1197 insertions(+), 52 deletions(-) create mode 100644 changelog.d/ITL-1019.md create mode 100644 tests/test_core/referential_tags/test_persistence.py create mode 100644 tests/test_pipeline/test_saved_job_with_referential_tags.py diff --git a/changelog.d/ITL-1019.md b/changelog.d/ITL-1019.md new file mode 100644 index 000000000..40ccb20a1 --- /dev/null +++ b/changelog.d/ITL-1019.md @@ -0,0 +1,27 @@ +### Added + +- A stored table states which of its tags are referential (ITL-1019): a function node's + pipeline table marks each referential tag column with Arrow field metadata + (`orcapod:tag_kind` = `referential`, `referential_tags.REFERENTIAL_TAG_METADATA_KEY`); a + plain tag carries no marker. The result table never records referential status. The + marker moves no hash and no stored path, and needs no new table layout. +- A function node reads its bindings back from its pipeline table and refuses + (`InputValidationError`, naming the table and both lists) a table whose marker disagrees + with what the node declares: its pod's fall-through tags, or a loaded job's saved record. +- A job or pipeline holding referential tags and fall-through pods can be saved and + loaded; it keeps its hashes and bindings. `ReferenceFallThroughPod.to_config` and + `from_config` work. + +### Changed + +- Saved files of format `0.3.0` state every binding: each source config that takes the + option writes `tag_only_columns` (`True` when every tag is plain), and each function + node writes `fall_through_references` (empty for a plain pod). The format version decides + what a file means, never a missing key: a file at `0.1.0` or `0.2.0` holds only plain + tags, and a `0.3.0` file without these keys, or whose keys disagree with its saved + schemas, is refused on load. + +### Known limitations + +- The SQLite and PostgreSQL databases keep no Arrow field metadata and refuse a pipeline + table with a referential tag, as they refuse logical types. diff --git a/design_issues/ITL-1023-RT1.md b/design_issues/ITL-1023-RT1.md index b791c065d..b2bd1f411 100644 --- a/design_issues/ITL-1023-RT1.md +++ b/design_issues/ITL-1023-RT1.md @@ -5,4 +5,4 @@ **Severity:** medium **Issue:** ITL-1023 -The workshop slice of referential tags (specification attached to ITL-1004) covers sources, function pods, `Join`, `GroupBy`, select and drop, renames, filters, `DropNulls`, `SemiJoin`, `Pick` and `Index`. Everything else raises "not supported with referential tags yet" when a referential tag reaches it, instead of giving a wrong answer: `Batch` (ITL-1017), `MergeJoin` (ITL-1018), storing an operator node's results in `LOG` or `REPLAY` mode, `DerivedSource` over a referential stream and saving a pipeline or job (ITL-1019), and any operator that does not declare `REFERENTIAL_TAGS_ISSUE = None` (ITL-1022). `PollingSource` keeps its tags plain. The identities of referential streams are not frozen yet. The project's issues (ITL-1009 onward) land the full design and close this entry. +The workshop slice of referential tags (specification attached to ITL-1004) covers sources, function pods, `Join`, `GroupBy`, select and drop, renames, filters, `DropNulls`, `SemiJoin`, `Pick` and `Index`. Everything else raises "not supported with referential tags yet" when a referential tag reaches it, instead of giving a wrong answer: `Batch` (ITL-1017), `MergeJoin` (ITL-1018), storing an operator node's results in `LOG` or `REPLAY` mode, and `DerivedSource` over a referential stream (ITL-1019; saving a pipeline or job and the function node's stored bindings are supported since ITL-1019), and any operator that does not declare `REFERENTIAL_TAGS_ISSUE = None` (ITL-1022). `PollingSource` keeps its tags plain. The identities of referential streams are not frozen yet. The project's issues (ITL-1009 onward) land the full design and close this entry. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 6fb99d58e..2208c914d 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -133,9 +133,52 @@ referential_tags.reference_materialization = "silent" referential_tags.reference_materialization = "warn" # the default ``` +## Stored tables and saved jobs + +A stored table states which of its tags are referential. A function node stores each +row's tags in its pipeline table, and a referential tag is stored there once, as a tag +column that carries Arrow field metadata: the key `orcapod:tag_kind` with the value +`referential` (`referential_tags.REFERENTIAL_TAG_METADATA_KEY`). A plain tag carries no +marker. The function's result table never records referential status, so plain and +referential runs of the same function share their results. The marker moves no hash, and +the stores keep it on every write (first write, append, merge, schema evolution; in +memory and on Delta). The SQLite and PostgreSQL databases keep no Arrow +metadata at all and refuse such a table, as they refuse logical types. + +```python +from orcapod.core.referential_tags import stored_references +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import PipelineJob + +database = InMemoryArrowDatabase() +job = PipelineJob(database=database) +with job: + scale.pod(spikes, label="scale") +job.run() +pipeline_table = next( + database.get_all_records(path) + for path in database.list_table_paths() + if "_result" not in path +) +print(stored_references(pipeline_table.schema)) +# ('unit', 'time') +``` + +A node reads the binding back from its table, and refuses a table whose marker disagrees +with what the node declares (its pod's fall-through tags, or for a loaded job the saved +file's). + +A saved job or pipeline (format `0.3.0`) states every binding: each source config writes +`tag_only_columns` (`True` when every tag is plain), and each function node writes +`fall_through_references` (empty for a plain pod). The format version, never a missing +key, decides what a file means: a file at `0.1.0` or `0.2.0` holds only plain tags, and a +`0.3.0` file without these keys is refused. A loaded job keeps its hashes and bindings, and +the file is checked against the stored tables when a node reads them. + ## Not supported yet These raise "not supported with referential tags yet" when a referential tag reaches them: `Batch`, `MergeJoin`, `SelectColumns`, `DropColumns`, storing an operator node's results (`LOG` and `REPLAY` cache modes), -`DerivedSource` over a stream with referential tags, and saving a pipeline or job. +and `DerivedSource` over a stream with referential tags (so `node.as_source()` of a node +with referential tags). `PollingSource` keeps its tags plain. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index a2e8e3498..adf888e8e 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -17,7 +17,6 @@ from orcapod.core.datagrams import Data from orcapod.core.referential_tags import ( materialization_diagnostic_off, - not_supported_yet, stream_references, ) from orcapod.core.streams.base import ( @@ -1925,21 +1924,47 @@ def plain_pod(self) -> FunctionPod: return self._plain_pod def to_config(self) -> dict[str, Any]: - """Raise: a pipeline holding referential tags cannot be saved yet (ITL-1019). + """Serialize this pod: the plain pod's config and the references (ITL-1019). - Raises: - NotImplementedError: Always. + Returns: + The plain pod's config with ``fall_through_references``, the referential + tags that fall through. """ - raise not_supported_yet("Saving a fall-through function pod", "ITL-1019") + from orcapod.pipeline.serialization import FALL_THROUGH_REFERENCES_KEY + + return { + **self._plain_pod.to_config(), + FALL_THROUGH_REFERENCES_KEY: list(self._references), + } @classmethod - def from_config(cls, config: dict[str, Any]) -> "ReferenceFallThroughPod": - """Raise: a fall-through pod is never saved, so never loaded (ITL-1019). + def from_config( + cls, config: dict[str, Any], *, fallback_to_proxy: bool = False + ) -> "ReferenceFallThroughPod": + """Rebuild the pod from ``to_config``'s output. + + Args: + config: A config written by ``to_config``. + fallback_to_proxy: Passed to ``FunctionPod.from_config``. + + Returns: + The fall-through pod over the rebuilt plain pod. Raises: - NotImplementedError: Always. + ValueError: If the config names no ``fall_through_references``. """ - raise not_supported_yet("Loading a fall-through function pod", "ITL-1019") + from orcapod.pipeline.serialization import FALL_THROUGH_REFERENCES_KEY + + if FALL_THROUGH_REFERENCES_KEY not in config: + raise ValueError( + f"a fall-through pod config names {FALL_THROUGH_REFERENCES_KEY}; " + "this one does not" + ) + plain = {k: v for k, v in config.items() if k != FALL_THROUGH_REFERENCES_KEY} + return cls( + FunctionPod.from_config(plain, fallback_to_proxy=fallback_to_proxy), + config[FALL_THROUGH_REFERENCES_KEY], + ) def identity_structure(self) -> Any: return (self._plain_pod, ("fall_through_references", self._references)) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index b644f58d3..880f47f11 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -36,6 +36,11 @@ result_pod, with_fall_through, ) +from orcapod.core.referential_tags import ( + mark_references, + stored_references, + without_reference_marks, +) from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig from orcapod.core.cached_function_pod import CachedFunctionPod @@ -431,6 +436,31 @@ def __init__( self._table_scope = table_scope self._node_identity_path_cache: tuple[str, ...] | None = None + def declared_references(self) -> tuple[str, ...]: + """Return the referential tags this node lets fall through, as it declares them. + + A node with a pod declares the pod's (``ReferenceFallThroughPod``); a node + loaded from a saved file declares the file's record (``fall_through_references`` + of its descriptor, ITL-1019). They are what the node's stored pipeline table + must mark. + + Returns: + The referential tags; empty for a plain pod. + + Raises: + ValueError: If the node has neither a pod nor a saved record. + """ + if self._function_pod is not None: + return fall_through_references(self._function_pod) + from orcapod.pipeline.serialization import FALL_THROUGH_REFERENCES_KEY + + if FALL_THROUGH_REFERENCES_KEY not in self._descriptor: + raise ValueError( + f"Function node {self.label!r} has no pod and no saved " + f"{FALL_THROUGH_REFERENCES_KEY}: its referential tags are unknown" + ) + return tuple(self._descriptor[FALL_THROUGH_REFERENCES_KEY]) + # ------------------------------------------------------------------ # load_status # ------------------------------------------------------------------ @@ -1142,12 +1172,15 @@ class _JoinedRecords(NamedTuple): empty_taginfo_rows: Mapping from ``base_entry_id`` bytes to the raw taginfo row dict for each entry in ``empty_data_tokens``. Used by ``_load_cached_entries`` to reconstruct the tag for each token. + references: The referential tags the pipeline table marks (ITL-1019), + checked against the node's pod when it has one. """ table: pa.Table taginfo_columns: tuple[str, ...] empty_data_tokens: dict[bytes, EmptyData] empty_taginfo_rows: dict[bytes, dict] + references: tuple[str, ...] class FunctionJobNode(RunHoldingNode, FunctionNodeBase): @@ -2911,11 +2944,16 @@ def _add_pipeline_record( } ) - # Combine: tag (with system tags) + input source columns + meta columns - combined_record = arrow_utils.hstack_tables( - tag.as_table(columns={"system_tags": True}), - input_source_table, - meta_table, + # Combine: tag (with system tags) + input source columns + meta columns. The + # table states which tags are referential: the ones that fall through this + # node's pod, marked on their stored column (ITL-1019). + combined_record = mark_references( + arrow_utils.hstack_tables( + tag.as_table(columns={"system_tags": True}), + input_source_table, + meta_table, + ), + fall_through_references(self._function_pod), ) try: @@ -3117,6 +3155,7 @@ def _fetch_joined_records( taginfo_columns = tuple(taginfo.column_names) taginfo_schema = taginfo.schema + references = self._stored_references(taginfo_schema) is_ephemeral_col = constants.IS_EPHEMERAL_COL taginfo_df = pl.DataFrame(taginfo) @@ -3275,6 +3314,7 @@ def _fetch_joined_records( taginfo_columns=taginfo_columns, empty_data_tokens=empty_data_tokens, empty_taginfo_rows=empty_taginfo_rows, + references=references, ) # Apply base_entry_id filter if requested @@ -3298,8 +3338,48 @@ def _fetch_joined_records( taginfo_columns=taginfo_columns, empty_data_tokens=empty_data_tokens, empty_taginfo_rows=empty_taginfo_rows, + references=references, ) + def _stored_references(self, taginfo_schema: "pa.Schema") -> tuple[str, ...]: + """Return the referential tags the pipeline table marks, checked against the pod. + + The stored table states its bindings (developer, November 16, 2026, ITL-1019): + a referential tag column carries the marker of ``referential_tags``. The node + reads the table only when it agrees with what the node declares + (``declared_references``: its pod's, or a loaded node's saved record, an + independent point of verification). + + Args: + taginfo_schema: The schema of the stored pipeline table. + + Returns: + The referential tags, in the order the node declares them. + + Raises: + InputValidationError: If the table's marker is invalid, or the table + and the node disagree on which tags are referential. + """ + table_path = self._require_pipeline_database().base_path + ( + self._versioned_pipeline_path + ) + try: + stored = stored_references(taginfo_schema) + except ValueError as exc: + raise InputValidationError( + f"Function node {self.label!r}, pipeline table {table_path}: {exc}" + ) from exc + expected = self.declared_references() + if set(stored) != set(expected): + source = "pod" if self._function_pod is not None else "saved job" + raise InputValidationError( + f"Function node {self.label!r}: the pipeline table {table_path} marks " + f"the referential tags {sorted(stored)}, but the node's {source} " + f"declares {sorted(expected)}. The table was not written by this " + "node's pipeline; nothing was read." + ) + return expected + def _stored_tag_keys(self, fetched: _JoinedRecords) -> tuple[str, ...]: """Return the user tag columns of the records in *fetched*. @@ -3376,11 +3456,13 @@ def _stored_rows_stream( uuid.UUID(bytes=bytes(cast(bytes, record_id))) for record_id in joined.column(constants.DATA_RECORD_ID).to_pylist() ] - # A fall-through column is the stored tag column in its data role (ITL-1023). + # A fall-through column is the stored tag column in its data role (ITL-1023): + # the table states which (ITL-1019). In the stream the binding is the shared + # name, so the marker stays with the stored table. stream = ArrowTableStream( - joined.drop(drop_cols), + without_reference_marks(joined.drop(drop_cols)), tag_columns=self._stored_tag_keys(fetched), - reference_tags=fall_through_references(self._function_pod), + reference_tags=fetched.references, ) return base_entry_ids, stream, data_ids diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index 61142366f..1dea111b9 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -25,12 +25,19 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from contextlib import contextmanager from contextvars import ContextVar -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from orcapod.errors import InputValidationError from orcapod.types import SourceInfoValue from orcapod.utils import arrow_utils +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import pyarrow as pa +else: + pa = LazyModule("pyarrow") + # Diagnostic emitted when a structural operator materializes a reference (section 7.4). # ``"warn"`` emits a ``ReferenceMaterializationWarning``; ``"silent"`` emits nothing. The # setting never changes a result, a schema or a hash. A module-level default for now. @@ -164,6 +171,109 @@ def warn_materialized(operation: object, names: Iterable[str]) -> None: ) +# The marker of a referential tag in a stored table (developer, November 16, 2026, +# ITL-1019): Arrow field metadata on the stored tag column, at the level of a logical +# type's extension metadata. Only a referential tag carries it; a plain tag carries no +# marker. It is not ``ARROW:extension:*`` metadata, so no schema or table hash reads it. +REFERENTIAL_TAG_METADATA_KEY = b"orcapod:tag_kind" +REFERENTIAL_TAG_METADATA_VALUE = b"referential" + + +def mark_references(table: "pa.Table", references: Iterable[str]) -> "pa.Table": + """Return ``table`` with exactly the columns ``references`` marked referential. + + A stored table states its own bindings: the writer marks the columns it knows to be + referential and removes a marker any other column carries (one that came in from + upstream metadata is not this writer's binding). Values and every other field + property (name, type, nullability, other metadata) are unchanged. + + Args: + table: The rows about to be stored. + references: The referential tags of the rows; each must be a column of + ``table``. + + Returns: + The table with the marker on exactly the ``references`` columns. + + Raises: + ValueError: If a reference is not a column of ``table``. + """ + refs = set(references) + if missing := refs - set(table.column_names): + raise ValueError( + f"referential tags {sorted(missing)} are not columns of the stored table" + ) + schema = table.schema + changed = False + for index, field in enumerate(schema): + metadata: dict[bytes | str, bytes | str] = { + key: value for key, value in (field.metadata or {}).items() + } + marked = metadata.get(REFERENTIAL_TAG_METADATA_KEY) is not None + if field.name in refs: + if metadata.get(REFERENTIAL_TAG_METADATA_KEY) == ( + REFERENTIAL_TAG_METADATA_VALUE + ): + continue + metadata[REFERENTIAL_TAG_METADATA_KEY] = REFERENTIAL_TAG_METADATA_VALUE + elif marked: + del metadata[REFERENTIAL_TAG_METADATA_KEY] + else: + continue + schema = schema.set( + index, + field.with_metadata(metadata) if metadata else field.remove_metadata(), + ) + changed = True + if not changed: + return table + # The same arrays under the new field metadata: no cast, no copy. + return pa.Table.from_arrays(table.columns, schema=schema) + + +def stored_references(schema: "pa.Schema") -> tuple[str, ...]: + """Return the columns a stored table marks as referential tags, in column order. + + Args: + schema: The Arrow schema of a stored table. + + Returns: + The names of the columns that carry the referential tag marker. + + Raises: + ValueError: If a column carries the marker key with another value. + """ + names: list[str] = [] + for field in schema: + value = (field.metadata or {}).get(REFERENTIAL_TAG_METADATA_KEY) + if value is None: + continue + if value != REFERENTIAL_TAG_METADATA_VALUE: + raise ValueError( + f"stored column {field.name!r} has {REFERENTIAL_TAG_METADATA_KEY!r} " + f"metadata {value!r}; the only valid value is " + f"{REFERENTIAL_TAG_METADATA_VALUE!r}" + ) + names.append(field.name) + return tuple(names) + + +def without_reference_marks(table: "pa.Table") -> "pa.Table": + """Return ``table`` with no referential tag marker on any column. + + A stored table read into a stream drops the marker: in a stream the binding is the + name intersection of the tag and data keys, and a marker carried further would be + written by a downstream store whose binding it is not. + + Args: + table: Rows read from a stored table. + + Returns: + The same rows; no field carries the marker. + """ + return mark_references(table, ()) + + def not_supported_yet(operation: str, issue: str) -> NotImplementedError: """Return the error for an operation that does not handle referential tags yet. diff --git a/src/orcapod/core/sources/arrow_table_source.py b/src/orcapod/core/sources/arrow_table_source.py index 890fbfd9f..801d57d2c 100644 --- a/src/orcapod/core/sources/arrow_table_source.py +++ b/src/orcapod/core/sources/arrow_table_source.py @@ -70,6 +70,8 @@ class ArrowTableSource(RootSource): and set ``record_id_column`` to a column that is unique per row. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # The config type key; also the kind prefix of the default source id. SOURCE_KIND: ClassVar[str] = "arrow_table" diff --git a/src/orcapod/core/sources/base.py b/src/orcapod/core/sources/base.py index e4fd05ed9..6ffa783eb 100644 --- a/src/orcapod/core/sources/base.py +++ b/src/orcapod/core/sources/base.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from orcapod import contexts from orcapod.config import OrcapodConfig @@ -16,6 +16,29 @@ from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +def saved_tag_only_columns(config: dict[str, Any]) -> bool | list[str]: + """Return the ``tag_only_columns`` of a saved source config (ITL-1019). + + A config of format 0.3.0 always holds it; the loader of an older file, whose tags + are all plain, puts ``True`` in first (``serialization.source_config_at_version``). + + Args: + config: A saved source config. + + Returns: + ``True`` (every tag plain) or the names of the plain tags. + + Raises: + ValueError: If the config does not hold ``tag_only_columns``. + """ + if "tag_only_columns" not in config: + raise ValueError( + f"source config {config.get('source_type')!r} has no tag_only_columns: " + "a saved source config states its plain tags (format 0.3.0)" + ) + return config["tag_only_columns"] + + class RootSource(StreamBase): """Abstract base class for all root sources in Orcapod. @@ -49,6 +72,10 @@ class RootSource(StreamBase): ``identity_structure()``. """ + # Whether the source takes ``tag_only_columns`` and saves it in its config + # (ITL-1019): from format 0.3.0 on such a config always holds it. + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = False + def __init__( self, source_id: str | None = None, @@ -111,16 +138,16 @@ def _identity_config(self) -> dict[str, Any]: } def _tag_only_config(self) -> dict[str, Any]: - """Return ``tag_only_columns`` for ``to_config``, when any tag is referential. + """Return ``tag_only_columns`` for ``to_config``: always written (ITL-1019). - A source whose tags are all plain writes nothing, so its config is the one - v0.3 wrote; a config without the key reads back with every tag plain - (``tag_only_columns=True``), as v0.3 configs mean. + The format version decides what a saved file means, never a missing key: a + config of format 0.3.0 states its plain tags, ``True`` when every tag is plain + and the names otherwise (``saved_tag_only_columns`` reads it back). """ tag_columns = tuple(getattr(self, "_tag_columns", ())) tag_only = tuple(getattr(self, "_tag_only_columns", tag_columns)) if tag_only == tag_columns: - return {} + return {"tag_only_columns": True} return {"tag_only_columns": list(tag_only)} def pipeline_identity_structure(self) -> Any: diff --git a/src/orcapod/core/sources/csv_source.py b/src/orcapod/core/sources/csv_source.py index f62761979..ee043a75f 100644 --- a/src/orcapod/core/sources/csv_source.py +++ b/src/orcapod/core/sources/csv_source.py @@ -2,11 +2,11 @@ import os from collections.abc import Collection -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override -from orcapod.core.sources.base import RootSource +from orcapod.core.sources.base import RootSource, saved_tag_only_columns from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.reload import keep_nullable_columns, same_rows from orcapod.core.sources.stream_builder import ( @@ -44,6 +44,8 @@ class CSVSource(RootSource): value, or the row's position. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + def __init__( self, file_path: str, @@ -171,7 +173,7 @@ def from_config(cls, config: dict[str, Any], db_registry=None) -> CSVSource: return cls( file_path=config["file_path"], tag_columns=config.get("tag_columns", ()), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/data_frame_source.py b/src/orcapod/core/sources/data_frame_source.py index c5b9af61e..3488e957b 100644 --- a/src/orcapod/core/sources/data_frame_source.py +++ b/src/orcapod/core/sources/data_frame_source.py @@ -29,6 +29,8 @@ class DataFrameSource(RootSource): ``SourceStreamBuilder`` (source-info, schema-hash, system tags). """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # The config type key; also the kind prefix of the default source id. SOURCE_KIND: ClassVar[str] = "data_frame" diff --git a/src/orcapod/core/sources/db_table_source.py b/src/orcapod/core/sources/db_table_source.py index 7572c13c4..18589f562 100644 --- a/src/orcapod/core/sources/db_table_source.py +++ b/src/orcapod/core/sources/db_table_source.py @@ -14,7 +14,7 @@ from collections.abc import Collection, Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override @@ -73,6 +73,8 @@ class DBTableSource(RootSource): provided, or is empty. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + def __init__( self, connector: DBConnectorProtocol, diff --git a/src/orcapod/core/sources/delta_table_source.py b/src/orcapod/core/sources/delta_table_source.py index 69143e4c1..159dcdacf 100644 --- a/src/orcapod/core/sources/delta_table_source.py +++ b/src/orcapod/core/sources/delta_table_source.py @@ -2,11 +2,11 @@ from collections.abc import Collection from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override -from orcapod.core.sources.base import RootSource +from orcapod.core.sources.base import RootSource, saved_tag_only_columns from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.reload import same_rows from orcapod.core.sources.stream_builder import ( @@ -41,6 +41,8 @@ class DeltaTableSource(RootSource): Record ids keep their rule: the ``record_id_column`` value, or the row's position. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + def __init__( self, delta_table_path: PathLike, @@ -168,7 +170,7 @@ def from_config( return cls( delta_table_path=config["delta_table_path"], tag_columns=config.get("tag_columns", ()), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/dict_source.py b/src/orcapod/core/sources/dict_source.py index 6c9b29a95..26dbeb971 100644 --- a/src/orcapod/core/sources/dict_source.py +++ b/src/orcapod/core/sources/dict_source.py @@ -26,6 +26,8 @@ class DictSource(RootSource): enriched by ``SourceStreamBuilder`` (source-info, schema-hash, system tags). """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # The config type key; also the kind prefix of the default source id. SOURCE_KIND: ClassVar[str] = "dict" diff --git a/src/orcapod/core/sources/postgresql_table_source.py b/src/orcapod/core/sources/postgresql_table_source.py index 2343f4262..a86fb13c3 100644 --- a/src/orcapod/core/sources/postgresql_table_source.py +++ b/src/orcapod/core/sources/postgresql_table_source.py @@ -20,6 +20,7 @@ from typing_extensions import override from orcapod.core.sources.db_table_source import DBTableSource +from orcapod.core.sources.base import saved_tag_only_columns from orcapod.databases.postgresql_connector import PostgreSQLConnector if TYPE_CHECKING: @@ -142,7 +143,7 @@ def from_config( dsn=config["dsn"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 686cd75e6..aa9cad100 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -38,11 +38,11 @@ import datetime from collections.abc import AsyncIterator, Collection, Generator, Iterator, Mapping from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple, ClassVar from typing_extensions import override -from orcapod.core.sources.base import RootSource +from orcapod.core.sources.base import RootSource, saved_tag_only_columns from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.derived_source import validate_tags_as_data from orcapod.core.sources.stream_builder import SourceStreamBuilder @@ -238,6 +238,8 @@ class SpiralDBTableSource(RootSource): given twice, or is reserved for system columns. """ + TAKES_TAG_ONLY_COLUMNS: ClassVar[bool] = True + # Whether a notebook display reads the first rows. False shows the schema and the # table's recorded row count only, with no scan (kept for the ITL-991 ruling on what # a preview may read). @@ -862,7 +864,7 @@ def from_config( table_name=config["table_name"], dataset=config.get("dataset", "default"), tag_columns=config.get("tag_columns"), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), where=_locationfrom_config(config.get("where") or {}), limit=config.get("limit"), diff --git a/src/orcapod/core/sources/sqlite_table_source.py b/src/orcapod/core/sources/sqlite_table_source.py index 934daa9a7..41a387125 100644 --- a/src/orcapod/core/sources/sqlite_table_source.py +++ b/src/orcapod/core/sources/sqlite_table_source.py @@ -30,6 +30,7 @@ from typing_extensions import override from orcapod.core.sources.db_table_source import DBTableSource +from orcapod.core.sources.base import saved_tag_only_columns from orcapod.databases.sqlite_connector import SQLiteConnector from orcapod.utils.schema_utils import _normalize_column_list @@ -186,7 +187,7 @@ def from_config( db_path=config["db_path"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), - tag_only_columns=config.get("tag_only_columns", True), + tag_only_columns=saved_tag_only_columns(config), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 70b068f9f..418d60a26 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -174,6 +174,7 @@ def save(self, path: str | Path) -> None: import json as _json from orcapod.pipeline.serialization import ( + FALL_THROUGH_REFERENCES_KEY, PIPELINE_FORMAT_VERSION, serialize_nullability, serialize_output_schema, @@ -182,15 +183,8 @@ def save(self, path: str | Path) -> None: from orcapod.core.nodes import OperatorNode, FunctionNode from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass - from orcapod.core.referential_tags import ( - not_supported_yet, - stream_references, - ) - nodes: dict[str, Any] = {} for content_hash_str, node in self._persistent_node_map.items(): - if stream_references(node): - raise not_supported_yet("Saving a pipeline", "ITL-1019") tag_schema, data_schema = node.output_schema() try: type_converter = node.data_context.type_converter @@ -264,6 +258,10 @@ def save(self, path: str | Path) -> None: if node._function_pod is not None: descriptor["function_config"] = node._function_pod.to_config() descriptor["table_scope"] = node._table_scope + # Written for every function node, also a plain one (ITL-1019). + descriptor[FALL_THROUGH_REFERENCES_KEY] = list( + node.declared_references() + ) case OperatorNode(): if node._operator is not None: @@ -309,8 +307,11 @@ def load(cls, path: str | Path) -> "Pipeline": """ import json as _json from orcapod.pipeline.serialization import ( + FALL_THROUGH_REFERENCES_KEY, SUPPORTED_FORMAT_VERSIONS, + check_saved_output_schema_is_plain, deserialize_schema, + saved_fall_through_references, ) from orcapod.core.nodes import FunctionNode, OperatorNode from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass @@ -372,6 +373,19 @@ def load(cls, path: str | Path) -> "Pipeline": deserialize_output_schema, ) + # The format version decides which tags may be referential (ITL-1019). + try: + check_saved_output_schema_is_plain(descriptor, version) + if descriptor.get("node_type") == "function": + # Stated explicitly from here on, whatever the version. + descriptor[FALL_THROUGH_REFERENCES_KEY] = list( + saved_fall_through_references(descriptor, version) + ) + except ValueError as exc: + raise ValueError( + f"{path}: saved node {descriptor.get('label')!r}: {exc}" + ) from exc + # Validate even a leaf node's saved identifiers before exposing the job. context = resolve_saved_context(descriptor.get("data_context_key")) try: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 605dbf415..7f830dc1f 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1586,6 +1586,7 @@ def load( from orcapod.pipeline.serialization import ( SUPPORTED_JOB_FORMAT_VERSIONS, resolve_source_from_config, + source_config_at_version, ) path = _Path(path) @@ -1619,6 +1620,12 @@ def load( data.get("bindings", {}).get("sources", {}).items() ): source_type = src_config.get("source_type") + # The file's version decides which tags are plain (ITL-1019): a refusal + # is the file's error, not a source that could not be rebuilt. + try: + src_config = source_config_at_version(src_config, version) + except ValueError as exc: + raise ValueError(f"{path}: source {spec_name!r}: {exc}") from exc try: source = resolve_source_from_config(src_config) if source is not None: diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index aae816224..64291d96c 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -34,8 +34,14 @@ # 0.3.0: a source's tags may be referential (ITL-1023). A file at 0.1.0 or 0.2.0 # holds v0.3 sources: their configs carry no ``tag_only_columns`` and read # back with every declared tag plain, so their saved hashes match. +# Plain tags are decided by this version, never by a missing key (ITL-1159, ITL-1019): +# everything in a file at a ``PLAIN_TAG_FORMAT_VERSIONS`` version is plain, and such a +# file never holds ``tag_only_columns`` or ``fall_through_references``. From 0.3.0 on, +# every source config that takes the option writes ``tag_only_columns`` and every +# function node writes ``fall_through_references``; a load refuses a file without them. PIPELINE_FORMAT_VERSION = "0.3.0" SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0", "0.3.0"}) +PLAIN_TAG_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0"}) # --------------------------------------------------------------------------- # LoadStatus @@ -1292,6 +1298,159 @@ def _get_arrow_primitive_types() -> dict[str, Any]: # --------------------------------------------------------------------------- # Same scheme as ``PIPELINE_FORMAT_VERSION``: the written version, and the set a -# load accepts (``PipelineJob.load``). +# load accepts (``PipelineJob.load``). The plain-tag rule stated there holds for a +# job file too (``PLAIN_TAG_FORMAT_VERSIONS``, ``source_config_at_version``). PIPELINE_JOB_FORMAT_VERSION = "0.3.0" SUPPORTED_JOB_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0", "0.3.0"}) + + +# --------------------------------------------------------------------------- +# Referential tags in saved files (ITL-1019) +# --------------------------------------------------------------------------- + +FALL_THROUGH_REFERENCES_KEY = "fall_through_references" +"""The function node descriptor key naming the referential tags its pod lets fall +through (a list; empty for a plain pod).""" + + +def saved_fall_through_references( + descriptor: dict[str, Any], version: str +) -> tuple[str, ...]: + """Return the referential tags a saved function node lets fall through. + + The file's format version decides what a file may hold (ITL-1019): at a + ``PLAIN_TAG_FORMAT_VERSIONS`` version nothing is referential and the key is never + written; from 0.3.0 on the key is always written. The saved output schema is + checked against it: its tag and data halves share exactly these names. When the + descriptor holds the pod's config, the config's list must agree too. + + Args: + descriptor: A function node descriptor of a saved pipeline. + version: The file's format version. + + Returns: + The referential tags, in the saved order. + + Raises: + ValueError: If the key is present in a file of a plain-tag version, missing + or not a list of names in a later one, or disagrees with the saved output + schema or the saved pod config. + """ + from orcapod.core.referential_tags import reference_names + + output_schema = descriptor.get("output_schema") or {} + shared = reference_names( + output_schema.get("tag", {}), output_schema.get("data", {}) + ) + if version in PLAIN_TAG_FORMAT_VERSIONS: + if FALL_THROUGH_REFERENCES_KEY in descriptor: + raise ValueError( + f"a file at format {version} holds only plain tags, but it names " + f"{FALL_THROUGH_REFERENCES_KEY} {descriptor[FALL_THROUGH_REFERENCES_KEY]!r}" + ) + references: tuple[str, ...] = () + else: + if FALL_THROUGH_REFERENCES_KEY not in descriptor: + raise ValueError( + f"a file at format {version} names the fall-through references of " + f"every function node, but this one has no {FALL_THROUGH_REFERENCES_KEY}" + ) + saved = descriptor[FALL_THROUGH_REFERENCES_KEY] + if not isinstance(saved, list) or not all(isinstance(n, str) for n in saved): + raise ValueError( + f"{FALL_THROUGH_REFERENCES_KEY} must be a list of names, got {saved!r}" + ) + references = tuple(saved) + if set(shared) != set(references): + raise ValueError( + f"the saved output schema has the referential tags {sorted(shared)}, but " + f"{FALL_THROUGH_REFERENCES_KEY} is {sorted(references)}" + ) + function_config = descriptor.get("function_config") + if function_config is not None and version not in PLAIN_TAG_FORMAT_VERSIONS: + configured = function_config.get(FALL_THROUGH_REFERENCES_KEY, []) + if set(configured) != set(references): + raise ValueError( + f"the saved pod lets {sorted(configured)} fall through, but " + f"{FALL_THROUGH_REFERENCES_KEY} is {sorted(references)}" + ) + return references + + +def check_saved_output_schema_is_plain( + descriptor: dict[str, Any], version: str +) -> None: + """Refuse a saved node with a referential tag in a file of a plain-tag version. + + Args: + descriptor: A node descriptor of a saved pipeline. + version: The file's format version. + + Raises: + ValueError: If the version holds only plain tags and the node's saved tag and + data halves share a name. + """ + from orcapod.core.referential_tags import reference_names + + if version not in PLAIN_TAG_FORMAT_VERSIONS: + return + output_schema = descriptor.get("output_schema") or {} + if shared := reference_names( + output_schema.get("tag", {}), output_schema.get("data", {}) + ): + raise ValueError( + f"a file at format {version} holds only plain tags, but the saved output " + f"schema has the referential tags {list(shared)}" + ) + + +def source_config_at_version(config: dict[str, Any], version: str) -> dict[str, Any]: + """Return a saved source config with its ``tag_only_columns`` as its version means. + + A source config of a ``PLAIN_TAG_FORMAT_VERSIONS`` file never holds + ``tag_only_columns``: every one of its tags is plain, which the returned config + states (``tag_only_columns=True``) for each source in it that takes the option, + nested sources (the inner source of a cached source, a derived source's origin) + included. A config of a later version holds the option for each such source and + is returned as it is. + + Args: + config: A saved source config. + version: The file's format version. + + Returns: + The config to rebuild the source from (a copy when anything was added). + + Raises: + ValueError: If a config of a plain-tag version holds ``tag_only_columns``, or + one of a later version lacks it. + """ + _ensure_registries() + plain_version = version in PLAIN_TAG_FORMAT_VERSIONS + + def upgrade(value: Any) -> Any: + if isinstance(value, list): + return [upgrade(item) for item in value] + if not isinstance(value, dict): + return value + upgraded = {key: upgrade(item) for key, item in value.items()} + cls = SOURCE_REGISTRY.get(upgraded.get("source_type")) # pyright: ignore[reportArgumentType] # any saved value + if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): + return upgraded + if plain_version and "tag_only_columns" in upgraded: + raise ValueError( + f"a file at format {version} holds only plain tags, but source " + f"config {upgraded.get('source_type')!r} names tag_only_columns " + f"{upgraded['tag_only_columns']!r}" + ) + if plain_version: + upgraded["tag_only_columns"] = True + elif "tag_only_columns" not in upgraded: + raise ValueError( + f"a file at format {version} states the plain tags of every source, " + f"but source config {upgraded.get('source_type')!r} has no " + "tag_only_columns" + ) + return upgraded + + return upgrade(config) diff --git a/tests/test_core/referential_tags/test_model_and_sources.py b/tests/test_core/referential_tags/test_model_and_sources.py index 81ff31369..f100accf4 100644 --- a/tests/test_core/referential_tags/test_model_and_sources.py +++ b/tests/test_core/referential_tags/test_model_and_sources.py @@ -104,23 +104,27 @@ def test_arrow_table_source_default_and_tag_only() -> None: assert data.arrow_schema().field("x").type == pa.int8() -def test_to_config_writes_tag_only_columns_only_for_referential_sources( - tmp_path: object, -) -> None: +def test_to_config_always_writes_tag_only_columns(tmp_path: object) -> None: + """Format 0.3.0 states every source's plain tags; the version, not a missing key, + decides an older file's meaning (ITL-1019).""" from pathlib import Path path = Path(str(tmp_path)) / "rows.csv" path.write_text("x,y\n1,0.5\n2,0.75\n") plain = op.sources.CSVSource(str(path), tag_columns=["x"], tag_only_columns=True) referential = op.sources.CSVSource(str(path), tag_columns=["x"]) - assert "tag_only_columns" not in plain.to_config() + assert plain.to_config()["tag_only_columns"] is True assert referential.to_config()["tag_only_columns"] == [] - # A v0.3 config (no key) reads back with every tag plain. assert op.sources.CSVSource.from_config(plain.to_config()).keys() == plain.keys() assert ( op.sources.CSVSource.from_config(referential.to_config()).keys() == referential.keys() ) + # A config without the key is refused: its meaning is not guessed. + config = plain.to_config() + del config["tag_only_columns"] + with pytest.raises(ValueError, match="has no tag_only_columns"): + op.sources.CSVSource.from_config(config) @pytest.mark.parametrize("tag_only", [(), ("k",)], ids=["referential", "plain"]) diff --git a/tests/test_core/referential_tags/test_persistence.py b/tests/test_core/referential_tags/test_persistence.py new file mode 100644 index 000000000..cd5ec6941 --- /dev/null +++ b/tests/test_core/referential_tags/test_persistence.py @@ -0,0 +1,354 @@ +"""A stored table states which of its tags are referential (ITL-1019). + +Developer decision, November 16, 2026: the stored table fully expresses what it stores. A +stored tag column that is a referential tag carries Arrow field metadata +(``referential_tags.REFERENTIAL_TAG_METADATA_KEY``); a plain tag carries none. A function +node marks the referential tags of its pipeline (tag) table, reads the binding back from +that table, and refuses a table that disagrees with its pod. The result table never records +referential status. The marker moves no hash. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.contexts import resolve_context +from orcapod.core.referential_tags import ( + REFERENTIAL_TAG_METADATA_KEY, + REFERENTIAL_TAG_METADATA_VALUE, + mark_references, + stored_references, + without_reference_marks, +) +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol + +CALLS: list[float] = [] + +ROWS = [ + {"unit": 1, "time": 10, "amp": 0.5}, + {"unit": 2, "time": 20, "amp": 1.5}, +] +MORE_ROWS = [*ROWS, {"unit": 3, "time": 30, "amp": 2.5}] + + +@op.function_pod(output_keys=["y"]) +def double(amp: float) -> float: + """Double the amplitude; reads no tag.""" + CALLS.append(amp) + return 2 * amp + + +def _source( + rows: list[dict[str, float]], tag_only: object = ("time",) +) -> op.sources.DictSource: + """``unit`` referential and ``time`` plain unless ``tag_only`` says otherwise.""" + return op.sources.DictSource( + rows, + tag_columns=["unit", "time"], + tag_only_columns=tag_only, # pyright: ignore[reportArgumentType] # the test varies it + source_id="spikes", + ) + + +def _in_memory(_: Path) -> ArrowDatabaseProtocol: + return InMemoryArrowDatabase() + + +def _delta(tmp_path: Path) -> ArrowDatabaseProtocol: + return DeltaTableDatabase(tmp_path / "store") + + +STORES = pytest.mark.parametrize( + "make_store", [_in_memory, _delta], ids=["memory", "delta"] +) +ORCHESTRATORS = pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) + + +def _run( + store: ArrowDatabaseProtocol, + rows: list[dict[str, float]], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], + tag_only: object = ("time",), + load_only: bool = False, +) -> PipelineJob: + job = PipelineJob(database=store) + with job: + double.pod(_source(rows, tag_only), label="double") + job.run(orchestrator=orchestrator(), load_only=load_only) + return job + + +def _tables(store: ArrowDatabaseProtocol) -> dict[str, pa.Table]: + """Return the stored tables by kind: ``pipeline`` (tag) and ``result``.""" + tables: dict[str, pa.Table] = {} + for path in store.list_table_paths(): + table = store.get_all_records(path) + assert table is not None + kind = "result" if "_result" in path else "pipeline" + assert kind not in tables, f"more than one {kind} table: {path}" + tables[kind] = table + return tables + + +def _marked(schema: pa.Schema) -> list[str]: + """The columns carrying the marker key, whatever its value.""" + return [ + f.name for f in schema if REFERENTIAL_TAG_METADATA_KEY in (f.metadata or {}) + ] + + +# --------------------------------------------------------------------------- +# The marker itself +# --------------------------------------------------------------------------- + + +def test_mark_references_marks_exactly_the_named_columns() -> None: + stray = pa.field("b", pa.int64(), metadata={REFERENTIAL_TAG_METADATA_KEY: b"x"}) + table = pa.Table.from_arrays( + [pa.array([1]), pa.array([2]), pa.array(["c"])], + schema=pa.schema( + [ + pa.field("a", pa.int64(), nullable=False), + stray, + pa.field("c", pa.string()), + ] + ), + ) + marked = mark_references(table, ["a"]) + # A marker that came in on another column is not this writer's binding. + assert _marked(marked.schema) == ["a"] + assert marked.schema.field("a").metadata == { + REFERENTIAL_TAG_METADATA_KEY: REFERENTIAL_TAG_METADATA_VALUE + } + assert marked.schema.field("b").metadata is None + # Values, types and nullability are untouched. + assert marked.schema.field("a").nullable is False + assert marked.equals(table) + assert stored_references(marked.schema) == ("a",) + # The read side drops every marker and gives back the unmarked schema. + assert without_reference_marks(marked).schema.equals( + mark_references(table, []).schema, check_metadata=True + ) + + +def test_mark_references_keeps_other_field_metadata() -> None: + field = pa.field("a", pa.int64(), metadata={b"other": b"kept"}) + table = pa.Table.from_arrays([pa.array([1])], schema=pa.schema([field])) + marked = mark_references(table, ["a"]) + assert marked.schema.field("a").metadata == { + b"other": b"kept", + REFERENTIAL_TAG_METADATA_KEY: REFERENTIAL_TAG_METADATA_VALUE, + } + assert without_reference_marks(marked).schema.field("a").metadata == { + b"other": b"kept" + } + + +def test_an_unmarked_table_is_returned_as_it_is() -> None: + table = pa.table({"a": [1]}) + assert mark_references(table, []) is table + + +def test_marking_a_column_the_table_lacks_raises() -> None: + with pytest.raises(ValueError, match="not columns of the stored table"): + mark_references(pa.table({"a": [1]}), ["b"]) + + +def test_a_marker_with_another_value_is_refused_on_read() -> None: + field = pa.field("a", pa.int64(), metadata={REFERENTIAL_TAG_METADATA_KEY: b"plain"}) + with pytest.raises(ValueError, match="only valid value"): + stored_references(pa.schema([field])) + + +def test_the_marker_moves_no_table_hash() -> None: + """Audit probe 6: the table hash ignores the marker (no golden value can move).""" + hasher = resolve_context(None).arrow_hasher + table = pa.table({"unit": [1, 2], "y": [0.5, 1.5]}) + marked = mark_references(table, ["unit"]) + assert marked.schema.field("unit").metadata + assert hasher.hash_table(marked) == hasher.hash_table(table) + + +# --------------------------------------------------------------------------- +# A function node's pipeline table +# --------------------------------------------------------------------------- + + +@STORES +@ORCHESTRATORS +def test_the_pipeline_table_marks_only_the_referential_tags( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + _run(store, ROWS, orchestrator) + tables = _tables(store) + pipeline = tables["pipeline"] + # ``unit`` is referential and is marked; ``time`` is plain and carries nothing. + assert _marked(pipeline.schema) == ["unit"] + assert REFERENTIAL_TAG_METADATA_KEY not in ( + pipeline.schema.field("time").metadata or {} + ) + assert stored_references(pipeline.schema) == ("unit",) + # The result table never records referential status: no marker, no tag column. + assert _marked(tables["result"].schema) == [] + assert "unit" not in tables["result"].column_names + + +@STORES +@ORCHESTRATORS +def test_a_plain_pipeline_table_carries_no_marker( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + _run(store, ROWS, orchestrator, tag_only=True) + for table in _tables(store).values(): + assert _marked(table.schema) == [] + + +@STORES +@ORCHESTRATORS +def test_the_marker_survives_an_append( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + _run(store, ROWS, orchestrator) + # More rows of the same pipeline: the same table, appended to. + CALLS.clear() + _run(store, MORE_ROWS, orchestrator) + assert CALLS == [2.5] + pipeline = _tables(store)["pipeline"] + assert pipeline.num_rows == 3 + assert stored_references(pipeline.schema) == ("unit",) + # A fresh handle on the same store reads the marker back from storage. + if isinstance(store, DeltaTableDatabase): + assert stored_references( + _tables(DeltaTableDatabase(tmp_path / "store"))["pipeline"].schema + ) == ("unit",) + + +@STORES +@ORCHESTRATORS +def test_the_binding_is_read_back_from_the_table( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + first = _run(store, ROWS, orchestrator).nodes["double"] + CALLS.clear() + # A load-only run reads every row from the store. + loaded = _run(store, ROWS, orchestrator, load_only=True).nodes["double"] + assert CALLS == [] + assert loaded.keys() == first.keys() == (("unit", "time"), ("unit", "y")) + table = loaded.as_table() + assert table.column_names.count("unit") == 1 + # The marker belongs to the stored table, not to the stream read from it. + assert _marked(table.schema) == [] + assert sorted(table.to_pylist(), key=lambda r: r["unit"]) == sorted( + first.as_table().to_pylist(), key=lambda r: r["unit"] + ) + for tag, data in loaded.iter_data(): + assert tag["unit"] == data["unit"] + + +@ORCHESTRATORS +def test_a_table_that_disagrees_with_the_pod_is_refused( + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + written = InMemoryArrowDatabase() + _run(written, ROWS, orchestrator) + # The same tables with the marker removed: as if another pipeline wrote them. + tampered = InMemoryArrowDatabase() + for path in written.list_table_paths(): + id_column = "__tampered_id" + table = written.get_all_records(path, record_id_column=id_column) + assert table is not None + tampered.add_records( + path, without_reference_marks(table), record_id_column=id_column, flush=True + ) + with pytest.raises(InputValidationError, match=r"marks the referential tags \[\]"): + _run(tampered, ROWS, orchestrator, load_only=True) + + +# --------------------------------------------------------------------------- +# Every write path of a store keeps the marker +# --------------------------------------------------------------------------- + + +def _rows(ids: list[int], extra: bool = False) -> pa.Table: + """Marked rows keyed by a binary record id; ``extra`` adds a column.""" + columns: dict[str, pa.Array[Any]] = { + "rid": pa.array([str(i).encode() for i in ids], pa.large_binary()), + "unit": pa.array(ids, pa.int64()), + "y": pa.array([float(i) for i in ids], pa.float64()), + } + if extra: + columns["z"] = pa.array(ids, pa.int64()) + return mark_references(pa.Table.from_pydict(columns), ["unit"]) + + +@pytest.mark.parametrize( + "make_store", + [ + _in_memory, + lambda tmp_path: DeltaTableDatabase( + tmp_path / "store", allow_schema_evolution=True + ), + ], + ids=["memory", "delta"], +) +def test_every_write_path_keeps_the_marker( + tmp_path: Path, make_store: Callable[[Path], ArrowDatabaseProtocol] +) -> None: + store = make_store(tmp_path) + path = ("node", "table") + + def references() -> tuple[str, ...]: + table = store.get_all_records(path) + assert table is not None + return stored_references(table.schema) + + store.add_records(path, _rows([1, 2]), record_id_column="rid", flush=True) + assert references() == ("unit",) # first write + store.add_records(path, _rows([3]), record_id_column="rid") + assert references() == ("unit",) # stored rows read together with pending ones + store.flush() + assert references() == ("unit",) # append + store.add_records( + path, _rows([3, 4]), record_id_column="rid", skip_duplicates=True, flush=True + ) + assert references() == ("unit",) # some ids stored already: a merge + unit = store.get_records_with_column_value(path, {"unit": 1}) + assert unit is not None and stored_references(unit.schema) == ("unit",) + if isinstance(store, DeltaTableDatabase): + # A new column: the stored schema evolves (in-memory tables do not evolve). + store.add_records( + path, _rows([5], extra=True), record_id_column="rid", flush=True + ) + assert references() == ("unit",) + reopened = DeltaTableDatabase(tmp_path / "store") + table = reopened.get_all_records(path) + assert table is not None and stored_references(table.schema) == ("unit",) diff --git a/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py b/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py index 93a1db6df..8710ff4cf 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py +++ b/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py @@ -227,6 +227,11 @@ def test_a_file_without_recorded_nullability_restores_side_effect_nodes( for descriptor in data["pipeline"]["nodes"].values(): del descriptor["output_schema"]["nullable"] descriptor.get("source_config", {}).pop("nullable", None) + # A 0.1.0 file holds only plain tags and never names them (ITL-1019). + descriptor.pop("fall_through_references", None) + descriptor.get("function_config", {}).pop("fall_through_references", None) + for config in data["bindings"]["sources"].values(): + config.pop("tag_only_columns", None) old = tmp_path / "format_0_1_0.json" old.write_text(json.dumps(data)) diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py index c548f5c57..e04528aa6 100644 --- a/tests/test_pipeline/test_loaded_slot_unregistered_class.py +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -147,6 +147,9 @@ def read_u(pp: cls) -> int: if descriptor["node_type"] == "source": descriptor.pop("pipeline_hash", None) descriptor.pop("content_hash", None) + # A 0.1.0 job file never names plain tags (ITL-1019). + for config in data["bindings"]["sources"].values(): + config.pop("tag_only_columns", None) with open(work + "/job.json", "w") as f: json.dump(data, f) # Nothing registers the class before the load. diff --git a/tests/test_pipeline/test_saved_job_before_referential_tags.py b/tests/test_pipeline/test_saved_job_before_referential_tags.py index c61997926..5249eff11 100644 --- a/tests/test_pipeline/test_saved_job_before_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_before_referential_tags.py @@ -23,6 +23,7 @@ SUPPORTED_FORMAT_VERSIONS, SUPPORTED_JOB_FORMAT_VERSIONS, resolve_database_from_config, + source_config_at_version, ) from orcapod.sources import CSVSource @@ -70,7 +71,10 @@ def test_a_v03_job_loads_with_unchanged_hashes(saved: dict[str, Any]) -> None: def test_a_v03_source_config_rebuilds_the_v03_source(saved: dict[str, Any]) -> None: config = saved["bindings"]["sources"]["subjects"] assert "tag_only_columns" not in config - source = CSVSource.from_config(config) + # The file's version (0.1.0) states that every tag is plain (ITL-1019). + upgraded = source_config_at_version(config, saved["orcapod_pipeline_job_version"]) + assert upgraded["tag_only_columns"] is True + source = CSVSource.from_config(upgraded) assert source.keys() == (("subject",), ("age",)) assert source.content_hash().to_string() == config["content_hash"] assert source.pipeline_hash().to_string() == config["pipeline_hash"] diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py new file mode 100644 index 000000000..f37c3af1a --- /dev/null +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -0,0 +1,265 @@ +"""Saving and loading a job or pipeline that holds referential tags (ITL-1019). + +A file of format 0.3.0 states its bindings: every source config that takes the option +writes ``tag_only_columns``, and every function node writes ``fall_through_references`` +(empty for a plain pod). Files at 0.1.0 and 0.2.0 hold only plain tags, decided by the +version, never by a missing key. The saved file is an independent point of verification: +a loaded node reads its stored table only when the table's marker agrees with the file. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import orcapod as op +from orcapod.core.function_pod import FunctionPod, ReferenceFallThroughPod +from orcapod.core.referential_tags import without_reference_marks +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + Pipeline, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.pipeline.serialization import ( + PLAIN_TAG_FORMAT_VERSIONS, + source_config_at_version, +) + +ORCHESTRATORS = pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) + + +@op.function_pod(output_keys=["y"]) +def double(amp: float) -> float: + """Double the amplitude; reads no tag, so ``unit`` falls through.""" + return 2 * amp + + +@op.function_pod(output_keys=["z"]) +def shift(y: float) -> float: + """A second step over the first one's output.""" + return y + 1 + + +def _csv(tmp_path: Path) -> str: + path = tmp_path / "spikes.csv" + path.write_text("unit,time,amp\n1,10,0.5\n2,20,1.5\n") + return str(path) + + +def _job(tmp_path: Path, tag_only: object = ("time",)) -> PipelineJob: + """A CSV source (``unit`` referential unless ``tag_only`` says otherwise), two pods.""" + source = op.sources.CSVSource( + _csv(tmp_path), + tag_columns=["unit", "time"], + tag_only_columns=tag_only, # pyright: ignore[reportArgumentType] # the test varies it + source_id="spikes", + ) + job = PipelineJob(database=DeltaTableDatabase(tmp_path / "store")) + with job: + doubled = double.pod(source, label="double") + shift.pod(doubled, label="shift") + return job + + +def _hashes(job: PipelineJob) -> dict[str, tuple[str, str, Any]]: + return { + label: ( + node.content_hash().to_string(), + node.pipeline_hash().to_string(), + node.keys(), + ) + for label, node in job.nodes.items() + } + + +def _saved(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def _function_descriptors(data: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + d["label"]: d + for d in data["pipeline"]["nodes"].values() + if d["node_type"] == "function" + } + + +@ORCHESTRATORS +def test_a_referential_job_saves_and_loads_with_its_hashes_and_bindings( + tmp_path: Path, + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + job = _job(tmp_path) + job.run(orchestrator=orchestrator()) + assert job.nodes["double"].keys() == (("unit", "time"), ("unit", "y")) + rows = sorted(job.nodes["shift"].as_table().to_pylist(), key=lambda r: r["unit"]) + job.save(tmp_path / "job.json") + + loaded = PipelineJob.load(tmp_path / "job.json") + assert _hashes(loaded) == _hashes(job) + # A loaded job reads its stored rows; the binding comes back with them. + loaded.run(load_only=True, orchestrator=orchestrator()) + table = loaded.nodes["shift"].as_table() + assert table.column_names.count("unit") == 1 + assert sorted(table.to_pylist(), key=lambda r: r["unit"]) == rows + records = loaded.nodes["double"].get_all_records() + assert records is not None and sorted(records.column("y").to_pylist()) == [1.0, 3.0] + + +def test_the_file_states_every_binding(tmp_path: Path) -> None: + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + assert data["orcapod_pipeline_job_version"] == "0.3.0" + assert data["bindings"]["sources"]["spikes"]["tag_only_columns"] == ["time"] + descriptors = _function_descriptors(data) + for label in ("double", "shift"): + descriptor = descriptors[label] + # Written for every function node, and the pod's config agrees. + assert descriptor["fall_through_references"] == ["unit"] + assert descriptor["function_config"]["fall_through_references"] == ["unit"] + + +def test_a_plain_job_states_its_plain_tags(tmp_path: Path) -> None: + job = _job(tmp_path, tag_only=True) + job.run() + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + assert data["bindings"]["sources"]["spikes"]["tag_only_columns"] is True + for descriptor in _function_descriptors(data).values(): + assert descriptor["fall_through_references"] == [] + assert "fall_through_references" not in descriptor["function_config"] + loaded = PipelineJob.load(tmp_path / "job.json") + assert _hashes(loaded) == _hashes(job) + + +def test_a_loaded_job_saved_again_keeps_its_bindings(tmp_path: Path) -> None: + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + PipelineJob.load(tmp_path / "job.json").save(tmp_path / "again.json") + for descriptor in _function_descriptors(_saved(tmp_path / "again.json")).values(): + assert descriptor["fall_through_references"] == ["unit"] + assert _hashes(PipelineJob.load(tmp_path / "again.json")) == _hashes(job) + + +def test_a_pipeline_saves_and_loads_its_bindings(tmp_path: Path) -> None: + job = _job(tmp_path) + job.compiled_pipeline.save(tmp_path / "pipeline.json") + loaded = Pipeline.load(tmp_path / "pipeline.json") + for label in ("spikes", "double", "shift"): + assert loaded.nodes[label].keys() == job.nodes[label].keys() + assert ( + loaded.nodes[label].pipeline_hash().to_string() + == job.nodes[label].pipeline_hash().to_string() + ) + + +def test_the_fall_through_pod_config_round_trips() -> None: + plain = double.pod + assert isinstance(plain, FunctionPod) + pod = ReferenceFallThroughPod(plain, ["unit"]) + config = pod.to_config() + assert config["fall_through_references"] == ["unit"] + rebuilt = ReferenceFallThroughPod.from_config(config) + assert rebuilt.references == ("unit",) + assert rebuilt.pipeline_hash() == pod.pipeline_hash() + del config["fall_through_references"] + with pytest.raises(ValueError, match="fall_through_references"): + ReferenceFallThroughPod.from_config(config) + + +# --------------------------------------------------------------------------- +# The format version decides; a missing or contradicting record is refused +# --------------------------------------------------------------------------- + + +def _edited(tmp_path: Path, edit: Any) -> Path: + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + edit(data) + path = tmp_path / "edited.json" + path.write_text(json.dumps(data)) + return path + + +def test_a_030_source_config_without_tag_only_columns_is_refused( + tmp_path: Path, +) -> None: + def edit(data: dict[str, Any]) -> None: + del data["bindings"]["sources"]["spikes"]["tag_only_columns"] + + # The file is malformed, not a source that moved: the load fails. + with pytest.raises(ValueError, match="source 'spikes'.*has no tag_only_columns"): + PipelineJob.load(_edited(tmp_path, edit)) + + +def test_a_030_function_node_without_its_references_is_refused(tmp_path: Path) -> None: + def edit(data: dict[str, Any]) -> None: + for descriptor in _function_descriptors(data).values(): + del descriptor["fall_through_references"] + + with pytest.raises(ValueError, match="has no fall_through_references"): + PipelineJob.load(_edited(tmp_path, edit)) + + +def test_references_that_disagree_with_the_saved_schema_are_refused( + tmp_path: Path, +) -> None: + def edit(data: dict[str, Any]) -> None: + _function_descriptors(data)["double"]["fall_through_references"] = [] + + with pytest.raises(ValueError, match="saved output schema has the referential"): + PipelineJob.load(_edited(tmp_path, edit)) + + +@pytest.mark.parametrize("version", sorted(PLAIN_TAG_FORMAT_VERSIONS)) +def test_an_older_file_holds_only_plain_tags(tmp_path: Path, version: str) -> None: + def edit(data: dict[str, Any]) -> None: + data["orcapod_pipeline_job_version"] = version + data["pipeline"]["orcapod_pipeline_version"] = version + + # A referential schema in a file of a plain-tag version is not that version. + with pytest.raises(ValueError, match=f"a file at format {version} holds only"): + PipelineJob.load(_edited(tmp_path, edit)) + with pytest.raises(ValueError, match="names tag_only_columns"): + source_config_at_version( + {"source_type": "csv", "tag_only_columns": []}, version + ) + assert source_config_at_version({"source_type": "csv"}, version) == { + "source_type": "csv", + "tag_only_columns": True, + } + + +def test_the_table_is_checked_against_the_saved_job(tmp_path: Path) -> None: + """The file and the store are independent: a loaded node refuses a disagreement.""" + job = _job(tmp_path) + job.run() + job.save(tmp_path / "job.json") + store = DeltaTableDatabase(tmp_path / "store") + # The same rows without the marker, as if another pipeline had written them. + tampered = InMemoryArrowDatabase() + for path in store.list_table_paths(): + table = store.get_all_records(path, record_id_column="__copy_id") + assert table is not None + tampered.add_records( + path, without_reference_marks(table), record_id_column="__copy_id" + ) + tampered.flush() + loaded = PipelineJob.load(tmp_path / "job.json", database=tampered) + with pytest.raises(InputValidationError, match="saved job declares \\['unit'\\]"): + loaded.nodes["double"].get_all_records() From 199221ccbca7300f9da60b39a5164c9bce8a04aa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 01:21:02 -0800 Subject: [PATCH 2/8] feat(referential_tags): refuse a database that cannot keep the marker before writing (ITL-1019) A function node with referential tags asks its pipeline database, when the database is attached, whether it can store the marker (RecordValidatingDatabaseProtocol: ConnectorArrowDatabase.validate_records). The SQLite and PostgreSQL connector databases keep no field metadata, so such a node is refused with an InputValidationError naming the database and the two ways out (a database that keeps field metadata, or tag_only_columns), before any result or record is written. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1019.md | 6 ++- docs/concepts/referential-tags.md | 4 +- src/orcapod/core/nodes/function_node.py | 6 +++ src/orcapod/core/referential_tags.py | 38 +++++++++++++++++++ .../databases/connector_arrow_database.py | 10 +++++ .../databases/extension_aware_database.py | 13 +++++++ src/orcapod/protocols/database_protocols.py | 15 ++++++++ .../referential_tags/test_persistence.py | 35 +++++++++++++++++ 8 files changed, 124 insertions(+), 3 deletions(-) diff --git a/changelog.d/ITL-1019.md b/changelog.d/ITL-1019.md index 40ccb20a1..f534c7756 100644 --- a/changelog.d/ITL-1019.md +++ b/changelog.d/ITL-1019.md @@ -23,5 +23,7 @@ ### Known limitations -- The SQLite and PostgreSQL databases keep no Arrow field metadata and refuse a pipeline - table with a referential tag, as they refuse logical types. +- The SQLite and PostgreSQL databases keep no Arrow field metadata, so a function node + that would store a referential tag there is refused (`InputValidationError`, naming the + database and the two ways out: a database that keeps field metadata, or + `tag_only_columns`) before anything is written. diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md index 2208c914d..fad4b0555 100644 --- a/docs/concepts/referential-tags.md +++ b/docs/concepts/referential-tags.md @@ -143,7 +143,9 @@ marker. The function's result table never records referential status, so plain a referential runs of the same function share their results. The marker moves no hash, and the stores keep it on every write (first write, append, merge, schema evolution; in memory and on Delta). The SQLite and PostgreSQL databases keep no Arrow -metadata at all and refuse such a table, as they refuse logical types. +metadata at all: a function node that would store a referential tag there is refused when +its database is attached, before anything is written. Use a database that keeps field +metadata, or keep those tags plain with `tag_only_columns`. ```python from orcapod.core.referential_tags import stored_references diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 880f47f11..62730b11a 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -38,6 +38,7 @@ ) from orcapod.core.referential_tags import ( mark_references, + require_database_keeps_marker, stored_references, without_reference_marks, ) @@ -1461,6 +1462,11 @@ def attach_databases( result_database = pipeline_database.at("_result") if self._function_pod is not None: + # A table must state its referential tags: refuse a database that + # cannot, before anything is written (ITL-1019). + require_database_keeps_marker( + pipeline_database, fall_through_references(self._function_pod) + ) # Normal path: wrap in CachedFunctionPod for compute + cache. # A fall-through pod's results are its plain pod's (ITL-1023). self._cached_function_pod = CachedFunctionPod( diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index 1dea111b9..614d586ca 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -258,6 +258,44 @@ def stored_references(schema: "pa.Schema") -> tuple[str, ...]: return tuple(names) +def require_database_keeps_marker(database: object, references: Iterable[str]) -> None: + """Refuse, before anything is written, a database that cannot store the marker. + + A stored table must state what it stores (developer, November 16, 2026, ITL-1019), + so a referential tag is never stored unmarked. A database that can check rows + before writing them (``RecordValidatingDatabaseProtocol``) is asked whether it + keeps the marker; one that cannot is refused. + + Args: + database: The database a node will store its referential tags in. + references: The referential tags it will store; nothing is checked when empty. + + Raises: + InputValidationError: If the database cannot keep the marker, naming the + database kind and the two ways out. + """ + from orcapod.protocols.database_protocols import RecordValidatingDatabaseProtocol + + references = tuple(references) + if not references or not isinstance(database, RecordValidatingDatabaseProtocol): + return + probe = mark_references( + pa.table({name: pa.array([], type=pa.null()) for name in references}), + references, + ) + try: + database.validate_records(probe) + except ValueError as exc: + raise InputValidationError( + f"{type(database).__name__} cannot store the referential tags " + f"{list(references)}: a stored referential tag carries Arrow field " + f"metadata, and this database keeps none ({exc}). Use a database that " + "keeps field metadata (DeltaTableDatabase, InMemoryArrowDatabase, the " + "SpiralDB database), or keep these tags plain with tag_only_columns on " + "the source. Nothing was written." + ) from exc + + def without_reference_marks(table: "pa.Table") -> "pa.Table": """Return ``table`` with no referential tag marker on any column. diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index af48cecd8..25bb2d8f3 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -269,6 +269,16 @@ def add_record( flush=flush, ) + def validate_records(self, records: pa.Table) -> None: + """Raise ``ValueError`` if the connector could not store ``records`` as given. + + The check ``add_records`` applies, without writing anything. + + Args: + records: The rows (any number, none included) to check. + """ + self._connector.validate_records(records) + def add_records( self, record_path: tuple[str, ...], diff --git a/src/orcapod/databases/extension_aware_database.py b/src/orcapod/databases/extension_aware_database.py index 01e4fa8af..74f719a1c 100644 --- a/src/orcapod/databases/extension_aware_database.py +++ b/src/orcapod/databases/extension_aware_database.py @@ -164,6 +164,19 @@ def add_records( flush=flush, ) + def validate_records(self, records: pa.Table) -> None: + """Raise ``ValueError`` if the wrapped database could not store ``records``. + + Delegates to the wrapped database when it can check rows before writing + (``RecordValidatingDatabaseProtocol``); otherwise accepts them. + """ + from orcapod.protocols.database_protocols import ( + RecordValidatingDatabaseProtocol, + ) + + if isinstance(self._db, RecordValidatingDatabaseProtocol): + self._db.validate_records(records) + def flush(self) -> None: self._db.flush() diff --git a/src/orcapod/protocols/database_protocols.py b/src/orcapod/protocols/database_protocols.py index fb585495c..7e533ae3c 100644 --- a/src/orcapod/protocols/database_protocols.py +++ b/src/orcapod/protocols/database_protocols.py @@ -227,6 +227,21 @@ def flush_batch(self, record_path: tuple[str, ...]) -> None: ... +@runtime_checkable +class RecordValidatingDatabaseProtocol(Protocol): + """A database that can say, before writing, whether it can store some rows. + + A database whose backend cannot keep every part of an Arrow table (the SQLite and + PostgreSQL connector databases keep no field metadata) refuses such rows on + write. ``validate_records`` applies the same check without writing, so a caller + can refuse up front (ITL-1019: a referential tag marker). + """ + + def validate_records(self, records: pa.Table) -> None: + """Raise ``ValueError`` if the database could not store ``records`` as given.""" + ... + + @runtime_checkable class DatabaseRegistryProtocol(Protocol): """Protocol for a database config registry used during pipeline save/load. diff --git a/tests/test_core/referential_tags/test_persistence.py b/tests/test_core/referential_tags/test_persistence.py index cd5ec6941..ebb103794 100644 --- a/tests/test_core/referential_tags/test_persistence.py +++ b/tests/test_core/referential_tags/test_persistence.py @@ -352,3 +352,38 @@ def references() -> tuple[str, ...]: reopened = DeltaTableDatabase(tmp_path / "store") table = reopened.get_all_records(path) assert table is not None and stored_references(table.schema) == ("unit",) + + +# --------------------------------------------------------------------------- +# A database that cannot keep the marker is refused before anything is written +# --------------------------------------------------------------------------- + + +def test_a_database_without_field_metadata_is_refused_before_any_write() -> None: + from orcapod.databases.connector_arrow_database import ConnectorArrowDatabase + from orcapod.databases.sqlite_connector import SQLiteConnector + + connector = SQLiteConnector(":memory:") + database = ConnectorArrowDatabase(connector) + CALLS.clear() + with pytest.raises(InputValidationError) as raised: + _run(database, ROWS, SyncPipelineOrchestrator) + message = str(raised.value) + # The database kind and both ways out are named. + assert "SQLiteConnector does not preserve Arrow metadata" in message + assert "keeps field metadata" in message and "tag_only_columns" in message + assert CALLS == [] + assert database.list_table_paths() == [] + + +def test_plain_tags_ask_nothing_of_the_database() -> None: + from orcapod.core.referential_tags import require_database_keeps_marker + from orcapod.databases.connector_arrow_database import ConnectorArrowDatabase + from orcapod.databases.sqlite_connector import SQLiteConnector + + database = ConnectorArrowDatabase(SQLiteConnector(":memory:")) + require_database_keeps_marker(database, ()) + with pytest.raises(InputValidationError, match="cannot store the referential"): + require_database_keeps_marker(database, ["unit"]) + # A database that keeps field metadata is accepted. + require_database_keeps_marker(InMemoryArrowDatabase(), ["unit"]) From fc1db0ab5d813896aa9230e65a34b0bec8a6a8db Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:06:23 -0800 Subject: [PATCH 3/8] fix(pipeline): a saved-node error of a loaded job names the job file (ITL-1019) PipelineJob.load reads its embedded pipeline from a temporary file; an error Pipeline.load raises for a saved node named that deleted file. It now names the job file, keeping the error's type. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 6 ++++++ .../test_pipeline/test_saved_job_with_referential_tags.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 7f830dc1f..8fe236939 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1606,6 +1606,12 @@ def load( _json.dump(data["pipeline"], tmp) try: pipeline = Pipeline.load(tmp_path) + except ValueError as exc: + # The embedded pipeline was read from a temporary file: name the job file. + message = str(exc) + if tmp_path in message: + exc.args = (message.replace(tmp_path, str(path)),) + raise finally: os.unlink(tmp_path) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index f37c3af1a..0e9894e1e 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -212,8 +212,11 @@ def edit(data: dict[str, Any]) -> None: for descriptor in _function_descriptors(data).values(): del descriptor["fall_through_references"] - with pytest.raises(ValueError, match="has no fall_through_references"): - PipelineJob.load(_edited(tmp_path, edit)) + path = _edited(tmp_path, edit) + with pytest.raises(ValueError, match="has no fall_through_references") as raised: + PipelineJob.load(path) + # The error names the job file, not the temporary file of its embedded pipeline. + assert str(raised.value).startswith(f"{path}: saved node") def test_references_that_disagree_with_the_saved_schema_are_refused( From 00818bd80a5c7d3739787c8eb5f8adcfe36b7c47 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:17:49 -0800 Subject: [PATCH 4/8] fix(pipeline): refuse a saved source config whose bindings contradict its slot (ITL-1019) A job file states a source's bindings twice: in the source config (tag_columns, tag_only_columns) and in its slot's saved schema. When they disagree the load now fails with an error naming the job file, the source and both lists, instead of logging a warning and leaving the source unbound (the path kept for a source whose data changed on disk). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 6 +++ src/orcapod/pipeline/serialization.py | 40 ++++++++++++++++++- .../test_saved_job_with_referential_tags.py | 13 ++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 8fe236939..1dbb845ea 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1585,6 +1585,7 @@ def load( from orcapod.pipeline.graph import Pipeline from orcapod.pipeline.serialization import ( SUPPORTED_JOB_FORMAT_VERSIONS, + check_source_config_binding, resolve_source_from_config, source_config_at_version, ) @@ -1630,6 +1631,11 @@ def load( # is the file's error, not a source that could not be rebuilt. try: src_config = source_config_at_version(src_config, version) + if spec_name in slots: + slot = slots[spec_name] + check_source_config_binding( + src_config, slot.tag_schema.keys(), slot.data_schema.keys() + ) except ValueError as exc: raise ValueError(f"{path}: source {spec_name!r}: {exc}") from exc try: diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 64291d96c..08a612823 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -5,7 +5,7 @@ import hashlib import json as _json import logging -from collections.abc import Collection +from collections.abc import Collection, Iterable from enum import Enum from typing import TYPE_CHECKING, Any, cast @@ -1454,3 +1454,41 @@ def upgrade(value: Any) -> Any: return upgraded return upgrade(config) + + +def check_source_config_binding( + config: dict[str, Any], tag_names: Iterable[str], data_names: Iterable[str] +) -> None: + """Refuse a saved source config whose referential tags contradict its slot. + + The source config (``tag_columns`` and ``tag_only_columns``) and the slot's saved + schema (a name in both halves is a referential tag) each state the source's + bindings, independently; a file in which they disagree is refused before the + source is rebuilt (ITL-1019). A config of a source that does not take the option, + or one written without ``tag_columns``, is not checked here. + + Args: + config: A saved source config, as its file version means it + (``source_config_at_version``). + tag_names: The slot's saved tag column names. + data_names: The slot's saved data column names. + + Raises: + ValueError: If the config and the slot name different referential tags. + """ + from orcapod.core.referential_tags import reference_names, source_reference_tags + + _ensure_registries() + cls = SOURCE_REGISTRY.get(config.get("source_type")) # pyright: ignore[reportArgumentType] # any saved value + if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): + return + tag_columns = config.get("tag_columns") + if not tag_columns or "tag_only_columns" not in config: + return + configured = source_reference_tags(tag_columns, config["tag_only_columns"]) + declared = reference_names(tag_names, data_names) + if set(configured) != set(declared): + raise ValueError( + f"its config makes {sorted(configured)} referential, but its slot's saved " + f"schema declares {sorted(declared)}" + ) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index 0e9894e1e..5a0d3137c 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -207,6 +207,19 @@ def edit(data: dict[str, Any]) -> None: PipelineJob.load(_edited(tmp_path, edit)) +def test_a_source_config_that_contradicts_its_slot_is_refused(tmp_path: Path) -> None: + def edit(data: dict[str, Any]) -> None: + # The slot declares ``unit`` referential; the config says every tag is plain. + data["bindings"]["sources"]["spikes"]["tag_only_columns"] = True + + path = _edited(tmp_path, edit) + with pytest.raises(ValueError, match="source 'spikes'") as raised: + PipelineJob.load(path) + message = str(raised.value) + assert str(path) in message + assert "['unit']" in message and "[]" in message + + def test_a_030_function_node_without_its_references_is_refused(tmp_path: Path) -> None: def edit(data: dict[str, Any]) -> None: for descriptor in _function_descriptors(data).values(): From ec5984a75ee285a03e6d3a457aa59c276269a0dd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 02:25:36 -0800 Subject: [PATCH 5/8] fix(pipeline): check an explicitly empty tag_columns against the slot too (ITL-1019) Only an absent or None tag_columns (a database source that takes its tags from the table's keys) skips the source config vs slot binding check. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/serialization.py | 6 ++++-- .../test_saved_job_with_referential_tags.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 08a612823..80c9f471d 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -1465,7 +1465,8 @@ def check_source_config_binding( schema (a name in both halves is a referential tag) each state the source's bindings, independently; a file in which they disagree is refused before the source is rebuilt (ITL-1019). A config of a source that does not take the option, - or one written without ``tag_columns``, is not checked here. + or one whose ``tag_columns`` is absent or ``None`` (a database source that takes + its tags from the table's keys), is not checked here. Args: config: A saved source config, as its file version means it @@ -1483,7 +1484,8 @@ def check_source_config_binding( if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): return tag_columns = config.get("tag_columns") - if not tag_columns or "tag_only_columns" not in config: + # ``None``: a database source takes its tags from the table's keys when read. + if tag_columns is None or "tag_only_columns" not in config: return configured = source_reference_tags(tag_columns, config["tag_only_columns"]) declared = reference_names(tag_names, data_names) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index 5a0d3137c..d155613b9 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -207,10 +207,17 @@ def edit(data: dict[str, Any]) -> None: PipelineJob.load(_edited(tmp_path, edit)) -def test_a_source_config_that_contradicts_its_slot_is_refused(tmp_path: Path) -> None: +@pytest.mark.parametrize( + "contradiction", + [{"tag_only_columns": True}, {"tag_columns": [], "tag_only_columns": True}], + ids=["all_plain", "no_tags"], +) +def test_a_source_config_that_contradicts_its_slot_is_refused( + tmp_path: Path, contradiction: dict[str, Any] +) -> None: def edit(data: dict[str, Any]) -> None: - # The slot declares ``unit`` referential; the config says every tag is plain. - data["bindings"]["sources"]["spikes"]["tag_only_columns"] = True + # The slot declares ``unit`` referential; the config makes nothing referential. + data["bindings"]["sources"]["spikes"].update(contradiction) path = _edited(tmp_path, edit) with pytest.raises(ValueError, match="source 'spikes'") as raised: From b4ad80c852933d8ca8db6002f0fc8d98fdf28238 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 12:58:01 -0800 Subject: [PATCH 6/8] fix(pipeline): read a bare-string tag_columns as one name in the binding check (ITL-1019) A source takes one tag name as a bare string, so a saved config may hold "tag_columns": "unit". check_source_config_binding passed it on as an iterable and compared its characters, refusing a file its source would rebuild. Also make the require_database_keeps_marker docstring say that a database that does not check rows is accepted. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/referential_tags.py | 8 +++-- src/orcapod/pipeline/serialization.py | 3 ++ .../test_saved_job_with_referential_tags.py | 31 +++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index 614d586ca..9835ceda2 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -262,9 +262,11 @@ def require_database_keeps_marker(database: object, references: Iterable[str]) - """Refuse, before anything is written, a database that cannot store the marker. A stored table must state what it stores (developer, November 16, 2026, ITL-1019), - so a referential tag is never stored unmarked. A database that can check rows - before writing them (``RecordValidatingDatabaseProtocol``) is asked whether it - keeps the marker; one that cannot is refused. + so a referential tag is never stored unmarked. A database that checks rows before + writing them (``RecordValidatingDatabaseProtocol``, the connector-backed databases) + is asked whether it keeps the marker and refused if it does not. Any other database + is accepted without a check: the in-memory and Delta databases keep Arrow field + metadata. Args: database: The database a node will store its referential tags in. diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 80c9f471d..a8149b51c 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -1487,6 +1487,9 @@ def check_source_config_binding( # ``None``: a database source takes its tags from the table's keys when read. if tag_columns is None or "tag_only_columns" not in config: return + # A source takes one tag name as a bare string: that name, never its characters. + if isinstance(tag_columns, str): + tag_columns = [tag_columns] configured = source_reference_tags(tag_columns, config["tag_only_columns"]) declared = reference_names(tag_names, data_names) if set(configured) != set(declared): diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index d155613b9..a00d5e8f7 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -286,3 +286,34 @@ def test_the_table_is_checked_against_the_saved_job(tmp_path: Path) -> None: loaded = PipelineJob.load(tmp_path / "job.json", database=tampered) with pytest.raises(InputValidationError, match="saved job declares \\['unit'\\]"): loaded.nodes["double"].get_all_records() + + +@pytest.mark.parametrize( + ("tag_only", "saved_tag_only"), + [((), []), (True, "unit")], + ids=["referential", "plain"], +) +def test_a_source_config_with_a_bare_string_tag_column_loads( + tmp_path: Path, tag_only: object, saved_tag_only: object +) -> None: + # A source takes one tag name as a bare string; the binding check reads it as that + # one name, never as its characters. + source = op.sources.CSVSource( + _csv(tmp_path), + tag_columns=["unit"], + tag_only_columns=tag_only, # pyright: ignore[reportArgumentType] # the test varies it + source_id="spikes", + ) + job = PipelineJob(database=DeltaTableDatabase(tmp_path / "store")) + with job: + double.pod(source, label="double") + job.save(tmp_path / "job.json") + data = _saved(tmp_path / "job.json") + data["bindings"]["sources"]["spikes"].update( + {"tag_columns": "unit", "tag_only_columns": saved_tag_only} + ) + path = tmp_path / "edited.json" + path.write_text(json.dumps(data)) + + loaded = PipelineJob.load(path) + assert _hashes(loaded) == _hashes(job) From 63dd6c737686abb0368cda8d06f9fe1a36690ae8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:03:25 -0800 Subject: [PATCH 7/8] test(pipeline): check the rebuilt source of a bare-string tag_columns config (ITL-1019) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_saved_job_with_referential_tags.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index a00d5e8f7..e9996a7a5 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -317,3 +317,8 @@ def test_a_source_config_with_a_bare_string_tag_column_loads( loaded = PipelineJob.load(path) assert _hashes(loaded) == _hashes(job) + # The source was rebuilt from the edited config, with its bindings and rows. + assert set(loaded.sources) == set(job.sources) == {"spikes"} + rebuilt, original = loaded.sources["spikes"], job.sources["spikes"] + assert rebuilt.keys() == original.keys() + assert rebuilt.as_table().to_pylist() == original.as_table().to_pylist() From 05caf89a6fd941ad9f12e0aba56204a479550f9f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 14:08:51 -0800 Subject: [PATCH 8/8] fix(pipeline): check restored source schemas, and read nested source configs only (ITL-1019) - A file of a plain-tag version (0.1.0, 0.2.0) is now also refused when a source node's source config restores a referential tag: Pipeline.load rebuilds the slot from source_config.tag_schema/data_schema, which the output-schema check did not read. - source_config_at_version recursed into every mapping, so a filter such as where={"source_type": "delta_table"} was read as a source config. It now follows only the nested source fields (inner_source, origin). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/serialization.py | 32 ++++++++++--- .../test_saved_job_with_referential_tags.py | 48 +++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index a8149b51c..50b369dbf 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -1386,9 +1386,13 @@ def check_saved_output_schema_is_plain( descriptor: A node descriptor of a saved pipeline. version: The file's format version. + A source node is rebuilt from its source config's ``tag_schema`` and + ``data_schema`` when the config holds them, so those are checked too. + Raises: ValueError: If the version holds only plain tags and the node's saved tag and - data halves share a name. + data halves share a name, in its output schema or in the schemas its + source config restores. """ from orcapod.core.referential_tags import reference_names @@ -1402,6 +1406,19 @@ def check_saved_output_schema_is_plain( f"a file at format {version} holds only plain tags, but the saved output " f"schema has the referential tags {list(shared)}" ) + source_config = descriptor.get("source_config") or {} + if shared := reference_names( + source_config.get("tag_schema") or {}, source_config.get("data_schema") or {} + ): + raise ValueError( + f"a file at format {version} holds only plain tags, but the saved source " + f"schema has the referential tags {list(shared)}" + ) + + +# The fields of a source config that hold a nested source config: a cached source's +# inner source and a derived source's origin. +NESTED_SOURCE_CONFIG_KEYS = ("inner_source", "origin") def source_config_at_version(config: dict[str, Any], version: str) -> dict[str, Any]: @@ -1428,12 +1445,13 @@ def source_config_at_version(config: dict[str, Any], version: str) -> dict[str, _ensure_registries() plain_version = version in PLAIN_TAG_FORMAT_VERSIONS - def upgrade(value: Any) -> Any: - if isinstance(value, list): - return [upgrade(item) for item in value] - if not isinstance(value, dict): - return value - upgraded = {key: upgrade(item) for key, item in value.items()} + def upgrade(value: dict[str, Any]) -> dict[str, Any]: + # Only the nested source fields hold source configs; any other mapping (a + # filter such as ``where``) is the source's own data, whatever its keys. + upgraded = dict(value) + for key in NESTED_SOURCE_CONFIG_KEYS: + if isinstance(upgraded.get(key), dict): + upgraded[key] = upgrade(upgraded[key]) cls = SOURCE_REGISTRY.get(upgraded.get("source_type")) # pyright: ignore[reportArgumentType] # any saved value if cls is None or not getattr(cls, "TAKES_TAG_ONLY_COLUMNS", False): return upgraded diff --git a/tests/test_pipeline/test_saved_job_with_referential_tags.py b/tests/test_pipeline/test_saved_job_with_referential_tags.py index e9996a7a5..552fab862 100644 --- a/tests/test_pipeline/test_saved_job_with_referential_tags.py +++ b/tests/test_pipeline/test_saved_job_with_referential_tags.py @@ -322,3 +322,51 @@ def test_a_source_config_with_a_bare_string_tag_column_loads( rebuilt, original = loaded.sources["spikes"], job.sources["spikes"] assert rebuilt.keys() == original.keys() assert rebuilt.as_table().to_pylist() == original.as_table().to_pylist() + + +@pytest.mark.parametrize("version", sorted(PLAIN_TAG_FORMAT_VERSIONS)) +def test_an_older_pipeline_with_a_referential_restored_source_schema_is_refused( + tmp_path: Path, version: str +) -> None: + # A source node is rebuilt from its source config's schemas, so those are what a + # plain-tag version must hold plain, not only the descriptor's output schema. + job = _job(tmp_path, tag_only=True) + job.compiled_pipeline.save(tmp_path / "pipeline.json") + data = _saved(tmp_path / "pipeline.json") + data["orcapod_pipeline_version"] = version + for descriptor in _function_descriptors({"pipeline": data}).values(): + del descriptor["fall_through_references"] + plain_path = tmp_path / "plain.json" + plain_path.write_text(json.dumps(data)) + Pipeline.load(plain_path) # the untouched file loads + + for descriptor in data["nodes"].values(): + if descriptor["node_type"] == "source": + descriptor["source_config"]["data_schema"]["unit"] = "int64" + path = tmp_path / "edited.json" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match=f"a file at format {version} holds only"): + Pipeline.load(path) + + +@pytest.mark.parametrize("version", [*sorted(PLAIN_TAG_FORMAT_VERSIONS), "0.3.0"]) +def test_a_mapping_inside_a_source_config_is_not_read_as_a_source( + version: str, +) -> None: + # A filter value may hold any key, ``source_type`` included; only the nested + # source fields (a cached source's inner source, a derived source's origin) are + # source configs. + where = {"source_type": "delta_table"} + tag_only = ( + {} if version in PLAIN_TAG_FORMAT_VERSIONS else {"tag_only_columns": True} + ) + config = {"source_type": "csv", "where": dict(where), **tag_only} + assert source_config_at_version(config, version)["where"] == where + cached = { + "source_type": "cached", + "inner_source": {"source_type": "csv", **tag_only}, + "cache_database": {"source_type": "delta_table"}, + } + upgraded = source_config_at_version(cached, version) + assert upgraded["inner_source"]["tag_only_columns"] is True + assert upgraded["cache_database"] == {"source_type": "delta_table"}