diff --git a/changelog.d/ITL-1138.md b/changelog.d/ITL-1138.md new file mode 100644 index 000000000..f29c3ed43 --- /dev/null +++ b/changelog.d/ITL-1138.md @@ -0,0 +1,3 @@ +### Fixed + +- A job over a column of a user class (an ordinary dataclass, an `orcapod.dataclass` or a pydantic model) loaded with `PipelineJob.load` in a process where nothing has registered the class yet reads like the job that saved it (ITL-1138). The load cannot restore such a column's saved type to a class, so the source slot held the saved type text: its pipeline hash was the hash of that text, and binding the same source was refused ("data column 'p' is declared extension<…> but the source has A") unless the class had been registered before the load. A loaded source slot now keeps its saved pipeline hash, and its saved content hash while no source is bound, as loaded function and operator nodes keep theirs, and restores such a column's type when a source is bound (building the source registers the class). Every node then has the saving process's hashes, schemas, system-tag column names and stored rows, and `run(load_only=True)` finds every stored result. A slot whose class is still not registered at bind refuses the source with an error that names the column and the saved class and says how to register it. Saving such a loaded job again (`job.save`), with or without a bind, keeps each such column's saved nullability: the type text cannot give it, so before this fix a nullable user-class column (`A | None`) was written as non-nullable and a later load restored it as `A`. A job file saved on a build before this fix from a job loaded before its class was registered may record a nullable user-class column as non-nullable; save it again from the code that defines the job. No hash, golden value or stored table changes; the job file format is unchanged, and a file without a slot's saved hashes still loads. diff --git a/design_issues/ITL-1138-LJ3.md b/design_issues/ITL-1138-LJ3.md new file mode 100644 index 000000000..af11979ef --- /dev/null +++ b/design_issues/ITL-1138-LJ3.md @@ -0,0 +1,21 @@ +## `src/orcapod/pipeline/job.py` — reading a loaded `PipelineJob` + +### LJ3 — a job loaded before its user class is registered held the class as text in its source slot +**Status:** resolved +**Severity:** high +**Issue:** ITL-1138 + +`Pipeline.load` restores a source slot's saved schema with `deserialize_schema`, which keeps the +saved type text (`extension`) for a column whose extension type the loading process +has not registered: a user class nothing has registered yet (no source of it built, no function +pod naming it defined). The loaded slot then hashed that text, so its pipeline hash differed from +the saved one, and with the bind check of ITL-1112 binding the same source was refused ("declared +extension<…> but the source has A"); a source node names its system-tag columns from the slot's +schema, so an accepted bind would have named them differently from the saving run. + +**Fix:** a slot rebuilt from a saved job keeps its saved pipeline hash, and its saved content hash +while unbound, as loaded function and operator nodes do (`SourceNodeBase._keep_saved`). A column +whose type was not restored is kept as pending and restored when a source is bound +(`_restore_pending_types`, called by the bind check), by which time the source has registered the +class; a column still not restorable makes the bind raise `SourceSpecMismatchError` naming the +column and the saved class. Tests: `tests/test_pipeline/test_loaded_slot_unregistered_class.py`. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 93872ea4b..c73d531bf 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -7,8 +7,10 @@ from __future__ import annotations +import contextlib import dataclasses import functools +import json import logging from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Collection, Iterator, Mapping @@ -35,7 +37,7 @@ from orcapod.logical_types.user_class_identity import equivalent_hints from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -from orcapod.types import ColumnConfig, Schema +from orcapod.types import ColumnConfig, ContentHash, Schema from orcapod.utils import arrow_utils from orcapod.utils.arrow_utils import system_tag_column_names from orcapod.utils.lazy_module import LazyModule @@ -58,6 +60,44 @@ def _type_text(data_type: Any) -> str: return data_type.__name__ if isinstance(data_type, type) else str(data_type) +@dataclasses.dataclass(frozen=True) +class _PendingType: + """A column of a loaded slot whose saved type could not be restored at load (ITL-1138). + + Attributes: + role: ``"tag"`` or ``"data"``. + text: The saved type text, as the job file holds it. + nullable: Whether the file records the column as nullable (``T | None``). + """ + + role: str + text: str + nullable: bool + + +def _saved_type_description(text: str) -> str: + """Name the type a saved type text stands for, for a message. + + A user class is named by its identity record (``user class 'mod.A' (dataclass)``), + any other extension type by its extension name; other text is quoted as it is. + + Args: + text: A saved type text, e.g. ``extension;>``. + """ + start = text.find("extension<") + if start < 0: + return f"type {text!r}" + name, _, rest = text[start + len("extension<") :].partition(";") + try: + metadata = json.loads(bytes.fromhex(rest.split(">", 1)[0])) + except ValueError: # not hex, or not JSON (UnicodeDecodeError is a ValueError) + metadata = None + identity = metadata.get("identity") if isinstance(metadata, dict) else None + if isinstance(identity, dict) and isinstance(identity.get("name"), str): + return f"user class {identity['name']!r} ({identity.get('kind', 'class')})" + return f"extension type {name!r}" + + def _is_narrower(declared: Any, given: Any) -> bool: """Return whether *given* is *declared* without its ``None`` member. @@ -241,6 +281,149 @@ def __init__( self._name = name self._tag_schema = tag_schema self._data_schema = data_schema + # A slot rebuilt from a saved job keeps the saved hashes, and the columns whose + # saved type could not be restored at load (ITL-1138); see ``_keep_saved``. + self._saved_pipeline_hash: ContentHash | None = None + self._saved_content_hash: ContentHash | None = None + self._pending_types: dict[str, _PendingType] = {} + + # ------------------------------------------------------------------ + # Rebuilt from a saved job + # ------------------------------------------------------------------ + + def _keep_saved( + self, + *, + pipeline_hash: str | None, + content_hash: str | None, + saved_types: Mapping[str, Mapping[str, str]], + nullable: Mapping[str, Collection[str]] | None, + ) -> None: + """Keep what a saved job recorded for this slot (ITL-1138). + + The slot returns the saved pipeline hash, and the saved content hash while no + source is bound, as loaded function and operator nodes return theirs, so its + stored address does not depend on what the loading process had registered. A + column whose saved type the load could not restore to a Python type (a user + class nothing in this process has registered yet) holds its saved text; it is + kept as pending and restored when a source is bound (``_fit``). + + Args: + pipeline_hash: The saved pipeline hash text, or ``None`` if not saved. + content_hash: The saved content hash text, or ``None`` if not saved. + saved_types: The saved type texts, ``{"tag": {...}, "data": {...}}``. + nullable: The saved nullable column names per role, or ``None`` when the + file does not record them. + """ + if pipeline_hash is not None: + self._saved_pipeline_hash = ContentHash.from_string(pipeline_hash) + if content_hash is not None: + self._saved_content_hash = ContentHash.from_string(content_hash) + for role, schema in (("tag", self._tag_schema), ("data", self._data_schema)): + for name, value in schema.items(): + text = saved_types.get(role, {}).get(name) + if isinstance(value, str) and text is not None: + self._pending_types[name] = _PendingType( + role=role, + text=text, + nullable=nullable is not None and name in nullable[role], + ) + + def _take_saved_state_of(self, other: "SourceNodeBase") -> None: + """Keep the saved state of *other*, a slot of the same schema. + + The pending types and the saved pipeline hash, which does not depend on the + slot's name, are always kept. The saved content hash covers the name too, so + it is kept only when this slot has *other*'s name. + """ + self._saved_pipeline_hash = other._saved_pipeline_hash + if self._name == other._name: + self._saved_content_hash = other._saved_content_hash + self._pending_types = dict(other._pending_types) + + def _pending_nullable_columns(self) -> dict[str, list[str]]: + """The pending columns the saved job records as nullable, by role. + + A column whose type is still the saved text has no Python type to read its + nullability from, so a job saved again records the saved nullability (ITL-1138). + """ + names: dict[str, list[str]] = {"tag": [], "data": []} + for name, pending in self._pending_types.items(): + if pending.nullable: + names[pending.role].append(name) + return names + + def _restore_pending_types(self) -> dict[str, str]: + """Restore the pending columns whose type this process can now restore. + + A source being bound has registered the class it holds, so a column saved with + that class restores now. The slot's schemas then hold the class, as they would + had the class been registered before the load. + + Returns: + For each column still not restorable, the description of its saved type. + """ + if not self._pending_types: + return {} + # Local import: orcapod.pipeline imports the node modules. + from orcapod.pipeline.serialization import deserialize_schema + + converter = self.data_context.type_converter + schemas = {"tag": dict(self._tag_schema), "data": dict(self._data_schema)} + still_pending: dict[str, _PendingType] = {} + for name, pending in self._pending_types.items(): + restored = deserialize_schema( + {name: pending.text}, + converter, + nullable=[name] if pending.nullable else [], + )[name] + if isinstance(restored, str): + still_pending[name] = pending + else: + schemas[pending.role][name] = restored + if len(still_pending) < len(self._pending_types): + self._tag_schema = Schema( + schemas["tag"], optional_fields=self._tag_schema.optional_fields + ) + self._data_schema = Schema( + schemas["data"], optional_fields=self._data_schema.optional_fields + ) + self._pending_types = still_pending + # The system-tag names come from the schema hash, now of the restored types. + with contextlib.suppress(AttributeError): # not computed yet + del self._schema_hash_str + self._invalidate_content_hash_cache() + # A slot without a saved pipeline hash computed one from the type text. + self._invalidate_pipeline_hash_cache() + return { + name: _saved_type_description(pending.text) + for name, pending in still_pending.items() + } + + def _bound_stream(self) -> "StreamProtocol | None": + """The concrete stream bound to this slot; a declaration has none.""" + return None + + def content_hash(self, hasher=None) -> ContentHash: + """Return the content hash; a slot rebuilt from a saved job returns the saved one. + + The saved value stands while no source is bound (ITL-1138); a bound source gives + its own identity, as for any slot. + """ + if self._saved_content_hash is not None and self._bound_stream() is None: + return self._saved_content_hash + return super().content_hash(hasher) + + def pipeline_hash(self, hasher=None) -> ContentHash: + """Return the pipeline hash; a slot rebuilt from a saved job returns the saved one. + + A slot's saved type of a user class this process has not registered is text + until a source is bound, so recomputing would give another value than the run's + (ITL-1138), as for loaded function and operator nodes. + """ + if self._saved_pipeline_hash is not None: + return self._saved_pipeline_hash + return super().pipeline_hash(hasher) # ------------------------------------------------------------------ # Identity @@ -440,6 +623,15 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] source_tag, source_data = source.output_schema() issues: list[str] = [] + unrestored = self._restore_pending_types() + for name, saved in unrestored.items(): + role = self._pending_types[name].role + issues.append( + f"{role} column {name!r} was saved as {saved}, which is not registered " + "in this process, so the source cannot be checked against it: bind a " + "source of that class (building one registers it) or define a function " + "pod that uses it before binding" + ) narrower: set[str] = set() for role, declared, given in ( ("tag", self._tag_schema, source_tag), @@ -452,6 +644,8 @@ def _fit(self, source: "StreamProtocol") -> tuple[Schema, Schema, frozenset[str] if extra: issues.append(f"unexpected {role} columns: {sorted(extra)}") for name in declared.keys(): + if name in unrestored: + continue if name not in given or equivalent_hints(given[name], declared[name]): continue if _is_narrower(declared[name], given[name]): @@ -726,11 +920,18 @@ def from_stream( slot_name = stream.source_id else: slot_name = f"{stream.label}:{stream.content_hash().to_hex()}" - return cls( + node = cls( name=slot_name, tag_schema=tag_schema, data_schema=data_schema, ) + if isinstance(stream, SourceNodeBase): + # A loaded job's slot keeps its pending types and its saved pipeline hash, + # which does not depend on the name (ITL-1138); its saved content hash + # covers the name, and the new slot has another name. + node._pending_types = dict(stream._pending_types) + node._saved_pipeline_hash = stream._saved_pipeline_hash + return node class SourceJobNode(SourceNodeBase): @@ -849,6 +1050,10 @@ def bound_source(self, value: StreamProtocol | None) -> None: self._bind(value) self._update_modified_time() + def _bound_stream(self) -> StreamProtocol | None: + """The concrete stream bound to this slot, or ``None``.""" + return self._bound_source + def _bind(self, value: StreamProtocol | None) -> None: """Check *value* and bind it, invalidating both hash caches. @@ -1213,20 +1418,24 @@ def from_stream( match stream: case SourceJobNode(): # Case 1: copy — preserve bound_source, do NOT wrap the SJN itself. - return cls( + node = cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, bound_source=stream.bound_source, ) + node._take_saved_state_of(stream) + return node case SourceNode(): # Case 2: unbound — schema placeholder; SJN hash = SourceNode hash. - return cls( + node = cls( name=name if name is not None else stream.name, tag_schema=stream.tag_schema, data_schema=stream.data_schema, bound_source=None, ) + node._take_saved_state_of(stream) + return node case _: # Case 3: concrete stream — bound SJN. # Slot name priority: @@ -1252,14 +1461,43 @@ def from_stream( bound_source=stream, ) + @classmethod + def _from_loaded_slot( + cls, slot: SourceNode, bound_source: StreamProtocol | None + ) -> "SourceJobNode": + """Build the job node of a slot rebuilt from a saved job (``PipelineJob.load``). + + The node takes the slot's saved hashes and pending types before *bound_source* + is checked, so a column saved with a class this process registers only now + (by building the source) is restored for the check (ITL-1138). + + Args: + slot: The slot ``Pipeline.load`` rebuilt from the job file. + bound_source: The source rebuilt from the saved config, if any. + + Raises: + SourceSpecMismatchError: If *bound_source* does not fit the slot. + """ + node = cls( + name=slot.name, + tag_schema=slot.tag_schema, + data_schema=slot.data_schema, + ) + node._take_saved_state_of(slot) + if bound_source is not None: + node._bind(bound_source) + return node + def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. Returns: A new ``SourceNode`` with the same name and schemas. """ - return SourceNode( + node = SourceNode( name=self._name, tag_schema=self._tag_schema, data_schema=self._data_schema, ) + node._take_saved_state_of(self) + return node diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index fe8092f3e..f101d1628 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -35,6 +35,33 @@ # --------------------------------------------------------------------------- +def _with_pending_nullability( + record: dict[str, list[str]], + pending: dict[str, list[str]], + slot: "cp.StreamProtocol", +) -> dict[str, list[str]]: + """Add a slot's pending nullable columns to a saved nullability record (ITL-1138). + + Args: + record: The nullable column names per role, as ``serialize_nullability`` + writes them. + pending: The slot's pending columns that its saved job records as nullable. + slot: The source slot, whose tag and data schemas give the column order. + + Returns: + The record with the pending names added, in the slot's column order. + """ + tag_schema, data_schema = slot.output_schema() + return { + role: [ + name + for name in schema + if name in record.get(role, []) or name in pending[role] + ] + for role, schema in (("tag", tag_schema), ("data", data_schema)) + } + + class Pipeline(AbstractPipelineBase[GraphNode]): """A pure computational blueprint recording operator and function pod invocations. @@ -211,6 +238,20 @@ def save(self, path: str | Path) -> None: ), } descriptor["reconstructable"] = True + # A column whose saved type is still text keeps its saved + # nullability, which the text cannot give (ITL-1138). + pending = node._pending_nullable_columns() # pyright: ignore[reportPrivateUsage] # the saver is the one caller + output_nullable = descriptor["output_schema"].get("nullable") + if output_nullable is not None: + descriptor["output_schema"]["nullable"] = ( + _with_pending_nullability(output_nullable, pending, node) + ) + source_nullable = serialize_nullability( + node.tag_schema, node.data_schema, type_converter + ) + descriptor["source_config"]["nullable"] = _with_pending_nullability( + source_nullable, pending, node + ) case FunctionNode(): if node._function_pod is not None: @@ -394,6 +435,14 @@ def load(cls, path: str | Path) -> "Pipeline": descriptor.get("data_context_key") ), ) + # The saved hashes, and the saved type text of a column whose + # class this process has not registered yet (ITL-1138). + node._keep_saved( # pyright: ignore[reportPrivateUsage] # the loader is the one caller + pipeline_hash=descriptor.get("pipeline_hash"), + content_hash=descriptor.get("content_hash"), + saved_types={"tag": saved_tag, "data": saved_data}, + nullable=saved_nullable, + ) # Restore label from descriptor if set explicitly stored_label = descriptor.get("label") if stored_label and stored_label != node_name: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 0227c3a22..eb44f9c4e 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1708,11 +1708,9 @@ def load( continue if isinstance(bp_node, _SourceNode): - sjn = SourceJobNode( - name=bp_node.name, - tag_schema=bp_node.tag_schema, - data_schema=bp_node.data_schema, - bound_source=sources.get(bp_node.name), + # Keeps the loaded slot's saved hashes and pending types (ITL-1138). + sjn = SourceJobNode._from_loaded_slot( # pyright: ignore[reportPrivateUsage] # the loader is the one caller + bp_node, bound_source=sources.get(bp_node.name) ) job._persistent_node_map[node_hash] = sjn if bp_node.name in sources: diff --git a/superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md b/superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md new file mode 100644 index 000000000..37811cdcc --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1138-loaded-slot-pending-type.md @@ -0,0 +1,34 @@ +# ITL-1138: a loaded source slot whose user class is not registered at load + +Branch `eywalker/itl-1138-a-job-loaded-before-its-user-class-is`, from PR #386's head +(`4ffdef06`), or `origin/main` if #386 has merged. Draft PR against `main`: +"Stacked on #386: merge right after it. Fixes ITL-1138". + +## Cause (measured for ITL-1137, sequence 4) + +`Pipeline.load` (`pipeline/graph.py`, `source_type == "node"`) restores a slot's saved schema +with `deserialize_schema`, which returns the raw saved text (`extension`) for a +column whose extension type is not registered in the loading process. `PipelineJob.load` copies +that schema into the `SourceJobNode`. Its pipeline hash is then the hash of the text; on #386 +the bind check compares the text with the source's class and refuses, and #386 names the +system-tag columns from the slot's pipeline hash. + +## Plan + +1. Failing fresh-process test first (public interface): sequence-4 job (s1 over `C`, s2; + `Join` -> `g(p: C, r: int)`; `MapData` -> `h(pp: C)`), exact and structural `C`; save; fresh + process loads without registering `C`, builds the sources, binds, `run(load_only=True)`; + every node's pipeline hash, content hash, `as_table(all_info=True)` schema (system-tag + names), rows and stored rows equal the saving process. Plus: pydantic; a class never + registered -> clear error at bind; a file saved before this change (the job file format + is unchanged, so an existing file is that file: write it with the base commit? no: the + saved descriptor already records `pipeline_hash`, so a file from before the change is the + same file; cover a descriptor without the saved hash if the format allows one). +2. Loaded slot keeps saved `pipeline_hash` (and `content_hash` while unbound) from its + descriptor, like the function and operator stubs. +3. Pending type text: the slot keeps a column it could not restore as text; at bind (the + `_fit` check) it restores it again through the data context's converter (the source has + registered the class by then). Restored -> the slot's schema holds the class and the check + compares types as usual. Not restored -> `SourceSpecMismatchError` naming the column and the + saved class identity (name and kind from the extension metadata) and what to do. +4. No hash, golden or stored-table change. diff --git a/tests/test_pipeline/test_loaded_slot_unregistered_class.py b/tests/test_pipeline/test_loaded_slot_unregistered_class.py new file mode 100644 index 000000000..6661ffdfe --- /dev/null +++ b/tests/test_pipeline/test_loaded_slot_unregistered_class.py @@ -0,0 +1,409 @@ +"""A job loaded before its user class is registered reads like the job that saved it (ITL-1138). + +A job over a column of a user class is saved; a fresh process loads it before anything has +registered the class (no source of it built, no function pod naming it defined). The saved type +of such a column cannot be restored to a class at load, so the loaded source slot keeps it as +pending text. The slot keeps its saved pipeline hash and content hash, and restores the type when +a source is bound, by which time the source has registered the class. Every node then has the +saving process's hashes, table schema (with its system-tag column names), rows and stored rows. +A slot whose class is still not registered at bind refuses the source with an error that names +the column and the saved class. + +Each process runs in a fresh interpreter: what a class's registration changes is per process. +""" + +import json +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any + +import pytest + +_CLASSES = textwrap.dedent( + """\ + import dataclasses + + import orcapod + import pydantic + + + @dataclasses.dataclass + class Exact: + u: int + v: str + + + @dataclasses.dataclass + class OtherExact: + u: int + v: str + + + @orcapod.dataclass + class Structural: + u: int + v: str + + + class Model(pydantic.BaseModel): + u: int + v: str + """ +) + +_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + mode, work, class_name = sys.argv[2], sys.argv[3], sys.argv[4] + + from orcapod import PipelineJob, function_pod + from orcapod.databases import DeltaTableDatabase + from orcapod.operators import Join, MapData + from orcapod.sources import DictSource + + + def sources(cls): + s1 = DictSource( + [{"id": 1, "p": cls(u=1, v="a")}, {"id": 2, "p": cls(u=2, v="b")}], + tag_columns=["id"], + data_schema={"id": int, "p": cls}, + source_id="s1", + ) + s2 = DictSource([{"id": 1, "r": 5}, {"id": 2, "r": 6}], tag_columns=["id"], source_id="s2") + return s1, s2 + + + def rows(table): + # The table's rows as sorted JSON text, so row order does not matter. + return sorted(json.dumps(row, sort_keys=True, default=str) for row in table.to_pylist()) + + + def describe(job): + out = {} + for label, node in sorted(job.nodes.items()): + # The schema with every column group: the system-tag names are compared. + schema = node.as_table(all_info=True).schema + # A source node stores nothing; a function or operator node has a stored view. + stored_view = getattr(node, "as_source", None) + out[label] = { + "pipeline_hash": node.pipeline_hash().to_string(), + "content_hash": node.content_hash().to_string(), + "schema": [ + f"{field.name}: {field.type} nullable={field.nullable}" for field in schema + ], + # Values of the user columns: a load-only run's status and execution + # columns differ from the saving run's by design. + "rows": rows(node.as_table()), + "stored_rows": None if stored_view is None else rows(stored_view().as_table()), + } + return out + + + if mode == "save": + import classes + + cls = getattr(classes, class_name) + s1, s2 = sources(cls) + + @function_pod(output_keys="q") + def add(p: cls, r: int) -> int: + return p.u + r + + @function_pod(output_keys="w") + def read_u(pp: cls) -> int: + return pp.u + + job = PipelineJob(name="demo", database=DeltaTableDatabase(work + "/store")) + with job: + joined = Join()(s1, s2, label="joined") + add.pod(joined, label="add") + renamed = MapData({"p": "pp"})(s1, label="renamed") + read_u.pod(renamed, label="read_u") + job.run() + job.save(work + "/job.json") + print(json.dumps(describe(job))) + else: + if mode == "load_old_file": + # A file as an earlier Orcapod wrote it: the format before orcapod_version, + # and no saved hashes on the source slots. + with open(work + "/job.json") as f: + data = json.load(f) + data["orcapod_pipeline_job_version"] = "0.1.0" + data.pop("orcapod_version", None) + for descriptor in data["pipeline"]["nodes"].values(): + if descriptor["node_type"] == "source": + descriptor.pop("pipeline_hash", None) + descriptor.pop("content_hash", None) + with open(work + "/job.json", "w") as f: + json.dump(data, f) + # Nothing registers the class before the load. + job = PipelineJob.load(work + "/job.json") + # The source slots' hashes while unbound, before the class is imported. + from orcapod.core.nodes.source_node import SourceJobNode + + unbound_slots = { + label: [node.pipeline_hash().to_string(), node.content_hash().to_string()] + for label, node in job.nodes.items() + if isinstance(node, SourceJobNode) + } + import classes + + bound_class = sys.argv[5] if len(sys.argv) > 5 else class_name + s1, s2 = sources(getattr(classes, bound_class)) # registers the class now + try: + job.bind(sources={"s1": s1, "s2": s2}) + except Exception as exc: + print(json.dumps({"bind_error": f"{type(exc).__name__}: {exc}"})) + sys.exit(0) + job.run(load_only=True) + out = describe(job) + out["_reused"] = {label: n.reused for label, n in job.last_run.nodes.items()} + out["_unbound_slots"] = unbound_slots + print(json.dumps(out)) + """ +) + + +def _run(tmp_path: Path, *args: str) -> dict[str, Any]: + """Run the script in a fresh interpreter and return the JSON it prints last.""" + result = subprocess.run( + [sys.executable, str(tmp_path / "script.py"), str(tmp_path), *args], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-3000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@pytest.fixture +def saved_job(tmp_path: Path): + """Return a function that saves the job over a class and returns the saving view.""" + _ = (tmp_path / "classes.py").write_text(_CLASSES) + _ = (tmp_path / "script.py").write_text(_SCRIPT) + + def save(class_name: str) -> dict[str, Any]: + return _run(tmp_path, "save", str(tmp_path / "work"), class_name) + + return save + + +@pytest.mark.parametrize("class_name", ["Exact", "Structural", "Model"]) +def test_a_job_loaded_before_its_class_is_registered_reads_like_the_saved_job( + tmp_path: Path, saved_job, class_name: str +) -> None: + """Hashes, schemas with system-tag names, rows and stored rows equal the saving process.""" + saved = saved_job(class_name) + loaded = _run(tmp_path, "load", str(tmp_path / "work"), class_name) + assert "bind_error" not in loaded, loaded.get("bind_error") + assert loaded.pop("_reused") == {"add": 2, "read_u": 2} + # Each source slot has its saved hashes as soon as it is loaded, before any bind: + # the saving run's pipeline hash, and the content hash the job file records for + # the slot (a bound slot's content hash is its source's, compared below). + job_file = json.loads((tmp_path / "work" / "job.json").read_text()) + file_content_hashes = { + descriptor["label"]: descriptor["content_hash"] + for descriptor in job_file["pipeline"]["nodes"].values() + if descriptor["node_type"] == "source" + } + assert loaded.pop("_unbound_slots") == { + label: [saved[label]["pipeline_hash"], file_content_hashes[label]] + for label in ("s1", "s2") + } + assert loaded == saved + + +def test_a_slot_whose_class_is_never_registered_refuses_the_source_clearly( + tmp_path: Path, saved_job +) -> None: + """Binding a source of a look-alike class names the column and the saved class.""" + _ = saved_job("Exact") + loaded = _run(tmp_path, "load", str(tmp_path / "work"), "Exact", "OtherExact") + error = loaded["bind_error"] + assert error.startswith("SourceSpecMismatchError: ") + assert "'p'" in error + assert "classes.Exact" in error + assert "not registered" in error + + +def test_a_file_without_the_slots_saved_hashes_loads_and_binds( + tmp_path: Path, saved_job +) -> None: + """With no saved hashes, the slot restores its type at bind and hashes as saved.""" + saved = saved_job("Exact") + loaded = _run(tmp_path, "load_old_file", str(tmp_path / "work"), "Exact") + assert "bind_error" not in loaded, loaded.get("bind_error") + assert loaded.pop("_reused") == {"add": 2, "read_u": 2} + _ = loaded.pop("_unbound_slots") # no saved hashes: recomputed once bound + assert loaded == saved + + +# --------------------------------------------------------------------------- +# A loaded slot's pending column: saving the job again, and copies of the slot +# --------------------------------------------------------------------------- + +_NULLABLE_SCRIPT = textwrap.dedent( + """\ + import json + import sys + import warnings + + warnings.simplefilter("ignore") + sys.path.insert(0, sys.argv[1]) + mode, work = sys.argv[2], sys.argv[3] + + from orcapod import PipelineJob, function_pod + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.databases import DeltaTableDatabase + from orcapod.sources import DictSource + + + def source(cls): + return DictSource( + [{"id": 1, "p": cls(u=1, v="a")}, {"id": 2, "p": None}], + tag_columns=["id"], + data_schema={"id": int, "p": cls | None}, + source_id="s", + ) + + + def slot_view(job): + node = job.nodes["s"] + return { + "data_schema": {k: str(v) for k, v in node.output_schema()[1].items()}, + "pipeline_hash": node.pipeline_hash().to_string(), + } + + + if mode == "save": + from classes import Exact + + @function_pod(output_keys="q") + def u_or_zero(p: Exact | None) -> int: + return 0 if p is None else p.u + + job = PipelineJob(name="demo", database=DeltaTableDatabase(work + "/store")) + with job: + u_or_zero.pod(source(Exact), label="u_or_zero") + job.run() + job.save(work + "/saved.json") + print(json.dumps(slot_view(job))) + elif mode in ("load_bind_save", "load_save"): + # Nothing registers the class before the load; the job is saved again. + job = PipelineJob.load(work + "/saved.json") + if mode == "load_bind_save": + from classes import Exact + + job.bind(sources={"s": source(Exact)}) + job.save(work + "/resaved.json") + print(json.dumps({})) + elif mode == "reload": + from classes import Exact + + _ = source(Exact) # registers the class before the load + job = PipelineJob.load(work + "/" + sys.argv[4]) + print(json.dumps(slot_view(job))) + elif mode == "legacy_validate": + # A file without the slots' saved hashes: the slot hashes the type text until + # the type is restored, here by validate() without a bind. + with open(work + "/saved.json") as f: + data = json.load(f) + for descriptor in data["pipeline"]["nodes"].values(): + if descriptor["node_type"] == "source": + descriptor.pop("pipeline_hash", None) + descriptor.pop("content_hash", None) + with open(work + "/legacy.json", "w") as f: + json.dump(data, f) + job = PipelineJob.load(work + "/legacy.json") + slot = job.nodes["s"] + _ = slot.pipeline_hash() # computed, and kept, from the type text + from classes import Exact + + slot.validate(source(Exact)) + print(json.dumps({"pipeline_hash": slot.pipeline_hash().to_string()})) + else: # copies of a loaded slot whose class is not registered + job = PipelineJob.load(work + "/saved.json") + slot = job.nodes["s"] + renamed = SourceJobNode.from_stream(slot, name="renamed") + same_name = SourceJobNode.from_stream(slot) + wrapped = SourceNode.from_stream(slot) + from classes import Exact + + exact = source(Exact) # registers the class now + bound = SourceJobNode.from_stream(wrapped) + bound.bound_source = exact + print(json.dumps({ + "renamed_keeps_content_hash": renamed.content_hash() == slot.content_hash(), + "same_name_keeps_content_hash": same_name.content_hash() == slot.content_hash(), + "renamed_keeps_pipeline_hash": renamed.pipeline_hash() == slot.pipeline_hash(), + "wrapped_keeps_pipeline_hash": wrapped.pipeline_hash() == slot.pipeline_hash(), + "wrapped_then_bound_schema": str(bound.output_schema()[1]["p"]), + })) + """ +) + + +@pytest.fixture +def nullable_job(tmp_path: Path): + """Save a job over ``p: Exact | None`` and return how to run the other modes.""" + _ = (tmp_path / "classes.py").write_text(_CLASSES) + _ = (tmp_path / "nullable.py").write_text(_NULLABLE_SCRIPT) + work = tmp_path / "work" + + def run(*args: str) -> dict[str, Any]: + result = subprocess.run( + [ + sys.executable, + str(tmp_path / "nullable.py"), + str(tmp_path), + args[0], + str(work), + *args[1:], + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr[-3000:] + return json.loads(result.stdout.strip().splitlines()[-1]) + + saved = run("save") + return saved, run + + +@pytest.mark.parametrize("mode", ["load_bind_save", "load_save"]) +def test_saving_a_loaded_job_again_keeps_a_pending_columns_nullability( + nullable_job, mode: str +) -> None: + """A job loaded before its class is registered and saved again reloads as saved.""" + saved, run = nullable_job + _ = run(mode) + assert run("reload", "resaved.json") == run("reload", "saved.json") == saved + + +def test_copies_of_a_loaded_slot_keep_its_pending_types_and_name_bound_hashes( + nullable_job, +) -> None: + """A copy keeps pending types; only a copy of the same name keeps its content hash.""" + _, run = nullable_job + copies = run("copies") + assert copies == { + "renamed_keeps_content_hash": False, + "same_name_keeps_content_hash": True, + "renamed_keeps_pipeline_hash": True, + "wrapped_keeps_pipeline_hash": True, + "wrapped_then_bound_schema": "classes.Exact | None", + } + + +def test_restoring_a_type_by_validation_rehashes_a_slot_without_saved_hashes( + nullable_job, +) -> None: + """A slot that hashed its type text hashes the restored class after validate().""" + saved, run = nullable_job + assert run("legacy_validate") == {"pipeline_hash": saved["pipeline_hash"]}