From f430c4c5c97c681060941955761b13ea6b48b5bf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 11:07:03 -0800 Subject: [PATCH 01/14] fix(databases): refuse a table a SQL connector database cannot store (ITL-1162) Each connector states which columns it stores, in one place (DBConnectorProtocol.unstorable_column_reason): a column it reads back as written, with the same Arrow type in its large variant. ConnectorArrowDatabase refuses any other column with UnstorableColumnError before anything is buffered, and a function job node (result table) and a side-effect job node (invocation log) ask the database when it is attached, so a job refuses them before anything runs, naming the node. The SQLite and PostgreSQL connectors no longer fall back to TEXT in CREATE TABLE. The flush schema comparison treats large and non-large variants as the same type. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1162.md | 7 + design_issues/ITL-1162-CA3.md | 30 ++ docs/concepts/pipeline-job.md | 33 ++ src/orcapod/contexts/stored.py | 20 + src/orcapod/core/cached_function_pod.py | 35 +- src/orcapod/core/nodes/function_node.py | 11 +- src/orcapod/core/result_cache.py | 7 +- .../databases/connector_arrow_database.py | 44 ++- src/orcapod/databases/postgresql_connector.py | 91 ++++- src/orcapod/databases/spiraldb_connector.py | 19 + src/orcapod/databases/sqlite_connector.py | 77 +++- src/orcapod/errors.py | 60 +++ src/orcapod/protocols/database_protocols.py | 15 + .../protocols/db_connector_protocol.py | 22 ++ src/orcapod/side_effects.py | 44 ++- .../test_core/sources/test_db_table_source.py | 5 + .../test_connector_arrow_database.py | 6 + .../test_postgresql_connector.py | 13 +- tests/test_databases/test_sqlite_connector.py | 16 +- .../test_databases/test_unstorable_columns.py | 352 ++++++++++++++++++ .../test_pipeline/test_store_write_failure.py | 18 +- 21 files changed, 875 insertions(+), 50 deletions(-) create mode 100644 changelog.d/ITL-1162.md create mode 100644 design_issues/ITL-1162-CA3.md create mode 100644 tests/test_databases/test_unstorable_columns.py diff --git a/changelog.d/ITL-1162.md b/changelog.d/ITL-1162.md new file mode 100644 index 000000000..6a0564364 --- /dev/null +++ b/changelog.d/ITL-1162.md @@ -0,0 +1,7 @@ +### Fixed + +- A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` now refuses a table it cannot store before anything of it is written, with `UnstorableColumnError` naming the connector, the column, its Arrow type and what to use instead (ITL-1162). A connector stores a column only when it reads it back as written: on SQLite `bool`, `int64`, `double`, `large_string` and `large_binary`; on PostgreSQL also `int32`, `float`, `date32`, `timestamp[us]` and `timestamp[us, tz=UTC]`. Before, a function node on such a database ran its function and then failed at commit (`sqlite3.ProgrammingError: ... type 'list' is not supported`), timestamps were stored as text and read back as strings, and narrower integers and floats read back widened, failing the next write. A job now refuses a node whose stored table such a database cannot hold (every function node; a side-effect node on SQLite) when the job is built, before anything runs, naming the node. The connectors no longer fall back to a `TEXT` column for a type they have no column type for. + +### Known limitations + +- No function node can run on a SQL connector job database (SQLite or PostgreSQL), and no side-effect node on SQLite: a function node's result table holds list-of-struct executor columns, which neither connector stores, and a side-effect log holds a UTC timestamp, which SQLite does not store. Use `InMemoryArrowDatabase` or `DeltaTableDatabase`. Storing such columns on SQL connectors is planned for a later release (ITL-1162). diff --git a/design_issues/ITL-1162-CA3.md b/design_issues/ITL-1162-CA3.md new file mode 100644 index 000000000..18cf178fb --- /dev/null +++ b/design_issues/ITL-1162-CA3.md @@ -0,0 +1,30 @@ +## `src/orcapod/databases/connector_arrow_database.py` + +### CA3 — SQL connector databases cannot store list, struct and (on SQLite) timestamp columns +**Status:** open +**Severity:** high +**Issue:** ITL-1162 + +A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` cannot store a +function node's result table. Its `__pf_exec_executor_info` and `__pf_exec_extra_info` +columns are `large_list>`. The SQLite connector logged "Unsupported Arrow +type ...; mapping to TEXT" and then failed to bind a Python list (`sqlite3.ProgrammingError`) +at commit, after the function had run. The same table's `__pod_ts` (`timestamp[us, tz=UTC]`) +and a side-effect log's `executed_at` were stored as TEXT and read back as `large_string`. +`int8`, `int32` and `float32` columns read back widened, so the second write to the table +failed its schema comparison. PostgreSQL raises for list columns at `CREATE TABLE` and has no +column type for structs. + +**Fix (refusal; developer ruling, November 16, 2026):** each connector states which columns +it stores, in one place (`DBConnectorProtocol.unstorable_column_reason(name, arrow_type, +primary_key=...)`): a column it reads back as written, with the same Arrow type in its large +variant. `ConnectorArrowDatabase.require_storable` raises `UnstorableColumnError` (connector, +column, Arrow type, reason, what to use instead) in `add_records`, before anything is +buffered. A function job node (its result table) and a side-effect job node (its invocation +log) ask the database when it is attached, so a job refuses them before anything runs, naming +the node. The connectors' silent TEXT fallback in `CREATE TABLE` now raises. Operator caches +and cached sources are refused at their write when a user column is unstorable. + +**Open (the real fix, a later release):** store list and struct columns (for example as JSON +text read back as the same Arrow type) and timestamps on SQLite, so a function node can run +on a SQL connector database. diff --git a/docs/concepts/pipeline-job.md b/docs/concepts/pipeline-job.md index f5bca8daa..415222076 100644 --- a/docs/concepts/pipeline-job.md +++ b/docs/concepts/pipeline-job.md @@ -120,6 +120,39 @@ run and computes nothing on a second `run()`. A job created without a database can be recorded but not run. Give it one later with `job.bind(database=...)`; `run()` without a database raises `ValueError`. +### SQL connector databases + +A `ConnectorArrowDatabase` over a `SQLiteConnector` or a `PostgreSQLConnector` stores a +column only when the connector reads it back as written, with the same Arrow type (text and +binary columns come back as `large_string` and `large_binary`, the types Orcapod uses): + +| Connector | Column types it stores | +|---|---| +| `SQLiteConnector` | `bool`, `int64`, `double`, `large_string`, `large_binary` | +| `PostgreSQLConnector` | `bool`, `int32`, `int64`, `float`, `double`, `large_string`, `large_binary`, `date32`, `timestamp[us]`, `timestamp[us, tz=UTC]` | + +Any other column type (lists, structs, dates or timestamps on SQLite, narrower integers and +floats, logical types) is refused with `UnstorableColumnError` before anything of the table is +written. A function node's result table always holds list-valued executor columns, and a +side-effect node's log a UTC timestamp, so a job refuses every function node on either +connector, and every side-effect node on SQLite, when the job is built, before anything runs. Use `InMemoryArrowDatabase` or +`DeltaTableDatabase` for those jobs. Storing list and struct columns on a SQL connector is +planned for a later release. + +```python +from orcapod.databases import ConnectorArrowDatabase, SQLiteConnector +from orcapod.errors import UnstorableColumnError + +sqlite_database = ConnectorArrowDatabase(SQLiteConnector(":memory:")) +sql_job = PipelineJob(name="bmi", database=sqlite_database) +try: + with sql_job: + compute_bmi.pod(mice(cohort_a), label="bmi") +except UnstorableColumnError as exc: + print(exc) +# node 'bmi': a SQLiteConnector database cannot store column '__pf_exec_executor_info' of Arrow type large_list<...>: SQLite has no column type for it. Use an InMemoryArrowDatabase or a DeltaTableDatabase instead. +``` + ## What a re-run computes A stored result is reused when the function's identity (its name, its output schema and its diff --git a/src/orcapod/contexts/stored.py b/src/orcapod/contexts/stored.py index e0ae1a58f..d2df47f3a 100644 --- a/src/orcapod/contexts/stored.py +++ b/src/orcapod/contexts/stored.py @@ -681,3 +681,23 @@ def stored_data_contexts( keys = cast("list[str]", per_value.field(0).to_pylist()) counts = cast("list[int]", per_value.field(1).to_pylist()) return dict(zip(keys, counts)) + + +def require_storable(database: object, schema: pa.Schema) -> None: + """Refuse ``schema`` if ``database`` states it cannot store one of its columns. + + The one check for ``ColumnCheckingDatabaseProtocol`` (ITL-1162): a job node asks + it about each table it stores when a database is attached, before anything runs. + A database that states no rule (in-memory, Delta) stores every column, and + nothing is checked. + + Args: + database: Any database object. + schema: The Arrow schema of a table the database is to hold. + + Raises: + UnstorableColumnError: If the database cannot store a column of ``schema``. + """ + check = getattr(database, "require_storable", None) + if callable(check): + _ = check(schema) diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index ea1fea3bf..8eac1650f 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -17,8 +17,8 @@ from orcapod.core.streams.view import StreamView from orcapod.system_constants import constants from orcapod.types import ColumnConfig, Schema -from orcapod.contexts.stored import StaleTables -from orcapod.core.result_cache import ResultCache +from orcapod.contexts.stored import StaleTables, require_storable +from orcapod.core.result_cache import ResultCache, pod_timestamp_type from orcapod.hooks import InvocationStatus from orcapod.protocols.core_protocols import ( FunctionPodProtocol, @@ -71,6 +71,37 @@ def __init__( name_path=self.uri[: 2 if function_pod.ctx_arg_name is not None else 1], ) + def require_storable_results(self) -> None: + """Refuse a result database that cannot store this pod's result table (ITL-1162). + + The result table holds the function's variation and execution columns + (``__pf_var_*``, ``__pf_exec_*``), its output columns and the ``__pod_ts`` + timestamp, typed from their declared schemas; nothing is computed. A + database that states which columns it stores (``ConnectorArrowDatabase``) + refuses here, before the function runs; any other database accepts. + + Raises: + UnstorableColumnError: If the result database cannot store a column. + """ + import pyarrow as pa + + pf = self._function_pod.data_function + python_schema = { + **{ + f"{constants.PF_VARIATION_PREFIX}{name}": hint + for name, hint in pf.get_function_variation_data_schema().items() + }, + **{ + f"{constants.PF_EXECUTION_PREFIX}{name}": hint + for name, hint in pf.get_execution_data_schema().items() + }, + **pf.output_data_schema, + } + schema = self.data_context.type_converter.python_schema_to_arrow_schema( + python_schema + ).append(pa.field(constants.POD_TIMESTAMP, pod_timestamp_type())) + require_storable(self._cache.result_database, schema) + @property def result_database(self) -> ArrowDatabaseProtocol: """The underlying result database.""" diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 0564bdd4d..c42c9df79 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -56,6 +56,7 @@ TypeIdentityError, OrcapodDatabaseReadError, OrcapodDatabaseWriteError, + UnstorableColumnError, ) from orcapod.contexts.stored import ( StaleTables, @@ -1413,10 +1414,18 @@ def attach_databases( if self._function_pod is not None: # Normal path: wrap in CachedFunctionPod for compute + cache. - self._cached_function_pod = CachedFunctionPod( + cached_function_pod = CachedFunctionPod( self._function_pod, result_database=result_database, ) + # A database that cannot store the result table refuses it now, + # before anything runs (ITL-1162). + try: + cached_function_pod.require_storable_results() + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise + self._cached_function_pod = cached_function_pod else: # Read-only stub path (loaded job without a live function pod). # Use the record path stored from the descriptor so that diff --git a/src/orcapod/core/result_cache.py b/src/orcapod/core/result_cache.py index fa3ee1de3..f89caaa83 100644 --- a/src/orcapod/core/result_cache.py +++ b/src/orcapod/core/result_cache.py @@ -41,6 +41,11 @@ logger = logging.getLogger(__name__) +def pod_timestamp_type() -> pa.DataType: + """The Arrow type of a result row's ``__pod_ts`` column: UTC microseconds.""" + return pa.timestamp("us", tz="UTC") + + class ResultCache: """Shared result caching backed by an ``ArrowDatabaseProtocol``. @@ -329,7 +334,7 @@ def store( data_table = data_table.append_column( constants.POD_TIMESTAMP, - pa.array([datetime.now(timezone.utc)], type=pa.timestamp("us", tz="UTC")), + pa.array([datetime.now(timezone.utc)], type=pod_timestamp_type()), ) data_table = with_data_context(data_table, data_context_key) diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index af48cecd8..7bdfcea79 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -24,6 +24,8 @@ from typing import TYPE_CHECKING, Any, cast from orcapod.databases.utils import coerce_record_id +from orcapod.errors import UnstorableColumnError +from orcapod.utils.arrow_utils import normalize_to_large_types from orcapod.protocols.db_connector_protocol import ColumnInfo, DBConnectorProtocol from orcapod.utils.lazy_module import LazyModule @@ -249,6 +251,39 @@ def table_exists(self, record_path: tuple[str, ...]) -> bool: return True return table_name in self._connector.get_table_names() + # ── Storable columns ────────────────────────────────────────────────────── + + def require_storable(self, schema: pa.Schema) -> None: + """Refuse a table schema with a column the connector cannot store (ITL-1162). + + The connector states which columns it stores + (``DBConnectorProtocol.unstorable_column_reason``): a column it would read + back as another type, or could not write at all, is refused. Called by + ``add_records`` before anything of the table is buffered, and by a job node + when the database is attached to it, before anything runs. + + Args: + schema: The Arrow schema of a table this database is to hold. The + ``__record_id`` column, when present, is checked as the key. + + Raises: + UnstorableColumnError: For the first column the connector cannot + store, naming the connector, the column, its Arrow type and why. + """ + for field in schema: + reason = self._connector.unstorable_column_reason( + field.name, + field.type, + primary_key=field.name == self.RECORD_ID_COLUMN, + ) + if reason is not None: + raise UnstorableColumnError( + database_kind=type(self._connector).__name__, + column=field.name, + arrow_type=field.type, + reason=reason, + ) + # ── Write methods ───────────────────────────────────────────────────────── def add_record( @@ -307,6 +342,7 @@ def add_records( ) self._connector.validate_records(records) + self.require_storable(records.schema) records = self._deduplicate_within_table(records) record_key = self._get_record_key(record_path) @@ -414,11 +450,15 @@ def flush(self) -> None: # cryptic DB-level error or a silent partial write. existing_table_names = self._connector.get_table_names() if table_name in existing_table_names: + # A connector reads text and binary back in their large variants, + # which are the same types (ITL-1162). existing_cols = { - c.name: c.arrow_type + c.name: normalize_to_large_types(c.arrow_type) for c in self._connector.get_column_info(table_name) } - pending_cols = {c.name: c.arrow_type for c in columns} + pending_cols = { + c.name: normalize_to_large_types(c.arrow_type) for c in columns + } if existing_cols != pending_cols: def _fmt(cols: dict) -> str: diff --git a/src/orcapod/databases/postgresql_connector.py b/src/orcapod/databases/postgresql_connector.py index 12885d607..49759875a 100644 --- a/src/orcapod/databases/postgresql_connector.py +++ b/src/orcapod/databases/postgresql_connector.py @@ -64,7 +64,7 @@ def _pg_type_to_arrow(pg_type_name: str, udt_name: str) -> pa.DataType: t = udt_name.lower() - if t == "bool": + if t in ("bool", "boolean"): return _pa.bool_() if t in ("int2", "smallint"): return _pa.int32() @@ -125,18 +125,16 @@ def _pg_type_to_arrow(pg_type_name: str, udt_name: str) -> pa.DataType: return _pa.large_string() -def _arrow_type_to_pg_sql(arrow_type: pa.DataType) -> str: - """Map an Arrow DataType to a PostgreSQL SQL type string for CREATE TABLE. +def _pg_sql_type(arrow_type: pa.DataType) -> str | None: + """Return the PostgreSQL column type for an Arrow type, or ``None`` when there is none. Args: arrow_type: The Arrow DataType to convert. Returns: - A PostgreSQL SQL type string (e.g. ``"TEXT"``, ``"BIGINT"``). - - Raises: - ValueError: If ``arrow_type`` is ``pa.large_list(...)`` — array columns - are not supported in CREATE TABLE (they are read-only from existing tables). + A PostgreSQL SQL type string (e.g. ``"TEXT"``, ``"BIGINT"``), or ``None`` + for a type with no column type here (lists, structs, extension types and + the rest). """ import pyarrow as _pa @@ -164,13 +162,63 @@ def _arrow_type_to_pg_sql(arrow_type: pa.DataType) -> str: return "DATE" if _pa.types.is_timestamp(arrow_type): return "TIMESTAMPTZ" if arrow_type.tz is not None else "TIMESTAMP" + return None + + +def _arrow_type_to_pg_sql(arrow_type: pa.DataType) -> str: + """Map an Arrow DataType to a PostgreSQL SQL type string for CREATE TABLE. + + Args: + arrow_type: The Arrow DataType to convert. + + Returns: + A PostgreSQL SQL type string (e.g. ``"TEXT"``, ``"BIGINT"``). + + Raises: + ValueError: If PostgreSQL has no column type here for ``arrow_type``: + array columns (``pa.large_list(...)``) are read-only from existing + tables, and there is no fallback to TEXT for any other type + (ITL-1162). + """ + import pyarrow as _pa + + sql_type = _pg_sql_type(arrow_type) + if sql_type is not None: + return sql_type if _pa.types.is_large_list(arrow_type) or _pa.types.is_list(arrow_type): raise ValueError( f"Arrow type {arrow_type!r} (list/array) is not supported for " "CREATE TABLE. Array columns can only be read from existing tables." ) - logger.warning("Unsupported Arrow type %r; mapping to TEXT", arrow_type) - return "TEXT" + raise ValueError( + f"Arrow type {arrow_type!r} is not supported for CREATE TABLE: " + "PostgreSQL has no column type for it here." + ) + + +def _pg_unstorable_reason(arrow_type: pa.DataType) -> str | None: + """Say why PostgreSQL cannot store a column of ``arrow_type``, or ``None`` when it can. + + A type is stored when the column type it is written as reads back as the same + type in its large variant (``_pg_type_to_arrow``): ``bool``, ``int32``, + ``int64``, ``float``, ``double``, ``large_string``, ``large_binary``, + ``date32``, ``timestamp[us]`` and ``timestamp[us, tz=UTC]``. + + Args: + arrow_type: The column's Arrow type. + + Returns: + A short reason, or ``None`` when PostgreSQL stores the type. + """ + from orcapod.utils.arrow_utils import normalize_to_large_types + + sql_type = _pg_sql_type(arrow_type) + if sql_type is None: + return "PostgreSQL has no column type for it here" + read_back = _pg_type_to_arrow(sql_type.lower(), sql_type.lower()) + if read_back != normalize_to_large_types(arrow_type): + return f"PostgreSQL stores it as {sql_type}, which reads back as {read_back}" + return None def _coerce_pg_value(value: Any, arrow_type: "pa.DataType") -> Any: @@ -527,6 +575,29 @@ def upsert_records( conn.rollback() raise + def unstorable_column_reason( + self, name: str, arrow_type: pa.DataType, *, primary_key: bool + ) -> str | None: + """Say why PostgreSQL cannot store a column, or ``None`` when it can (ITL-1162). + + PostgreSQL stores a column only when it reads it back as written: + ``bool``, ``int32``, ``int64``, ``float``, ``double``, ``large_string``, + ``large_binary``, ``date32``, ``timestamp[us]`` and + ``timestamp[us, tz=UTC]`` (``string`` and ``binary`` read back in their + large variants). Other integer widths, and timestamps of another unit or + time zone, read back as another type, and lists, structs and extension types have no column type here. + The name and the key role do not matter to PostgreSQL. + + Args: + name: The column's name. + arrow_type: The column's Arrow type. + primary_key: Whether the column is the table's primary key. + + Returns: + A short reason, or ``None`` when PostgreSQL stores the column. + """ + return _pg_unstorable_reason(arrow_type) + def validate_records(self, records: pa.Table) -> None: """Reject tables carrying any Arrow field or schema metadata. diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index 68c39783c..3e570c7a5 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -947,6 +947,25 @@ def write_rows(rows: pa.Table) -> None: if len(novel) > 0: write_rows(novel) + def unstorable_column_reason( + self, name: str, arrow_type: pa.DataType, *, primary_key: bool + ) -> str | None: + """Say why SpiralDB cannot store a column, or ``None`` when it can (ITL-1162). + + SpiralDB stores Arrow tables natively, so no column is refused here yet. + Its column-name rules (a reserved ``__`` prefix on non-key columns, + ITL-1164) are not stated here yet. + + Args: + name: The column's name. + arrow_type: The column's Arrow type. + primary_key: Whether the column is the table's primary key. + + Returns: + Always ``None``. + """ + return None + def validate_records(self, records: pa.Table) -> None: """No-op: ``SpiralDBConnector`` preserves Arrow field metadata via the native KV store.""" diff --git a/src/orcapod/databases/sqlite_connector.py b/src/orcapod/databases/sqlite_connector.py index bd0e4253f..484bd26f7 100644 --- a/src/orcapod/databases/sqlite_connector.py +++ b/src/orcapod/databases/sqlite_connector.py @@ -65,14 +65,16 @@ def _sqlite_type_to_arrow(declared_type: str) -> pa.DataType: return pa.float64() -def _arrow_type_to_sqlite_sql(arrow_type: pa.DataType) -> str: - """Map an Arrow DataType to a SQLite SQL type string for CREATE TABLE. +def _sqlite_sql_type(arrow_type: pa.DataType) -> str | None: + """Return the SQLite column type for an Arrow type, or ``None`` when there is none. Args: arrow_type: The Arrow DataType to convert. Returns: - A SQLite SQL type string (e.g. "INTEGER", "TEXT"). + A SQLite SQL type string (e.g. "INTEGER", "TEXT"), or ``None`` for a type + SQLite has no column type for (lists, structs, dates, timestamps, extension + types and the rest). """ import pyarrow as _pa # noqa: PLC0415 — needed at call time @@ -86,8 +88,51 @@ def _arrow_type_to_sqlite_sql(arrow_type: pa.DataType) -> str: return "TEXT" if _pa.types.is_binary(arrow_type) or _pa.types.is_large_binary(arrow_type): return "BLOB" - logger.warning("Unsupported Arrow type %r; mapping to TEXT", arrow_type) - return "TEXT" + return None + + +def _arrow_type_to_sqlite_sql(arrow_type: pa.DataType) -> str: + """Map an Arrow DataType to a SQLite SQL type string for CREATE TABLE. + + Args: + arrow_type: The Arrow DataType to convert. + + Returns: + A SQLite SQL type string (e.g. "INTEGER", "TEXT"). + + Raises: + ValueError: If SQLite has no column type for ``arrow_type``. There is no + fallback to TEXT: a value of such a type would fail to bind or read + back as text (ITL-1162). + """ + sql_type = _sqlite_sql_type(arrow_type) + if sql_type is None: + raise ValueError(f"SQLite has no column type for Arrow type {arrow_type}") + return sql_type + + +def _sqlite_unstorable_reason(arrow_type: pa.DataType) -> str | None: + """Say why SQLite cannot store a column of ``arrow_type``, or ``None`` when it can. + + A type is stored when the column type it is written as reads back as the same + type in its large variant: ``bool``, ``int64``, ``double``, ``large_string`` + and ``large_binary`` (and ``string`` / ``binary``, read back large). + + Args: + arrow_type: The column's Arrow type. + + Returns: + A short reason, or ``None`` when SQLite stores the type. + """ + from orcapod.utils.arrow_utils import normalize_to_large_types # noqa: PLC0415 + + sql_type = _sqlite_sql_type(arrow_type) + if sql_type is None: + return "SQLite has no column type for it" + read_back = _sqlite_type_to_arrow(sql_type) + if read_back != normalize_to_large_types(arrow_type): + return f"SQLite stores it as {sql_type}, which reads back as {read_back}" + return None def _coerce_column(values: list[Any], arrow_type: pa.DataType) -> list[Any]: @@ -391,6 +436,28 @@ def upsert_records( rows = (tuple(row[c] for c in cols) for row in records.to_pylist()) conn.executemany(sql, rows) + def unstorable_column_reason( + self, name: str, arrow_type: pa.DataType, *, primary_key: bool + ) -> str | None: + """Say why SQLite cannot store a column, or ``None`` when it can (ITL-1162). + + SQLite stores a column only when it reads it back as written: + ``bool``, ``int64``, ``double``, ``large_string`` and ``large_binary`` + (``string`` and ``binary`` read back in their large variants). Narrower + integers and floats read back widened, and lists, structs, dates, + timestamps and extension types have no SQLite column type. The name and + the key role do not matter to SQLite. + + Args: + name: The column's name. + arrow_type: The column's Arrow type. + primary_key: Whether the column is the table's primary key. + + Returns: + A short reason, or ``None`` when SQLite stores the column. + """ + return _sqlite_unstorable_reason(arrow_type) + def validate_records(self, records: pa.Table) -> None: """Reject tables carrying any Arrow field or schema metadata. diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index e3d1fef1e..4a8f7833f 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -76,6 +76,66 @@ class UnsupportedValueError(ValueError): """ +class UnstorableColumnError(ValueError): + """A database cannot store a column of a table it is asked to hold (ITL-1162). + + Raised by a database whose backend states which columns it stores faithfully (a + ``ConnectorArrowDatabase``: the SQLite and PostgreSQL connectors), before + anything of the table is written. A job raises it when the database is attached + to a node whose stored tables hold such a column, before anything runs. + + Attributes: + database_kind: The kind of database that refused, e.g. ``"SQLiteConnector"``. + column: Name of the column. + arrow_type: The column's Arrow type. + reason: Why the database cannot store it. + node_label: Label of the node whose table holds the column, or ``None`` when + the refusal is not tied to one node. + """ + + def __init__( + self, + *, + database_kind: str, + column: str, + arrow_type: object, + reason: str, + node_label: str | None = None, + ) -> None: + """Create the error. + + Args: + database_kind: The kind of database that refused. + column: Name of the column. + arrow_type: The column's Arrow type. + reason: Why the database cannot store it. + node_label: Label of the node whose table holds the column, if known. + """ + self.database_kind = database_kind + self.column = column + self.arrow_type = arrow_type + self.reason = reason + self.node_label = node_label + super().__init__(database_kind, column, arrow_type, reason) + + def add_context(self, node_label: str | None = None) -> None: + """Fill in the node label where the raising site did not know it. + + Args: + node_label: Label of the node whose table holds the column. + """ + if self.node_label is None: + self.node_label = node_label + + def __str__(self) -> str: + node = f"node {self.node_label!r}: " if self.node_label is not None else "" + return ( + f"{node}a {self.database_kind} database cannot store column " + f"{self.column!r} of Arrow type {self.arrow_type}: {self.reason}. " + "Use an InMemoryArrowDatabase or a DeltaTableDatabase instead." + ) + + class MissingOptionalDependencyError(ImportError): """A stored value needs an optional dependency that is not installed. diff --git a/src/orcapod/protocols/database_protocols.py b/src/orcapod/protocols/database_protocols.py index fb585495c..a8c15765b 100644 --- a/src/orcapod/protocols/database_protocols.py +++ b/src/orcapod/protocols/database_protocols.py @@ -227,6 +227,21 @@ def flush_batch(self, record_path: tuple[str, ...]) -> None: ... +class ColumnCheckingDatabaseProtocol(Protocol): + """A database that states which columns it can store (ITL-1162). + + Such a database (``ConnectorArrowDatabase``) refuses a table with a column its + backend cannot store faithfully, before anything of it is written. A job node + asks it about its stored tables when the database is attached, so a pipeline + the database cannot hold is refused before anything runs. Check with + ``orcapod.contexts.stored.require_storable``. + """ + + def require_storable(self, schema: pa.Schema) -> None: + """Raise ``UnstorableColumnError`` for a column of ``schema`` it cannot store.""" + ... + + @runtime_checkable class DatabaseRegistryProtocol(Protocol): """Protocol for a database config registry used during pipeline save/load. diff --git a/src/orcapod/protocols/db_connector_protocol.py b/src/orcapod/protocols/db_connector_protocol.py index 7cd0ee8d3..45bd94d6d 100644 --- a/src/orcapod/protocols/db_connector_protocol.py +++ b/src/orcapod/protocols/db_connector_protocol.py @@ -118,6 +118,28 @@ def upsert_records( """ ... + def unstorable_column_reason( + self, name: str, arrow_type: pa.DataType, *, primary_key: bool + ) -> str | None: + """Say why this connector cannot store a column, or ``None`` when it can. + + The one place a connector states which columns it stores (ITL-1162). A + connector stores a column only when it reads it back as written: the same + Arrow type, in its large variant (``large_string``, ``large_binary``). A + type the backend would widen, turn into text or fail to bind is refused. + ``ConnectorArrowDatabase.require_storable`` asks this for every column of + a table before anything of it is written. + + Args: + name: The column's name. + arrow_type: The column's Arrow type. + primary_key: Whether the column is the table's primary key. + + Returns: + A short reason, or ``None`` when the connector stores the column. + """ + ... + def validate_records(self, records: pa.Table) -> None: """Validate that ``records`` are safe to write through this connector. diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index fad8d7f1a..314abf631 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -37,12 +37,14 @@ OrcapodDatabaseError, OrcapodDatabaseReadError, OrcapodDatabaseWriteError, + UnstorableColumnError, ) from orcapod.contexts.stored import ( StaleTables, account_for_stale_tables, find_stale_tables, require_same_data_context, + require_storable, with_data_context, ) from orcapod.invocation import ( @@ -470,6 +472,22 @@ def _execute_side_effect_row( return (tag, data) +def _invocation_log_schema() -> pa.Schema: + """The columns of an invocation log row, before ``__data_context`` is appended.""" + return pa.schema( + [ + pa.field("record_id_hash", pa.large_binary()), + pa.field("pipeline_run_id", pa.large_string()), + pa.field("executed_at", pa.timestamp("us", tz="UTC")), + ] + ) + + +def _stored_invocation_log_schema() -> pa.Schema: + """The columns of an invocation log row as stored, ``__data_context`` included.""" + return with_data_context(_invocation_log_schema().empty_table(), "").schema + + def _write_invocation_row( *, pipeline_database: ArrowDatabaseProtocol, @@ -497,15 +515,15 @@ def _write_invocation_row( ``record_id``; stored on the row as ``__data_context``. """ executed_at = datetime.datetime.now(datetime.timezone.utc) - record = pa.table( - { - "record_id_hash": pa.array([record_id_hash_bytes], type=pa.large_binary()), - "pipeline_run_id": pa.array([run_id], type=pa.large_string()), - "executed_at": pa.array( - [executed_at], - type=pa.timestamp("us", tz="UTC"), - ), - } + record = pa.Table.from_pylist( + [ + { + "record_id_hash": record_id_hash_bytes, + "pipeline_run_id": run_id, + "executed_at": executed_at, + } + ], + schema=_invocation_log_schema(), ) try: pipeline_database.add_record( @@ -1125,6 +1143,14 @@ def attach_databases( pipeline_database: Pre-scoped pipeline DB (at pipeline root), or ``None`` to detach. """ + if pipeline_database is not None: + # A database that cannot store the invocation log refuses it now, + # before anything is delivered (ITL-1162). + try: + require_storable(pipeline_database, _stored_invocation_log_schema()) + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise self._pipeline_database = pipeline_database self._unlisted_store_reported = False self._current_log_checked = False diff --git a/tests/test_core/sources/test_db_table_source.py b/tests/test_core/sources/test_db_table_source.py index c5e1ba8f5..d0ec67946 100644 --- a/tests/test_core/sources/test_db_table_source.py +++ b/tests/test_core/sources/test_db_table_source.py @@ -77,6 +77,11 @@ def create_table_if_not_exists(self, *args: Any, **kwargs: Any) -> None: def upsert_records(self, *args: Any, **kwargs: Any) -> None: pass # not used by DBTableSource + def unstorable_column_reason( + self, name: str, arrow_type: Any, *, primary_key: bool + ) -> str | None: + return None # not used by DBTableSource + def validate_records(self, records: Any) -> None: pass # not used by DBTableSource diff --git a/tests/test_databases/test_connector_arrow_database.py b/tests/test_databases/test_connector_arrow_database.py index f2d83f3f6..47ce59548 100644 --- a/tests/test_databases/test_connector_arrow_database.py +++ b/tests/test_databases/test_connector_arrow_database.py @@ -155,6 +155,12 @@ def upsert_records( kept = existing.filter(mask) self._tables[table_name] = pa.concat_tables([kept, records]) + def unstorable_column_reason( + self, name: str, arrow_type: pa.DataType, *, primary_key: bool + ) -> str | None: + """Store every column: this in-memory mock keeps Arrow tables as they are.""" + return None + def validate_records(self, records: pa.Table) -> None: """Reject any Arrow field or schema metadata — mirrors real SQL connectors.""" problem_fields = [f.name for f in records.schema if _field_has_metadata(f)] diff --git a/tests/test_databases/test_postgresql_connector.py b/tests/test_databases/test_postgresql_connector.py index 41045c3c0..545d7b096 100644 --- a/tests/test_databases/test_postgresql_connector.py +++ b/tests/test_databases/test_postgresql_connector.py @@ -139,10 +139,15 @@ def test_large_list_raises(self): with pytest.raises(ValueError, match="not supported"): _arrow_type_to_pg_sql(pa.large_list(pa.int32())) - def test_unknown_falls_back_to_text(self): - # pa.null() is an unusual type not in the mapping - result = _arrow_type_to_pg_sql(pa.null()) - assert result == "TEXT" + def test_unknown_raises_instead_of_text(self): + """No silent TEXT fallback (ITL-1162).""" + for arrow_type in (pa.null(), pa.struct([("a", pa.int64())])): + with pytest.raises(ValueError, match="not supported"): + _arrow_type_to_pg_sql(arrow_type) + + def test_boolean_information_schema_name(self): + """``information_schema`` names a bool column ``boolean``.""" + assert _pg_type_to_arrow("boolean", "boolean") == pa.bool_() class TestCoercePgValue: diff --git a/tests/test_databases/test_sqlite_connector.py b/tests/test_databases/test_sqlite_connector.py index 729706111..b3fc9c908 100644 --- a/tests/test_databases/test_sqlite_connector.py +++ b/tests/test_databases/test_sqlite_connector.py @@ -93,13 +93,15 @@ def test_binary(self): def test_bool(self): assert _arrow_type_to_sqlite_sql(pa.bool_()) == "BOOLEAN" - def test_unsupported_type_logs_warning_and_returns_text(self, caplog): - import logging - - with caplog.at_level(logging.WARNING): - result = _arrow_type_to_sqlite_sql(pa.list_(pa.int64())) - assert result == "TEXT" - assert "Unsupported Arrow type" in caplog.text + def test_unsupported_type_raises_instead_of_text(self): + """No silent TEXT fallback: such a value would fail to bind (ITL-1162).""" + for arrow_type in ( + pa.list_(pa.int64()), + pa.timestamp("us", tz="UTC"), + pa.date32(), + ): + with pytest.raises(ValueError, match="no column type"): + _arrow_type_to_sqlite_sql(arrow_type) class TestSQLiteConnectorScaffold: diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py new file mode 100644 index 000000000..ff8a6fcb1 --- /dev/null +++ b/tests/test_databases/test_unstorable_columns.py @@ -0,0 +1,352 @@ +"""A connector database refuses a table it cannot store, before anything is written (ITL-1162). + +Each connector states which columns it stores: those it reads back as written +(``unstorable_column_reason``). ``ConnectorArrowDatabase`` refuses any other column +before anything of the table is buffered, and a job refuses a function or +side-effect node whose stored table holds one when the database is attached, before +anything runs. A function node's result table always holds list-of-struct executor +columns and a UTC timestamp, so no function node runs on SQLite or PostgreSQL. +""" + +from __future__ import annotations + +import datetime +import pathlib +from typing import Any +from unittest.mock import MagicMock, patch + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import SelectDataColumns +from orcapod.databases import ( + ConnectorArrowDatabase, + InMemoryArrowDatabase, + PostgreSQLConnector, + SQLiteConnector, +) +from orcapod.errors import OrcapodDatabaseWriteError, UnstorableColumnError +from orcapod.side_effects import SideEffectPod, SideEffectPodConfig +from orcapod.sources import ArrowTableSource, DictSource +from orcapod.types import CacheMode + +CALLS: list[int] = [] +DELIVERED: list[int] = [] + +EXECUTOR_INFO = "__pf_exec_executor_info" +EXECUTOR_INFO_TYPE = pa.large_list( + pa.struct([("key", pa.large_string()), ("value", pa.large_string())]) +) + + +@pytest.fixture(autouse=True) +def _reset() -> None: + """Start every test with no recorded calls or deliveries.""" + CALLS.clear() + DELIVERED.clear() + + +@function_pod(output_keys="double") +def double(x: int) -> int: + """Double ``x``, recording the call.""" + CALLS.append(x) + return 2 * x + + +def _deliver(x: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(x) + + +deliver = SideEffectPod(_deliver, SideEffectPodConfig(track_completion=True)) + + +def _source(rows: list[dict[str, Any]] | None = None) -> DictSource: + """A small in-memory source with tag ``k``.""" + return DictSource( + data=rows if rows is not None else [{"k": "a", "x": 1}, {"k": "b", "x": 2}], + tag_columns=["k"], + tag_only_columns=True, + source_id="src", + ) + + +def _sqlite_database(tmp_path: pathlib.Path) -> ConnectorArrowDatabase: + return ConnectorArrowDatabase(SQLiteConnector(str(tmp_path / "store.db"))) + + +# --------------------------------------------------------------------------- +# A job refuses a node up front +# --------------------------------------------------------------------------- + + +class TestJobRefusesUpFront: + def test_function_node_on_sqlite_refused_before_anything_runs( + self, tmp_path: pathlib.Path + ): + """The reported case: one function node on a SQLite job database.""" + database = _sqlite_database(tmp_path) + job = PipelineJob(name="p", database=database) + + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = double.pod(_source(), label="dbl") + + err = exc_info.value + assert err.node_label == "dbl" + assert err.database_kind == "SQLiteConnector" + assert err.column == EXECUTOR_INFO + assert err.arrow_type == EXECUTOR_INFO_TYPE + message = str(err) + assert "node 'dbl'" in message + assert "SQLiteConnector" in message + assert EXECUTOR_INFO in message + assert "InMemoryArrowDatabase" in message + assert "DeltaTableDatabase" in message + # Nothing ran and nothing was written. + assert CALLS == [] + assert database.list_table_paths() == [] + + def test_side_effect_node_on_sqlite_refused_before_delivery( + self, tmp_path: pathlib.Path + ): + """The invocation log's UTC timestamp is refused before anything is delivered.""" + database = _sqlite_database(tmp_path) + job = PipelineJob(name="p", database=database) + + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = deliver.process(_source(), label="out") + + err = exc_info.value + assert err.node_label == "out" + assert err.column == "executed_at" + assert err.arrow_type == pa.timestamp("us", tz="UTC") + assert DELIVERED == [] + assert database.list_table_paths() == [] + + def test_function_node_on_postgresql_refused(self): + """PostgreSQL has no column type for the list-of-struct executor columns.""" + with patch("psycopg.connect") as mock_connect: + mock_connect.return_value = MagicMock() + connector = PostgreSQLConnector("postgresql://localhost/test") + try: + job = PipelineJob(name="p", database=ConnectorArrowDatabase(connector)) + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = double.pod(_source(), label="dbl") + finally: + connector._conn = None # nothing to close + err = exc_info.value + assert err.node_label == "dbl" + assert err.database_kind == "PostgreSQLConnector" + assert err.column == EXECUTOR_INFO + assert CALLS == [] + + def test_in_memory_database_still_runs_function_node(self): + """A database that states no rule stores every column.""" + job = PipelineJob(name="p", database=InMemoryArrowDatabase()) + with job: + _ = double.pod(_source(), label="dbl") + _ = job.run() + assert sorted(CALLS) == [1, 2] + + +# --------------------------------------------------------------------------- +# The write itself refuses, before anything is buffered +# --------------------------------------------------------------------------- + + +UNSTORABLE_ON_SQLITE = [ + pytest.param(pa.int8(), [1], id="int8"), + pytest.param(pa.int32(), [1], id="int32"), + pytest.param(pa.uint64(), [1], id="uint64"), + pytest.param(pa.float32(), [1.5], id="float32"), + pytest.param(pa.date32(), [datetime.date(2026, 1, 1)], id="date32"), + pytest.param( + pa.timestamp("us", tz="UTC"), + [datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)], + id="timestamp_utc", + ), + pytest.param( + pa.timestamp("us"), [datetime.datetime(2026, 1, 1)], id="timestamp_naive" + ), + pytest.param(pa.large_list(pa.int64()), [[1]], id="large_list"), + pytest.param(EXECUTOR_INFO_TYPE, [[{"key": "a", "value": "b"}]], id="list_struct"), + pytest.param(pa.struct([("a", pa.int64())]), [{"a": 1}], id="struct"), + pytest.param(pa.null(), [None], id="null"), +] + +STORABLE_ON_SQLITE = [ + pytest.param(pa.bool_(), [True], id="bool"), + pytest.param(pa.int64(), [1], id="int64"), + pytest.param(pa.float64(), [1.5], id="float64"), + pytest.param(pa.string(), ["a"], id="string"), + pytest.param(pa.large_string(), ["a"], id="large_string"), + pytest.param(pa.binary(), [b"a"], id="binary"), + pytest.param(pa.large_binary(), [b"a"], id="large_binary"), +] + + +class TestWriteRefuses: + @pytest.mark.parametrize("arrow_type, values", UNSTORABLE_ON_SQLITE) + def test_sqlite_refuses_column_before_buffering( + self, tmp_path: pathlib.Path, arrow_type, values + ): + database = _sqlite_database(tmp_path) + table = pa.table({"v": pa.array(values, type=arrow_type)}) + + with pytest.raises(UnstorableColumnError) as exc_info: + database.add_record(("t",), b"1", table) + + err = exc_info.value + assert err.column == "v" + assert err.arrow_type == arrow_type + assert err.node_label is None + assert str(err).startswith("a SQLiteConnector database cannot store column") + # Nothing was buffered, so nothing is written or read back. + database.flush() + assert database.get_all_records(("t",)) is None + assert database.list_table_paths() == [] + + @pytest.mark.parametrize("arrow_type, values", STORABLE_ON_SQLITE) + def test_sqlite_stores_and_reads_back( + self, tmp_path: pathlib.Path, arrow_type, values + ): + """What SQLite accepts reads back as written, and a second write succeeds.""" + from orcapod.utils.arrow_utils import normalize_to_large_types + + database = _sqlite_database(tmp_path) + table = pa.table({"v": pa.array(values, type=arrow_type)}) + database.add_record(("t",), b"1", table, flush=True) + database.add_record(("t",), b"2", table, flush=True) + + stored = database.get_all_records(("t",)) + assert stored is not None + assert stored.schema.field("v").type == normalize_to_large_types(arrow_type) + assert stored["v"].to_pylist() == values * 2 + + def test_operator_cache_with_storable_columns_runs_on_sqlite( + self, tmp_path: pathlib.Path + ): + """Operator caches hold only the operator's columns plus binary/text ones.""" + database = _sqlite_database(tmp_path) + job = PipelineJob(name="p", database=database) + with job: + _ = SelectDataColumns(["x"])(_source(), label="sel") + job.nodes["sel"].attach_databases( + pipeline_database=database.at("p"), cache_mode=CacheMode.LOG + ) + _ = job.run() + job.nodes["sel"].clear_cache() + _ = job.run() # the second write meets the stored table's read-back types + assert sorted(job.nodes["sel"].as_table()["x"].to_pylist()) == [1, 2] + + def test_operator_cache_with_unstorable_column_refused_at_write( + self, tmp_path: pathlib.Path + ): + """An operator cache holding an int32 column is refused when it is written.""" + database = _sqlite_database(tmp_path) + source = ArrowTableSource( + pa.table( + { + "k": pa.array(["a"], type=pa.large_string()), + "x": pa.array([1], type=pa.int32()), + } + ), + tag_columns=["k"], + tag_only_columns=True, + source_id="src", + ) + job = PipelineJob(name="p", database=database) + with job: + _ = SelectDataColumns(["x"])(source, label="sel") + job.nodes["sel"].attach_databases( + pipeline_database=database.at("p"), cache_mode=CacheMode.LOG + ) + + with pytest.raises(OrcapodDatabaseWriteError) as exc_info: + _ = job.run() + + assert exc_info.value.node_label == "sel" + cause = exc_info.value.__cause__ + assert isinstance(cause, UnstorableColumnError) + assert cause.column == "x" + assert cause.arrow_type == pa.int32() + assert database.list_table_paths() == [] + + +# --------------------------------------------------------------------------- +# The rule each connector states +# --------------------------------------------------------------------------- + + +class TestConnectorRule: + def test_sqlite_rule(self): + connector = SQLiteConnector(":memory:") + try: + for arrow_type in ( + pa.bool_(), + pa.int64(), + pa.float64(), + pa.string(), + pa.large_string(), + pa.binary(), + pa.large_binary(), + ): + assert ( + connector.unstorable_column_reason( + "c", arrow_type, primary_key=False + ) + is None + ), arrow_type + reason = connector.unstorable_column_reason( + "c", pa.int32(), primary_key=False + ) + assert reason == "SQLite stores it as INTEGER, which reads back as int64" + reason = connector.unstorable_column_reason( + "c", pa.timestamp("us", tz="UTC"), primary_key=False + ) + assert reason == "SQLite has no column type for it" + finally: + connector.close() + + @pytest.mark.parametrize( + "arrow_type, storable", + [ + (pa.bool_(), True), + (pa.int8(), False), + (pa.int16(), False), + (pa.int32(), True), + (pa.int64(), True), + (pa.uint32(), False), + (pa.float32(), True), + (pa.float64(), True), + (pa.string(), True), + (pa.large_string(), True), + (pa.binary(), True), + (pa.large_binary(), True), + (pa.date32(), True), + (pa.timestamp("us"), True), + (pa.timestamp("us", tz="UTC"), True), + (pa.timestamp("ns", tz="UTC"), False), + (pa.timestamp("us", tz="America/New_York"), False), + (pa.large_list(pa.int64()), False), + (EXECUTOR_INFO_TYPE, False), + (pa.struct([("a", pa.int64())]), False), + (pa.null(), False), + ], + ) + def test_postgresql_rule(self, arrow_type, storable): + """By code reading: the type it writes reads back as the same type.""" + with patch("psycopg.connect") as mock_connect: + mock_connect.return_value = MagicMock() + connector = PostgreSQLConnector("postgresql://localhost/test") + try: + reason = connector.unstorable_column_reason( + "c", arrow_type, primary_key=False + ) + finally: + connector._conn = None + assert (reason is None) is storable, reason diff --git a/tests/test_pipeline/test_store_write_failure.py b/tests/test_pipeline/test_store_write_failure.py index fb9a7cfe8..d02388ffe 100644 --- a/tests/test_pipeline/test_store_write_failure.py +++ b/tests/test_pipeline/test_store_write_failure.py @@ -20,7 +20,7 @@ from orcapod import PipelineJob, function_pod from orcapod.channels import Channel from orcapod.core.operators import SelectDataColumns -from orcapod.errors import OrcapodDatabaseWriteError +from orcapod.errors import OrcapodDatabaseWriteError, UnstorableColumnError from orcapod.databases import ( ConnectorArrowDatabase, InMemoryArrowDatabase, @@ -460,24 +460,24 @@ def test_final_flush_failure_warns_pending_output(self, orchestrator_cls): # --------------------------------------------------------------------------- -# The reported trigger: a logical-type output on SQLite (ITL-877 is the limit itself) +# The reported trigger: a logical-type output on SQLite (ITL-877 is the limit itself). +# Since ITL-1162 a SQLite database refuses every function node when the job is built. # --------------------------------------------------------------------------- class TestSQLiteTrigger: - """The reported trigger: a logical-type output on SQLite.""" + """The reported trigger: a function node on SQLite, refused before it runs.""" - def test_logical_type_output_on_sqlite_raises(self, orchestrator_cls, tmp_path): - """A ``pathlib.Path`` output on SQLite raises, naming the node.""" + def test_function_node_on_sqlite_refused_naming_the_node(self, tmp_path): + """The result table's executor columns cannot be stored; the node is named.""" store = ConnectorArrowDatabase(SQLiteConnector(str(tmp_path / "store.db"))) - job = build_job(store, ROWS, pod=to_path) - with pytest.raises(OrcapodDatabaseWriteError) as exc_info: - job.run(orchestrator=orchestrator_cls()) + with pytest.raises(UnstorableColumnError) as exc_info: + build_job(store, ROWS, pod=to_path) assert exc_info.value.node_label == "dbl" assert "node 'dbl'" in str(exc_info.value) - assert "does not preserve Arrow metadata" in str(exc_info.value) + assert calls == [] # --------------------------------------------------------------------------- From 859a5bfe43c476e3c3cf3685ed1cefa04a236dc4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 11:25:47 -0800 Subject: [PATCH 02/14] fix(databases): refuse SpiralDB reserved names; check every stored table up front (ITL-1162, ITL-1164) SpiralDBConnector states its name rule through unstorable_column_reason: a value column named with a '__' prefix, or any column whose name contains '.', is refused (developer ruling, 2026-11-16); the pyspiral fake refuses the same names on write with SpiralDB's messages. A function node now also checks its pipeline records (one schema definition for writer and check), an operator node with caching on and a cached source check their bookkeeping columns, and ExtensionAwareDatabase forwards the check to the database it wraps (CodeRabbit CLI findings on f430c4c5). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1162.md | 3 +- design_issues/ITL-1162-CA3.md | 18 +- docs/concepts/pipeline-job.md | 35 ++- src/orcapod/contexts/stored.py | 7 +- src/orcapod/core/cached_function_pod.py | 25 +- src/orcapod/core/nodes/function_node.py | 81 ++++--- src/orcapod/core/nodes/operator_node.py | 22 ++ src/orcapod/core/sources/cached_source.py | 11 + .../databases/extension_aware_database.py | 15 ++ src/orcapod/databases/spiraldb_connector.py | 13 +- src/orcapod/errors.py | 7 +- .../protocols/db_connector_protocol.py | 5 +- src/orcapod/side_effects.py | 3 +- tests/test_databases/fake_pyspiral.py | 15 +- .../test_databases/test_unstorable_columns.py | 213 ++++++++++++++++++ 15 files changed, 401 insertions(+), 72 deletions(-) diff --git a/changelog.d/ITL-1162.md b/changelog.d/ITL-1162.md index 6a0564364..4ea470e09 100644 --- a/changelog.d/ITL-1162.md +++ b/changelog.d/ITL-1162.md @@ -1,7 +1,8 @@ ### Fixed - A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` now refuses a table it cannot store before anything of it is written, with `UnstorableColumnError` naming the connector, the column, its Arrow type and what to use instead (ITL-1162). A connector stores a column only when it reads it back as written: on SQLite `bool`, `int64`, `double`, `large_string` and `large_binary`; on PostgreSQL also `int32`, `float`, `date32`, `timestamp[us]` and `timestamp[us, tz=UTC]`. Before, a function node on such a database ran its function and then failed at commit (`sqlite3.ProgrammingError: ... type 'list' is not supported`), timestamps were stored as text and read back as strings, and narrower integers and floats read back widened, failing the next write. A job now refuses a node whose stored table such a database cannot hold (every function node; a side-effect node on SQLite) when the job is built, before anything runs, naming the node. The connectors no longer fall back to a `TEXT` column for a type they have no column type for. +- A `ConnectorArrowDatabase` over `SpiralDBConnector` now refuses, by the same rule, a column SpiralDB cannot store: a value column whose name starts with `__` (the `__record_id` key is allowed) or any column whose name contains `.` (ITL-1164). Before, a function node on such a database ran and then failed at commit (`Column name '__input_data_hash' has reserved prefix`). A job refuses every function node, side-effect node, operator cache and cached source on such a database when it is built. ### Known limitations -- No function node can run on a SQL connector job database (SQLite or PostgreSQL), and no side-effect node on SQLite: a function node's result table holds list-of-struct executor columns, which neither connector stores, and a side-effect log holds a UTC timestamp, which SQLite does not store. Use `InMemoryArrowDatabase` or `DeltaTableDatabase`. Storing such columns on SQL connectors is planned for a later release (ITL-1162). +- A connector job database cannot hold function nodes in v0.3.0: neither SQL connector (SQLite, PostgreSQL) nor SpiralDB. A function node's result table holds list-of-struct executor columns, which neither SQL connector stores, and `__`-prefixed value columns, which SpiralDB reserves (ITL-1164). A side-effect node is refused on SQLite (its log's UTC timestamp) and on SpiralDB, and an operator cache or cached source on SpiralDB (`__data_context`). Use a Delta database (`DeltaTableDatabase`) or `InMemoryArrowDatabase`. Storing such columns on connector databases is planned for a later release (ITL-1162, ITL-1164). diff --git a/design_issues/ITL-1162-CA3.md b/design_issues/ITL-1162-CA3.md index 18cf178fb..966a003a5 100644 --- a/design_issues/ITL-1162-CA3.md +++ b/design_issues/ITL-1162-CA3.md @@ -1,6 +1,6 @@ ## `src/orcapod/databases/connector_arrow_database.py` -### CA3 — SQL connector databases cannot store list, struct and (on SQLite) timestamp columns +### CA3 — Connector databases cannot store Orcapod's own stored tables (list, struct and timestamp columns; SpiralDB reserved names) **Status:** open **Severity:** high **Issue:** ITL-1162 @@ -13,7 +13,9 @@ at commit, after the function had run. The same table's `__pod_ts` (`timestamp[u and a side-effect log's `executed_at` were stored as TEXT and read back as `large_string`. `int8`, `int32` and `float32` columns read back widened, so the second write to the table failed its schema comparison. PostgreSQL raises for list columns at `CREATE TABLE` and has no -column type for structs. +column type for structs. SpiralDB stores every type but refuses value columns named with a +`__` prefix and any column name containing `.` (ITL-1164), so it cannot hold a function +result table (`__input_data_hash`, ...) or any table's `__data_context`. **Fix (refusal; developer ruling, November 16, 2026):** each connector states which columns it stores, in one place (`DBConnectorProtocol.unstorable_column_reason(name, arrow_type, @@ -22,9 +24,13 @@ variant. `ConnectorArrowDatabase.require_storable` raises `UnstorableColumnError column, Arrow type, reason, what to use instead) in `add_records`, before anything is buffered. A function job node (its result table) and a side-effect job node (its invocation log) ask the database when it is attached, so a job refuses them before anything runs, naming -the node. The connectors' silent TEXT fallback in `CREATE TABLE` now raises. Operator caches -and cached sources are refused at their write when a user column is unstorable. +the node. An operator job node with a cache and a cached source ask about their bookkeeping +columns (`__node_content_hash`, `__data_context`) when attached or built; their user columns +are checked at the write. `SpiralDBConnector` states its name rule the same way (developer +ruling, November 16, 2026). The connectors' silent TEXT fallback in `CREATE TABLE` now +raises. **Open (the real fix, a later release):** store list and struct columns (for example as JSON -text read back as the same Arrow type) and timestamps on SQLite, so a function node can run -on a SQL connector database. +text read back as the same Arrow type) and timestamps on SQLite, and rename SpiralDB's +reserved column names on write and recover them on read (ITL-1164), so a function node can +run on a connector database. diff --git a/docs/concepts/pipeline-job.md b/docs/concepts/pipeline-job.md index 415222076..6ea2b6ef5 100644 --- a/docs/concepts/pipeline-job.md +++ b/docs/concepts/pipeline-job.md @@ -120,24 +120,37 @@ run and computes nothing on a second `run()`. A job created without a database can be recorded but not run. Give it one later with `job.bind(database=...)`; `run()` without a database raises `ValueError`. -### SQL connector databases +### Connector databases -A `ConnectorArrowDatabase` over a `SQLiteConnector` or a `PostgreSQLConnector` stores a -column only when the connector reads it back as written, with the same Arrow type (text and -binary columns come back as `large_string` and `large_binary`, the types Orcapod uses): +A `ConnectorArrowDatabase` stores a column only when its connector can store it as written. +Each connector states which columns those are, and anything else is refused with +`UnstorableColumnError` before anything of the table is written. + +The SQL connectors store a column only when they read it back with the same Arrow type (text +and binary columns come back as `large_string` and `large_binary`, the types Orcapod uses): | Connector | Column types it stores | |---|---| | `SQLiteConnector` | `bool`, `int64`, `double`, `large_string`, `large_binary` | | `PostgreSQLConnector` | `bool`, `int32`, `int64`, `float`, `double`, `large_string`, `large_binary`, `date32`, `timestamp[us]`, `timestamp[us, tz=UTC]` | -Any other column type (lists, structs, dates or timestamps on SQLite, narrower integers and -floats, logical types) is refused with `UnstorableColumnError` before anything of the table is -written. A function node's result table always holds list-valued executor columns, and a -side-effect node's log a UTC timestamp, so a job refuses every function node on either -connector, and every side-effect node on SQLite, when the job is built, before anything runs. Use `InMemoryArrowDatabase` or -`DeltaTableDatabase` for those jobs. Storing list and struct columns on a SQL connector is -planned for a later release. +Any other type (lists, structs, dates or timestamps on SQLite, narrower integers and floats, +logical types) is refused. `SpiralDBConnector` stores every type, but SpiralDB reserves the +`__` prefix on value columns (the `__record_id` key is allowed) and `.` in any column name. + +Orcapod's own stored tables hold such columns. A function node's result table holds +list-valued executor columns and `__`-prefixed columns. A side-effect node's log holds a UTC +timestamp and `__data_context`, and an operator cache and a cached source hold +`__data_context` too. So a job refuses these nodes when it is built, before anything runs: + +- every function node, on all three connectors; +- every side-effect node, on SQLite and SpiralDB; +- every operator cache (an operator node with caching on) and cached source, on SpiralDB. + +An operator cache or cached source on a SQL connector is refused when it is written, before +anything is stored, if one of its columns has a type the connector does not store. Use +`InMemoryArrowDatabase` or `DeltaTableDatabase` for such jobs. Storing these columns on +connector databases is planned for a later release. ```python from orcapod.databases import ConnectorArrowDatabase, SQLiteConnector diff --git a/src/orcapod/contexts/stored.py b/src/orcapod/contexts/stored.py index d2df47f3a..ecb5b4f4f 100644 --- a/src/orcapod/contexts/stored.py +++ b/src/orcapod/contexts/stored.py @@ -108,11 +108,16 @@ def with_data_context(table: "pa.Table", context_key: str) -> "pa.Table": ``large_string`` holding ``context_key`` on every row. """ return table.append_column( - pa.field(constants.DATA_CONTEXT_COL, pa.large_string(), nullable=False), + data_context_field(), pa.array([context_key] * table.num_rows, type=pa.large_string()), ) +def data_context_field() -> "pa.Field[pa.DataType]": + """The ``__data_context`` field every stored table ends with (``with_data_context``).""" + return pa.field(constants.DATA_CONTEXT_COL, pa.large_string(), nullable=False) + + def require_same_data_context(input_context_key: str, own_context_key: str) -> None: """Refuse an input that was hashed under another data context than the writer's. diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index 8eac1650f..f5fc21ea2 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -17,7 +17,11 @@ from orcapod.core.streams.view import StreamView from orcapod.system_constants import constants from orcapod.types import ColumnConfig, Schema -from orcapod.contexts.stored import StaleTables, require_storable +from orcapod.contexts.stored import ( + StaleTables, + data_context_field, + require_storable, +) from orcapod.core.result_cache import ResultCache, pod_timestamp_type from orcapod.hooks import InvocationStatus from orcapod.protocols.core_protocols import ( @@ -74,9 +78,10 @@ def __init__( def require_storable_results(self) -> None: """Refuse a result database that cannot store this pod's result table (ITL-1162). - The result table holds the function's variation and execution columns - (``__pf_var_*``, ``__pf_exec_*``), its output columns and the ``__pod_ts`` - timestamp, typed from their declared schemas; nothing is computed. A + The result table holds the input data hash, the function's variation and + execution columns (``__pf_var_*``, ``__pf_exec_*``), its output columns, + the ``__pod_ts`` timestamp and ``__data_context``, typed from their declared + schemas; nothing is computed. (The source-info columns are text.) A database that states which columns it stores (``ConnectorArrowDatabase``) refuses here, before the function runs; any other database accepts. @@ -97,9 +102,17 @@ def require_storable_results(self) -> None: }, **pf.output_data_schema, } - schema = self.data_context.type_converter.python_schema_to_arrow_schema( + declared = self.data_context.type_converter.python_schema_to_arrow_schema( python_schema - ).append(pa.field(constants.POD_TIMESTAMP, pod_timestamp_type())) + ) + schema = pa.schema( + [ + pa.field(constants.INPUT_DATA_HASH_COL, pa.large_binary()), + *declared, + pa.field(constants.POD_TIMESTAMP, pod_timestamp_type()), + data_context_field(), + ] + ) require_storable(self._cache.result_database, schema) @property diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index c42c9df79..f9044900b 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -67,6 +67,7 @@ find_stale_tables, report_rows_of_other_data_contexts, report_unexpected_row_count, + require_storable, ) from orcapod.utils.database_errors import ( commit_best_effort, @@ -126,6 +127,28 @@ _PIPELINE_RECOMPUTATION_INDEX_COL = "__pipeline_recomputation_index" +def _pipeline_meta_schema() -> pa.Schema: + """The bookkeeping columns of a function node's pipeline record (``pdb_v2``). + + The record holds the tags, system tags and input source-info columns, then + these. One definition for the writer (``add_pipeline_record``) and for the + check of the pipeline database when it is attached (ITL-1162). + """ + return pa.schema( + [ + pa.field(constants.DATA_RECORD_ID, pa.large_binary()), + pa.field(constants.NODE_CONTENT_HASH_COL, pa.large_binary()), + pa.field(constants.INPUT_DATA_HASH_COL, pa.large_binary()), + pa.field(constants.OUTPUT_DATA_HASH_COL, pa.large_binary()), + pa.field(constants.DATA_CONTEXT_COL, pa.large_string()), + pa.field(f"{constants.META_PREFIX}computed", pa.bool_()), + pa.field(constants.IS_EPHEMERAL_COL, pa.bool_()), + pa.field(_PIPELINE_BASE_ENTRY_ID_COL, pa.large_binary()), + pa.field(_PIPELINE_RECOMPUTATION_INDEX_COL, pa.int32()), + ] + ) + + def _build_record_id_preimage( tag: "TagProtocol", input_data: "DataProtocol", @@ -1418,10 +1441,11 @@ def attach_databases( self._function_pod, result_database=result_database, ) - # A database that cannot store the result table refuses it now, - # before anything runs (ITL-1162). + # A database that cannot store the result table or the pipeline + # records refuses them now, before anything runs (ITL-1162). try: cached_function_pod.require_storable_results() + require_storable(pipeline_database, _pipeline_meta_schema()) except UnstorableColumnError as exc: exc.add_context(node_label=self.label) raise @@ -2851,40 +2875,27 @@ def _add_pipeline_record( input_source_table = input_table_with_source.select(source_col_names) # Build the meta columns table. - meta_table = pa.table( + meta_table = pa.Table.from_pydict( { - constants.DATA_RECORD_ID: pa.array( - [data_record_id.bytes], type=pa.large_binary() - ), - constants.NODE_CONTENT_HASH_COL: pa.array( - [self.content_hash().to_prefixed_digest()], type=pa.large_binary() - ), - constants.INPUT_DATA_HASH_COL: pa.array( - [input_data.content_hash().to_prefixed_digest()], - type=pa.large_binary(), - ), - constants.OUTPUT_DATA_HASH_COL: pa.array( - [ - output_data_hash.to_prefixed_digest() - if output_data_hash is not None - else None - ], - type=pa.large_binary(), - ), - constants.DATA_CONTEXT_COL: pa.array( - [self._stored_data_context_key], type=pa.large_string() - ), - f"{constants.META_PREFIX}computed": pa.array( - [computed], type=pa.bool_() - ), - constants.IS_EPHEMERAL_COL: pa.array([is_ephemeral], type=pa.bool_()), - _PIPELINE_BASE_ENTRY_ID_COL: pa.array( - [base_entry_id], type=pa.large_binary() - ), - _PIPELINE_RECOMPUTATION_INDEX_COL: pa.array( - [new_index], type=pa.int32() - ), - } + constants.DATA_RECORD_ID: [data_record_id.bytes], + constants.NODE_CONTENT_HASH_COL: [ + self.content_hash().to_prefixed_digest() + ], + constants.INPUT_DATA_HASH_COL: [ + input_data.content_hash().to_prefixed_digest() + ], + constants.OUTPUT_DATA_HASH_COL: [ + output_data_hash.to_prefixed_digest() + if output_data_hash is not None + else None + ], + constants.DATA_CONTEXT_COL: [self._stored_data_context_key], + f"{constants.META_PREFIX}computed": [computed], + constants.IS_EPHEMERAL_COL: [is_ephemeral], + _PIPELINE_BASE_ENTRY_ID_COL: [base_entry_id], + _PIPELINE_RECOMPUTATION_INDEX_COL: [new_index], + }, + schema=_pipeline_meta_schema(), ) # Combine: tag (with system tags) + input source columns + meta columns diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 83b7949c8..576638e32 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -50,8 +50,11 @@ PipelineJobRequiredError, OrcapodDatabaseReadError, OrcapodDatabaseWriteError, + UnstorableColumnError, ) from orcapod.contexts.stored import ( + data_context_field, + require_storable, keep_rows_of_data_context, require_same_data_context, account_for_stale_tables, @@ -843,6 +846,25 @@ def attach_databases( pipeline_database: Database for pipeline records. cache_mode: Caching behaviour (OFF, LOG, or REPLAY). """ + if cache_mode != CacheMode.OFF: + # A database that cannot store the cache's own columns refuses it now, + # before anything runs (ITL-1162). The operator's output columns are + # checked when they are written. + try: + require_storable( + pipeline_database, + pa.schema( + [ + pa.field( + constants.NODE_CONTENT_HASH_COL, pa.large_binary() + ), + data_context_field(), + ] + ), + ) + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise # Another database has its own tables of other generations to account for. self._stale_tables_checked = False self._skipped_rows_reported = False diff --git a/src/orcapod/core/sources/cached_source.py b/src/orcapod/core/sources/cached_source.py index 4549b9fa7..b87ae790e 100644 --- a/src/orcapod/core/sources/cached_source.py +++ b/src/orcapod/core/sources/cached_source.py @@ -11,6 +11,7 @@ from orcapod import contexts from orcapod.config import OrcapodConfig from orcapod.core.sources.base import RootSource +from orcapod.errors import UnstorableColumnError from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ( check_head_size, @@ -23,6 +24,8 @@ head_of, ) from orcapod.contexts.stored import ( + data_context_field, + require_storable, keep_rows_of_data_context, require_same_data_context, account_for_stale_tables, @@ -99,6 +102,14 @@ def __init__( data_context=data_context, config=config, ) + # A database that cannot store the cache's own columns refuses it now, + # before anything is read (ITL-1162). The source's columns are checked + # when they are written. + try: + require_storable(cache_database, pa.schema([data_context_field()])) + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise self._source: SourceProtocol = source self._cache_database = cache_database self._cache_path_prefix = cache_path_prefix diff --git a/src/orcapod/databases/extension_aware_database.py b/src/orcapod/databases/extension_aware_database.py index 01e4fa8af..23cf95166 100644 --- a/src/orcapod/databases/extension_aware_database.py +++ b/src/orcapod/databases/extension_aware_database.py @@ -187,6 +187,21 @@ def flush_batch(self) -> Callable[[tuple[str, ...]], None]: """ return getattr(self._db, "flush_batch") # noqa: B009 # AttributeError when absent, by design + @property + def require_storable(self) -> Callable[[pa.Schema], None]: + """The wrapped database's ``require_storable`` (ITL-1162). + + A job node asks it about its stored tables when the database is attached, + so a wrapped connector database refuses a table it cannot store before + anything runs. Extension-typed columns are checked as given: a connector + that cannot store their storage type with its metadata cannot store them. + + Raises: + AttributeError: If the wrapped database has none, so this wrapper + checks columns only when the wrapped database does. + """ + return getattr(self._db, "require_storable") # noqa: B009 # AttributeError when absent, by design + # ── Structural delegation ───────────────────────────────────────────────── @property diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index 3e570c7a5..81e869b59 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -952,9 +952,10 @@ def unstorable_column_reason( ) -> str | None: """Say why SpiralDB cannot store a column, or ``None`` when it can (ITL-1162). - SpiralDB stores Arrow tables natively, so no column is refused here yet. - Its column-name rules (a reserved ``__`` prefix on non-key columns, - ITL-1164) are not stated here yet. + SpiralDB stores Arrow tables natively, so every type is stored; its limits + are on column names (ITL-1164). It reserves ``.`` as its column-group + separator in every name, and the ``__`` prefix on value (column group) + columns; a key column may start with ``__`` (``__record_id``). Args: name: The column's name. @@ -962,8 +963,12 @@ def unstorable_column_reason( primary_key: Whether the column is the table's primary key. Returns: - Always ``None``. + A short reason, or ``None`` when SpiralDB stores the column. """ + if "." in name: + return "SpiralDB reserves '.' in column names as its column-group separator" + if name.startswith("__") and not primary_key: + return "SpiralDB reserves the '__' prefix on value columns" return None def validate_records(self, records: pa.Table) -> None: diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 4a8f7833f..f6fa6f9f5 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -80,9 +80,10 @@ class UnstorableColumnError(ValueError): """A database cannot store a column of a table it is asked to hold (ITL-1162). Raised by a database whose backend states which columns it stores faithfully (a - ``ConnectorArrowDatabase``: the SQLite and PostgreSQL connectors), before - anything of the table is written. A job raises it when the database is attached - to a node whose stored tables hold such a column, before anything runs. + ``ConnectorArrowDatabase``: column types on the SQLite and PostgreSQL connectors, + reserved column names on SpiralDB), before anything of the table is written. A + job raises it when the database is attached to a node whose stored tables hold + such a column, before anything runs. Attributes: database_kind: The kind of database that refused, e.g. ``"SQLiteConnector"``. diff --git a/src/orcapod/protocols/db_connector_protocol.py b/src/orcapod/protocols/db_connector_protocol.py index 45bd94d6d..9c6555da6 100644 --- a/src/orcapod/protocols/db_connector_protocol.py +++ b/src/orcapod/protocols/db_connector_protocol.py @@ -125,8 +125,9 @@ def unstorable_column_reason( The one place a connector states which columns it stores (ITL-1162). A connector stores a column only when it reads it back as written: the same - Arrow type, in its large variant (``large_string``, ``large_binary``). A - type the backend would widen, turn into text or fail to bind is refused. + name, and the same Arrow type in its large variant (``large_string``, + ``large_binary``). A type the backend would widen, turn into text or fail to + bind is refused, and so is a name the backend reserves. ``ConnectorArrowDatabase.require_storable`` asks this for every column of a table before anything of it is written. diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index 314abf631..a33e44f4d 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -42,6 +42,7 @@ from orcapod.contexts.stored import ( StaleTables, account_for_stale_tables, + data_context_field, find_stale_tables, require_same_data_context, require_storable, @@ -485,7 +486,7 @@ def _invocation_log_schema() -> pa.Schema: def _stored_invocation_log_schema() -> pa.Schema: """The columns of an invocation log row as stored, ``__data_context`` included.""" - return with_data_context(_invocation_log_schema().empty_table(), "").schema + return _invocation_log_schema().append(data_context_field()) def _write_invocation_row( diff --git a/tests/test_databases/fake_pyspiral.py b/tests/test_databases/fake_pyspiral.py index 887baff8c..88dd25328 100644 --- a/tests/test_databases/fake_pyspiral.py +++ b/tests/test_databases/fake_pyspiral.py @@ -557,14 +557,25 @@ def write( **kwargs: Accepted for signature compatibility; unused. Raises: - ValueError: If a key column is missing, a struct value is null without - ``push_down_nulls``, or the column set differs from the stored table. + ValueError: If a key column is missing, a column name is one SpiralDB + reserves (a ``.`` anywhere, or a ``__`` prefix on a value column), a + struct value is null without ``push_down_nulls``, or the column set + differs from the stored table. """ b = self._store.behaviour keys = self.key_schema missing = [k for k in keys.names if k not in table.column_names] if missing: raise ValueError(f"write is missing key columns {missing}") + for name in table.column_names: + # SpiralDB's own refusals, with its messages as observed live (ITL-1164). + if "." in name: + raise ValueError(f"Column '{name}' contains a reserved dot-separator") + if name.startswith("__") and name not in keys.names: + raise ValueError( + f"Column name '{name}' has reserved prefix: column group columns " + "must not start with '__'" + ) if b.reject_struct_nulls and not push_down_nulls: for name in table.column_names: if _has_struct_nulls(table.column(name)): diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index ff8a6fcb1..812f5e0b1 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -350,3 +350,216 @@ def test_postgresql_rule(self, arrow_type, storable): finally: connector._conn = None assert (reason is None) is storable, reason + + +# --------------------------------------------------------------------------- +# SpiralDB: reserved column names (ITL-1164), through the pyspiral fake +# --------------------------------------------------------------------------- + + +@pytest.fixture +def spiral_database(tmp_path: pathlib.Path): + """A ``ConnectorArrowDatabase`` over ``SpiralDBConnector``, stored by the fake.""" + from orcapod.databases import SpiralDBConnector + from tests.test_databases import fake_pyspiral + + with fake_pyspiral.install(tmp_path / "spiral"): + connector = SpiralDBConnector("p") + try: + yield ConnectorArrowDatabase(connector) + finally: + connector.close() + + +class TestSpiralDBRefusesReservedNames: + def test_function_node_refused_before_anything_runs(self, spiral_database): + job = PipelineJob(name="p", database=spiral_database) + + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = double.pod(_source(), label="dbl") + + err = exc_info.value + assert err.node_label == "dbl" + assert err.database_kind == "SpiralDBConnector" + assert err.column == "__input_data_hash" + assert "'__' prefix" in err.reason + assert "DeltaTableDatabase" in str(err) + assert CALLS == [] + assert spiral_database.list_table_paths() == [] + + def test_side_effect_node_refused_before_delivery(self, spiral_database): + job = PipelineJob(name="p", database=spiral_database) + + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = deliver.process(_source(), label="out") + + assert exc_info.value.node_label == "out" + assert exc_info.value.column == "__data_context" + assert DELIVERED == [] + + def test_operator_cache_refused_when_attached(self, spiral_database): + job = PipelineJob(name="p", database=spiral_database) + with job: + _ = SelectDataColumns(["x"])(_source(), label="sel") + + with pytest.raises(UnstorableColumnError) as exc_info: + job.nodes["sel"].attach_databases( + pipeline_database=spiral_database.at("p"), cache_mode=CacheMode.LOG + ) + + assert exc_info.value.node_label == "sel" + assert exc_info.value.column == "__node_content_hash" + + def test_operator_without_cache_runs(self, spiral_database): + """An operator whose cache is off stores nothing, so nothing is refused.""" + job = PipelineJob(name="p", database=spiral_database) + with job: + _ = SelectDataColumns(["x"])(_source(), label="sel") + _ = job.run() + assert sorted(job.nodes["sel"].as_table()["x"].to_pylist()) == [1, 2] + + def test_cached_source_refused_when_built(self, spiral_database): + from orcapod.core.sources.cached_source import CachedSource + + with pytest.raises(UnstorableColumnError) as exc_info: + _ = CachedSource(_source(), cache_database=spiral_database) + + assert exc_info.value.column == "__data_context" + + @pytest.mark.parametrize( + "name, reason", + [ + ("__value", "'__' prefix"), + ("a.b", "'.'"), + ], + ) + def test_write_refuses_reserved_name(self, spiral_database, name, reason): + table = pa.table({name: pa.array([1], type=pa.int64())}) + + with pytest.raises(UnstorableColumnError) as exc_info: + spiral_database.add_record(("t",), b"1", table) + + assert exc_info.value.column == name + assert reason in exc_info.value.reason + spiral_database.flush() + assert spiral_database.list_table_paths() == [] + + def test_key_column_may_start_with_double_underscore(self, spiral_database): + """``__record_id`` is the key, which SpiralDB accepts; every type is stored.""" + table = pa.table( + { + "v": pa.array([[{"key": "a", "value": "b"}]], type=EXECUTOR_INFO_TYPE), + "ts": pa.array( + [datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)], + type=pa.timestamp("us", tz="UTC"), + ), + } + ) + spiral_database.add_record(("t",), b"1", table, flush=True) + stored = spiral_database.get_all_records(("t",)) + assert stored is not None + assert stored.num_rows == 1 + + +class TestSpiralFakeRefusesLikeSpiralDB: + """The fake refuses the names SpiralDB refuses, with its messages (ITL-1164).""" + + @pytest.mark.parametrize( + "name, message", + [ + ( + "__input_data_hash", + "Column name '__input_data_hash' has reserved prefix: column group " + "columns must not start with '__'", + ), + ("a.b", "Column 'a.b' contains a reserved dot-separator"), + ], + ) + def test_connector_write_refused(self, tmp_path: pathlib.Path, name, message): + from orcapod.databases import SpiralDBConnector + from orcapod.types import ColumnInfo + from tests.test_databases import fake_pyspiral + + with fake_pyspiral.install(tmp_path / "spiral"): + connector = SpiralDBConnector("p") + connector.create_table_if_not_exists( + "t", + [ + ColumnInfo("k", pa.large_string(), nullable=False), + ColumnInfo(name, pa.int64(), nullable=True), + ], + pk_column="k", + ) + rows = pa.table( + { + "k": pa.array(["a"], type=pa.large_string()), + name: pa.array([1], type=pa.int64()), + } + ) + with pytest.raises(ValueError, match="reserved") as exc_info: + connector.upsert_records("t", rows, id_column="k") + connector.close() + assert message in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# Every stored table of a node is checked, also through a wrapper +# --------------------------------------------------------------------------- + + +class TestEveryTableChecked: + def test_function_node_pipeline_database_checked(self, tmp_path: pathlib.Path): + """A SQLite pipeline database is refused even when results go elsewhere.""" + job = PipelineJob(name="p", database=InMemoryArrowDatabase()) + with job: + _ = double.pod(_source(), label="dbl") + node = job.nodes["dbl"] + + with pytest.raises(UnstorableColumnError) as exc_info: + node.attach_databases( + pipeline_database=_sqlite_database(tmp_path), + result_database=InMemoryArrowDatabase(), + ) + + assert exc_info.value.node_label == "dbl" + assert exc_info.value.column == "__pipeline_recomputation_index" + assert exc_info.value.arrow_type == pa.int32() + + def test_extension_aware_wrapper_checked_before_delivery( + self, tmp_path: pathlib.Path + ): + """The check reaches a connector database inside ``ExtensionAwareDatabase``.""" + from orcapod.contexts import get_default_context + from orcapod.databases import ExtensionAwareDatabase + + # The wrapper's annotations do not match the protocols (existing debt). + converter: Any = get_default_context().type_converter + wrapped: Any = ExtensionAwareDatabase( + _sqlite_database(tmp_path), converter=converter + ) + job = PipelineJob(name="p", database=wrapped) + + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = deliver.process(_source(), label="out") + + assert exc_info.value.node_label == "out" + assert exc_info.value.column == "executed_at" + assert DELIVERED == [] + + def test_extension_aware_wrapper_over_in_memory_not_checked(self): + """A wrapped database that states no rule stores every column.""" + from orcapod.contexts import get_default_context + from orcapod.databases import ExtensionAwareDatabase + + converter: Any = get_default_context().type_converter + wrapped: Any = ExtensionAwareDatabase( + InMemoryArrowDatabase(), converter=converter + ) + job = PipelineJob(name="p", database=wrapped) + with job: + _ = double.pod(_source(), label="dbl") + _ = job.run() + assert sorted(CALLS) == [1, 2] From 92432304223456d6e4cf370fdc491663dd578382 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 11:42:43 -0800 Subject: [PATCH 03/14] fix(pipeline): refuse an unstorable job database before anything changes; SpiralDB timestamps (ITL-1162) Each node kind states its check once (require_storable_tables), used by its attach_databases; the job checks every node before attaching any, in bind and when distributing, so a refused database leaves the job as it was. SpiralDB refuses a time-zone-aware timestamp column, which it reads back without the zone (coordinator decision on a CodeRabbit CLI finding on 859a5bfe). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1162.md | 2 +- design_issues/ITL-1162-CA3.md | 7 ++- docs/concepts/pipeline-job.md | 5 +- src/orcapod/core/nodes/function_node.py | 49 ++++++++++++---- src/orcapod/core/nodes/operator_node.py | 57 ++++++++++++------- src/orcapod/databases/spiraldb_connector.py | 15 +++-- src/orcapod/pipeline/job.py | 34 +++++++++++ src/orcapod/side_effects.py | 26 +++++++-- .../test_databases/test_unstorable_columns.py | 53 +++++++++++++++-- 9 files changed, 199 insertions(+), 49 deletions(-) diff --git a/changelog.d/ITL-1162.md b/changelog.d/ITL-1162.md index 4ea470e09..bd8c5ffd5 100644 --- a/changelog.d/ITL-1162.md +++ b/changelog.d/ITL-1162.md @@ -1,7 +1,7 @@ ### Fixed - A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` now refuses a table it cannot store before anything of it is written, with `UnstorableColumnError` naming the connector, the column, its Arrow type and what to use instead (ITL-1162). A connector stores a column only when it reads it back as written: on SQLite `bool`, `int64`, `double`, `large_string` and `large_binary`; on PostgreSQL also `int32`, `float`, `date32`, `timestamp[us]` and `timestamp[us, tz=UTC]`. Before, a function node on such a database ran its function and then failed at commit (`sqlite3.ProgrammingError: ... type 'list' is not supported`), timestamps were stored as text and read back as strings, and narrower integers and floats read back widened, failing the next write. A job now refuses a node whose stored table such a database cannot hold (every function node; a side-effect node on SQLite) when the job is built, before anything runs, naming the node. The connectors no longer fall back to a `TEXT` column for a type they have no column type for. -- A `ConnectorArrowDatabase` over `SpiralDBConnector` now refuses, by the same rule, a column SpiralDB cannot store: a value column whose name starts with `__` (the `__record_id` key is allowed) or any column whose name contains `.` (ITL-1164). Before, a function node on such a database ran and then failed at commit (`Column name '__input_data_hash' has reserved prefix`). A job refuses every function node, side-effect node, operator cache and cached source on such a database when it is built. +- A `ConnectorArrowDatabase` over `SpiralDBConnector` now refuses, by the same rule, a column SpiralDB cannot store: a value column whose name starts with `__` (the `__record_id` key is allowed) or any column whose name contains `.`, and a timestamp column with a time zone, which SpiralDB reads back without it (ITL-1164). Before, a function node on such a database ran and then failed at commit (`Column name '__input_data_hash' has reserved prefix`). A job refuses every function node, side-effect node, operator cache and cached source on such a database when it is built. ### Known limitations diff --git a/design_issues/ITL-1162-CA3.md b/design_issues/ITL-1162-CA3.md index 966a003a5..c5107b181 100644 --- a/design_issues/ITL-1162-CA3.md +++ b/design_issues/ITL-1162-CA3.md @@ -13,8 +13,8 @@ at commit, after the function had run. The same table's `__pod_ts` (`timestamp[u and a side-effect log's `executed_at` were stored as TEXT and read back as `large_string`. `int8`, `int32` and `float32` columns read back widened, so the second write to the table failed its schema comparison. PostgreSQL raises for list columns at `CREATE TABLE` and has no -column type for structs. SpiralDB stores every type but refuses value columns named with a -`__` prefix and any column name containing `.` (ITL-1164), so it cannot hold a function +column type for structs. SpiralDB refuses value columns named with a `__` prefix and any column +name containing `.`, and drops a timestamp's time zone (ITL-1164), so it cannot hold a function result table (`__input_data_hash`, ...) or any table's `__data_context`. **Fix (refusal; developer ruling, November 16, 2026):** each connector states which columns @@ -32,5 +32,6 @@ raises. **Open (the real fix, a later release):** store list and struct columns (for example as JSON text read back as the same Arrow type) and timestamps on SQLite, and rename SpiralDB's -reserved column names on write and recover them on read (ITL-1164), so a function node can +reserved column names on write and recover them on read, and restore a timestamp's time +zone (ITL-1164), so a function node can run on a connector database. diff --git a/docs/concepts/pipeline-job.md b/docs/concepts/pipeline-job.md index 6ea2b6ef5..e01345064 100644 --- a/docs/concepts/pipeline-job.md +++ b/docs/concepts/pipeline-job.md @@ -135,8 +135,9 @@ and binary columns come back as `large_string` and `large_binary`, the types Orc | `PostgreSQLConnector` | `bool`, `int32`, `int64`, `float`, `double`, `large_string`, `large_binary`, `date32`, `timestamp[us]`, `timestamp[us, tz=UTC]` | Any other type (lists, structs, dates or timestamps on SQLite, narrower integers and floats, -logical types) is refused. `SpiralDBConnector` stores every type, but SpiralDB reserves the -`__` prefix on value columns (the `__record_id` key is allowed) and `.` in any column name. +logical types) is refused. `SpiralDBConnector` stores other types as written, except a +timestamp with a time zone (SpiralDB drops the zone), and SpiralDB reserves the `__` prefix on +value columns (the `__record_id` key is allowed) and `.` in any column name. Orcapod's own stored tables hold such columns. A function node's result table holds list-valued executor columns and `__`-prefixed columns. A side-effect node's log holds a UTC diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index f9044900b..72b00f13e 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1386,6 +1386,40 @@ def _account_for_stale_tables(self) -> None: # attach_databases # ------------------------------------------------------------------ + def require_storable_tables( + self, + pipeline_database: ArrowDatabaseProtocol, + result_database: ArrowDatabaseProtocol | None = None, + ) -> None: + """Refuse databases that cannot store this node's tables (ITL-1162). + + Checks the result table (``CachedFunctionPod.require_storable_results``) + against ``result_database`` and the pipeline records' own columns against + ``pipeline_database``. Nothing is computed and nothing changes. A node + loaded without its function stores nothing and is not checked. + + Args: + pipeline_database: The database for pipeline records. + result_database: The database for results; defaults to + ``pipeline_database.at("_result")``, as in ``attach_databases``. + + Raises: + UnstorableColumnError: If a database cannot store a column, naming + this node. + """ + if self._function_pod is None: + return + if result_database is None: + result_database = pipeline_database.at("_result") + try: + CachedFunctionPod( + self._function_pod, result_database=result_database + ).require_storable_results() + require_storable(pipeline_database, _pipeline_meta_schema()) + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise + def attach_databases( self, pipeline_database: ArrowDatabaseProtocol | None = None, @@ -1422,6 +1456,10 @@ def attach_databases( ``set_ephemeral_database(ephemeral_database)`` after ``attach_databases()``. """ + if pipeline_database is not None: + # A database that cannot store this node's tables refuses them now, + # before anything runs or changes (ITL-1162). + self.require_storable_tables(pipeline_database, result_database) # Another database has its own tables of other generations to account for. self._stale_tables_checked = False self._current_tables_checked = False @@ -1437,19 +1475,10 @@ def attach_databases( if self._function_pod is not None: # Normal path: wrap in CachedFunctionPod for compute + cache. - cached_function_pod = CachedFunctionPod( + self._cached_function_pod = CachedFunctionPod( self._function_pod, result_database=result_database, ) - # A database that cannot store the result table or the pipeline - # records refuses them now, before anything runs (ITL-1162). - try: - cached_function_pod.require_storable_results() - require_storable(pipeline_database, _pipeline_meta_schema()) - except UnstorableColumnError as exc: - exc.add_context(node_label=self.label) - raise - self._cached_function_pod = cached_function_pod else: # Read-only stub path (loaded job without a live function pod). # Use the record path stored from the descriptor so that diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 576638e32..58dcf7560 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -835,6 +835,41 @@ def clear_cache(self) -> None: # attach_databases # ------------------------------------------------------------------ + def require_storable_tables( + self, + pipeline_database: ArrowDatabaseProtocol, + cache_mode: CacheMode = CacheMode.OFF, + ) -> None: + """Refuse a database that cannot store this node's cache (ITL-1162). + + With caching on, checks the cache's own columns (``__node_content_hash``, + ``__data_context``); the operator's output columns are checked when they + are written. Nothing is computed and nothing changes. + + Args: + pipeline_database: The database for the cache. + cache_mode: The caching behaviour; ``OFF`` stores nothing. + + Raises: + UnstorableColumnError: If the database cannot store a column, naming + this node. + """ + if cache_mode == CacheMode.OFF: + return + try: + require_storable( + pipeline_database, + pa.schema( + [ + pa.field(constants.NODE_CONTENT_HASH_COL, pa.large_binary()), + data_context_field(), + ] + ), + ) + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise + def attach_databases( self, pipeline_database: ArrowDatabaseProtocol, @@ -846,25 +881,9 @@ def attach_databases( pipeline_database: Database for pipeline records. cache_mode: Caching behaviour (OFF, LOG, or REPLAY). """ - if cache_mode != CacheMode.OFF: - # A database that cannot store the cache's own columns refuses it now, - # before anything runs (ITL-1162). The operator's output columns are - # checked when they are written. - try: - require_storable( - pipeline_database, - pa.schema( - [ - pa.field( - constants.NODE_CONTENT_HASH_COL, pa.large_binary() - ), - data_context_field(), - ] - ), - ) - except UnstorableColumnError as exc: - exc.add_context(node_label=self.label) - raise + # A database that cannot store the cache refuses it now, before anything + # runs or changes (ITL-1162). + self.require_storable_tables(pipeline_database, cache_mode) # Another database has its own tables of other generations to account for. self._stale_tables_checked = False self._skipped_rows_reported = False diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index 81e869b59..146134c68 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -952,10 +952,12 @@ def unstorable_column_reason( ) -> str | None: """Say why SpiralDB cannot store a column, or ``None`` when it can (ITL-1162). - SpiralDB stores Arrow tables natively, so every type is stored; its limits - are on column names (ITL-1164). It reserves ``.`` as its column-group - separator in every name, and the ``__`` prefix on value (column group) - columns; a key column may start with ``__`` (``__record_id``). + SpiralDB stores Arrow tables natively, but not every column as written + (ITL-1164). It reserves ``.`` as its column-group separator in every name, + and the ``__`` prefix on value (column group) columns; a key column may + start with ``__`` (``__record_id``). It drops a timestamp's time zone, so a + time-zone-aware timestamp column (``timestamp[us, tz=UTC]``) would read + back as another type and is refused. Args: name: The column's name. @@ -969,6 +971,11 @@ def unstorable_column_reason( return "SpiralDB reserves '.' in column names as its column-group separator" if name.startswith("__") and not primary_key: return "SpiralDB reserves the '__' prefix on value columns" + if pa.types.is_timestamp(arrow_type) and arrow_type.tz is not None: + return ( + "SpiralDB drops a timestamp's time zone, so it reads back as " + f"timestamp[{arrow_type.unit}]" + ) return None def validate_records(self, records: pa.Table) -> None: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 605dbf415..719729e3b 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -419,6 +419,8 @@ def bind( SourceSpecMismatchError: If any source's columns differ from its slot's declared names, roles or types. ValueError: If a source key has no matching ``SourceJobNode`` slot. + UnstorableColumnError: If *database* cannot store a table of a node + (ITL-1162); the job is left as it was. """ from orcapod.core.nodes.source_node import SourceNodeBase @@ -441,6 +443,11 @@ def bind( if isinstance(node, SourceNodeBase) and node.name in sources: node.validate(sources[node.name]) + # A database that cannot store a node's tables is refused before anything + # changes too (ITL-1162). + if database_changed and database is not None: + self._require_storable_tables(database) + # Every source is validated before anything changes: a refused source leaves # the job as it was, its database included (ITL-1112). if database is not None: @@ -476,6 +483,30 @@ def bind( # _distribute_databases() # ------------------------------------------------------------------ + def _require_storable_tables(self, database: "ArrowDatabaseProtocol") -> None: + """Refuse ``database`` if it cannot store a table of one of the job's nodes. + + The same views as ``_distribute_databases`` gives the nodes. Nothing + changes (ITL-1162). + + Args: + database: The job database to check. + + Raises: + UnstorableColumnError: For the first node whose table the database + cannot store, naming the node. + """ + pipeline_db = database.at(*self._name) + result_db = pipeline_db.at("_result") + for node in (self._persistent_node_map or {}).values(): + if isinstance(node, FunctionJobNode): + node.require_storable_tables(pipeline_db, result_db) + elif isinstance(node, OperatorJobNode): + op_cache_mode = getattr(node, "_cache_mode", None) or CacheMode.OFF + node.require_storable_tables(pipeline_db, op_cache_mode) + elif isinstance(node, SideEffectJobNode): + node.require_storable_tables(pipeline_db) + def _distribute_databases(self) -> None: """Wire live DB references to all FunctionJobNode and OperatorJobNode objects. @@ -491,6 +522,9 @@ def _distribute_databases(self) -> None: "Call bind(database=...) or from_pipeline(..., database=...) first." ) + # Every node is checked before any is attached, so a refused database + # leaves every node as it was (ITL-1162). + self._require_storable_tables(self._database) pipeline_db = self._database.at(*self._name) result_db = pipeline_db.at("_result") diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index a33e44f4d..91368f447 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -1130,6 +1130,24 @@ def _refuse_log_of_another_data_context(self) -> None: # DB attachment # ------------------------------------------------------------------ + def require_storable_tables(self, pipeline_database: ArrowDatabaseProtocol) -> None: + """Refuse a database that cannot store this node's invocation log (ITL-1162). + + Nothing is delivered and nothing changes. + + Args: + pipeline_database: The database for the invocation log. + + Raises: + UnstorableColumnError: If the database cannot store a column, naming + this node. + """ + try: + require_storable(pipeline_database, _stored_invocation_log_schema()) + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise + def attach_databases( self, pipeline_database: ArrowDatabaseProtocol | None = None, @@ -1146,12 +1164,8 @@ def attach_databases( """ if pipeline_database is not None: # A database that cannot store the invocation log refuses it now, - # before anything is delivered (ITL-1162). - try: - require_storable(pipeline_database, _stored_invocation_log_schema()) - except UnstorableColumnError as exc: - exc.add_context(node_label=self.label) - raise + # before anything is delivered or changes (ITL-1162). + self.require_storable_tables(pipeline_database) self._pipeline_database = pipeline_database self._unlisted_store_reported = False self._current_log_checked = False diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 812f5e0b1..f56242dcb 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -396,7 +396,8 @@ def test_side_effect_node_refused_before_delivery(self, spiral_database): _ = deliver.process(_source(), label="out") assert exc_info.value.node_label == "out" - assert exc_info.value.column == "__data_context" + # The first column it cannot store: the time-zone-aware timestamp. + assert exc_info.value.column == "executed_at" assert DELIVERED == [] def test_operator_cache_refused_when_attached(self, spiral_database): @@ -446,14 +447,33 @@ def test_write_refuses_reserved_name(self, spiral_database, name, reason): spiral_database.flush() assert spiral_database.list_table_paths() == [] - def test_key_column_may_start_with_double_underscore(self, spiral_database): - """``__record_id`` is the key, which SpiralDB accepts; every type is stored.""" + def test_write_refuses_time_zone_aware_timestamp(self, spiral_database): + """SpiralDB drops the time zone, so the column would read back as another type.""" table = pa.table( { - "v": pa.array([[{"key": "a", "value": "b"}]], type=EXECUTOR_INFO_TYPE), "ts": pa.array( [datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)], type=pa.timestamp("us", tz="UTC"), + ) + } + ) + + with pytest.raises(UnstorableColumnError) as exc_info: + spiral_database.add_record(("t",), b"1", table) + + assert exc_info.value.column == "ts" + assert "time zone" in exc_info.value.reason + assert "timestamp[us]" in exc_info.value.reason + spiral_database.flush() + assert spiral_database.list_table_paths() == [] + + def test_key_column_may_start_with_double_underscore(self, spiral_database): + """``__record_id`` is the key, which SpiralDB accepts; lists are stored.""" + table = pa.table( + { + "v": pa.array([[{"key": "a", "value": "b"}]], type=EXECUTOR_INFO_TYPE), + "ts": pa.array( + [datetime.datetime(2026, 1, 1)], type=pa.timestamp("us") ), } ) @@ -563,3 +583,28 @@ def test_extension_aware_wrapper_over_in_memory_not_checked(self): _ = double.pod(_source(), label="dbl") _ = job.run() assert sorted(CALLS) == [1, 2] + + +class TestRefusedBindLeavesJobAsItWas: + def test_refused_database_bind_changes_nothing(self, tmp_path: pathlib.Path): + """Every node is checked before any is attached (CodeRabbit, ITL-1162).""" + in_memory = InMemoryArrowDatabase() + job = PipelineJob(name="p", database=in_memory) + with job: + out = double.pod(_source(), label="dbl") + _ = SelectDataColumns(["double"])(out, label="sel") + _ = job.run() + assert sorted(CALLS) == [1, 2] + sqlite = _sqlite_database(tmp_path) + + with pytest.raises(UnstorableColumnError) as exc_info: + job.bind(database=sqlite) + + assert exc_info.value.node_label == "dbl" + # The job keeps its database: a new run reads the stored results. + job.nodes["dbl"].clear_cache() + CALLS.clear() + _ = job.run() + assert CALLS == [] + assert sorted(job.nodes["dbl"].as_table()["double"].to_pylist()) == [2, 4] + assert sqlite.list_table_paths() == [] From d03ef6f18d61184cafa8b8d989bc08288dff1573 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 12:00:16 -0800 Subject: [PATCH 04/14] fix(databases): check column types before metadata; keep null in the schema comparison (ITL-1162) An extension-typed column on a SQL connector now raises the documented UnstorableColumnError (a ValueError) rather than the metadata check's plain ValueError, and the flush schema comparison keeps null distinct from large_string (CodeRabbit CLI findings on 92432304). Co-Authored-By: Claude Opus 5.5 --- .../databases/connector_arrow_database.py | 7 +++--- .../test_databases/test_unstorable_columns.py | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index 7bdfcea79..4e9f02725 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -341,8 +341,8 @@ def add_records( f"got {rid_type}. Encode the column to bytes before calling add_records()." ) - self._connector.validate_records(records) self.require_storable(records.schema) + self._connector.validate_records(records) records = self._deduplicate_within_table(records) record_key = self._get_record_key(record_path) @@ -453,11 +453,12 @@ def flush(self) -> None: # A connector reads text and binary back in their large variants, # which are the same types (ITL-1162). existing_cols = { - c.name: normalize_to_large_types(c.arrow_type) + c.name: normalize_to_large_types(c.arrow_type, keep_null=True) for c in self._connector.get_column_info(table_name) } pending_cols = { - c.name: normalize_to_large_types(c.arrow_type) for c in columns + c.name: normalize_to_large_types(c.arrow_type, keep_null=True) + for c in columns } if existing_cols != pending_cols: diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index f56242dcb..032991b22 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -608,3 +608,25 @@ def test_refused_database_bind_changes_nothing(self, tmp_path: pathlib.Path): assert CALLS == [] assert sorted(job.nodes["dbl"].as_table()["double"].to_pylist()) == [2, 4] assert sqlite.list_table_paths() == [] + + +class TestWriteChecksOrder: + def test_extension_type_refused_as_unstorable_column(self, tmp_path: pathlib.Path): + """The type rule runs before the metadata check (CodeRabbit, ITL-1162).""" + database = _sqlite_database(tmp_path) + table = pa.table({"u": pa.array([b"0123456789abcdef"], type=pa.uuid())}) + + with pytest.raises(UnstorableColumnError) as exc_info: + database.add_record(("t",), b"1", table) + + assert exc_info.value.column == "u" + + def test_null_column_is_not_a_large_string_column(self, spiral_database): + """The schema comparison keeps ``null``: a later ``large_string`` is a change.""" + null_table = pa.table({"v": pa.array([None], type=pa.null())}) + text_table = pa.table({"v": pa.array(["a"], type=pa.large_string())}) + spiral_database.add_record(("t",), b"1", null_table, flush=True) + + spiral_database.add_record(("t",), b"2", text_table) + with pytest.raises(ValueError, match="Schema mismatch"): + spiral_database.flush() From c0a43bb320916a4b28d5059fd0a541af9f2374e6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 12:13:45 -0800 Subject: [PATCH 05/14] fix(databases): buffer connector writes in large variants; live PostgreSQL round-trip tests (ITL-1162) ConnectorArrowDatabase buffers records in their large variants, so a read that joins buffered and committed rows sees one layout (CodeRabbit CLI finding on d03ef6f1). The PostgreSQL integration suite (CI's live server) checks that every type the connector states it stores reads back as written, twice, and that a function node job is refused. Co-Authored-By: Claude Opus 5.5 --- .../databases/connector_arrow_database.py | 8 +- .../test_postgresql_connector_integration.py | 73 +++++++++++++++++++ .../test_databases/test_unstorable_columns.py | 13 ++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index 4e9f02725..d87e2c9c6 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -25,7 +25,10 @@ from orcapod.databases.utils import coerce_record_id from orcapod.errors import UnstorableColumnError -from orcapod.utils.arrow_utils import normalize_to_large_types +from orcapod.utils.arrow_utils import ( + normalize_table_to_large_types, + normalize_to_large_types, +) from orcapod.protocols.db_connector_protocol import ColumnInfo, DBConnectorProtocol from orcapod.utils.lazy_module import LazyModule @@ -343,6 +346,9 @@ def add_records( self.require_storable(records.schema) self._connector.validate_records(records) + # Buffered in the large variants every connector reads back, so a read that + # joins buffered and committed rows sees one layout (ITL-1122, ITL-1162). + records = normalize_table_to_large_types(records, keep_null=True) records = self._deduplicate_within_table(records) record_key = self._get_record_key(record_path) diff --git a/tests/test_databases/test_postgresql_connector_integration.py b/tests/test_databases/test_postgresql_connector_integration.py index e37c379ea..79f379586 100644 --- a/tests/test_databases/test_postgresql_connector_integration.py +++ b/tests/test_databases/test_postgresql_connector_integration.py @@ -21,6 +21,7 @@ from __future__ import annotations +import datetime import os import uuid from collections.abc import Iterator @@ -475,3 +476,75 @@ def test_schema_mismatch_raises(self, db: ConnectorArrowDatabase) -> None: db.add_record(("fn", "mismatch"), record_id="r2", record=r2) with pytest.raises(ValueError, match="Schema mismatch"): db.flush() + + +# --------------------------------------------------------------------------- +# What the connector states it stores reads back as written (ITL-1162) +# --------------------------------------------------------------------------- + + +@pytest.mark.postgres +@pytest.mark.parametrize( + "arrow_type, values", + [ + pytest.param(pa.bool_(), [True, None], id="bool"), + pytest.param(pa.int32(), [1, None], id="int32"), + pytest.param(pa.int64(), [1, None], id="int64"), + pytest.param(pa.float32(), [1.5, None], id="float32"), + pytest.param(pa.float64(), [1.5, None], id="float64"), + pytest.param(pa.large_string(), ["a", None], id="large_string"), + pytest.param(pa.large_binary(), [b"a", None], id="large_binary"), + pytest.param(pa.date32(), [datetime.date(2026, 1, 1), None], id="date32"), + pytest.param( + pa.timestamp("us"), + [datetime.datetime(2026, 1, 1, 12), None], + id="timestamp", + ), + pytest.param( + pa.timestamp("us", tz="UTC"), + [datetime.datetime(2026, 1, 1, 12, tzinfo=datetime.timezone.utc), None], + id="timestamp_utc", + ), + ], +) +def test_storable_types_read_back_as_written( + connector: PostgreSQLConnector, arrow_type: pa.DataType, values: list[object] +) -> None: + """Each type ``unstorable_column_reason`` accepts round-trips, twice (ITL-1162).""" + assert ( + connector.unstorable_column_reason("v", arrow_type, primary_key=False) is None + ) + db = ConnectorArrowDatabase(connector) + table = pa.table({"v": pa.array(values, type=arrow_type)}) + db.add_record(("t",), b"1", table.slice(0, 1), flush=True) + db.add_record(("t",), b"2", table.slice(1, 1), flush=True) + + stored = db.get_all_records(("t",)) + assert stored is not None + assert stored.schema.field("v").type == arrow_type + assert sorted(stored["v"].to_pylist(), key=lambda v: v is None) == values + + +@pytest.mark.postgres +def test_function_node_job_refused(connector: PostgreSQLConnector) -> None: + """A function node's result table cannot be stored; the job refuses it (ITL-1162).""" + from orcapod import PipelineJob, function_pod + from orcapod.errors import UnstorableColumnError + from orcapod.sources import DictSource + + @function_pod(output_keys="double") + def double(x: int) -> int: + return 2 * x + + job = PipelineJob(name="p", database=ConnectorArrowDatabase(connector)) + source = DictSource( + data=[{"k": "a", "x": 1}], + tag_columns=["k"], + tag_only_columns=True, + source_id="src", + ) + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = double.pod(source, label="dbl") + assert exc_info.value.column == "__pf_exec_executor_info" + assert connector.get_table_names() == [] diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 032991b22..cab682be0 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -630,3 +630,16 @@ def test_null_column_is_not_a_large_string_column(self, spiral_database): spiral_database.add_record(("t",), b"2", text_table) with pytest.raises(ValueError, match="Schema mismatch"): spiral_database.flush() + + def test_buffered_text_reads_with_committed_text(self, tmp_path: pathlib.Path): + """Buffered ``string`` rows read together with committed ``large_string`` rows.""" + database = _sqlite_database(tmp_path) + table = pa.table({"v": pa.array(["a"], type=pa.string())}) + database.add_record(("t",), b"1", table, flush=True) + database.add_record(("t",), b"2", table) + + stored = database.get_all_records(("t",)) + + assert stored is not None + assert stored.schema.field("v").type == pa.large_string() + assert stored["v"].to_pylist() == ["a", "a"] From 7307f9581b4502298a28eb37009e49a0989ad10b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 12:42:58 -0800 Subject: [PATCH 06/14] fix(databases): refuse PostgreSQL column names over 63 bytes; normalize views before the check (ITL-1162) PostgreSQL truncates identifiers to 63 bytes, so a longer column name reads back as another name: PostgreSQLConnector refuses it (coordinator ruling; the real fix is ITL-1166). On PostgreSQL an operator cache with caching on is refused at its write, before anything is stored, because its system-tag column names exceed the limit. ConnectorArrowDatabase normalizes records (view types included) before the column check, and the SQL rules normalize the type they are asked about (CodeRabbit CLI findings on c0a43bb3). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1162.md | 2 + design_issues/ITL-1162-CA3.md | 5 +- docs/concepts/pipeline-job.md | 7 +- .../databases/connector_arrow_database.py | 7 +- src/orcapod/databases/postgresql_connector.py | 17 +++- src/orcapod/databases/sqlite_connector.py | 3 +- .../test_postgresql_connector_integration.py | 48 ++++++++++ .../test_databases/test_unstorable_columns.py | 87 +++++++++++++++++++ 8 files changed, 166 insertions(+), 10 deletions(-) diff --git a/changelog.d/ITL-1162.md b/changelog.d/ITL-1162.md index bd8c5ffd5..eca57ad46 100644 --- a/changelog.d/ITL-1162.md +++ b/changelog.d/ITL-1162.md @@ -1,8 +1,10 @@ ### Fixed - A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` now refuses a table it cannot store before anything of it is written, with `UnstorableColumnError` naming the connector, the column, its Arrow type and what to use instead (ITL-1162). A connector stores a column only when it reads it back as written: on SQLite `bool`, `int64`, `double`, `large_string` and `large_binary`; on PostgreSQL also `int32`, `float`, `date32`, `timestamp[us]` and `timestamp[us, tz=UTC]`. Before, a function node on such a database ran its function and then failed at commit (`sqlite3.ProgrammingError: ... type 'list' is not supported`), timestamps were stored as text and read back as strings, and narrower integers and floats read back widened, failing the next write. A job now refuses a node whose stored table such a database cannot hold (every function node; a side-effect node on SQLite) when the job is built, before anything runs, naming the node. The connectors no longer fall back to a `TEXT` column for a type they have no column type for. +- `PostgreSQLConnector` now refuses a column name longer than 63 bytes (UTF-8), which PostgreSQL truncates, so it would read back as another name (ITL-1162). Before, such a column was stored under the truncated name. - A `ConnectorArrowDatabase` over `SpiralDBConnector` now refuses, by the same rule, a column SpiralDB cannot store: a value column whose name starts with `__` (the `__record_id` key is allowed) or any column whose name contains `.`, and a timestamp column with a time zone, which SpiralDB reads back without it (ITL-1164). Before, a function node on such a database ran and then failed at commit (`Column name '__input_data_hash' has reserved prefix`). A job refuses every function node, side-effect node, operator cache and cached source on such a database when it is built. ### Known limitations - A connector job database cannot hold function nodes in v0.3.0: neither SQL connector (SQLite, PostgreSQL) nor SpiralDB. A function node's result table holds list-of-struct executor columns, which neither SQL connector stores, and `__`-prefixed value columns, which SpiralDB reserves (ITL-1164). A side-effect node is refused on SQLite (its log's UTC timestamp) and on SpiralDB, and an operator cache or cached source on SpiralDB (`__data_context`). Use a Delta database (`DeltaTableDatabase`) or `InMemoryArrowDatabase`. Storing such columns on connector databases is planned for a later release (ITL-1162, ITL-1164). +- On PostgreSQL, an operator cache with caching on is refused in v0.3.0 when it is written (before anything is stored): its system-tag column names (`_tag_source_id::` and a 64-character hash) are longer than PostgreSQL's 63-byte limit. Storing them under a mapped name is planned for a later release (ITL-1166). diff --git a/design_issues/ITL-1162-CA3.md b/design_issues/ITL-1162-CA3.md index c5107b181..9151eaaa7 100644 --- a/design_issues/ITL-1162-CA3.md +++ b/design_issues/ITL-1162-CA3.md @@ -13,7 +13,8 @@ at commit, after the function had run. The same table's `__pod_ts` (`timestamp[u and a side-effect log's `executed_at` were stored as TEXT and read back as `large_string`. `int8`, `int32` and `float32` columns read back widened, so the second write to the table failed its schema comparison. PostgreSQL raises for list columns at `CREATE TABLE` and has no -column type for structs. SpiralDB refuses value columns named with a `__` prefix and any column +column type for structs, and truncates a column name to 63 bytes, so an operator cache's +system-tag columns (`_tag_source_id::<64-hex>`) read back under other names. SpiralDB refuses value columns named with a `__` prefix and any column name containing `.`, and drops a timestamp's time zone (ITL-1164), so it cannot hold a function result table (`__input_data_hash`, ...) or any table's `__data_context`. @@ -33,5 +34,5 @@ raises. **Open (the real fix, a later release):** store list and struct columns (for example as JSON text read back as the same Arrow type) and timestamps on SQLite, and rename SpiralDB's reserved column names on write and recover them on read, and restore a timestamp's time -zone (ITL-1164), so a function node can +zone (ITL-1164); map PostgreSQL names over 63 bytes the same way (ITL-1166), so a function node can run on a connector database. diff --git a/docs/concepts/pipeline-job.md b/docs/concepts/pipeline-job.md index e01345064..ec89854c0 100644 --- a/docs/concepts/pipeline-job.md +++ b/docs/concepts/pipeline-job.md @@ -135,7 +135,8 @@ and binary columns come back as `large_string` and `large_binary`, the types Orc | `PostgreSQLConnector` | `bool`, `int32`, `int64`, `float`, `double`, `large_string`, `large_binary`, `date32`, `timestamp[us]`, `timestamp[us, tz=UTC]` | Any other type (lists, structs, dates or timestamps on SQLite, narrower integers and floats, -logical types) is refused. `SpiralDBConnector` stores other types as written, except a +logical types) is refused. PostgreSQL also refuses a column name longer than 63 bytes, which it +would truncate. `SpiralDBConnector` stores other types as written, except a timestamp with a time zone (SpiralDB drops the zone), and SpiralDB reserves the `__` prefix on value columns (the `__record_id` key is allowed) and `.` in any column name. @@ -149,7 +150,9 @@ timestamp and `__data_context`, and an operator cache and a cached source hold - every operator cache (an operator node with caching on) and cached source, on SpiralDB. An operator cache or cached source on a SQL connector is refused when it is written, before -anything is stored, if one of its columns has a type the connector does not store. Use +anything is stored, if one of its columns has a type or name the connector does not store. On +PostgreSQL that is every operator cache with caching on: its system-tag column names +(`_tag_source_id::` and a 64-character hash) are longer than 63 bytes. Use `InMemoryArrowDatabase` or `DeltaTableDatabase` for such jobs. Storing these columns on connector databases is planned for a later release. diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index d87e2c9c6..4e10bf13f 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -344,11 +344,12 @@ def add_records( f"got {rid_type}. Encode the column to bytes before calling add_records()." ) + # Checked and buffered in the large variants every connector reads back + # (view types included), so a read that joins buffered and committed rows + # sees one layout (ITL-1122, ITL-1162). + records = normalize_table_to_large_types(records, keep_null=True) self.require_storable(records.schema) self._connector.validate_records(records) - # Buffered in the large variants every connector reads back, so a read that - # joins buffered and committed rows sees one layout (ITL-1122, ITL-1162). - records = normalize_table_to_large_types(records, keep_null=True) records = self._deduplicate_within_table(records) record_key = self._get_record_key(record_path) diff --git a/src/orcapod/databases/postgresql_connector.py b/src/orcapod/databases/postgresql_connector.py index 49759875a..67b9dcc41 100644 --- a/src/orcapod/databases/postgresql_connector.py +++ b/src/orcapod/databases/postgresql_connector.py @@ -125,6 +125,11 @@ def _pg_type_to_arrow(pg_type_name: str, udt_name: str) -> pa.DataType: return _pa.large_string() +# PostgreSQL truncates an identifier (a quoted one too) to NAMEDATALEN - 1 bytes, 63 on a +# default build, so a longer column name reads back as another name (ITL-1162). +_PG_MAX_IDENTIFIER_BYTES = 63 + + def _pg_sql_type(arrow_type: pa.DataType) -> str | None: """Return the PostgreSQL column type for an Arrow type, or ``None`` when there is none. @@ -212,11 +217,12 @@ def _pg_unstorable_reason(arrow_type: pa.DataType) -> str | None: """ from orcapod.utils.arrow_utils import normalize_to_large_types + arrow_type = normalize_to_large_types(arrow_type, keep_null=True) sql_type = _pg_sql_type(arrow_type) if sql_type is None: return "PostgreSQL has no column type for it here" read_back = _pg_type_to_arrow(sql_type.lower(), sql_type.lower()) - if read_back != normalize_to_large_types(arrow_type): + if read_back != arrow_type: return f"PostgreSQL stores it as {sql_type}, which reads back as {read_back}" return None @@ -586,7 +592,8 @@ def unstorable_column_reason( ``timestamp[us, tz=UTC]`` (``string`` and ``binary`` read back in their large variants). Other integer widths, and timestamps of another unit or time zone, read back as another type, and lists, structs and extension types have no column type here. - The name and the key role do not matter to PostgreSQL. + A name longer than 63 bytes (UTF-8) is refused: PostgreSQL truncates it, + so it would read back as another name. The key role does not matter. Args: name: The column's name. @@ -596,6 +603,12 @@ def unstorable_column_reason( Returns: A short reason, or ``None`` when PostgreSQL stores the column. """ + size = len(name.encode("utf-8")) + if size > _PG_MAX_IDENTIFIER_BYTES: + return ( + f"PostgreSQL truncates a column name to {_PG_MAX_IDENTIFIER_BYTES} " + f"bytes, and this name has {size}" + ) return _pg_unstorable_reason(arrow_type) def validate_records(self, records: pa.Table) -> None: diff --git a/src/orcapod/databases/sqlite_connector.py b/src/orcapod/databases/sqlite_connector.py index 484bd26f7..7983f8203 100644 --- a/src/orcapod/databases/sqlite_connector.py +++ b/src/orcapod/databases/sqlite_connector.py @@ -126,11 +126,12 @@ def _sqlite_unstorable_reason(arrow_type: pa.DataType) -> str | None: """ from orcapod.utils.arrow_utils import normalize_to_large_types # noqa: PLC0415 + arrow_type = normalize_to_large_types(arrow_type, keep_null=True) sql_type = _sqlite_sql_type(arrow_type) if sql_type is None: return "SQLite has no column type for it" read_back = _sqlite_type_to_arrow(sql_type) - if read_back != normalize_to_large_types(arrow_type): + if read_back != arrow_type: return f"SQLite stores it as {sql_type}, which reads back as {read_back}" return None diff --git a/tests/test_databases/test_postgresql_connector_integration.py b/tests/test_databases/test_postgresql_connector_integration.py index 79f379586..473560085 100644 --- a/tests/test_databases/test_postgresql_connector_integration.py +++ b/tests/test_databases/test_postgresql_connector_integration.py @@ -548,3 +548,51 @@ def double(x: int) -> int: _ = double.pod(source, label="dbl") assert exc_info.value.column == "__pf_exec_executor_info" assert connector.get_table_names() == [] + + +@pytest.mark.postgres +def test_long_column_name_refused_and_nothing_written( + connector: PostgreSQLConnector, +) -> None: + """A name PostgreSQL would truncate is refused before the table exists (ITL-1162).""" + from orcapod.errors import UnstorableColumnError + + db = ConnectorArrowDatabase(connector) + name = "_tag_source_id::" + "0" * 64 + table = pa.table({name: pa.array(["s"], type=pa.large_string())}) + with pytest.raises(UnstorableColumnError): + db.add_record(("t",), b"1", table) + db.flush() + assert connector.get_table_names() == [] + + +@pytest.mark.postgres +def test_operator_cache_refused_at_write_and_nothing_written( + connector: PostgreSQLConnector, +) -> None: + """An operator cache's system-tag column names exceed 63 bytes (ITL-1162).""" + from orcapod import PipelineJob + from orcapod.core.operators import SelectDataColumns + from orcapod.errors import OrcapodDatabaseWriteError, UnstorableColumnError + from orcapod.sources import DictSource + from orcapod.types import CacheMode + + db = ConnectorArrowDatabase(connector) + job = PipelineJob(name="p", database=db) + source = DictSource( + data=[{"k": "a", "x": 1, "y": 2}], + tag_columns=["k"], + tag_only_columns=True, + source_id="src", + ) + with job: + _ = SelectDataColumns(["x"])(source, label="sel") + job.nodes["sel"].attach_databases( + pipeline_database=db.at("p"), cache_mode=CacheMode.LOG + ) + with pytest.raises(OrcapodDatabaseWriteError) as exc_info: + _ = job.run() + cause = exc_info.value.__cause__ + assert isinstance(cause, UnstorableColumnError) + assert cause.column.startswith("_tag_") + assert connector.get_table_names() == [] diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index cab682be0..58236c435 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -643,3 +643,90 @@ def test_buffered_text_reads_with_committed_text(self, tmp_path: pathlib.Path): assert stored is not None assert stored.schema.field("v").type == pa.large_string() assert stored["v"].to_pylist() == ["a", "a"] + + @pytest.mark.parametrize( + "arrow_type, values", + [ + pytest.param(pa.string_view(), ["a"], id="string_view"), + pytest.param(pa.binary_view(), [b"a"], id="binary_view"), + ], + ) + def test_view_types_stored_in_large_variant( + self, tmp_path: pathlib.Path, arrow_type, values + ): + """View layouts are normalized before the check, as every table is.""" + database = _sqlite_database(tmp_path) + table = pa.table({"v": pa.array(values, type=arrow_type)}) + database.add_record(("t",), b"1", table, flush=True) + + stored = database.get_all_records(("t",)) + + assert stored is not None + assert stored["v"].to_pylist() == values + + +# --------------------------------------------------------------------------- +# PostgreSQL truncates identifiers to 63 bytes; SQLite has no such limit +# --------------------------------------------------------------------------- + + +def _mocked_postgresql() -> tuple[PostgreSQLConnector, MagicMock]: + """A ``PostgreSQLConnector`` over a mocked connection (no server).""" + with patch("psycopg.connect") as mock_connect: + conn = MagicMock() + mock_connect.return_value = conn + return PostgreSQLConnector("postgresql://localhost/test"), conn + + +class TestIdentifierLength: + @pytest.mark.parametrize( + "name, storable", + [ + ("a" * 63, True), + ("a" * 64, False), + ("é" * 31, True), # 62 bytes + ("é" * 32, False), # 64 bytes + ("_tag_source_id::" + "0" * 64, False), # a system-tag column name + ], + ) + def test_postgresql_rule(self, name, storable): + connector, _ = _mocked_postgresql() + try: + reason = connector.unstorable_column_reason( + name, pa.int64(), primary_key=False + ) + finally: + connector._conn = None + assert (reason is None) is storable, reason + if not storable: + assert reason is not None and "63 bytes" in reason + + def test_postgresql_refuses_long_name_before_anything_is_written(self): + connector, conn = _mocked_postgresql() + try: + database = ConnectorArrowDatabase(connector) + name = "_tag_source_id::" + "0" * 64 + table = pa.table({name: pa.array(["s"], type=pa.large_string())}) + calls_before = len(conn.mock_calls) + + with pytest.raises(UnstorableColumnError) as exc_info: + database.add_record(("t",), b"1", table) + database.flush() # nothing was buffered + + assert exc_info.value.column == name + assert exc_info.value.database_kind == "PostgreSQLConnector" + assert len(conn.mock_calls) == calls_before # no statement was sent + finally: + connector._conn = None + + def test_sqlite_keeps_long_names(self, tmp_path: pathlib.Path): + """SQLite has no identifier length limit: a long name reads back as written.""" + database = _sqlite_database(tmp_path) + name = "_tag_source_id::" + "0" * 200 + table = pa.table({name: pa.array(["s"], type=pa.large_string())}) + database.add_record(("t",), b"1", table, flush=True) + + stored = database.get_all_records(("t",)) + + assert stored is not None + assert name in stored.column_names From 42e4377d8104f991c798c3b4ae3a28daab2f1043 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:12:40 -0800 Subject: [PATCH 07/14] fix(function_node): check an ephemeral result database before anything changes (ITL-1162) FunctionJobNode.set_ephemeral_database and Pipeline.set_ephemeral_database refuse a database that cannot store the result table before any node changes (CodeRabbit CLI finding on 7307f958); the result-table check is one method on FunctionNodeBase. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 32 +++++++++++++++++-- src/orcapod/pipeline/base.py | 11 +++++++ .../test_databases/test_unstorable_columns.py | 19 +++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 72b00f13e..58885f96e 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -852,6 +852,30 @@ def __repr__(self) -> str: f"input_stream={short_stream_repr(self._input_stream)})" ) + def require_storable_results(self, result_database: ArrowDatabaseProtocol) -> None: + """Refuse a result database that cannot store this node's result table (ITL-1162). + + Nothing is computed and nothing changes. A node without its function stores + no results and is not checked. + + Args: + result_database: The database results would be stored in (the + persistent result database or an ephemeral one). + + Raises: + UnstorableColumnError: If the database cannot store a column, naming + this node. + """ + if self._function_pod is None: + return + try: + CachedFunctionPod( + self._function_pod, result_database=result_database + ).require_storable_results() + except UnstorableColumnError as exc: + exc.add_context(node_label=self.label) + raise + @abstractmethod def set_ephemeral_database(self, database: "ArrowDatabaseProtocol | None") -> None: """Assign or remove the ephemeral result database for this node. @@ -1411,10 +1435,8 @@ def require_storable_tables( return if result_database is None: result_database = pipeline_database.at("_result") + self.require_storable_results(result_database) try: - CachedFunctionPod( - self._function_pod, result_database=result_database - ).require_storable_results() require_storable(pipeline_database, _pipeline_meta_schema()) except UnstorableColumnError as exc: exc.add_context(node_label=self.label) @@ -1931,6 +1953,10 @@ def set_ephemeral_database(self, database: "ArrowDatabaseProtocol | None") -> No database: The ``ArrowDatabaseProtocol`` to use for ephemeral result storage, or ``None`` to detach and revert to persistent-only writes. """ + if database is not None: + # A database that cannot store the result table refuses it now, before + # anything runs or changes (ITL-1162). + self.require_storable_results(database) # Another database has its own tables of other generations to account for, # as in ``attach_databases``. self._stale_tables_checked = False diff --git a/src/orcapod/pipeline/base.py b/src/orcapod/pipeline/base.py index 3236c1c2c..c9ee88e93 100644 --- a/src/orcapod/pipeline/base.py +++ b/src/orcapod/pipeline/base.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Generic, TypeVar +from orcapod.core.nodes.function_node import FunctionNodeBase from orcapod.core.tracker import AutoRegisteringContextBasedTracker from orcapod.pipeline.dag import OrcaDAG from orcapod.pipeline.pod_invocation import ( @@ -154,7 +155,17 @@ def set_ephemeral_database(self, database: "ArrowDatabaseProtocol | None") -> No Args: database: An ``ArrowDatabaseProtocol`` instance to attach, or ``None`` to detach. + + Raises: + UnstorableColumnError: If ``database`` cannot store a function node's + result table (ITL-1162); no node is changed. """ + if database is not None: + # Every function node is checked before any changes, so a database that + # cannot store a result table leaves every node as it was (ITL-1162). + for node in self._nodes.values(): + if isinstance(node, FunctionNodeBase): + node.require_storable_results(database) for node in self._nodes.values(): if node.node_type not in ("source", "side_effect"): node.set_ephemeral_database(database) diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 58236c435..27b3d208a 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -730,3 +730,22 @@ def test_sqlite_keeps_long_names(self, tmp_path: pathlib.Path): assert stored is not None assert name in stored.column_names + + +class TestPipelineRecordsAndEphemeralResults: + def test_ephemeral_database_checked_before_anything_changes( + self, tmp_path: pathlib.Path + ): + job = PipelineJob(name="p", database=InMemoryArrowDatabase()) + with job: + _ = double.pod(_source(), label="dbl") + + with pytest.raises(UnstorableColumnError) as exc_info: + job.set_ephemeral_database(_sqlite_database(tmp_path)) + + assert exc_info.value.node_label == "dbl" + assert exc_info.value.column == EXECUTOR_INFO + # No node took the database: a run computes and stores as before. + assert job.nodes["dbl"].ephemeral_result_database is None + _ = job.run() + assert sorted(CALLS) == [1, 2] From 50cec098434c8d2a1a1ca3ba98d04ab5cf75337d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:39:32 -0800 Subject: [PATCH 08/14] fix(function_node): check an ephemeral database before attach changes the node; state the up-front scope (ITL-1162) attach_databases refuses an ephemeral database that cannot store the result table before any state changes. UnstorableColumnError states which columns are refused up front (a node's own columns and a function's outputs) and which when their table is first written (carried input columns, operator outputs) (CodeRabbit CLI findings on 42e4377d). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 2 ++ src/orcapod/errors.py | 6 ++++- .../test_databases/test_unstorable_columns.py | 22 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 58885f96e..e0cd42ed6 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1482,6 +1482,8 @@ def attach_databases( # A database that cannot store this node's tables refuses them now, # before anything runs or changes (ITL-1162). self.require_storable_tables(pipeline_database, result_database) + if ephemeral_database is not None: + self.require_storable_results(ephemeral_database) # Another database has its own tables of other generations to account for. self._stale_tables_checked = False self._current_tables_checked = False diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index f6fa6f9f5..b5371ba8a 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -83,7 +83,11 @@ class UnstorableColumnError(ValueError): ``ConnectorArrowDatabase``: column types on the SQLite and PostgreSQL connectors, reserved column names on SpiralDB), before anything of the table is written. A job raises it when the database is attached to a node whose stored tables hold - such a column, before anything runs. + such a column, before anything runs, for the columns known from declared + schemas alone: a node's own columns and a function's outputs. The columns a + node carries from its input (tags, system tags, source info) and an operator's + output columns are refused when their table is first written, before anything + of it is stored. Attributes: database_kind: The kind of database that refused, e.g. ``"SQLiteConnector"``. diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 27b3d208a..9c9b29cfd 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -749,3 +749,25 @@ def test_ephemeral_database_checked_before_anything_changes( assert job.nodes["dbl"].ephemeral_result_database is None _ = job.run() assert sorted(CALLS) == [1, 2] + + def test_attach_with_refused_ephemeral_database_changes_nothing( + self, tmp_path: pathlib.Path + ): + in_memory = InMemoryArrowDatabase() + job = PipelineJob(name="p", database=in_memory) + with job: + _ = double.pod(_source(), label="dbl") + _ = job.run() + node = job.nodes["dbl"] + + with pytest.raises(UnstorableColumnError): + node.attach_databases( + pipeline_database=InMemoryArrowDatabase(), + ephemeral_database=_sqlite_database(tmp_path), + ) + + # The node kept its databases and its run: nothing is recomputed. + CALLS.clear() + assert sorted(node.as_table()["double"].to_pylist()) == [2, 4] + assert node.ephemeral_result_database is None + assert CALLS == [] From 807166aa52c9aaea4e116b3afdb546eadbe31ee5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 13:55:57 -0800 Subject: [PATCH 09/14] test(store_write_failure): record calls of to_path so the refusal test sees them (ITL-1162) CodeRabbit CLI finding on 06b8f8b6. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_store_write_failure.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline/test_store_write_failure.py b/tests/test_pipeline/test_store_write_failure.py index d02388ffe..ffd8e1757 100644 --- a/tests/test_pipeline/test_store_write_failure.py +++ b/tests/test_pipeline/test_store_write_failure.py @@ -158,7 +158,8 @@ def double_or_fail(x: int) -> int: @function_pod(output_keys="where") def to_path(x: int) -> pathlib.Path: - """Return a ``pathlib.Path`` output, a logical type SQLite cannot store.""" + """Return a ``pathlib.Path`` output, a logical type SQLite cannot store; record the call.""" + calls.append(x) return pathlib.Path(f"/data/{x}.txt") From a8ecac15b17fde08796e616f47ecc5f41c27158c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 14:48:52 -0800 Subject: [PATCH 10/14] fix(databases): address in-PR CodeRabbit review on 807166aa (ITL-1162) - check the normalized schema before the table cast, so a refused column raises UnstorableColumnError, not an Arrow cast error; - SpiralDB refuses a time-zone-aware timestamp nested in a list, struct or map; - UnstorableColumnError survives pickling and copying; - a pod with its own cache database (pod_cache_database) is checked too; - PostgreSQL refuses its system column names (xmin, ctid, ...). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/cached_function_pod.py | 3 + .../databases/connector_arrow_database.py | 5 +- src/orcapod/databases/postgresql_connector.py | 9 +- src/orcapod/databases/spiraldb_connector.py | 24 +++++ src/orcapod/errors.py | 30 ++++++ .../test_databases/test_unstorable_columns.py | 91 +++++++++++++++++++ 6 files changed, 160 insertions(+), 2 deletions(-) diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index f5fc21ea2..216b38c89 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -114,6 +114,9 @@ def require_storable_results(self) -> None: ] ) require_storable(self._cache.result_database, schema) + # A pod with its own cache database (``pod_cache_database``) stores there too. + if isinstance(self._function_pod, CachedFunctionPod): + self._function_pod.require_storable_results() @property def result_database(self) -> ArrowDatabaseProtocol: diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index 4e10bf13f..83092d920 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -26,6 +26,7 @@ from orcapod.databases.utils import coerce_record_id from orcapod.errors import UnstorableColumnError from orcapod.utils.arrow_utils import ( + normalize_schema_to_large_types, normalize_table_to_large_types, normalize_to_large_types, ) @@ -347,8 +348,10 @@ def add_records( # Checked and buffered in the large variants every connector reads back # (view types included), so a read that joins buffered and committed rows # sees one layout (ITL-1122, ITL-1162). + self.require_storable( + normalize_schema_to_large_types(records.schema, keep_null=True) + ) records = normalize_table_to_large_types(records, keep_null=True) - self.require_storable(records.schema) self._connector.validate_records(records) records = self._deduplicate_within_table(records) diff --git a/src/orcapod/databases/postgresql_connector.py b/src/orcapod/databases/postgresql_connector.py index be71e55f3..50ed0d968 100644 --- a/src/orcapod/databases/postgresql_connector.py +++ b/src/orcapod/databases/postgresql_connector.py @@ -129,6 +129,10 @@ def _pg_type_to_arrow(pg_type_name: str, udt_name: str) -> pa.DataType: # default build, so a longer column name reads back as another name (ITL-1162). _PG_MAX_IDENTIFIER_BYTES = 63 +# Every PostgreSQL table has these system columns; a user column of the same name is +# refused by CREATE TABLE, quoted or not (ITL-1162). +_PG_SYSTEM_COLUMNS = frozenset({"tableoid", "xmin", "cmin", "xmax", "cmax", "ctid"}) + def _pg_sql_type(arrow_type: pa.DataType) -> str | None: """Return the PostgreSQL column type for an Arrow type, or ``None`` when there is none. @@ -611,7 +615,8 @@ def unstorable_column_reason( large variants). Other integer widths, and timestamps of another unit or time zone, read back as another type, and lists, structs and extension types have no column type here. A name longer than 63 bytes (UTF-8) is refused: PostgreSQL truncates it, - so it would read back as another name. The key role does not matter. + so it would read back as another name. So is a system column name + (``xmin``, ``ctid``, ...). The key role does not matter. Args: name: The column's name. @@ -621,6 +626,8 @@ def unstorable_column_reason( Returns: A short reason, or ``None`` when PostgreSQL stores the column. """ + if name in _PG_SYSTEM_COLUMNS: + return f"PostgreSQL reserves {name!r} as a system column name" size = len(name.encode("utf-8")) if size > _PG_MAX_IDENTIFIER_BYTES: return ( diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index 146134c68..bcb0454e6 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -377,6 +377,25 @@ def _about_table(table: str) -> Iterator[None]: # --------------------------------------------------------------------------- +def _holds_zone_aware_timestamp(arrow_type: pa.DataType) -> bool: + """Whether ``arrow_type`` is or nests a timestamp with a time zone (ITL-1162). + + Args: + arrow_type: The column's Arrow type. + + Returns: + ``True`` for a zone-aware timestamp at any depth of lists, structs and maps. + """ + if pa.types.is_timestamp(arrow_type): + return arrow_type.tz is not None + if isinstance(arrow_type, pa.ExtensionType): + return _holds_zone_aware_timestamp(arrow_type.storage_type) + return any( + _holds_zone_aware_timestamp(arrow_type.field(i).type) + for i in range(arrow_type.num_fields) + ) + + class SpiralDBConnector: """DBConnectorProtocol implementation backed by SpiralDB (pyspiral). @@ -976,6 +995,11 @@ def unstorable_column_reason( "SpiralDB drops a timestamp's time zone, so it reads back as " f"timestamp[{arrow_type.unit}]" ) + if _holds_zone_aware_timestamp(arrow_type): + return ( + "SpiralDB drops the time zone of a timestamp nested in it, so it " + "reads back as another type" + ) return None def validate_records(self, records: pa.Table) -> None: diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 7b14bd685..fedbe14f7 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -132,6 +132,19 @@ def __init__( self.node_label = node_label super().__init__(database_kind, column, arrow_type, reason) + def __reduce__(self) -> tuple[Any, ...]: + """Rebuild from the keyword fields, so pickling and copying keep them.""" + return ( + _rebuild_unstorable_column_error, + ( + self.database_kind, + self.column, + self.arrow_type, + self.reason, + self.node_label, + ), + ) + def add_context(self, node_label: str | None = None) -> None: """Fill in the node label where the raising site did not know it. @@ -150,6 +163,23 @@ def __str__(self) -> str: ) +def _rebuild_unstorable_column_error( + database_kind: str, + column: str, + arrow_type: object, + reason: str, + node_label: str | None, +) -> UnstorableColumnError: + """Rebuild an ``UnstorableColumnError`` from its fields (``__reduce__``).""" + return UnstorableColumnError( + database_kind=database_kind, + column=column, + arrow_type=arrow_type, + reason=reason, + node_label=node_label, + ) + + class MissingOptionalDependencyError(ImportError): """A stored value needs an optional dependency that is not installed. diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 9c9b29cfd..0af92155b 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -771,3 +771,94 @@ def test_attach_with_refused_ephemeral_database_changes_nothing( assert sorted(node.as_table()["double"].to_pylist()) == [2, 4] assert node.ephemeral_result_database is None assert CALLS == [] + + +# --------------------------------------------------------------------------- +# In-PR CodeRabbit review on 807166aa +# --------------------------------------------------------------------------- + + +class TestReviewFollowUps: + def test_error_survives_pickling(self): + import copy + import pickle + + err = UnstorableColumnError( + database_kind="SQLiteConnector", + column="v", + arrow_type=pa.int32(), + reason="r", + node_label="dbl", + ) + for again in (pickle.loads(pickle.dumps(err)), copy.copy(err)): + assert isinstance(again, UnstorableColumnError) + assert (again.column, again.arrow_type, again.node_label) == ( + "v", + pa.int32(), + "dbl", + ) + assert str(again) == str(err) + + @pytest.mark.parametrize( + "name", ["xmin", "xmax", "cmin", "cmax", "ctid", "tableoid"] + ) + def test_postgresql_refuses_system_column_names(self, name): + connector, _ = _mocked_postgresql() + try: + reason = connector.unstorable_column_reason( + name, pa.int64(), primary_key=False + ) + finally: + connector._conn = None + assert reason is not None and "system column" in reason + + @pytest.mark.parametrize( + "arrow_type", + [ + pa.large_list(pa.timestamp("us", tz="UTC")), + pa.struct([("t", pa.timestamp("us", tz="UTC"))]), + pa.map_(pa.large_string(), pa.timestamp("ms", tz="UTC")), + ], + ) + def test_spiraldb_refuses_nested_time_zone_aware_timestamp( + self, spiral_database, arrow_type + ): + table = pa.table({"v": pa.array([None], type=arrow_type)}) + + with pytest.raises(UnstorableColumnError) as exc_info: + spiral_database.add_record(("t",), b"1", table) + + assert "time zone" in exc_info.value.reason + + def test_union_refused_before_any_cast(self, tmp_path: pathlib.Path): + """A refused column raises the refusal, not an Arrow cast error.""" + database = _sqlite_database(tmp_path) + union = pa.UnionArray.from_dense( + pa.array([0], type=pa.int8()), + pa.array([0], type=pa.int32()), + [pa.array(["a"], type=pa.string())], + ) + table = pa.table({"u": union}) + + with pytest.raises(UnstorableColumnError) as exc_info: + database.add_record(("t",), b"1", table) + + assert exc_info.value.column == "u" + + def test_nested_cached_pod_checked(self, tmp_path: pathlib.Path): + """A pod with its own cache database is checked with the job's (ITL-1162).""" + sqlite = _sqlite_database(tmp_path) + + @function_pod(output_keys="double", pod_cache_database=sqlite) + def cached_double(x: int) -> int: + CALLS.append(x) + return 2 * x + + job = PipelineJob(name="p", database=InMemoryArrowDatabase()) + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = cached_double.pod(_source(), label="dbl") + + assert exc_info.value.node_label == "dbl" + assert exc_info.value.database_kind == "SQLiteConnector" + assert CALLS == [] From d464ace19257145c5f6423c47aa1bd5994fc288c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 14:59:18 -0800 Subject: [PATCH 11/14] test(spiraldb): opt-in live checks of the reserved-name refusal (ITL-1162, ITL-1164) Gated by SPIRAL_INTEGRATION_TESTS like the rest of the file: a function node job on a SpiralDB database is refused before the function runs, and SpiralDB itself refuses the names the connector refuses (in-PR CodeRabbit pre-merge check on a8ecac15). Co-Authored-By: Claude Opus 5.5 --- .../test_spiraldb_connector_integration.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_databases/test_spiraldb_connector_integration.py b/tests/test_databases/test_spiraldb_connector_integration.py index adb8833da..5a5d34c3a 100644 --- a/tests/test_databases/test_spiraldb_connector_integration.py +++ b/tests/test_databases/test_spiraldb_connector_integration.py @@ -386,3 +386,65 @@ def test_nested_struct_field_metadata_round_trip(self, connector): assert inner.metadata.get(b"unit") == b"volts" finally: connector.delete_table(table_name) + + +class TestReservedNamesRefusedLive: + """ITL-1162 / ITL-1164 against the live service (opt-in, like every test here). + + The refusal itself is decided by ``SpiralDBConnector.unstorable_column_reason`` + before any request; the second test checks the rule against SpiralDB's own + refusals, so the fake's model of them stays honest. + """ + + def test_function_node_job_refused_before_anything_runs(self, connector): + from orcapod import PipelineJob, function_pod + from orcapod.errors import UnstorableColumnError + from orcapod.sources import DictSource + + calls: list[int] = [] + + @function_pod(output_keys="double") + def double(x: int) -> int: + calls.append(x) + return 2 * x + + job = PipelineJob( + name=_unique_table("itl1162"), database=ConnectorArrowDatabase(connector) + ) + source = DictSource( + data=[{"k": "a", "x": 1}], + tag_columns=["k"], + tag_only_columns=True, + source_id="src", + ) + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = double.pod(source, label="dbl") + assert exc_info.value.column == "__input_data_hash" + assert calls == [] + + @pytest.mark.parametrize("name", ["__value", "a.b"]) + def test_spiraldb_itself_refuses_the_names_the_connector_refuses( + self, connector, name + ): + assert ( + connector.unstorable_column_reason(name, pa.int64(), primary_key=False) + is not None + ) + table = _unique_table("itl1162_names") + connector.create_table_if_not_exists( + table, + [ + ColumnInfo("k", pa.large_string(), nullable=False), + ColumnInfo(name, pa.int64(), nullable=True), + ], + pk_column="k", + ) + rows = pa.table( + { + "k": pa.array(["a"], type=pa.large_string()), + name: pa.array([1], type=pa.int64()), + } + ) + with pytest.raises(Exception, match="reserved"): + connector.upsert_records(table, rows, id_column="k") From 5e92c00143b693bd709b97ce10b51efd0d82eed7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 15:11:40 -0800 Subject: [PATCH 12/14] fix(databases): CREATE TABLE maps view layouts as the rule reads them; clean up live tables (ITL-1162) The SQLite and PostgreSQL type maps normalize view types (string_view, binary_view) to their large variants, as unstorable_column_reason does, and the live SpiralDB name test deletes its table (CodeRabbit CLI findings on d464ace1). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/postgresql_connector.py | 4 +++ src/orcapod/databases/sqlite_connector.py | 4 +++ .../test_spiraldb_connector_integration.py | 36 ++++++++++--------- .../test_databases/test_unstorable_columns.py | 23 ++++++++++++ 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/orcapod/databases/postgresql_connector.py b/src/orcapod/databases/postgresql_connector.py index 50ed0d968..2ae12f527 100644 --- a/src/orcapod/databases/postgresql_connector.py +++ b/src/orcapod/databases/postgresql_connector.py @@ -147,6 +147,10 @@ def _pg_sql_type(arrow_type: pa.DataType) -> str | None: """ import pyarrow as _pa + from orcapod.utils.arrow_utils import normalize_to_large_types + + # View layouts map as their large variants, as the storability rule reads them. + arrow_type = normalize_to_large_types(arrow_type, keep_null=True) if arrow_type == _pa.bool_(): return "BOOLEAN" if _pa.types.is_integer(arrow_type): diff --git a/src/orcapod/databases/sqlite_connector.py b/src/orcapod/databases/sqlite_connector.py index 7983f8203..88fd35375 100644 --- a/src/orcapod/databases/sqlite_connector.py +++ b/src/orcapod/databases/sqlite_connector.py @@ -78,6 +78,10 @@ def _sqlite_sql_type(arrow_type: pa.DataType) -> str | None: """ import pyarrow as _pa # noqa: PLC0415 — needed at call time + from orcapod.utils.arrow_utils import normalize_to_large_types # noqa: PLC0415 + + # View layouts map as their large variants, as the storability rule reads them. + arrow_type = normalize_to_large_types(arrow_type, keep_null=True) if arrow_type == _pa.bool_(): return "BOOLEAN" if _pa.types.is_integer(arrow_type): diff --git a/tests/test_databases/test_spiraldb_connector_integration.py b/tests/test_databases/test_spiraldb_connector_integration.py index 5a5d34c3a..c04b1b82d 100644 --- a/tests/test_databases/test_spiraldb_connector_integration.py +++ b/tests/test_databases/test_spiraldb_connector_integration.py @@ -432,19 +432,23 @@ def test_spiraldb_itself_refuses_the_names_the_connector_refuses( is not None ) table = _unique_table("itl1162_names") - connector.create_table_if_not_exists( - table, - [ - ColumnInfo("k", pa.large_string(), nullable=False), - ColumnInfo(name, pa.int64(), nullable=True), - ], - pk_column="k", - ) - rows = pa.table( - { - "k": pa.array(["a"], type=pa.large_string()), - name: pa.array([1], type=pa.int64()), - } - ) - with pytest.raises(Exception, match="reserved"): - connector.upsert_records(table, rows, id_column="k") + try: + connector.create_table_if_not_exists( + table, + [ + ColumnInfo("k", pa.large_string(), nullable=False), + ColumnInfo(name, pa.int64(), nullable=True), + ], + pk_column="k", + ) + rows = pa.table( + { + "k": pa.array(["a"], type=pa.large_string()), + name: pa.array([1], type=pa.int64()), + } + ) + with pytest.raises(Exception, match="reserved"): + connector.upsert_records(table, rows, id_column="k") + finally: + if table in connector.get_table_names(): + connector.delete_table(table) diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 0af92155b..3d05da6da 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -862,3 +862,26 @@ def cached_double(x: int) -> int: assert exc_info.value.node_label == "dbl" assert exc_info.value.database_kind == "SQLiteConnector" assert CALLS == [] + + def test_sql_type_maps_accept_view_types(self, tmp_path: pathlib.Path): + """CREATE TABLE maps view layouts as the rule reads them (CLI run 12).""" + from orcapod.types import ColumnInfo + + connector = SQLiteConnector(str(tmp_path / "views.db")) + try: + connector.create_table_if_not_exists( + "t", + [ + ColumnInfo("k", pa.string_view(), nullable=False), + ColumnInfo("b", pa.binary_view(), nullable=True), + ], + pk_column="k", + ) + types = {c.name: c.arrow_type for c in connector.get_column_info("t")} + finally: + connector.close() + assert types == {"k": pa.large_string(), "b": pa.large_binary()} + from orcapod.databases.postgresql_connector import _arrow_type_to_pg_sql + + assert _arrow_type_to_pg_sql(pa.string_view()) == "TEXT" + assert _arrow_type_to_pg_sql(pa.binary_view()) == "BYTEA" From 25e8d9340c168fefde0ef7b1dda2f94b75e01c96 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 15:37:51 -0800 Subject: [PATCH 13/14] fix(databases): refuse PostgreSQL table names over 63 bytes at every table entry (ITL-1162) PostgreSQL truncates a table name past 63 bytes, so two Orcapod tables sharing their first 63 bytes would silently be one. Each connector states which table names it holds (unstorable_table_name_reason); ConnectorArrowDatabase refuses any other name with UnstorableTableNameError at every entry that creates, writes, reads or checks a table, and PostgreSQLConnector guards its own table entries, so a truncated name is never sent (coordinator ruling; real fix in ITL-1166). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1162.md | 3 +- design_issues/ITL-1162-CA3.md | 10 +- docs/concepts/pipeline-job.md | 10 +- .../databases/connector_arrow_database.py | 25 ++++- src/orcapod/databases/postgresql_connector.py | 33 ++++++ src/orcapod/databases/spiraldb_connector.py | 13 +++ src/orcapod/databases/sqlite_connector.py | 13 +++ src/orcapod/errors.py | 53 +++++++++ .../protocols/db_connector_protocol.py | 15 +++ .../test_core/sources/test_db_table_source.py | 3 + .../test_connector_arrow_database.py | 3 + .../test_postgresql_connector_integration.py | 35 +++++- .../test_databases/test_unstorable_columns.py | 104 ++++++++++++++++++ 13 files changed, 307 insertions(+), 13 deletions(-) diff --git a/changelog.d/ITL-1162.md b/changelog.d/ITL-1162.md index eca57ad46..145553228 100644 --- a/changelog.d/ITL-1162.md +++ b/changelog.d/ITL-1162.md @@ -1,10 +1,11 @@ ### Fixed - A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` now refuses a table it cannot store before anything of it is written, with `UnstorableColumnError` naming the connector, the column, its Arrow type and what to use instead (ITL-1162). A connector stores a column only when it reads it back as written: on SQLite `bool`, `int64`, `double`, `large_string` and `large_binary`; on PostgreSQL also `int32`, `float`, `date32`, `timestamp[us]` and `timestamp[us, tz=UTC]`. Before, a function node on such a database ran its function and then failed at commit (`sqlite3.ProgrammingError: ... type 'list' is not supported`), timestamps were stored as text and read back as strings, and narrower integers and floats read back widened, failing the next write. A job now refuses a node whose stored table such a database cannot hold (every function node; a side-effect node on SQLite) when the job is built, before anything runs, naming the node. The connectors no longer fall back to a `TEXT` column for a type they have no column type for. -- `PostgreSQLConnector` now refuses a column name longer than 63 bytes (UTF-8), which PostgreSQL truncates, so it would read back as another name (ITL-1162). Before, such a column was stored under the truncated name. +- `PostgreSQLConnector` now refuses a column name longer than 63 bytes (UTF-8), which PostgreSQL truncates, so it would read back as another name (ITL-1162). Before, such a column was stored under the truncated name. A table name longer than 63 bytes is refused too, with `UnstorableTableNameError`, at every entry that creates, writes, reads or checks a table, so a truncated name is never sent: two tables whose names share their first 63 bytes would otherwise silently be one. - A `ConnectorArrowDatabase` over `SpiralDBConnector` now refuses, by the same rule, a column SpiralDB cannot store: a value column whose name starts with `__` (the `__record_id` key is allowed) or any column whose name contains `.`, and a timestamp column with a time zone, which SpiralDB reads back without it (ITL-1164). Before, a function node on such a database ran and then failed at commit (`Column name '__input_data_hash' has reserved prefix`). A job refuses every function node, side-effect node, operator cache and cached source on such a database when it is built. ### Known limitations - A connector job database cannot hold function nodes in v0.3.0: neither SQL connector (SQLite, PostgreSQL) nor SpiralDB. A function node's result table holds list-of-struct executor columns, which neither SQL connector stores, and `__`-prefixed value columns, which SpiralDB reserves (ITL-1164). A side-effect node is refused on SQLite (its log's UTC timestamp) and on SpiralDB, and an operator cache or cached source on SpiralDB (`__data_context`). Use a Delta database (`DeltaTableDatabase`) or `InMemoryArrowDatabase`. Storing such columns on connector databases is planned for a later release (ITL-1162, ITL-1164). - On PostgreSQL, an operator cache with caching on is refused in v0.3.0 when it is written (before anything is stored): its system-tag column names (`_tag_source_id::` and a 64-character hash) are longer than PostgreSQL's 63-byte limit. Storing them under a mapped name is planned for a later release (ITL-1166). +- On PostgreSQL in v0.3.0, a table whose Orcapod path gives a name longer than 63 bytes is refused. Every node table and cached source path holds a 64-character hash, so only tables written directly through `ConnectorArrowDatabase` under short paths fit; reading an existing table with `PostgreSQLTableSource` is unaffected. Mapping long names is planned for a later release (ITL-1166). diff --git a/design_issues/ITL-1162-CA3.md b/design_issues/ITL-1162-CA3.md index 9151eaaa7..73e1c1ee6 100644 --- a/design_issues/ITL-1162-CA3.md +++ b/design_issues/ITL-1162-CA3.md @@ -14,7 +14,9 @@ and a side-effect log's `executed_at` were stored as TEXT and read back as `larg `int8`, `int32` and `float32` columns read back widened, so the second write to the table failed its schema comparison. PostgreSQL raises for list columns at `CREATE TABLE` and has no column type for structs, and truncates a column name to 63 bytes, so an operator cache's -system-tag columns (`_tag_source_id::<64-hex>`) read back under other names. SpiralDB refuses value columns named with a `__` prefix and any column +system-tag columns (`_tag_source_id::<64-hex>`) read back under other names. Its table names +(the node path joined by `__`, 109-173 characters) are truncated the same way, so two tables +sharing their first 63 bytes would silently be one. SpiralDB refuses value columns named with a `__` prefix and any column name containing `.`, and drops a timestamp's time zone (ITL-1164), so it cannot hold a function result table (`__input_data_hash`, ...) or any table's `__data_context`. @@ -28,11 +30,13 @@ log) ask the database when it is attached, so a job refuses them before anything the node. An operator job node with a cache and a cached source ask about their bookkeeping columns (`__node_content_hash`, `__data_context`) when attached or built; their user columns are checked at the write. `SpiralDBConnector` states its name rule the same way (developer -ruling, November 16, 2026). The connectors' silent TEXT fallback in `CREATE TABLE` now +ruling, November 16, 2026). `PostgreSQLConnector` also states which table names it holds +(`unstorable_table_name_reason`, 63 bytes), checked at every table entry +(`UnstorableTableNameError`). The connectors' silent TEXT fallback in `CREATE TABLE` now raises. **Open (the real fix, a later release):** store list and struct columns (for example as JSON text read back as the same Arrow type) and timestamps on SQLite, and rename SpiralDB's reserved column names on write and recover them on read, and restore a timestamp's time -zone (ITL-1164); map PostgreSQL names over 63 bytes the same way (ITL-1166), so a function node can +zone (ITL-1164); map PostgreSQL column and table names over 63 bytes the same way (ITL-1166), so a function node can run on a connector database. diff --git a/docs/concepts/pipeline-job.md b/docs/concepts/pipeline-job.md index ec89854c0..745e875ce 100644 --- a/docs/concepts/pipeline-job.md +++ b/docs/concepts/pipeline-job.md @@ -135,8 +135,8 @@ and binary columns come back as `large_string` and `large_binary`, the types Orc | `PostgreSQLConnector` | `bool`, `int32`, `int64`, `float`, `double`, `large_string`, `large_binary`, `date32`, `timestamp[us]`, `timestamp[us, tz=UTC]` | Any other type (lists, structs, dates or timestamps on SQLite, narrower integers and floats, -logical types) is refused. PostgreSQL also refuses a column name longer than 63 bytes, which it -would truncate. `SpiralDBConnector` stores other types as written, except a +logical types) is refused. PostgreSQL also refuses a column name or a table name longer than +63 bytes, which it would truncate (`UnstorableTableNameError` for a table). `SpiralDBConnector` stores other types as written, except a timestamp with a time zone (SpiralDB drops the zone), and SpiralDB reserves the `__` prefix on value columns (the `__record_id` key is allowed) and `.` in any column name. @@ -152,7 +152,11 @@ timestamp and `__data_context`, and an operator cache and a cached source hold An operator cache or cached source on a SQL connector is refused when it is written, before anything is stored, if one of its columns has a type or name the connector does not store. On PostgreSQL that is every operator cache with caching on: its system-tag column names -(`_tag_source_id::` and a 64-character hash) are longer than 63 bytes. Use +(`_tag_source_id::` and a 64-character hash) are longer than 63 bytes. +A table's name is its Orcapod path joined by `__`, and every node table and cached source path +holds a 64-character hash, so on PostgreSQL only tables you write through `ConnectorArrowDatabase` +yourself under short paths fit; reading an existing table with `PostgreSQLTableSource` is +unaffected. Use `InMemoryArrowDatabase` or `DeltaTableDatabase` for such jobs. Storing these columns on connector databases is planned for a later release. diff --git a/src/orcapod/databases/connector_arrow_database.py b/src/orcapod/databases/connector_arrow_database.py index 83092d920..de5df5ca2 100644 --- a/src/orcapod/databases/connector_arrow_database.py +++ b/src/orcapod/databases/connector_arrow_database.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, cast from orcapod.databases.utils import coerce_record_id -from orcapod.errors import UnstorableColumnError +from orcapod.errors import UnstorableColumnError, UnstorableTableNameError from orcapod.utils.arrow_utils import ( normalize_schema_to_large_types, normalize_table_to_large_types, @@ -191,6 +191,7 @@ def _deduplicate_within_table(self, table: pa.Table) -> pa.Table: def _get_committed_table(self, record_path: tuple[str, ...]) -> pa.Table | None: """Fetch all committed records for a path from the connector.""" table_name = self._path_to_table_name(self._path_prefix + record_path) + self._require_table_name(table_name) if table_name not in self._connector.get_table_names(): return None batches = list(self._connector.iter_batches(f'SELECT * FROM "{table_name}"')) @@ -250,6 +251,7 @@ def table_exists(self, record_path: tuple[str, ...]) -> bool: ``True`` if a table exists at the given path, ``False`` otherwise. """ table_name = self._path_to_table_name(self._path_prefix + record_path) + self._require_table_name(table_name) record_key = self._get_record_key(record_path) if record_key in self._pending_batches: return True @@ -288,6 +290,23 @@ def require_storable(self, schema: pa.Schema) -> None: reason=reason, ) + def _require_table_name(self, table_name: str) -> None: + """Refuse a table name the connector would not keep (ITL-1162). + + Called at every entry that creates, writes, reads or checks a table, so a + name the backend would change (truncate) is never sent to it. + + Raises: + UnstorableTableNameError: Naming the connector, the name and why. + """ + reason = self._connector.unstorable_table_name_reason(table_name) + if reason is not None: + raise UnstorableTableNameError( + database_kind=type(self._connector).__name__, + table_name=table_name, + reason=reason, + ) + # ── Write methods ───────────────────────────────────────────────────────── def add_record( @@ -351,6 +370,9 @@ def add_records( self.require_storable( normalize_schema_to_large_types(records.schema, keep_null=True) ) + self._require_table_name( + self._path_to_table_name(self._path_prefix + record_path) + ) records = normalize_table_to_large_types(records, keep_null=True) self._connector.validate_records(records) @@ -448,6 +470,7 @@ def flush(self) -> None: for record_key in list(self._pending_batches.keys()): record_path = tuple(record_key.split("/")) table_name = self._path_to_table_name(record_path) + self._require_table_name(table_name) pending = self._pending_batches.pop(record_key) self._pending_record_ids.pop(record_key, None) skip_existing = self._pending_skip_existing.pop(record_key, False) diff --git a/src/orcapod/databases/postgresql_connector.py b/src/orcapod/databases/postgresql_connector.py index 2ae12f527..e8befed1b 100644 --- a/src/orcapod/databases/postgresql_connector.py +++ b/src/orcapod/databases/postgresql_connector.py @@ -21,6 +21,7 @@ from collections.abc import Iterator from typing import TYPE_CHECKING, Any +from orcapod.errors import UnstorableTableNameError from orcapod.types import ColumnInfo from orcapod.utils.lazy_module import LazyModule @@ -422,6 +423,7 @@ def current_location(self) -> tuple[str, str]: def get_pk_columns(self, table_name: str) -> list[str]: """Return primary-key column names in key-sequence order.""" + self._require_table_name(table_name) with self._lock: conn = self._require_open() self._validate_table_name(table_name) @@ -445,6 +447,7 @@ def get_pk_columns(self, table_name: str) -> list[str]: def get_column_info(self, table_name: str) -> list[ColumnInfo]: """Return column metadata with Arrow-mapped types.""" + self._require_table_name(table_name) with self._lock: conn = self._require_open() self._validate_table_name(table_name) @@ -536,6 +539,7 @@ def create_table_if_not_exists( pk_column: str, ) -> None: """Create a table with the given columns if it does not already exist.""" + self._require_table_name(table_name) col_names = [col.name for col in columns] if pk_column not in col_names: raise ValueError( @@ -570,6 +574,7 @@ def upsert_records( skip_existing: bool = False, ) -> None: """Write records to a table using upsert semantics.""" + self._require_table_name(table_name) with self._lock: conn = self._require_open() self._validate_table_name(table_name) @@ -640,6 +645,34 @@ def unstorable_column_reason( ) return _pg_unstorable_reason(arrow_type) + def unstorable_table_name_reason(self, table_name: str) -> str | None: + """Say why PostgreSQL cannot hold a table of this name, or ``None`` (ITL-1162). + + PostgreSQL truncates a name longer than 63 bytes (UTF-8), so two such + names with the same first 63 bytes would be one table. + + Args: + table_name: The table name. + + Returns: + A short reason, or ``None`` when PostgreSQL holds the name. + """ + size = len(table_name.encode("utf-8")) + if size > _PG_MAX_IDENTIFIER_BYTES: + return ( + f"PostgreSQL truncates a table name to {_PG_MAX_IDENTIFIER_BYTES} " + f"bytes, and this name has {size}" + ) + return None + + def _require_table_name(self, table_name: str) -> None: + """Refuse a table name PostgreSQL would truncate, before it is sent.""" + reason = self.unstorable_table_name_reason(table_name) + if reason is not None: + raise UnstorableTableNameError( + database_kind=type(self).__name__, table_name=table_name, reason=reason + ) + def validate_records(self, records: pa.Table) -> None: """Reject tables carrying any Arrow field or schema metadata. diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index bcb0454e6..f88e5c455 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -1002,6 +1002,19 @@ def unstorable_column_reason( ) return None + def unstorable_table_name_reason(self, table_name: str) -> str | None: + """Say why SpiralDB cannot hold a table of this name: it holds every name. + + The database's table names are letters, digits and ``_``. + + Args: + table_name: The table name. + + Returns: + Always ``None``. + """ + return None + def validate_records(self, records: pa.Table) -> None: """No-op: ``SpiralDBConnector`` preserves Arrow field metadata via the native KV store.""" diff --git a/src/orcapod/databases/sqlite_connector.py b/src/orcapod/databases/sqlite_connector.py index 88fd35375..4656d8fd0 100644 --- a/src/orcapod/databases/sqlite_connector.py +++ b/src/orcapod/databases/sqlite_connector.py @@ -463,6 +463,19 @@ def unstorable_column_reason( """ return _sqlite_unstorable_reason(arrow_type) + def unstorable_table_name_reason(self, table_name: str) -> str | None: + """Say why SQLite cannot hold a table of this name: it holds every name. + + SQLite has no identifier length limit. + + Args: + table_name: The table name. + + Returns: + Always ``None``. + """ + return None + def validate_records(self, records: pa.Table) -> None: """Reject tables carrying any Arrow field or schema metadata. diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index fedbe14f7..be3fcd042 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -163,6 +163,59 @@ def __str__(self) -> str: ) +class UnstorableTableNameError(ValueError): + """A database cannot hold a table under the name it is asked to use (ITL-1162). + + PostgreSQL truncates a table name longer than 63 bytes, so two Orcapod tables + whose names share their first 63 bytes would silently become one. The + connector states which names it holds (``unstorable_table_name_reason``), and + a ``ConnectorArrowDatabase`` refuses any other name at every entry that + creates, writes, reads or checks a table, so a truncated name is never sent. + + Attributes: + database_kind: The kind of database that refused, e.g. + ``"PostgreSQLConnector"``. + table_name: The table name the database was asked to use. + reason: Why the database cannot hold it. + """ + + def __init__(self, *, database_kind: str, table_name: str, reason: str) -> None: + """Create the error. + + Args: + database_kind: The kind of database that refused. + table_name: The table name the database was asked to use. + reason: Why the database cannot hold it. + """ + self.database_kind = database_kind + self.table_name = table_name + self.reason = reason + super().__init__(database_kind, table_name, reason) + + def __reduce__(self) -> tuple[Any, ...]: + """Rebuild from the keyword fields, so pickling and copying keep them.""" + return ( + _rebuild_unstorable_table_name_error, + (self.database_kind, self.table_name, self.reason), + ) + + def __str__(self) -> str: + return ( + f"a {self.database_kind} database cannot hold a table named " + f"{self.table_name!r}: {self.reason}. Use an InMemoryArrowDatabase or a " + "DeltaTableDatabase instead." + ) + + +def _rebuild_unstorable_table_name_error( + database_kind: str, table_name: str, reason: str +) -> UnstorableTableNameError: + """Rebuild an ``UnstorableTableNameError`` from its fields (``__reduce__``).""" + return UnstorableTableNameError( + database_kind=database_kind, table_name=table_name, reason=reason + ) + + def _rebuild_unstorable_column_error( database_kind: str, column: str, diff --git a/src/orcapod/protocols/db_connector_protocol.py b/src/orcapod/protocols/db_connector_protocol.py index 9c6555da6..6ed2f94bd 100644 --- a/src/orcapod/protocols/db_connector_protocol.py +++ b/src/orcapod/protocols/db_connector_protocol.py @@ -141,6 +141,21 @@ def unstorable_column_reason( """ ... + def unstorable_table_name_reason(self, table_name: str) -> str | None: + """Say why this connector cannot hold a table of this name, or ``None``. + + The one place a connector states which table names it holds (ITL-1162): a + name the backend would change (PostgreSQL truncates past 63 bytes) is + refused, so two tables can never collide under one stored name. + + Args: + table_name: The table name. + + Returns: + A short reason, or ``None`` when the connector holds the name. + """ + ... + def validate_records(self, records: pa.Table) -> None: """Validate that ``records`` are safe to write through this connector. diff --git a/tests/test_core/sources/test_db_table_source.py b/tests/test_core/sources/test_db_table_source.py index b130f069c..22b4d1d3f 100644 --- a/tests/test_core/sources/test_db_table_source.py +++ b/tests/test_core/sources/test_db_table_source.py @@ -77,6 +77,9 @@ def create_table_if_not_exists(self, *args: Any, **kwargs: Any) -> None: def upsert_records(self, *args: Any, **kwargs: Any) -> None: pass # not used by DBTableSource + def unstorable_table_name_reason(self, table_name: str) -> str | None: + return None # every name is held + def unstorable_column_reason( self, name: str, arrow_type: Any, *, primary_key: bool ) -> str | None: diff --git a/tests/test_databases/test_connector_arrow_database.py b/tests/test_databases/test_connector_arrow_database.py index 47ce59548..5fdb3934a 100644 --- a/tests/test_databases/test_connector_arrow_database.py +++ b/tests/test_databases/test_connector_arrow_database.py @@ -155,6 +155,9 @@ def upsert_records( kept = existing.filter(mask) self._tables[table_name] = pa.concat_tables([kept, records]) + def unstorable_table_name_reason(self, table_name: str) -> str | None: + return None # every name is held + def unstorable_column_reason( self, name: str, arrow_type: pa.DataType, *, primary_key: bool ) -> str | None: diff --git a/tests/test_databases/test_postgresql_connector_integration.py b/tests/test_databases/test_postgresql_connector_integration.py index 473560085..ae5436de4 100644 --- a/tests/test_databases/test_postgresql_connector_integration.py +++ b/tests/test_databases/test_postgresql_connector_integration.py @@ -573,7 +573,7 @@ def test_operator_cache_refused_at_write_and_nothing_written( """An operator cache's system-tag column names exceed 63 bytes (ITL-1162).""" from orcapod import PipelineJob from orcapod.core.operators import SelectDataColumns - from orcapod.errors import OrcapodDatabaseWriteError, UnstorableColumnError + from orcapod.errors import UnstorableColumnError, UnstorableTableNameError from orcapod.sources import DictSource from orcapod.types import CacheMode @@ -590,9 +590,34 @@ def test_operator_cache_refused_at_write_and_nothing_written( job.nodes["sel"].attach_databases( pipeline_database=db.at("p"), cache_mode=CacheMode.LOG ) - with pytest.raises(OrcapodDatabaseWriteError) as exc_info: + with pytest.raises(Exception) as exc_info: _ = job.run() - cause = exc_info.value.__cause__ - assert isinstance(cause, UnstorableColumnError) - assert cause.column.startswith("_tag_") + # The cache's table name (over 63 bytes) or its system-tag column names are + # refused, whichever entry comes first; nothing reaches the server. + chain: list[BaseException] = [] + err: BaseException | None = exc_info.value + while err is not None: + chain.append(err) + err = err.__cause__ or err.__context__ + assert any( + isinstance(e, (UnstorableColumnError, UnstorableTableNameError)) for e in chain + ), chain + assert connector.get_table_names() == [] + + +@pytest.mark.postgres +def test_long_table_name_refused_and_nothing_written( + connector: PostgreSQLConnector, +) -> None: + """A table name PostgreSQL would truncate is never sent to it (ITL-1162).""" + from orcapod.errors import UnstorableTableNameError + + db = ConnectorArrowDatabase(connector) + path = ("fn", "x" * 70) + table = pa.table({"v": pa.array([1], type=pa.int64())}) + with pytest.raises(UnstorableTableNameError): + db.add_record(path, b"1", table) + with pytest.raises(UnstorableTableNameError): + db.get_all_records(path) + db.flush() assert connector.get_table_names() == [] diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 3d05da6da..0c1dbd062 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -885,3 +885,107 @@ def test_sql_type_maps_accept_view_types(self, tmp_path: pathlib.Path): assert _arrow_type_to_pg_sql(pa.string_view()) == "TEXT" assert _arrow_type_to_pg_sql(pa.binary_view()) == "BYTEA" + + +# --------------------------------------------------------------------------- +# PostgreSQL table names over 63 bytes (coordinator ruling, ITL-1162) +# --------------------------------------------------------------------------- + + +class TestTableNameLength: + def test_connector_rules(self, tmp_path: pathlib.Path): + connector, _ = _mocked_postgresql() + try: + assert connector.unstorable_table_name_reason("t" * 63) is None + reason = connector.unstorable_table_name_reason("t" * 64) + assert reason is not None and "63 bytes" in reason + finally: + connector._conn = None + sqlite = SQLiteConnector(str(tmp_path / "names.db")) + try: + assert sqlite.unstorable_table_name_reason("t" * 500) is None + finally: + sqlite.close() + + def test_every_table_entry_refuses_a_long_name(self): + """Write, read and check all refuse before anything is sent to the server.""" + from orcapod.errors import UnstorableTableNameError + + connector, conn = _mocked_postgresql() + try: + database = ConnectorArrowDatabase(connector) + path = ("fn", "x" * 70) + table = pa.table({"v": pa.array([1], type=pa.int64())}) + calls_before = len(conn.mock_calls) + + with pytest.raises(UnstorableTableNameError) as exc_info: + database.add_record(path, b"1", table) + with pytest.raises(UnstorableTableNameError): + database.get_all_records(path) + with pytest.raises(UnstorableTableNameError): + database.table_exists(path) + database.flush() # nothing was buffered + + assert len(conn.mock_calls) == calls_before + err = exc_info.value + assert err.database_kind == "PostgreSQLConnector" + assert err.table_name == "fn__" + "x" * 70 + assert "DeltaTableDatabase" in str(err) + finally: + connector._conn = None + + def test_connector_entries_refuse_a_long_name(self): + from orcapod.errors import UnstorableTableNameError + from orcapod.types import ColumnInfo + + connector, conn = _mocked_postgresql() + name = "t" * 64 + try: + calls_before = len(conn.mock_calls) + with pytest.raises(UnstorableTableNameError): + connector.create_table_if_not_exists( + name, [ColumnInfo("k", pa.large_string(), False)], pk_column="k" + ) + with pytest.raises(UnstorableTableNameError): + connector.get_column_info(name) + with pytest.raises(UnstorableTableNameError): + connector.get_pk_columns(name) + with pytest.raises(UnstorableTableNameError): + connector.upsert_records(name, pa.table({"k": ["a"]}), id_column="k") + assert len(conn.mock_calls) == calls_before + finally: + connector._conn = None + + def test_cached_source_on_postgresql_refused(self): + """A cached source's table name holds a hash: over 63 bytes on PostgreSQL.""" + from orcapod.core.sources.cached_source import CachedSource + from orcapod.errors import UnstorableTableNameError + + connector, conn = _mocked_postgresql() + try: + cached = CachedSource( + _source(), cache_database=ConnectorArrowDatabase(connector) + ) + calls_before = len(conn.mock_calls) + with pytest.raises(Exception) as exc_info: + _ = cached.as_table() + chain: list[BaseException] = [] + err: BaseException | None = exc_info.value + while err is not None: + chain.append(err) + err = err.__cause__ or err.__context__ + assert any(isinstance(e, UnstorableTableNameError) for e in chain), chain + assert len(conn.mock_calls) == calls_before + finally: + connector._conn = None + + def test_pickling(self): + import pickle + + from orcapod.errors import UnstorableTableNameError + + err = UnstorableTableNameError( + database_kind="PostgreSQLConnector", table_name="t", reason="r" + ) + again = pickle.loads(pickle.dumps(err)) + assert str(again) == str(err) From 70a02b0fedd6f2c304ea0304492463a6fafdfa4a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 16:09:55 -0800 Subject: [PATCH 14/14] fix(databases): check a cached data function's result database; SQLite reserved table prefix (ITL-1162) A function with its own result database (function_pod(result_database=...)) is checked with the same result-table schema before anything runs, and SQLiteConnector refuses a table name starting with 'sqlite_', which SQLite reserves (in-PR CodeRabbit review on 45e5e44e). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1162.md | 2 +- src/orcapod/core/cached_function_pod.py | 5 +++ src/orcapod/core/data_function.py | 5 +++ src/orcapod/databases/sqlite_connector.py | 9 ++++-- .../test_databases/test_unstorable_columns.py | 32 +++++++++++++++++++ 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/changelog.d/ITL-1162.md b/changelog.d/ITL-1162.md index 145553228..cfcd589f6 100644 --- a/changelog.d/ITL-1162.md +++ b/changelog.d/ITL-1162.md @@ -1,7 +1,7 @@ ### Fixed - A `ConnectorArrowDatabase` over `SQLiteConnector` or `PostgreSQLConnector` now refuses a table it cannot store before anything of it is written, with `UnstorableColumnError` naming the connector, the column, its Arrow type and what to use instead (ITL-1162). A connector stores a column only when it reads it back as written: on SQLite `bool`, `int64`, `double`, `large_string` and `large_binary`; on PostgreSQL also `int32`, `float`, `date32`, `timestamp[us]` and `timestamp[us, tz=UTC]`. Before, a function node on such a database ran its function and then failed at commit (`sqlite3.ProgrammingError: ... type 'list' is not supported`), timestamps were stored as text and read back as strings, and narrower integers and floats read back widened, failing the next write. A job now refuses a node whose stored table such a database cannot hold (every function node; a side-effect node on SQLite) when the job is built, before anything runs, naming the node. The connectors no longer fall back to a `TEXT` column for a type they have no column type for. -- `PostgreSQLConnector` now refuses a column name longer than 63 bytes (UTF-8), which PostgreSQL truncates, so it would read back as another name (ITL-1162). Before, such a column was stored under the truncated name. A table name longer than 63 bytes is refused too, with `UnstorableTableNameError`, at every entry that creates, writes, reads or checks a table, so a truncated name is never sent: two tables whose names share their first 63 bytes would otherwise silently be one. +- `PostgreSQLConnector` now refuses a column name longer than 63 bytes (UTF-8), which PostgreSQL truncates, so it would read back as another name (ITL-1162). Before, such a column was stored under the truncated name. A table name longer than 63 bytes is refused too, with `UnstorableTableNameError`, at every entry that creates, writes, reads or checks a table, so a truncated name is never sent: two tables whose names share their first 63 bytes would otherwise silently be one. `SQLiteConnector` refuses a table name starting with `sqlite_`, which SQLite reserves. - A `ConnectorArrowDatabase` over `SpiralDBConnector` now refuses, by the same rule, a column SpiralDB cannot store: a value column whose name starts with `__` (the `__record_id` key is allowed) or any column whose name contains `.`, and a timestamp column with a time zone, which SpiralDB reads back without it (ITL-1164). Before, a function node on such a database ran and then failed at commit (`Column name '__input_data_hash' has reserved prefix`). A job refuses every function node, side-effect node, operator cache and cached source on such a database when it is built. ### Known limitations diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py index 216b38c89..0ca3094e7 100644 --- a/src/orcapod/core/cached_function_pod.py +++ b/src/orcapod/core/cached_function_pod.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from typing import TYPE_CHECKING, Any +from orcapod.core.data_function import CachedDataFunction from orcapod.core.datagrams import Datagram from orcapod.core.function_pod import ( FunctionPodStream, @@ -114,6 +115,10 @@ def require_storable_results(self) -> None: ] ) require_storable(self._cache.result_database, schema) + # A function with its own result database (``function_pod(result_database=)``) + # stores the same table there. + if isinstance(pf, CachedDataFunction): + require_storable(pf.result_database, schema) # A pod with its own cache database (``pod_cache_database``) stores there too. if isinstance(self._function_pod, CachedFunctionPod): self._function_pod.require_storable_results() diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index 83c4a7fd1..4c6213d74 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -1115,6 +1115,11 @@ def __init__( name_path=record_path_prefix + self.uri[:1], ) + @property + def result_database(self) -> ArrowDatabaseProtocol: + """The database this function's results are cached in.""" + return self._cache.result_database + def _build_metadata_datagrams(self) -> tuple[Datagram, Datagram]: """Build variation and execution Datagrams for cache storage.""" variation_datagram = Datagram( diff --git a/src/orcapod/databases/sqlite_connector.py b/src/orcapod/databases/sqlite_connector.py index 4656d8fd0..72c89236b 100644 --- a/src/orcapod/databases/sqlite_connector.py +++ b/src/orcapod/databases/sqlite_connector.py @@ -464,16 +464,19 @@ def unstorable_column_reason( return _sqlite_unstorable_reason(arrow_type) def unstorable_table_name_reason(self, table_name: str) -> str | None: - """Say why SQLite cannot hold a table of this name: it holds every name. + """Say why SQLite cannot hold a table of this name, or ``None`` (ITL-1162). - SQLite has no identifier length limit. + SQLite has no identifier length limit, but reserves table names that + start with ``sqlite_`` (in any case) for itself. Args: table_name: The table name. Returns: - Always ``None``. + A short reason, or ``None`` when SQLite holds the name. """ + if table_name.lower().startswith("sqlite_"): + return "SQLite reserves table names that start with 'sqlite_'" return None def validate_records(self, records: pa.Table) -> None: diff --git a/tests/test_databases/test_unstorable_columns.py b/tests/test_databases/test_unstorable_columns.py index 0c1dbd062..833d2e113 100644 --- a/tests/test_databases/test_unstorable_columns.py +++ b/tests/test_databases/test_unstorable_columns.py @@ -989,3 +989,35 @@ def test_pickling(self): ) again = pickle.loads(pickle.dumps(err)) assert str(again) == str(err) + + +class TestInPRReviewOn45e5e44e: + def test_data_function_result_database_checked(self, tmp_path: pathlib.Path): + """``function_pod(result_database=...)`` caches in its own database too.""" + sqlite = _sqlite_database(tmp_path) + + @function_pod(output_keys="double", result_database=sqlite) + def cached_double(x: int) -> int: + CALLS.append(x) + return 2 * x + + job = PipelineJob(name="p", database=InMemoryArrowDatabase()) + with pytest.raises(UnstorableColumnError) as exc_info: + with job: + _ = cached_double.pod(_source(), label="dbl") + + assert exc_info.value.node_label == "dbl" + assert exc_info.value.database_kind == "SQLiteConnector" + assert CALLS == [] + + @pytest.mark.parametrize("name", ["sqlite_results", "SQLite_x", "SQLITE_"]) + def test_sqlite_reserved_table_prefix(self, tmp_path: pathlib.Path, name): + from orcapod.errors import UnstorableTableNameError + + database = _sqlite_database(tmp_path) + table = pa.table({"v": pa.array([1], type=pa.int64())}) + + with pytest.raises(UnstorableTableNameError): + database.add_record((name,), b"1", table) + database.flush() + assert database.list_table_paths() == []