From b2a3a69801d3bff2bcc07ff6d1402cde7e092ea8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 20:49:13 -0800 Subject: [PATCH 01/30] test(databases): failing regression tests for Delta read failures reported as no rows (ITL-884) Adds StoreReadError and tests showing that an unreadable Delta table reads as None, that a table which fails to load can be overwritten, that a changed column type is appended (silently cast or left unreadable), and that both orchestrators finish a run over an unreadable result table without raising. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/errors.py | 52 +++ .../2026-11-10-itl-884-delta-read-failures.md | 189 ++++++++++ .../test_delta_read_failures.py | 340 ++++++++++++++++++ .../test_pipeline/test_store_read_failure.py | 133 +++++++ 4 files changed, 714 insertions(+) create mode 100644 superpowers/plans/2026-11-10-itl-884-delta-read-failures.md create mode 100644 tests/test_databases/test_delta_read_failures.py create mode 100644 tests/test_pipeline/test_store_read_failure.py diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 81c653627..d27e850fc 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -195,3 +195,55 @@ def __init__( f"node={node_identity_path!r} " f"cached_content_hash={cached_content_hash!r}" ) + + +class StoreReadError(RuntimeError): + """Raised when a stored table exists but could not be read. + + A table that does not exist reads as empty. A table that exists but cannot be + read (a missing or damaged data file, a log the backend cannot load, columns + whose stored data no longer matches the table schema) raises this error + instead, so stored results never look dropped. Nothing is recomputed or + overwritten. A failed store read always fails the run, whatever the + orchestrator's ``error_policy``. + + The backend's own exception is chained as ``__cause__``. + + Attributes: + table: Location of the table that could not be read (a directory or a + cloud URI). + record_path: Path of the table, including the database view's prefix. + """ + + def __init__(self, table: str, record_path: tuple[str, ...]) -> None: + self.table = table + self.record_path = tuple(record_path) + super().__init__(table, self.record_path) + + def __str__(self) -> str: + cause = self.__cause__ + reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" + return ( + f"Could not read the stored table at {self.table!r}{reason}. The table " + "exists, so its rows are not treated as missing: nothing was recomputed " + "or overwritten." + ) + + @classmethod + def in_group(cls, group: BaseExceptionGroup) -> "list[StoreReadError]": + """Return every ``StoreReadError`` in an exception group, nested ones included. + + Args: + group: An exception group, such as the one an ``asyncio.TaskGroup`` + raises. + + Returns: + The ``StoreReadError`` leaves in the order the group holds them. + """ + found: "list[StoreReadError]" = [] + for exc in group.exceptions: + if isinstance(exc, BaseExceptionGroup): + found.extend(cls.in_group(exc)) + elif isinstance(exc, cls): + found.append(exc) + return found diff --git a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md new file mode 100644 index 000000000..71ef7fa94 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md @@ -0,0 +1,189 @@ +# ITL-884: Delta read failures are never reported as "no rows" — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A Delta table that exists but cannot be read raises `StoreReadError`, never an +empty result. A flush that would make an existing table unreadable, or silently cast the +stored column type, is refused before anything is written. + +**Architecture:** Every read path in `DeltaTableDatabase` separates "table absent" +(`TableNotFoundError` → `None`, unchanged) from "table present but unreadable" (raise +`StoreReadError` naming the table URI, with the backend exception chained). `_flush_by_key` +creates a table only on `TableNotFoundError`. Before it merges into an existing table, it compares +the incoming batch's column types with the stored Delta schema. Both sides go through Delta's +own type mapping, with extension types stripped to storage, large-normalized and +nullability-blind. A difference raises `InputValidationError` naming the table, column and both +types. A store read failure fails the run whatever `error_policy` is (same shape as PR #287's +`StoreWriteError`). + +**Tech Stack:** deltalake 1.0.2, pyarrow, pytest. + +Approved by the coordinator (2026-11-10): contract as proposed. D1 adds a sibling +`StoreReadError(RuntimeError)`, and the append refusal raises `InputValidationError`. D2 is +option A: re-raise in the two per-row handlers and unwrap in the async orchestrator, with the +same shape as #287. D3 makes the comparison nullability-blind, plus a test that pins +nullability-only behaviour. D4 fixes the `DeltaProtocolError` → overwrite path. + +No hash, path or stored-schema changes. + +--- + +## File map + +- `src/orcapod/errors.py` gets `StoreReadError` (`table`, `record_path`, `__str__` from + `__cause__`, `in_group`). +- `src/orcapod/databases/delta_lake_databases.py` changes the read paths and + `_flush_by_key`, and adds `_require_compatible_append` and the module helper + `_comparable_storage_type`. +- `src/orcapod/core/nodes/function_node.py` (`execute` per-row handler) and + `src/orcapod/core/function_pod.py` (async `process_one`) both get + `except StoreReadError: ...; raise`. +- `src/orcapod/pipeline/async_orchestrator.py` gets `except* StoreReadError` and raises the + first leaf. +- Tests: `tests/test_databases/test_delta_read_failures.py` and + `tests/test_pipeline/test_store_read_failure.py`. +- Docs: `docs/concepts/function-pods.md` gets an admonition, `CHANGELOG.md` gets Fixed and + Breaking entries, and `DESIGN_ISSUES.md` gets a new entry at the end. + +## Task 1: Failing regression tests (database level) + +`tests/test_databases/test_delta_read_failures.py`, public `DeltaTableDatabase` API only: + +- `unreadable_table` fixture: write 2 rows, flush, delete the table's `*.parquet` files (the + log stays intact). Use a fresh `DeltaTableDatabase` instance for the reads. +- `test_get_all_records_raises` checks `StoreReadError`: `exc.table` ends with the table + directory, `str(exc)` contains the URI, and `__cause__` is set. +- `test_get_records_by_ids_raises`, `test_get_record_by_id_raises`, + `test_get_records_with_column_value_raises`, and + `test_add_records_skip_duplicates_raises` (existing-ids refresh). Each raises + `StoreReadError`. +- `test_absent_table_reads_return_none` covers all four reads on a path never written: + `None`, `table_exists` is False. +- `test_unloadable_log_raises_on_read_and_flush_never_overwrites`: the table's log names an + unsupported reader feature (the load fails). Reads raise `StoreReadError`. `add_records(..., + flush=True)` raises, the log files and parquet files are byte-identical afterwards, and no + new log version exists. +- `test_protocol_error_on_load_is_not_treated_as_absent`: monkeypatch + `deltalake.DeltaTable` to raise `DeltaProtocolError` for this table (the error deltalake + 1.0.2 does not raise naturally from a local table). Flush raises and the stored files are + untouched. The comment explains the monkeypatch. +- Append refusal: + - `test_castable_type_change_is_refused`: int64 stored, float64 appended. Raises (flush + wraps it in `RuntimeError`, so assert that `InputValidationError` is the `__cause__`). + The message names the column `x`, `int64` and `double`. A re-read gives the original row + only, with type int64. + - `test_narrowing_change_is_refused`: int64 stored, int32 appended. + - `test_extension_storage_change_is_refused`: `ext` stored, `ext>` + appended. Refused, and the table is still readable afterwards (1 row). + - Legitimate cases still pass: identical schema; `string` vs `large_string` (and + `list` vs `large_list`); a new column; a missing column; an extension type + with unchanged storage (2 rows read back). + - `test_nullability_only_difference_is_accepted`: a nested non-null struct field vs a + nullable one is accepted. This pins today's nullability-blind behaviour (ITL-842 audit). + It also records what Delta stores for nullability after the merge, as an assertion on + the read-back schema. + +Run `uv run pytest tests/test_databases/test_delta_read_failures.py -v`. Expected: the read +tests FAIL (`None` / no raise / raw errors), and the refusal tests FAIL (the write is +accepted). + +## Task 2: Failing regression tests (pipeline level, both orchestrators) + +`tests/test_pipeline/test_store_read_failure.py`, parametrized over +`SyncPipelineOrchestrator()` and `AsyncPipelineOrchestrator()`: + +- Run `source -> FunctionPod(double) label="dbl"` on a `DeltaTableDatabase`, then delete + the parquet files of the `_result` table. +- `test_unreadable_result_table_fails_the_run`: a fresh store and job, `job.run(orchestrator=...)` + raises `StoreReadError`, and the function is not called. +- `test_unreadable_result_table_fails_the_run_under_continue`: also delete the pipeline + (tag) table, so that Phase 1 sees no tag rows and the per-row lookup is the read that fails. + With `error_policy="continue"` it still raises `StoreReadError` and makes 0 calls. +- `test_absent_tables_still_compute`: control case. Delete both tables entirely, then the + run computes all 3 rows. + +Run the tests and expect FAIL. + +## Task 3: `StoreReadError` + +Add the class to `src/orcapod/errors.py`: + +```python +class StoreReadError(RuntimeError): + """Raised when a stored table exists but could not be read. ...""" + + def __init__(self, table: str, record_path: tuple[str, ...]) -> None: + self.table = table + self.record_path = tuple(record_path) + super().__init__(table, self.record_path) + + def __str__(self) -> str: + cause = self.__cause__ + reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" + return ( + f"Could not read the stored table at {self.table!r}{reason}. The table exists, " + "so its rows are not treated as missing: nothing was recomputed or overwritten." + ) + + @classmethod + def in_group(cls, group: BaseExceptionGroup) -> "list[StoreReadError]": ... +``` + +## Task 4: Delta read paths + +- `_get_delta_table`: on `TableNotFoundError` return `None`. On any other exception, drop + the cache entry and `raise StoreReadError(table_uri, full_path) from exc`. +- Add the helper `_read_failed(record_path, exc) -> StoreReadError`, built from + `_get_table_uri(record_path)` and `self._path_prefix + record_path`. +- `get_all_records`: `except Exception as exc: raise self._read_failed(...) from exc` + (replaces the warning-and-skip). +- `get_records_by_ids`, `get_record_by_id`, `get_records_with_column_value`: narrow each + `try` to the read itself, then `raise self._read_failed(...) from exc` (replaces + `return None` / the raw re-raise). +- `_refresh_existing_ids_cache`: wrap the `to_pyarrow_table()` failure the same way. + +## Task 5: Delta flush path + +- In `_flush_by_key`, load only on `TableNotFoundError`, where `delta_table = None`. Any + other exception does `raise StoreReadError(table_uri, abs_path_components) from exc`. The + existing outer handler restores the pending batch. +- Before `merge`: `self._require_compatible_append(table_uri, delta_table, combined_table)`. +- `_comparable_storage_type(t)` is a module-level function. It strips extension types to + storage at every depth, rebuilds struct/list/fixed-size-list/map with nullable children + (comparison only, never stored), and recurses. +- `_require_compatible_append` builds the stored types from + `pa.schema(delta_table.schema().to_arrow())`. The incoming types go through + `deltalake.Schema.from_arrow(pa.schema([pa.field(name, _comparable_storage_type(type))]))` + and `.to_arrow()`. Both sides are passed through `normalize_to_large_types(..., + keep_null=True)` and `_comparable_storage_type`. Only columns present on both sides are + compared. On a mismatch it raises `InputValidationError(f"Cannot append to the stored table + {table_uri!r}: column {name!r} is stored as {stored} but the new rows have {incoming}. ...")`. + +Run the Task 1 and Task 2 tests and expect PASS. + +## Task 6: Run-level handling (same shape as #287) + +- `FunctionNode.execute` per-row handler: put `except StoreReadError as exc:` before + `except Exception`. It calls `ctx_obs.on_data_crash(...)` and `ctx_obs.on_node_end(...)`, + then `raise`. +- `FunctionPod` async `process_one`: put `except StoreReadError as exc:` before + `except Exception`. It calls `obs.on_data_crash(...)` and then `raise`. +- `AsyncPipelineOrchestrator._run_async`: add `except* StoreReadError as group: raise + StoreReadError.in_group(group)[0]`. + +Run the Task 2 tests and expect PASS. + +## Task 7: Docs, changelog, design issues, full suite + +- `docs/concepts/function-pods.md` gets a `!!! note "When the store cannot be read"` + admonition after "Re-runs compute only what is new". It has no python block. +- `CHANGELOG.md`: a Fixed entry (ITL-884: read failures raise `StoreReadError`; an unloadable + table is never overwritten). Breaking Changes: a castable column type change appended to an + existing Delta table is now refused with `InputValidationError` instead of being silently + cast to the stored type. +- `DESIGN_ISSUES.md`: one self-contained entry at the end with status resolved and a Fix note. +- Then run `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code 0) and `uv run pytest tests/`. +- Commit in Conventional Commits style: `test(databases): ...` for the failing tests first, + then `fix(databases): ...`, then `docs: ...`. diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py new file mode 100644 index 000000000..597312934 --- /dev/null +++ b/tests/test_databases/test_delta_read_failures.py @@ -0,0 +1,340 @@ +"""Regression tests for ITL-884: a Delta table that cannot be read is never "no rows". + +Covers: +- A table that exists but cannot be read raises ``StoreReadError`` on every read + path, naming the table; a table that does not exist still reads as ``None``. +- A table whose log cannot be loaded is never overwritten by a flush. +- An append whose column types differ from the stored table is refused before + anything is written, naming the column and both types; legitimate appends + (same types in another layout, new or missing columns, unchanged extension + storage) still succeed. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path + +import deltalake +import pyarrow as pa +import pytest + +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import InputValidationError, StoreReadError + +PATH = ("results", "t") + + +def rows(ids: list[bytes], x: pa.Array, **extra: pa.Array) -> pa.Table: + """Build a table keyed by ``rid`` with an ``x`` column and optional extras.""" + return pa.table({"rid": pa.array(ids, pa.large_binary()), "x": x, **extra}) + + +def table_dir(base: Path) -> Path: + return base.joinpath(*PATH) + + +def snapshot(directory: Path) -> dict[str, bytes]: + """Every file under ``directory`` with its bytes, keyed by relative path.""" + return { + str(p.relative_to(directory)): p.read_bytes() + for p in sorted(directory.rglob("*")) + if p.is_file() + } + + +class _Ext(pa.ExtensionType): + """A minimal extension type standing in for a logical type's storage.""" + + def __init__(self, storage: pa.DataType) -> None: + super().__init__(storage, "orcapod.test.itl884") + + def __arrow_ext_serialize__(self) -> bytes: + return b"" + + @classmethod + def __arrow_ext_deserialize__( + cls, storage_type: pa.DataType, serialized: bytes + ) -> "_Ext": + return cls(storage_type) + + +def ext_array(values: list, storage: pa.DataType) -> pa.ExtensionArray: + return pa.ExtensionArray.from_storage(_Ext(storage), pa.array(values, storage)) + + +@pytest.fixture +def base(tmp_path: Path) -> Path: + return tmp_path / "store" + + +@pytest.fixture +def stored(base: Path) -> DeltaTableDatabase: + """A store holding two rows (``a``, ``b``) of ``x: int64`` at ``PATH``.""" + db = DeltaTableDatabase(base_path=base) + db.add_records( + PATH, + rows([b"a", b"b"], pa.array([1, 2], pa.int64())), + record_id_column="rid", + flush=True, + ) + return db + + +@pytest.fixture +def unreadable(base: Path, stored: DeltaTableDatabase) -> DeltaTableDatabase: + """A fresh view of a table whose log is intact but whose data files are gone.""" + data_files = list(table_dir(base).glob("*.parquet")) + assert data_files + for f in data_files: + f.unlink() + return DeltaTableDatabase(base_path=base) + + +def unsupported_reader_feature(base: Path) -> None: + """Make the table's log require a reader feature deltalake does not know.""" + log = table_dir(base) / "_delta_log" / "00000000000000000000.json" + actions = [json.loads(line) for line in log.read_text().splitlines()] + for action in actions: + if "protocol" in action: + action["protocol"] = { + "minReaderVersion": 3, + "minWriterVersion": 7, + "readerFeatures": ["orcapodMadeUpFeature"], + "writerFeatures": ["orcapodMadeUpFeature"], + } + log.write_text("\n".join(json.dumps(a) for a in actions)) + + +def assert_names_table(exc: StoreReadError, base: Path) -> None: + assert Path(exc.table) == table_dir(base) + assert exc.table in str(exc) + assert exc.__cause__ is not None + + +# --------------------------------------------------------------------------- +# Reads: unreadable table raises, absent table is None +# --------------------------------------------------------------------------- + +READS: dict[str, Callable[[DeltaTableDatabase], object]] = { + "get_all_records": lambda db: db.get_all_records(PATH), + "get_records_by_ids": lambda db: db.get_records_by_ids(PATH, [b"a"]), + "get_record_by_id": lambda db: db.get_record_by_id(PATH, b"a"), + "get_records_with_column_value": lambda db: db.get_records_with_column_value( + PATH, {"x": 1} + ), +} + + +class TestUnreadableTable: + @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) + def test_read_raises_store_read_error(self, base, unreadable, read): + with pytest.raises(StoreReadError) as info: + read(unreadable) + assert_names_table(info.value, base) + + def test_skip_duplicates_append_raises_store_read_error(self, base, unreadable): + # skip_duplicates reads the stored record ids before buffering the rows. + with pytest.raises(StoreReadError) as info: + unreadable.add_records( + PATH, + rows([b"c"], pa.array([3], pa.int64())), + record_id_column="rid", + skip_duplicates=True, + ) + assert_names_table(info.value, base) + + def test_table_still_exists(self, unreadable): + assert unreadable.table_exists(PATH) + + +class TestAbsentTable: + @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) + def test_read_returns_none(self, base, read): + db = DeltaTableDatabase(base_path=base) + assert read(db) is None + assert not db.table_exists(PATH) + + +# --------------------------------------------------------------------------- +# Flush never overwrites a table it could not load +# --------------------------------------------------------------------------- + + +class TestUnloadableTableIsNeverOverwritten: + def test_unsupported_reader_feature(self, base, stored): + unsupported_reader_feature(base) + before = snapshot(table_dir(base)) + db = DeltaTableDatabase(base_path=base) + + with pytest.raises(StoreReadError): + db.get_all_records(PATH) + with pytest.raises(RuntimeError) as info: + db.add_records( + PATH, + rows([b"c"], pa.array([3], pa.int64())), + record_id_column="rid", + flush=True, + ) + assert isinstance(info.value.__cause__, StoreReadError) + assert snapshot(table_dir(base)) == before + + def test_protocol_error_on_load_is_not_treated_as_absent( + self, base, stored, monkeypatch + ): + # deltalake 1.0.2 does not raise DeltaProtocolError for a local table we can + # build here, so the load is replaced by one that raises it for this table. + # Before ITL-884 the flush treated this error as "no table" and wrote the + # batch with mode="overwrite", replacing the stored rows. + real = deltalake.DeltaTable + target = table_dir(base) + + def failing_load(table_uri, *args, **kwargs): + if Path(str(table_uri)) == target: + raise deltalake.exceptions.DeltaProtocolError("simulated") + return real(table_uri, *args, **kwargs) + + before = snapshot(target) + db = DeltaTableDatabase(base_path=base) + monkeypatch.setattr(deltalake, "DeltaTable", failing_load) + with pytest.raises(RuntimeError) as info: + db.add_records( + PATH, + rows([b"c"], pa.array([3], pa.int64())), + record_id_column="rid", + flush=True, + ) + monkeypatch.undo() + + assert isinstance(info.value.__cause__, StoreReadError) + assert snapshot(target) == before + stored_rows = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert stored_rows is not None + assert stored_rows.num_rows == 2 + + +# --------------------------------------------------------------------------- +# Append refusal: a column's type differs from the stored table +# --------------------------------------------------------------------------- + + +def append(db: DeltaTableDatabase, table: pa.Table) -> None: + db.add_records(PATH, table, record_id_column="rid", flush=True) + + +def refusal(info: pytest.ExceptionInfo[RuntimeError]) -> InputValidationError: + """The ``InputValidationError`` behind a refused flush.""" + cause = info.value.__cause__ + assert isinstance(cause, InputValidationError), repr(info.value) + return cause + + +class TestChangedColumnTypeIsRefused: + @pytest.mark.parametrize( + ("new_type", "value", "named"), + [ + (pa.float64(), 2.0, "double"), + (pa.int32(), 2, "int32"), + (pa.large_string(), "2", "string"), + ], + ids=["int64->float64", "int64->int32", "int64->large_string"], + ) + def test_plain_type_change(self, base, stored, new_type, value, named): + with pytest.raises(RuntimeError) as info: + append(stored, rows([b"c"], pa.array([value], new_type))) + message = str(refusal(info)) + assert "'x'" in message + assert "int64" in message + assert named in message + assert str(table_dir(base)) in message + + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + assert back.num_rows == 2 + assert back.schema.field("x").type == pa.int64() + + def test_extension_storage_change(self, base): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) + + with pytest.raises(RuntimeError) as info: + append( + db, + rows([b"b"], ext_array([[1.0]], pa.large_list(pa.float32()))), + ) + message = str(refusal(info)) + assert "'x'" in message + assert "binary" in message + assert "list" in message + + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + assert back.num_rows == 1 + + +class TestLegitimateAppendsStillPass: + def read_back(self, base: Path) -> pa.Table: + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + return back + + def test_identical_schema(self, base, stored): + append(stored, rows([b"c"], pa.array([3], pa.int64()))) + assert self.read_back(base).num_rows == 3 + + @pytest.mark.parametrize( + ("first", "second"), + [ + (pa.array(["a"], pa.string()), pa.array(["b"], pa.large_string())), + (pa.array(["a"], pa.large_string()), pa.array(["b"], pa.string())), + ( + pa.array([[1]], pa.list_(pa.int64())), + pa.array([[2]], pa.large_list(pa.int64())), + ), + (pa.array([b"a"], pa.binary()), pa.array([b"b"], pa.large_binary())), + ], + ids=["string->large", "large->string", "list->large", "binary->large"], + ) + def test_large_and_non_large_layouts(self, base, first, second): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], first)) + append(db, rows([b"b"], second)) + assert self.read_back(base).num_rows == 2 + + def test_new_column(self, base, stored): + append( + stored, + rows([b"c"], pa.array([3], pa.int64()), y=pa.array(["new"])), + ) + back = self.read_back(base) + assert back.num_rows == 3 + assert "y" in back.column_names + + def test_missing_column(self, base): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], pa.array([1], pa.int64()), y=pa.array(["old"]))) + append(db, rows([b"b"], pa.array([2], pa.int64()))) + assert self.read_back(base).num_rows == 2 + + def test_extension_type_with_unchanged_storage(self, base): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) + append(db, rows([b"b"], ext_array([b"y"], pa.large_binary()))) + assert self.read_back(base).num_rows == 2 + + def test_nullability_only_difference_is_accepted(self, base): + # Pins today's behaviour: the append check ignores nullability (ITL-884, + # decision D3). Whether a nullability change should be refused belongs to + # the nullability audit, ITL-842. The read-back assertion records what + # Delta does on merge: the stored non-null child becomes nullable, so the + # rows written first now read back with a different (looser) type. + strict = pa.struct([pa.field("a", pa.int64(), nullable=False)]) + loose = pa.struct([pa.field("a", pa.int64(), nullable=True)]) + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], pa.array([{"a": 1}], strict))) + append(db, rows([b"b"], pa.array([{"a": 2}], loose))) + + back = self.read_back(base) + assert back.num_rows == 2 + assert back.schema.field("x").type == loose diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py new file mode 100644 index 000000000..4010ae2cb --- /dev/null +++ b/tests/test_pipeline/test_store_read_failure.py @@ -0,0 +1,133 @@ +"""Regression tests for ITL-884 at the job level: an unreadable store fails the run. + +Before ITL-884 a Delta result table that existed but could not be read was +returned as "no rows": the job warned that the data "may have been deleted +externally", swallowed the per-row read errors under ``error_policy="continue"``, +finished without raising, and the node showed no records. Both orchestrators +must now raise ``StoreReadError`` instead, and a table that does not exist must +still be computed. +""" + +from __future__ import annotations + +import shutil +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.function_pod import FunctionPod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import StoreReadError +from orcapod.pipeline import PipelineJob +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +_CALLS: list[int] = [] + + +def double(x: int) -> int: + _CALLS.append(x) + return 2 * x + + +ORCHESTRATORS: dict[str, Callable[[], object]] = { + "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), + "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), +} + + +@pytest.fixture(autouse=True) +def _reset_calls(): + _CALLS.clear() + yield + _CALLS.clear() + + +@pytest.fixture(params=list(ORCHESTRATORS)) +def make_orchestrator(request) -> Callable[[], object]: + return ORCHESTRATORS[request.param] + + +def build_job(store_path: Path) -> PipelineJob: + source = ArrowTableSource( + pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + tag_columns=["k"], + infer_nullable=True, + ) + job = PipelineJob(name="itl884", store=DeltaTableDatabase(base_path=store_path)) + with job: + FunctionPod(PythonDataFunction(double, output_keys="y"))(source, label="dbl") + return job + + +def table_dirs(store_path: Path) -> tuple[Path, Path]: + """The (result table, pipeline table) directories the job wrote.""" + dirs = {p.parent for p in store_path.rglob("*.parquet")} + result = [d for d in dirs if "_result" in d.parts] + pipeline = [d for d in dirs if "_result" not in d.parts] + assert len(result) == 1 and len(pipeline) == 1, sorted(dirs) + return result[0], pipeline[0] + + +def delete_data_files(table: Path) -> None: + for f in table.glob("*.parquet"): + f.unlink() + + +@pytest.fixture +def stored(tmp_path: Path, make_orchestrator) -> Path: + """A store in which the job has computed and stored all three rows.""" + store_path = tmp_path / "store" + build_job(store_path).run(orchestrator=make_orchestrator()) + assert len(_CALLS) == 3 + _CALLS.clear() + return store_path + + +def test_unreadable_result_table_fails_the_run(stored, make_orchestrator): + result_table, _ = table_dirs(stored) + delete_data_files(result_table) + + job = build_job(stored) + with pytest.raises(StoreReadError) as info: + job.run(orchestrator=make_orchestrator()) + + assert Path(info.value.table) == result_table + assert _CALLS == [] + + +def test_unreadable_result_table_fails_the_run_under_continue( + stored, make_orchestrator +): + # With the pipeline table gone, the job has no stored tag rows to join, so the + # first read of the result table is the per-row cache lookup, which runs inside + # the error_policy="continue" handler. + result_table, pipeline_table = table_dirs(stored) + delete_data_files(result_table) + shutil.rmtree(pipeline_table) + + job = build_job(stored) + with pytest.raises(StoreReadError) as info: + job.run(orchestrator=make_orchestrator()) + + assert Path(info.value.table) == result_table + assert _CALLS == [] + + +def test_absent_tables_are_computed(stored, make_orchestrator): + result_table, pipeline_table = table_dirs(stored) + shutil.rmtree(result_table) + shutil.rmtree(pipeline_table) + + build_job(stored).run(orchestrator=make_orchestrator()) + assert sorted(_CALLS) == [1, 2, 3] + + # The recomputed rows were stored: a fresh job over the same store computes + # nothing. (Reading the node right after its own run can miss rows, ITL-864.) + _CALLS.clear() + build_job(stored).run(orchestrator=make_orchestrator()) + assert _CALLS == [] From 79141a595832013d5c85db14a49e659b172bb1cb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 20:55:08 -0800 Subject: [PATCH 02/30] fix(databases): a Delta table that cannot be read raises StoreReadError, never no rows (ITL-884) Only TableNotFoundError means a table is absent. Every other load or read error in DeltaTableDatabase raises StoreReadError naming the table, and a flush that cannot load an existing table raises instead of overwriting it. Before merging into an existing table the flush compares column types with the stored schema through Delta's own type mapping (extension types by storage, large-normalized, nullability ignored) and refuses a mismatch with InputValidationError naming the table, column and both types. A store read failure fails the run under both orchestrators whatever the error_policy. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 6 + src/orcapod/core/nodes/function_node.py | 7 + src/orcapod/databases/delta_lake_databases.py | 265 ++++++++++++------ src/orcapod/pipeline/async_orchestrator.py | 5 + .../test_delta_read_failures.py | 14 +- .../test_delta_table_database.py | 2 +- .../test_pipeline/test_store_read_failure.py | 6 +- 7 files changed, 220 insertions(+), 85 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index ba7e3b1d4..4db5ed7bb 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -14,6 +14,7 @@ from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.streams.base import StreamBase from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.errors import StoreReadError from orcapod.protocols.core_protocols import ( ArgumentGroup, FunctionPodProtocol, @@ -587,6 +588,11 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: out_tag, result_data = await self._async_invoke_with_hooks( tag, data, logger=pkt_logger, run_id=run_id ) + except StoreReadError as exc: + # A failed store read fails the run; only rows whose + # function raised are skipped. + obs.on_data_crash(pod_label, tag, data, exc) + raise except Exception as exc: logger.debug( "Data processing failed, skipping: %s", exc, exc_info=True diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 7af980049..ff6fed40f 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -35,6 +35,7 @@ EphemeralResultMissingError, PipelineJobRequiredError, SchemaVersionError, + StoreReadError, ) from orcapod.protocols.core_protocols import ( FunctionPodProtocol, @@ -1294,6 +1295,12 @@ def execute( tag_out, result = self._process_data_internal( tag, data, logger=pkt_logger, run_id=run_id ) + except StoreReadError as exc: + # A failed store read fails the run whatever the error policy: + # error_policy only decides about rows whose function raised. + ctx_obs.on_data_crash(node_label, tag, data, exc) + ctx_obs.on_node_end(node_label, node_hash) + raise except Exception as exc: logger.warning( "Data execution failed in %s: %s", diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index c7dec5dcf..e767854cd 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from orcapod.databases.utils import coerce_record_id +from orcapod.errors import InputValidationError, StoreReadError from orcapod.databases.storage_utils import is_cloud_uri, parse_base_path from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -27,6 +28,49 @@ logger = logging.getLogger(__name__) +def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: + """Return ``arrow_type`` reduced to what the append check compares. + + Extension types are replaced by their storage type and every nested field is + made nullable, at every depth. The result is used only to compare a batch + with a stored table (ITL-884); it is never stored, so dropping nested + non-null flags here changes no schema or hash. + + Args: + arrow_type: The Arrow type to reduce. + + Returns: + The storage type with all nested fields nullable. + """ + if isinstance(arrow_type, pa.ExtensionType): + return _comparable_storage_type(arrow_type.storage_type) + if pa.types.is_struct(arrow_type): + return pa.struct( + [pa.field(f.name, _comparable_storage_type(f.type)) for f in arrow_type] + ) + if pa.types.is_large_list(arrow_type): + field = arrow_type.value_field + return pa.large_list(pa.field(field.name, _comparable_storage_type(field.type))) + if pa.types.is_list(arrow_type): + field = arrow_type.value_field + return pa.list_(pa.field(field.name, _comparable_storage_type(field.type))) + if pa.types.is_fixed_size_list(arrow_type): + field = arrow_type.value_field + return pa.list_( + pa.field(field.name, _comparable_storage_type(field.type)), + arrow_type.list_size, + ) + if pa.types.is_map(arrow_type): + key, item = arrow_type.key_field, arrow_type.item_field + # pyarrow accepts key and item Fields here; its stubs only list types. + return pa.map_( # pyright: ignore[reportCallIssue] + pa.field(key.name, _comparable_storage_type(key.type), nullable=False), + pa.field(item.name, _comparable_storage_type(item.type)), + keys_sorted=arrow_type.keys_sorted, + ) + return arrow_type + + class DeltaTableDatabase: """ A Delta table database with clear insert vs update semantics. @@ -213,11 +257,23 @@ def _get_delta_table( # Table doesn't exist return None except Exception as e: - logger.error(f"Error loading Delta table for {record_key}: {e}") - # Try to clear any corrupted cache - if record_key in self._delta_table_cache: - self._delta_table_cache.pop(record_key) - raise + # The table exists but could not be loaded: never report it as absent. + self._delta_table_cache.pop(record_key, None) + raise self._read_failed(record_path) from e + + def _read_failed(self, record_path: tuple[str, ...]) -> StoreReadError: + """Build the error for a table at ``record_path`` that exists but cannot be read. + + Args: + record_path: Path components relative to this database view. + + Returns: + A ``StoreReadError`` naming the table; the caller raises it from the + backend's exception. + """ + return StoreReadError( + self._get_table_uri(record_path), self._path_prefix + record_path + ) def table_exists(self, record_path: tuple[str, ...]) -> bool: """Return ``True`` if a Delta table exists at ``record_path``. @@ -317,28 +373,26 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: # Get all existing IDs from Delta table using standard RECORD_ID_COLUMN # TODO: replace this with more targetted loading of only the target column and in batches arrow_table = delta_table.to_pyarrow_table() - if arrow_table.num_rows == 0: - self._existing_ids_cache[record_key] = set() - elif self.RECORD_ID_COLUMN not in arrow_table.column_names: - # TODO: replace this with proper checking of the table schema first! - logger.warning(f"Delta table missing {self.RECORD_ID_COLUMN} column") - self._existing_ids_cache[record_key] = set() - else: - existing_ids = cast( - set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) - ) - self._existing_ids_cache[record_key] = existing_ids + except Exception as e: + # Leave the cache dirty so the next call reads the table again. + raise self._read_failed(record_path) from e - self._cache_dirty[record_key] = False - logger.debug( - f"Refreshed existing IDs cache: {len(self._existing_ids_cache)} IDs" + if arrow_table.num_rows == 0: + self._existing_ids_cache[record_key] = set() + elif self.RECORD_ID_COLUMN not in arrow_table.column_names: + # TODO: replace this with proper checking of the table schema first! + logger.warning(f"Delta table missing {self.RECORD_ID_COLUMN} column") + self._existing_ids_cache[record_key] = set() + else: + existing_ids = cast( + set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) ) + self._existing_ids_cache[record_key] = existing_ids - except Exception as e: - logger.error(f"Failed to refresh existing IDs cache: {e}") - self._existing_ids_cache[record_key] = set() - self._cache_dirty[record_key] = False - raise + self._cache_dirty[record_key] = False + logger.debug( + f"Refreshed existing IDs cache: {len(self._existing_ids_cache)} IDs" + ) def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[str]: """Get the set of existing IDs in the Delta table, using cache when possible.""" @@ -631,6 +685,9 @@ def get_all_records( Returns: Combined Arrow table with all records, or None if no records exist + + Raises: + StoreReadError: If the Delta table exists but could not be read. """ record_key = self._get_record_key(record_path) @@ -642,10 +699,10 @@ def get_all_records( delta_table_data = delta_table.to_pyarrow_dataset( as_large_types=True ).to_table() - if delta_table_data.num_rows > 0: - tables_to_combine.append(delta_table_data) except Exception as e: - logger.warning(f"Error reading Delta table: {e}") + raise self._read_failed(record_path) from e + if delta_table_data.num_rows > 0: + tables_to_combine.append(delta_table_data) # Add pending batch data if ( @@ -700,21 +757,16 @@ def get_records_with_column_value( if delta_table is None: return None - try: - # Use schema-preserving read - result = self._read_delta_table(delta_table, expression=combined_expression) - - if len(result) == 0: - return None + # Use schema-preserving read + result = self._read_delta_table( + delta_table, record_path, expression=combined_expression + ) - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) + if len(result) == 0: + return None - except Exception as e: - logger.error( - f"Error getting record with {column_values} from {'/'.join(record_path)}: {e}" - ) - raise e + # Handle (remove/rename) the record id column before returning + return self._handle_record_id_column(result, record_id_column) def get_record_by_id( self, @@ -758,22 +810,15 @@ def get_record_by_id( if delta_table is None: return None - try: - # Use schema-preserving read - filter_expr = self._create_record_id_filter(record_id) - result = self._read_delta_table(delta_table, filters=filter_expr) - - if len(result) == 0: - return None + # Use schema-preserving read + filter_expr = self._create_record_id_filter(record_id) + result = self._read_delta_table(delta_table, record_path, filters=filter_expr) - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) + if len(result) == 0: + return None - except Exception as e: - logger.error( - f"Error getting record {record_id} from {'/'.join(record_path)}: {e}" - ) - raise e + # Handle (remove/rename) the record id column before returning + return self._handle_record_id_column(result, record_id_column) def get_records_by_ids( self, @@ -815,26 +860,20 @@ def get_records_by_ids( delta_table = self._get_delta_table(record_path) if delta_table is None: return None - try: - # Use schema-preserving read with filters - filter_expr = self._create_record_ids_filter(record_ids_list) - result = self._read_delta_table(delta_table, filters=filter_expr) - - if len(result) == 0: - return None - - # Handle record_id column based on parameter - return self._handle_record_id_column(result, record_id_column) + # Use schema-preserving read with filters + filter_expr = self._create_record_ids_filter(record_ids_list) + result = self._read_delta_table(delta_table, record_path, filters=filter_expr) - except Exception as e: - logger.error( - f"Error getting records by IDs from {'/'.join(record_path)}: {e}" - ) + if len(result) == 0: return None + # Handle record_id column based on parameter + return self._handle_record_id_column(result, record_id_column) + def _read_delta_table( self, delta_table: deltalake.DeltaTable, + record_path: tuple[str, ...], filters: list | None = None, expression: pc.Expression | None = None, ) -> pa.Table: @@ -843,14 +882,23 @@ def _read_delta_table( Args: delta_table: The Delta table to read from + record_path: Path of the table relative to this view, for errors filters: Optional filters to apply + expression: Optional filter expression, used instead of ``filters`` Returns: Arrow table with preserved schema + + Raises: + StoreReadError: If the table's data could not be read. An error in + the filter itself (an unknown column, say) is raised as is. """ filter_expr = None # Use to_pyarrow_dataset with as_large_types for Polars compatible arrow table loading - dataset = delta_table.to_pyarrow_dataset(as_large_types=True) + try: + dataset = delta_table.to_pyarrow_dataset(as_large_types=True) + except Exception as e: + raise self._read_failed(record_path) from e if filters and expression is None: for filt in filters: if len(filt) == 3: @@ -876,7 +924,11 @@ def _read_delta_table( # Convert view types before filtering: PyArrow has no comparison/sort # kernels for string_view, so filtering the scan directly crashes # (ENG-601). Trades predicate pushdown for correctness. - table = arrow_utils.normalize_table_view_types(dataset.to_table()) + try: + scanned = dataset.to_table() + except Exception as e: + raise self._read_failed(record_path) from e + table = arrow_utils.normalize_table_view_types(scanned) if filter_expr is not None: return table.filter(filter_expr) return table @@ -991,18 +1043,19 @@ def _flush_by_key(self, record_key: str) -> None: os.makedirs(table_uri, exist_ok=True) - # Check if Delta table exists at this URI - try: - import deltalake as _dl + # Check if Delta table exists at this URI. Only TableNotFoundError + # means "absent": a table that exists but cannot be loaded must never + # be replaced by the mode="overwrite" write below (ITL-884). + from deltalake.exceptions import TableNotFoundError - delta_table = _dl.DeltaTable( + try: + delta_table = deltalake.DeltaTable( table_uri, storage_options=self._storage_options or None ) - except ( - _dl.exceptions.TableNotFoundError, - _dl.exceptions.DeltaProtocolError, - ): + except TableNotFoundError: delta_table = None + except Exception as load_error: + raise StoreReadError(table_uri, abs_path_components) from load_error if delta_table is None: # TODO: reconsider mode="overwrite" here @@ -1017,6 +1070,7 @@ def _flush_by_key(self, record_key: str) -> None: f"Created new Delta table for {record_key} with {len(combined_table)} records" ) else: + self._require_compatible_append(table_uri, delta_table, combined_table) delta_table.merge( source=combined_table, predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}", @@ -1046,6 +1100,61 @@ def _flush_by_key(self, record_key: str) -> None: self._pending_record_ids[record_key] = pending_ids raise + @staticmethod + def _require_compatible_append( + table_uri: str, delta_table: deltalake.DeltaTable, batch: pa.Table + ) -> None: + """Refuse a batch whose column types differ from the stored table's. + + Delta's merge either casts such a batch to the stored type (``int32`` or + ``float64`` rows into an ``int64`` column) or, when an extension type is + involved, writes data files the table can no longer read. Both sides are + compared through Delta's own type mapping, with extension types reduced to + storage, large-normalized and nullability ignored, so a type Delta stores + differently (``uint64`` as ``int64``) is not refused. Columns present on + only one side are allowed (schema evolution). The stored schema comes from + the loaded log; no data is read. + + Args: + table_uri: Location of the stored table, for the error message. + delta_table: The loaded stored table. + batch: The rows about to be merged into it. + + Raises: + InputValidationError: If a column's type in ``batch`` differs from its + stored type. Nothing is written. + """ + stored = pa.schema(delta_table.schema().to_arrow()) # pyright: ignore[reportArgumentType] # arro3 schema; pyarrow reads it through the C data interface + incoming = pa.schema( + deltalake.Schema.from_arrow( + pa.schema( + [ + pa.field(f.name, _comparable_storage_type(f.type)) + for f in batch.schema + ] + ) + ).to_arrow() # pyright: ignore[reportArgumentType] # arro3 schema, as above + ) + for field in incoming: + index = stored.get_field_index(field.name) + if index < 0: + continue + stored_type = _comparable_storage_type( + arrow_utils.normalize_to_large_types( + stored.field(index).type, keep_null=True + ) + ) + incoming_type = _comparable_storage_type( + arrow_utils.normalize_to_large_types(field.type, keep_null=True) + ) + if stored_type != incoming_type: + raise InputValidationError( + f"Cannot append to the stored table {table_uri!r}: column " + f"{field.name!r} is stored as {stored_type} but the new rows have " + f"{incoming_type} (input type {batch.schema.field(field.name).type}). " + "The stored rows are unchanged and nothing was written." + ) + @property def base_path(self) -> tuple[str, ...]: """The current relative root of this database view (always () for root instances).""" diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 4a75564c5..ecb9b1b42 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Literal from orcapod.channels import BroadcastChannel, Channel +from orcapod.errors import StoreReadError from orcapod.pipeline.result import OrchestratorResult from orcapod.protocols.node_protocols import ( is_function_node, @@ -250,6 +251,10 @@ async def _run_async( return OrchestratorResult( node_outputs=collectors if materialize_results else {} ) + except* StoreReadError as group: + # A failed store read stops the run. Raise it on its own, as the sync + # orchestrator does, rather than inside the TaskGroup's exception group. + raise StoreReadError.in_group(group)[0] finally: effective_observer.on_run_end(run_id) diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index 597312934..aaf1f77ca 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -15,10 +15,12 @@ import json from collections.abc import Callable from pathlib import Path +from typing import Any import deltalake import pyarrow as pa import pytest +from deltalake.exceptions import DeltaProtocolError from orcapod.databases import DeltaTableDatabase from orcapod.errors import InputValidationError, StoreReadError @@ -26,7 +28,7 @@ PATH = ("results", "t") -def rows(ids: list[bytes], x: pa.Array, **extra: pa.Array) -> pa.Table: +def rows(ids: list[bytes], x: pa.Array[Any], **extra: pa.Array[Any]) -> pa.Table: """Build a table keyed by ``rid`` with an ``x`` column and optional extras.""" return pa.table({"rid": pa.array(ids, pa.large_binary()), "x": x, **extra}) @@ -60,7 +62,7 @@ def __arrow_ext_deserialize__( return cls(storage_type) -def ext_array(values: list, storage: pa.DataType) -> pa.ExtensionArray: +def ext_array(values: list[Any], storage: pa.DataType) -> pa.ExtensionArray[Any]: return pa.ExtensionArray.from_storage(_Ext(storage), pa.array(values, storage)) @@ -149,6 +151,12 @@ def test_table_still_exists(self, unreadable): assert unreadable.table_exists(PATH) +def test_bad_filter_is_not_reported_as_unreadable_table(stored): + # The table reads fine; the filter names a column it does not have. + with pytest.raises(pa.ArrowInvalid): + stored.get_records_with_column_value(PATH, {"no_such_column": 1}) + + class TestAbsentTable: @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) def test_read_returns_none(self, base, read): @@ -192,7 +200,7 @@ def test_protocol_error_on_load_is_not_treated_as_absent( def failing_load(table_uri, *args, **kwargs): if Path(str(table_uri)) == target: - raise deltalake.exceptions.DeltaProtocolError("simulated") + raise DeltaProtocolError("simulated") return real(table_uri, *args, **kwargs) before = snapshot(target) diff --git a/tests/test_databases/test_delta_table_database.py b/tests/test_databases/test_delta_table_database.py index 4a5296ac1..27142c3fc 100644 --- a/tests/test_databases/test_delta_table_database.py +++ b/tests/test_databases/test_delta_table_database.py @@ -561,7 +561,7 @@ def test_read_delta_table_no_filter_normalizes_views(self, db): tbl = make_string_view_table(__record_id=["p", "q"], category=["A", "B"]) deltalake.write_deltalake(uri, tbl, mode="overwrite") - result = db._read_delta_table(db._get_delta_table(self.PATH)) + result = db._read_delta_table(db._get_delta_table(self.PATH), self.PATH) assert result.num_rows == 2 assert not any(pa.types.is_string_view(f.type) for f in result.schema) diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index 4010ae2cb..f36b97d1c 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -17,8 +17,7 @@ import pyarrow as pa import pytest -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod +from orcapod import function_pod from orcapod.core.sources import ArrowTableSource from orcapod.databases import DeltaTableDatabase from orcapod.errors import StoreReadError @@ -29,6 +28,7 @@ _CALLS: list[int] = [] +@function_pod(output_keys="y") def double(x: int) -> int: _CALLS.append(x) return 2 * x @@ -60,7 +60,7 @@ def build_job(store_path: Path) -> PipelineJob: ) job = PipelineJob(name="itl884", store=DeltaTableDatabase(base_path=store_path)) with job: - FunctionPod(PythonDataFunction(double, output_keys="y"))(source, label="dbl") + _ = double.pod(source, label="dbl") return job From c7ffc7dc6c3c3324138b220026bb2a710bfcbf08 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 20:55:08 -0800 Subject: [PATCH 03/30] docs: state the Delta read-failure and append type-check contract (ITL-884) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ DESIGN_ISSUES.md | 34 ++++++++++++++++++++++++++++++++++ docs/concepts/function-pods.md | 13 +++++++++++++ 3 files changed, 75 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a71b50c0..ecd675777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,22 @@ info. Hashes of pipelines that already worked are unchanged. ### Fixed +#### A Delta table that cannot be read is no longer reported as having no rows (ITL-884) + +When a `DeltaTableDatabase` table existed but could not be read (a missing or damaged data +file, a log the backend cannot load, data files whose column type no longer matches the +table), `get_all_records` and `get_records_by_ids` returned `None`, the same answer as for a +table that does not exist. A `PipelineJob` then warned that the stored results "may have been +deleted externally", swallowed the per-row read errors under `error_policy="continue"`, finished +without raising, and its nodes showed no records. Every read now raises +`orcapod.errors.StoreReadError`, which names the table and chains the backend's error, and a +store read failure fails the run under both orchestrators whatever the `error_policy`. A table +that does not exist still reads as empty. + +A flush that could not load an existing table no longer treats it as absent. Before this fix a +`DeltaProtocolError` while loading the table made the flush write the batch with +`mode="overwrite"`, replacing the stored rows. The flush now raises and writes nothing. + #### Materialized rows keep their own source info (ITL-845) When rows were collected and turned back into a stream, every row got the source-info @@ -115,6 +131,18 @@ replaced by `_`. Correcting stored values is tracked in ITL-853. ### Breaking Changes +#### A changed column type is refused when appending to a Delta table (ITL-884) + +Before writing to an existing `DeltaTableDatabase` table, the flush now compares the new rows' +column types with the stored ones. If a column's type differs, the flush raises +`InputValidationError` naming the table, the column and both types, and writes nothing. Before, +Delta's merge silently cast the new rows to the stored type (`int32` or `float64` rows written +into an `int64` column, `large_binary` into `large_string`), or, when an extension (logical) type +was involved, accepted rows the table could then no longer read. The comparison treats types the +way Delta stores them: `string` and `large_string` (and the other large and non-large layouts) +are the same type, an extension type is compared by its storage type, and nullability is not +compared. New columns and missing columns are still accepted. + #### Operator outputs keep their inputs' Arrow types, normalized to large variants (ITL-832, ITL-813) `MergeJoin` no longer infers Arrow types from Python values, and it no longer fails on a diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index a6f9b59ed..7bc13f001 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2171,3 +2171,37 @@ equals the materialized one. Hashes of pipelines that already worked are unchang Regressions: `tests/test_core/operators/test_chained_aggregation.py`. Open: on the async barrier path, per-row source info is still row 0's (ITL-845); that assertion is a strict xfail. + +--- + +## `src/orcapod/databases/delta_lake_databases.py` — read failures + +### DR1 — A Delta table that could not be read was reported as having no rows +**Status:** resolved +**Severity:** critical +**Issue:** ITL-884 + +`DeltaTableDatabase.get_all_records` logged a read error at WARNING and returned `None`, and +`get_records_by_ids` returned `None` on any error. Both conflated "the table does not exist" +with "the table exists but cannot be read". Callers treat `None` as no rows. A function node +recomputed with a "deleted externally" warning, its per-row lookups failed inside the +`error_policy="continue"` handler, and the run finished without raising, so stored results +looked dropped. `_flush_by_key` also treated a `DeltaProtocolError` on load as "no table" and +wrote the batch with `mode="overwrite"`. The case that exposed it: an append whose column type +differs from the stored table, which Delta's merge either casts silently to the stored type or, +with an extension type involved, accepts and then cannot read. Other backends were not +affected: the in-memory and connector databases (SQLite, PostgreSQL, SpiralDB) return `None` +only for a table that does not exist and let read errors propagate. + +**Fix:** only `TableNotFoundError` means absent. Every other load or read error raises +`orcapod.errors.StoreReadError`, which names the table and chains the cause. `FunctionNode.execute`, the +async `FunctionPod` row handler and the async orchestrator let it fail the run whatever the +`error_policy`. Before merging into an existing table, `_flush_by_key` compares the batch with +the stored Delta schema through Delta's own type mapping (extension types by storage, +large-normalized, nullability ignored). On a difference it raises `InputValidationError` naming the +table, column and both types, and nothing is written. Regressions: +`tests/test_databases/test_delta_read_failures.py` and +`tests/test_pipeline/test_store_read_failure.py`. Open: whether a nullability-only change +should be refused (Delta's merge relaxes a non-null nested field to nullable) belongs to the +nullability audit, ITL-842. + diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8618052c..602056047 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -218,6 +218,19 @@ print("function calls:", len(calls)) version, e.g. `@function_pod(output_keys="bmi", version="v1.0")` (the default is `"v0.0"`; a minor bump such as `"v0.1"` is not enough), or rename the function. +!!! note "When the store cannot be read" + A table that does not exist yet reads as empty, and its rows are computed. A table that + exists but cannot be read (for example, a data file was deleted or damaged) is never + treated as empty: the run stops with `orcapod.errors.StoreReadError`, which names the + table and the underlying error, whatever the orchestrator's `error_policy`. Nothing is + recomputed or overwritten, so you can repair the table or restore the file and run again. + + In a `DeltaTableDatabase`, rows whose column types differ from the stored table's + (`float64` values for a column stored as `int64`, say) are refused with + `InputValidationError`, which names the table, the column and both types. They are not + converted to the stored type. `string` and `large_string`, and the other large and + non-large layouts of one type, count as the same type. + A node from a job that has run also provides: - **`get_all_records()`** -- every record this step of the job has stored, as a PyArrow From 5a067645d6bc8053a1cedc66965024619b528836 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:29:14 -0800 Subject: [PATCH 04/30] fix(databases): flush raises StoreReadError and the append refusal as themselves (ITL-884) DeltaTableDatabase.flush() wrapped every failed batch in a RuntimeError. A StoreReadError raised by a result-cache flush during per-row processing was therefore skipped as a row failure under error_policy="continue": the next row was computed and the run ended with RuntimeError. Callers catching InputValidationError also missed refused appends. flush() now raises the first StoreReadError, or else the first InputValidationError, as itself, with a note for every other batch that failed. Every failure is still logged and every failed batch stays queued. A flush that fails only for other reasons still raises RuntimeError. Addresses CodeRabbit comment 4534230764 on PR #290. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 +- docs/concepts/function-pods.md | 7 + src/orcapod/databases/delta_lake_databases.py | 51 +++++- ...tl-884-coderabbit-flush-and-async-group.md | 159 ++++++++++++++++++ .../test_delta_read_failures.py | 75 +++++++-- .../test_pipeline/test_store_read_failure.py | 53 +++++- 6 files changed, 337 insertions(+), 20 deletions(-) create mode 100644 superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd675777..306ea1d00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,8 +92,9 @@ table that does not exist. A `PipelineJob` then warned that the stored results " deleted externally", swallowed the per-row read errors under `error_policy="continue"`, finished without raising, and its nodes showed no records. Every read now raises `orcapod.errors.StoreReadError`, which names the table and chains the backend's error, and a -store read failure fails the run under both orchestrators whatever the `error_policy`. A table -that does not exist still reads as empty. +store read failure fails the run under both orchestrators whatever the `error_policy`, also +when it happens while a row's result is being stored. A table that does not exist still reads as +empty. A flush that could not load an existing table no longer treats it as absent. Before this fix a `DeltaProtocolError` while loading the table made the flush write the batch with @@ -143,6 +144,13 @@ way Delta stores them: `string` and `large_string` (and the other large and non- are the same type, an extension type is compared by its storage type, and nullability is not compared. New columns and missing columns are still accepted. +`DeltaTableDatabase.flush()` (and `add_records(..., flush=True)`) now raises a +`StoreReadError` or this `InputValidationError` as itself. Before, it wrapped every failure in a +`RuntimeError`. When several tables fail in one flush, the `StoreReadError` is raised, or else the +first `InputValidationError`, with each other failure attached as a note and logged. Failed rows +stay queued for the next flush. A flush that fails only for other reasons still raises +`RuntimeError`, with the first failure as its cause. + #### Operator outputs keep their inputs' Arrow types, normalized to large variants (ITL-832, ITL-813) `MergeJoin` no longer infers Arrow types from Python values, and it no longer fails on a diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 602056047..2d8bdd384 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -231,6 +231,13 @@ print("function calls:", len(calls)) converted to the stored type. `string` and `large_string`, and the other large and non-large layouts of one type, count as the same type. + Both errors reach you as themselves, also from `flush()` and from + `add_records(..., flush=True)`. When several pending tables fail in one flush, the + `StoreReadError` (or else the first `InputValidationError`) is raised, and each other + failure is attached to it as a note and logged. Failed rows stay queued, so a later + flush tries them again. A flush that fails for any other reason raises `RuntimeError`, + with the first failure as its cause. + A node from a job that has run also provides: - **`get_all_records()`** -- every record this step of the job has stored, as a PyArrow diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index e767854cd..afb121aa5 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -71,6 +71,38 @@ def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: return arrow_type +def _store_error_to_raise(errors: list[tuple[str, Exception]]) -> Exception | None: + """Pick the store error a failed flush raises as itself. + + A ``StoreReadError`` is preferred, because it is the error that fails a run + whatever the orchestrator's ``error_policy``; failing that, an + ``InputValidationError`` from a refused append. Every other failed batch is + added to the chosen error as a note, so no failure is hidden. + + Args: + errors: The ``(record_key, error)`` pair of every batch that failed, in + flush order. + + Returns: + The chosen error with its notes added, or ``None`` when no batch failed + with a store error. + """ + chosen: Exception | None = None + for kind in (StoreReadError, InputValidationError): + chosen = next((e for _, e in errors if isinstance(e, kind)), None) + if chosen is not None: + break + if chosen is None: + return None + for record_key, other in errors: + if other is not chosen: + chosen.add_note( + f"Another batch also failed to flush ({record_key}): " + f"{type(other).__name__}: {other}" + ) + return chosen + + class DeltaTableDatabase: """ A Delta table database with clear insert vs update semantics. @@ -977,7 +1009,21 @@ def from_config(cls, config: dict[str, Any]) -> "DeltaTableDatabase": ) def flush(self) -> None: - """Flush all pending batches.""" + """Flush all pending batches. + + Every pending batch is attempted, even after one fails. A batch that fails + stays queued, so a later flush tries it again, and each failure is logged. + + Raises: + StoreReadError: If a batch's existing table could not be loaded. It is + raised as itself; every other batch that failed in this flush is + added to it as a note. + InputValidationError: If no batch failed with ``StoreReadError`` but a + batch's column types differ from its stored table's. Other failed + batches are added to it as notes. + RuntimeError: If batches failed with other errors only, naming the + failed batches, with the first failure as its ``__cause__``. + """ errors: list[tuple[str, Exception]] = [] for record_key in list(self._pending_batches.keys()): try: @@ -986,6 +1032,9 @@ def flush(self) -> None: logger.error(f"Error flushing batch for {record_key}: {e}") errors.append((record_key, e)) if errors: + store_error = _store_error_to_raise(errors) + if store_error is not None: + raise store_error failed_keys = [k for k, _ in errors] raise RuntimeError( f"Failed to flush {len(errors)} batch(es): {failed_keys}" diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md new file mode 100644 index 000000000..caaebe4b1 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -0,0 +1,159 @@ +# ITL-884 follow-up: store errors keep their type through `flush()` and out of the async TaskGroup + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Answer the two open CodeRabbit threads on PR #290 (ids 4534230764 and 4534230775). +A `StoreReadError` or an append-refusal `InputValidationError` raised while flushing reaches the +caller as itself, so a store read failure in a per-row result-cache flush fails the run under +`error_policy="continue"`. The async orchestrator raises a lone `StoreReadError` even when an +unrelated node failed in the same `TaskGroup`. + +**Architecture:** `DeltaTableDatabase.flush()` keeps attempting every pending batch and logging +each failure. When at least one batch failed with a store error, it raises the first +`StoreReadError` (else the first `InputValidationError`) itself, with a note for every other +failed batch. Only non-store failures keep the `RuntimeError` wrapper. The async orchestrator +replaces `except* StoreReadError` with a plain `except ExceptionGroup` that selects one +`StoreReadError` (`_single_store_read_error`), notes and logs the rest, and raises it, in the same +shape as PR #287's `_single_store_error` for `StoreWriteError` so the second PR to merge has a +trivial conflict. + +**Tech Stack:** deltalake 1.0.2, pyarrow, asyncio `TaskGroup`, pytest. + +Approved by the coordinator (2026-11-10) on condition that each finding reproduces. Both do: + +- (1b) Run a job, then run it again over two new rows, with the function corrupting the result + table's log (a new log version that requires an unknown reader feature) during its first call. + The per-row lookup still succeeds, the result-cache flush raises `StoreReadError` wrapped in + `RuntimeError`, the sync per-row handler under `continue` skips the row, the next row is + computed, and the run ends with `RuntimeError` from the job's end-of-run flush. +- (2) CI's 3.11 is 3.11.18, so the 3.11.0-3.11.3 single-leaf wrapping does not occur there. The + concurrent case reproduces deterministically on 3.11.18 and 3.13.18: node `dbl` raises + `StoreReadError` in `_load_cached_entries` and node `tri` fails in the observer's + `on_node_start`, both before their first `await`, so both land in one group and `job.run` + raises `ExceptionGroup([StoreReadError, ExceptionGroup([ValueError])])`. + +No hash, path or stored-schema changes. No public API added. + +--- + +## File map + +- `src/orcapod/databases/delta_lake_databases.py`: `flush()` and a module helper + `_store_error_to_raise(errors)`. +- `src/orcapod/pipeline/async_orchestrator.py`: plain `except ExceptionGroup`, helpers + `_single_store_read_error(group)` and `_leaves(group)` (identical to #287's `_leaves`). +- Tests: `tests/test_databases/test_delta_read_failures.py` (assert the bare error types, add a + multi-batch test), `tests/test_pipeline/test_store_read_failure.py` (flush during per-row + processing, both orchestrators; concurrent unrelated failure, async). +- Docs: `docs/concepts/function-pods.md` admonition, `CHANGELOG.md` ITL-884 entries, + `DESIGN_ISSUES.md` DR1 fix note. + +## Task 1: Failing tests for finding (1) + +- [ ] `tests/test_databases/test_delta_read_failures.py`: the `refusal` helper is replaced by + `pytest.raises(InputValidationError)` at each refusal site; the two unloadable-table flush tests + use `pytest.raises(StoreReadError)`; the absent `__cause__` assertions go. +- [ ] New `test_flush_raises_the_store_error_and_notes_every_other_failed_batch`: two tables + with pending batches (one whose stored `x` is int64 and receives float64, one whose log requires + an unknown reader feature); `flush()` raises the `StoreReadError`, one note names the other + table's key and `InputValidationError`, and both batches stay queued (a second `flush()` raises + again). +- [ ] New `test_flush_keeps_runtime_error_for_other_failures`: control case. Monkeypatch + `deltalake.write_deltalake` to raise `OSError` for a new table; `flush()` raises `RuntimeError` + with the `OSError` as `__cause__` (unchanged behaviour). +- [ ] `tests/test_pipeline/test_store_read_failure.py`: + `test_store_read_failure_in_a_row_flush_fails_the_run`, both orchestrators with + `error_policy="continue"`. Rerun over `a..e` with the function corrupting the result table on + its first call; `job.run` raises `StoreReadError` naming the result table, and the function ran + for exactly one new row. +- [ ] Run `uv run pytest tests/test_databases/test_delta_read_failures.py + tests/test_pipeline/test_store_read_failure.py -v`. Expected: the new and changed tests FAIL with + `RuntimeError`, and the per-row test also shows two calls. + +## Task 2: Fix `flush()` + +```python +def flush(self) -> None: + errors: list[tuple[str, Exception]] = [] + for record_key in list(self._pending_batches.keys()): + try: + self._flush_by_key(record_key) + except Exception as e: + logger.error(f"Error flushing batch for {record_key}: {e}") + errors.append((record_key, e)) + if errors: + store_error = _store_error_to_raise(errors) + if store_error is not None: + raise store_error + failed_keys = [k for k, _ in errors] + raise RuntimeError( + f"Failed to flush {len(errors)} batch(es): {failed_keys}" + ) from errors[0][1] + + +def _store_error_to_raise(errors): + """First StoreReadError, else first InputValidationError, with notes; else None.""" + for kind in (StoreReadError, InputValidationError): + chosen = next((e for _, e in errors if isinstance(e, kind)), None) + if chosen is not None: + break + else: + return None + for key, other in errors: + if other is not chosen: + chosen.add_note( + f"Another batch also failed to flush ({key}): " + f"{type(other).__name__}: {other}" + ) + return chosen +``` + +`StoreReadError` is preferred because it is the one the per-row handlers treat as fatal. The +docstring gets a Raises section. Run Task 1's tests: PASS. + +## Task 3: Failing test for finding (2) + +- [ ] `test_concurrent_unrelated_failure_still_raises_store_read_error` (async only): an + observer (a `NoOpObserver` subclass whose `contextualize` returns itself and whose + `on_node_start` raises `ValueError` for `tri`), two function nodes over one source, the `dbl` + result table's data files deleted. `job.run` raises `StoreReadError` (not a group) naming the + result table, `__cause__` is set, and a note names `ValueError: observer failed for tri`. +- [ ] Run it: FAIL with `ExceptionGroup`. + +## Task 4: Fix the async orchestrator + +```python + except ExceptionGroup as group: + # A failed store read stops the run. Raise it on its own, as the sync + # orchestrator does, rather than inside the TaskGroup's exception group. + # A plain except (not except*) so the raise is never wrapped again. + store_error = _single_store_read_error(group) + if store_error is None: + raise + raise store_error +``` + +`_single_store_read_error(group)`: `StoreReadError.in_group(group)`; none gives `None`; the first +is chosen; every other leaf (`_leaves`) is noted on it ("Another store read failed in the same +run: ..." also logged at WARNING; "Another node failed in the same run: : "). Its +`__cause__` stays the backend error; the group is its `__context__`. Run: PASS. + +## Task 5: Docs, design log, verification, commit, replies + +- [ ] Concept page and CHANGELOG: `flush()` / `add_records(..., flush=True)` raise + `StoreReadError` and `InputValidationError` themselves; when several batches fail, the first + store error is raised with a note per other batch, every failed batch stays queued and each + failure is logged; other flush failures still raise `RuntimeError`. +- [ ] `DESIGN_ISSUES.md` DR1: one sentence on the flush and async-group fix. +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, basedpyright (exit code), the full + suite on 3.11, 3.12 and 3.13, `tests/test_hashing/golden` unchanged. +- [ ] One commit per finding, push, signed replies on both threads, wait for CI and CodeRabbit. + +## Interaction with PR #287 (report, do not fix here) + +#287's `ResultCache.store` wraps every `add_record`/`flush` failure in `StoreWriteError`, and its +`PipelineJob.run` wraps the end-of-run flush the same way. Once both PRs are in, a +`StoreReadError` from a flush reaches the caller as the `__cause__` of a `StoreWriteError`. It +is still fatal, but `except StoreReadError` misses it. Whichever PR merges second should decide +whether those wrappers let `StoreReadError` through. diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index aaf1f77ca..94ae2d228 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -4,6 +4,9 @@ - A table that exists but cannot be read raises ``StoreReadError`` on every read path, naming the table; a table that does not exist still reads as ``None``. - A table whose log cannot be loaded is never overwritten by a flush. +- ``flush`` raises ``StoreReadError`` and the append refusal's + ``InputValidationError`` themselves, not wrapped in ``RuntimeError``, and + notes every other batch that failed in the same flush. - An append whose column types differ from the stored table is refused before anything is written, naming the column and both types; legitimate appends (same types in another layout, new or missing columns, unchanged extension @@ -178,14 +181,14 @@ def test_unsupported_reader_feature(self, base, stored): with pytest.raises(StoreReadError): db.get_all_records(PATH) - with pytest.raises(RuntimeError) as info: + with pytest.raises(StoreReadError) as info: db.add_records( PATH, rows([b"c"], pa.array([3], pa.int64())), record_id_column="rid", flush=True, ) - assert isinstance(info.value.__cause__, StoreReadError) + assert_names_table(info.value, base) assert snapshot(table_dir(base)) == before def test_protocol_error_on_load_is_not_treated_as_absent( @@ -206,7 +209,7 @@ def failing_load(table_uri, *args, **kwargs): before = snapshot(target) db = DeltaTableDatabase(base_path=base) monkeypatch.setattr(deltalake, "DeltaTable", failing_load) - with pytest.raises(RuntimeError) as info: + with pytest.raises(StoreReadError) as info: db.add_records( PATH, rows([b"c"], pa.array([3], pa.int64())), @@ -215,13 +218,62 @@ def failing_load(table_uri, *args, **kwargs): ) monkeypatch.undo() - assert isinstance(info.value.__cause__, StoreReadError) + assert isinstance(info.value.__cause__, DeltaProtocolError) assert snapshot(target) == before stored_rows = DeltaTableDatabase(base_path=base).get_all_records(PATH) assert stored_rows is not None assert stored_rows.num_rows == 2 +class TestFlushRaisesStoreErrorsThemselves: + def test_store_read_error_is_raised_with_a_note_per_other_failed_batch( + self, base, stored + ): + other = ("results", "u") + stored.add_records( + other, + rows([b"a"], pa.array([1], pa.int64())), + record_id_column="rid", + flush=True, + ) + unsupported_reader_feature(base) + db = DeltaTableDatabase(base_path=base) + # Queued first: refused at flush, because x is stored as int64. + db.add_records(other, rows([b"b"], pa.array([2.0])), record_id_column="rid") + # Queued second: its table's log cannot be loaded. + db.add_records( + PATH, rows([b"c"], pa.array([3], pa.int64())), record_id_column="rid" + ) + + with pytest.raises(StoreReadError) as info: + db.flush() + + # The StoreReadError wins over the earlier refusal, because it is the one + # that fails a run whatever the error policy; the refusal is noted on it. + assert_names_table(info.value, base) + notes = getattr(info.value, "__notes__", []) + assert len(notes) == 1 + assert "results/u" in notes[0] + assert "InputValidationError" in notes[0] + # Both failed batches stay queued, so the next flush fails the same way. + with pytest.raises(StoreReadError): + db.flush() + + def test_other_flush_failures_still_raise_runtime_error(self, base, monkeypatch): + def failing_write(*args, **kwargs): + raise OSError("simulated disk failure") + + db = DeltaTableDatabase(base_path=base) + db.add_records( + PATH, rows([b"a"], pa.array([1], pa.int64())), record_id_column="rid" + ) + monkeypatch.setattr(deltalake, "write_deltalake", failing_write) + + with pytest.raises(RuntimeError, match="Failed to flush 1 batch") as info: + db.flush() + assert isinstance(info.value.__cause__, OSError) + + # --------------------------------------------------------------------------- # Append refusal: a column's type differs from the stored table # --------------------------------------------------------------------------- @@ -231,13 +283,6 @@ def append(db: DeltaTableDatabase, table: pa.Table) -> None: db.add_records(PATH, table, record_id_column="rid", flush=True) -def refusal(info: pytest.ExceptionInfo[RuntimeError]) -> InputValidationError: - """The ``InputValidationError`` behind a refused flush.""" - cause = info.value.__cause__ - assert isinstance(cause, InputValidationError), repr(info.value) - return cause - - class TestChangedColumnTypeIsRefused: @pytest.mark.parametrize( ("new_type", "value", "named"), @@ -249,9 +294,9 @@ class TestChangedColumnTypeIsRefused: ids=["int64->float64", "int64->int32", "int64->large_string"], ) def test_plain_type_change(self, base, stored, new_type, value, named): - with pytest.raises(RuntimeError) as info: + with pytest.raises(InputValidationError) as info: append(stored, rows([b"c"], pa.array([value], new_type))) - message = str(refusal(info)) + message = str(info.value) assert "'x'" in message assert "int64" in message assert named in message @@ -266,12 +311,12 @@ def test_extension_storage_change(self, base): db = DeltaTableDatabase(base_path=base) append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) - with pytest.raises(RuntimeError) as info: + with pytest.raises(InputValidationError) as info: append( db, rows([b"b"], ext_array([[1.0]], pa.large_list(pa.float32()))), ) - message = str(refusal(info)) + message = str(info.value) assert "'x'" in message assert "binary" in message assert "list" in message diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index f36b97d1c..3311033df 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -10,6 +10,7 @@ from __future__ import annotations +import json import shutil from collections.abc import Callable from pathlib import Path @@ -26,11 +27,16 @@ from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator _CALLS: list[int] = [] +# Tables the function damages on its next call, standing in for another process +# breaking a table while the job runs. +_DAMAGE_ON_NEXT_CALL: list[Path] = [] @function_pod(output_keys="y") def double(x: int) -> int: _CALLS.append(x) + while _DAMAGE_ON_NEXT_CALL: + require_unknown_reader_feature(_DAMAGE_ON_NEXT_CALL.pop()) return 2 * x @@ -43,8 +49,10 @@ def double(x: int) -> int: @pytest.fixture(autouse=True) def _reset_calls(): _CALLS.clear() + _DAMAGE_ON_NEXT_CALL.clear() yield _CALLS.clear() + _DAMAGE_ON_NEXT_CALL.clear() @pytest.fixture(params=list(ORCHESTRATORS)) @@ -52,9 +60,9 @@ def make_orchestrator(request) -> Callable[[], object]: return ORCHESTRATORS[request.param] -def build_job(store_path: Path) -> PipelineJob: +def build_job(store_path: Path, keys: str = "abc") -> PipelineJob: source = ArrowTableSource( - pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + pa.table({"k": list(keys), "x": list(range(1, len(keys) + 1))}), tag_columns=["k"], infer_nullable=True, ) @@ -78,6 +86,23 @@ def delete_data_files(table: Path) -> None: f.unlink() +def require_unknown_reader_feature(table: Path) -> None: + """Commit a log version that requires a reader feature deltalake does not know. + + Data files already loaded stay readable through the job's open table handle, + but loading the table again, as every flush does, fails. + """ + log = table / "_delta_log" + version = max(int(p.stem) for p in log.glob("*.json")) + 1 + protocol = { + "minReaderVersion": 3, + "minWriterVersion": 7, + "readerFeatures": ["orcapodMadeUpFeature"], + "writerFeatures": ["orcapodMadeUpFeature"], + } + (log / f"{version:020d}.json").write_text(json.dumps({"protocol": protocol})) + + @pytest.fixture def stored(tmp_path: Path, make_orchestrator) -> Path: """A store in which the job has computed and stored all three rows.""" @@ -118,6 +143,30 @@ def test_unreadable_result_table_fails_the_run_under_continue( assert _CALLS == [] +def test_store_read_failure_in_a_row_flush_fails_the_run(stored, make_orchestrator): + # Rows d and e are new. The result table breaks while d is computed, after + # d's cache lookup succeeded, so the failure surfaces in the flush that stores + # d's result, inside the error_policy="continue" per-row handler. Before the + # fix the flush wrapped the StoreReadError in a RuntimeError, the handler + # skipped d as a failed row, computed e, and the run ended with RuntimeError. + result_table, _ = table_dirs(stored) + _DAMAGE_ON_NEXT_CALL.append(result_table) + + orchestrator = make_orchestrator() + job = build_job(stored, keys="abcde") + with pytest.raises(StoreReadError) as info: + job.run(orchestrator=orchestrator) + + assert Path(info.value.table) == result_table + if isinstance(orchestrator, SyncPipelineOrchestrator): + # The run stops at d: e is never computed. + assert _CALLS == [4] + else: + # The async pod computes rows concurrently, so e may already be in flight + # when d's flush fails. + assert 4 in _CALLS and len(_CALLS) <= 2 + + def test_absent_tables_are_computed(stored, make_orchestrator): result_table, pipeline_table = table_dirs(stored) shutil.rmtree(result_table) From 3c75c095804a9eff1c594565627af1109984ffa3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:32:28 -0800 Subject: [PATCH 05/30] fix(pipeline): raise StoreReadError on its own when another node also failed (ITL-884) The async orchestrator re-raised the StoreReadError inside an "except* StoreReadError" handler. When an unrelated node failed in the same TaskGroup, the unmatched exception stayed in the group and the caller got an ExceptionGroup instead of the StoreReadError (on any Python version; on 3.11.0-3.11.3 even a lone error was re-wrapped). A plain "except ExceptionGroup" now selects the first StoreReadError and raises it, noting every other failure on it (other store read errors are also logged), with the group as its __context__ and the backend error kept as __cause__. Same shape as PR #287's handling of StoreWriteError. Addresses CodeRabbit comment 4534230775 on PR #290. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 7 ++- src/orcapod/pipeline/async_orchestrator.py | 58 ++++++++++++++++++- .../test_pipeline/test_store_read_failure.py | 53 +++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 7ce70ad6b..9de98f4fc 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2236,7 +2236,12 @@ async `FunctionPod` row handler and the async orchestrator let it fail the run w `error_policy`. Before merging into an existing table, `_flush_by_key` compares the batch with the stored Delta schema through Delta's own type mapping (extension types by storage, large-normalized, nullability ignored). On a difference it raises `InputValidationError` naming the -table, column and both types, and nothing is written. Regressions: +table, column and both types, and nothing is written. `flush()` raises a `StoreReadError` or +that `InputValidationError` as itself, not wrapped in `RuntimeError`, with every other failed +batch noted on it, so a read failure in a per-row result-cache flush still fails the run. The +async orchestrator selects the `StoreReadError` from the `TaskGroup`'s exception group with a +plain `except`, noting the other failures, so a concurrent unrelated failure cannot turn it into +an `ExceptionGroup`. Regressions: `tests/test_databases/test_delta_read_failures.py` and `tests/test_pipeline/test_store_read_failure.py`. Open: whether a nullability-only change should be refused (Delta's merge relaxes a non-null nested field to nullable) belongs to the diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index ecb9b1b42..4fb3a0db9 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -251,14 +251,68 @@ async def _run_async( return OrchestratorResult( node_outputs=collectors if materialize_results else {} ) - except* StoreReadError as group: + except ExceptionGroup as group: # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. - raise StoreReadError.in_group(group)[0] + # A plain except, not except*: raising inside except* can wrap the + # error in a new group again. + store_error = _single_store_read_error(group) + if store_error is None: + raise + raise store_error finally: effective_observer.on_run_end(run_id) +def _single_store_read_error(group: ExceptionGroup) -> StoreReadError | None: + """Pick the one ``StoreReadError`` an async run raises from its exception group. + + Concurrent node tasks can fail together, so the ``TaskGroup``'s group may hold + several store read errors and other exceptions. The caller gets a single + ``StoreReadError``, as from the sync orchestrator: the first one in the group. + Every other exception stays attached for diagnosis: each is noted on the + chosen error, other store read errors are also logged at WARNING, and the + whole group is the chosen error's ``__context__``. Its ``__cause__`` stays the + backend error. + + Args: + group: The exception group the run's ``TaskGroup`` raised. + + Returns: + The store read error to raise, or ``None`` when the group holds none. + """ + store_errors = StoreReadError.in_group(group) + if not store_errors: + return None + chosen = store_errors[0] + for other in _leaves(group): + if other is chosen: + continue + if isinstance(other, StoreReadError): + logger.warning( + "Another store read failed in the same run: %s", + other, + exc_info=other, + ) + chosen.add_note(f"Another store read failed in the same run: {other}") + else: + chosen.add_note( + f"Another node failed in the same run: {type(other).__name__}: {other}" + ) + return chosen + + +def _leaves(group: BaseExceptionGroup) -> list[BaseException]: + """Return every non-group exception in ``group``, nested groups flattened.""" + found: list[BaseException] = [] + for exc in group.exceptions: + if isinstance(exc, BaseExceptionGroup): + found.extend(_leaves(exc)) + else: + found.append(exc) + return found + + class _CollectingWriter: """Wrapper that collects items while forwarding to real writer.""" diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index 3311033df..c1eaa2b86 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -24,6 +24,7 @@ from orcapod.errors import StoreReadError from orcapod.pipeline import PipelineJob from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator _CALLS: list[int] = [] @@ -40,6 +41,11 @@ def double(x: int) -> int: return 2 * x +@function_pod(output_keys="z") +def triple(x: int) -> int: + return 3 * x + + ORCHESTRATORS: dict[str, Callable[[], object]] = { "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), @@ -180,3 +186,50 @@ def test_absent_tables_are_computed(stored, make_orchestrator): _CALLS.clear() build_job(stored).run(orchestrator=make_orchestrator()) assert _CALLS == [] + + +class _FailsToStartNode(NoOpObserver): + """An observer whose ``on_node_start`` raises for one node label.""" + + def __init__(self, label: str) -> None: + self.label = label + + def on_node_start( + self, node_label: str, node_hash: str, tag_schema: object = None + ) -> None: + if node_label == self.label: + raise ValueError(f"observer failed for {node_label}") + + +def test_unrelated_concurrent_failure_does_not_hide_store_read_error(tmp_path): + # Node "dbl" raises StoreReadError while loading its stored results and node + # "tri" fails in the observer's on_node_start. Both fail before their first + # await, so both land in the async TaskGroup's exception group. Before the fix, + # re-raising inside "except* StoreReadError" left the unmatched ValueError in + # the group and the caller got an ExceptionGroup, whatever the Python version. + stored = tmp_path / "store" + _ = build_job(stored).run() + result_table, _ = table_dirs(stored) + delete_data_files(result_table) + + source = ArrowTableSource( + pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + tag_columns=["k"], + infer_nullable=True, + ) + job = PipelineJob(name="itl884", store=DeltaTableDatabase(base_path=stored)) + with job: + _ = double.pod(source, label="dbl") + _ = triple.pod(source, label="tri") + + with pytest.raises(StoreReadError) as info: + job.run( + orchestrator=AsyncPipelineOrchestrator(), + observer=_FailsToStartNode("tri"), + ) + + assert type(info.value) is StoreReadError + assert Path(info.value.table) == result_table + assert info.value.__cause__ is not None + notes = getattr(info.value, "__notes__", []) + assert any("ValueError: observer failed for tri" in note for note in notes), notes From 2e238c695e612d4e43c883baa8bcb515edc25815 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:37:53 -0800 Subject: [PATCH 06/30] test(databases): regression tests for Delta reads missing their own writes (ITL-864) Database-level tests for each read method (sibling at() handles, stored plus pending rows, id overlap under skip_duplicates True and False, record_id_column) and a PipelineJob test on both orchestrators. They fail before the fix. Co-Authored-By: Claude Opus 5.5 --- ...26-11-10-itl-864-delta-read-your-writes.md | 158 +++++++++++++++ .../test_delta_read_your_writes.py | 183 ++++++++++++++++++ .../test_delta_node_records_after_run.py | 46 +++++ 3 files changed, 387 insertions(+) create mode 100644 superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md create mode 100644 tests/test_databases/test_delta_read_your_writes.py create mode 100644 tests/test_pipeline/test_delta_node_records_after_run.py diff --git a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md new file mode 100644 index 000000000..993c7eaa2 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md @@ -0,0 +1,158 @@ +# ITL-864 DeltaTableDatabase read-your-writes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every read method of `DeltaTableDatabase` returns exactly the rows a flush would leave, through every handle of an `at()` family, so `node.get_all_records()` returns every row right after the first run on a Delta store. + +**Architecture:** Two defects, one contract. (1) `at()` handles share the root's pending batches, and any handle's `flush()` writes every pending key, but each handle keeps its own `_delta_table_cache`, so a handle that loaded a table earlier keeps serving a stale `DeltaTable` snapshot after a sibling flushed. The fix shares `_delta_table_cache`, `_existing_ids_cache` and `_cache_dirty` across the family, exactly as pending state is already shared. (2) The read methods merge pending and stored rows inconsistently (`get_records_by_ids` ignores pending; `get_records_with_column_value` ignores stored whenever a pending batch exists; overlapping ids show twice). The fix: every read returns stored rows plus pending rows whose record id is not stored (Delta's flush is `when_not_matched_insert_all`, so the stored row wins), `None` when empty. + +**Tech Stack:** Python, pyarrow, deltalake, pytest, `uv run`. + +Approved by the coordinator (option A, Nov 10, 2026). No public API, hash, stored schema or dependency change. Error handling is untouched (PR #290 / ITL-884 rewrites it in the same file; merge main after it lands). + +--- + +## Reproduction (Step 1 of the brief) + +`PipelineJob`, `DictSource` of 2 or 3 ints, `int -> int` pod, `DeltaTableDatabase`: + +| store | orchestrator | n | `get_all_records` | `iter_data` | +|---|---|---|---|---| +| delta | sync | 2 / 3 | 1 / 1 | 2 / 3 | +| delta | async | 2 / 3 | 1 / 2 | 2 / 3 | +| memory | both | 2 / 3 | 2 / 3 | 2 / 3 | + +Trace: the result-store handle (`at("pipeline", "_result")`) flushes the pipeline table's rows; the node's pipeline handle (`at("pipeline")`) cached the table at version 0 and still reads version 0 while the table is at version 2. Its pending batch is empty; a fresh handle reads all rows. + +## Contract and per-method findings (database level) + +Contract: a read through a handle reflects all writes of its `at()` family, pending and flushed together, and equals what a flush would leave: no duplicates, no missing rows. + +| case | `get_all_records` | `get_record_by_id` | `get_records_by_ids` | `get_records_with_column_value` | +|---|---|---|---|---| +| A sibling handle flushed | stale | stale | stale | stale | +| B stored + pending, one handle | ok | ok | misses pending | misses stored when pending exists; empty table instead of `None` | +| C pending id already stored, `skip_duplicates=False` | duplicate | returns pending row, flush keeps stored | (dup) | (dup) | +| D same, `skip_duplicates=True` | ok | ok | ok | ok | +| `record_id_column=` | returns (no hang) | | | | + +Other backends: `InMemoryArrowDatabase` shares `_tables` across `at()` and `ConnectorArrowDatabase` re-reads committed rows on every call, so neither goes stale; both merge pending + committed in every method. Both flush with the NEW row winning on id overlap (Delta keeps the stored row) and show both rows before flush under `skip_duplicates=False`. Report only. + +Thread safety: the async orchestrator runs only the user function in `run_in_executor` (`core/executors/local.py`); every database read and write happens on the event-loop thread, as for the already-shared pending dicts, so sharing the caches adds no cross-thread access. + +## File structure + +- Modify: `src/orcapod/databases/delta_lake_databases.py` — `__init__` / `at()` share caches; new helpers `_pending_not_stored` and `_combine_stored_and_pending`; the four read methods use them. +- Create: `tests/test_databases/test_delta_read_your_writes.py` — database-level regression tests. +- Create: `tests/test_pipeline/test_delta_node_records_after_run.py` — `PipelineJob` regression, both orchestrators. +- Modify: `DESIGN_ISSUES.md` — one entry appended at the end. + +--- + +### Task 1: Failing database-level tests + +**Files:** Create `tests/test_databases/test_delta_read_your_writes.py` + +- [ ] **Step 1: Write the tests** + +Helpers: `_rows(db, ids, vals, groups)` builds a table with `__record_id` (`large_binary`), `v` (`int64`), `g` (`int64`); `_values(table)` returns sorted `v` or `None`. A `READS` parametrization maps each read method to a call and its expected values: + +```python +READS = { + "get_all_records": (lambda db: db.get_all_records(PATH), [1, 2]), + "get_record_by_id": (lambda db: db.get_record_by_id(PATH, b"b"), [2]), + "get_records_by_ids": (lambda db: db.get_records_by_ids(PATH, [b"a", b"b"]), [1, 2]), + "get_records_with_column_value": ( + lambda db: db.get_records_with_column_value(PATH, {"g": 0}), [1, 2]), +} +``` + +Tests: +1. `test_sibling_handle_sees_rows_flushed_by_another_handle[method][flusher]`: `root = DeltaTableDatabase(tmp_path)`, `h1, h2 = root.at("s"), root.at("s")`; `h1` writes `a=1` with `flush=True` and reads once (loads its snapshot); `h2` writes `b=2`; read through `h1` and through `root` (path `("s",) + PATH`) before flush; flush through `flusher` in (`h2`, `root`, `h1`); read through `h1`, `root` and a new `root.at("s")` after flush. All equal the expected values. +2. `test_pending_and_stored_rows_are_both_returned[method]`: one handle, stored `a=1 (g=0), b=2 (g=1)`, pending `c=3 (g=1)`: `get_records_by_ids([a, c]) == [1, 3]`, `get_records_with_column_value({"g": 1}) == [2, 3]`, `{"g": 0} == [1]`, `get_all_records == [1, 2, 3]`, `get_record_by_id(c) == [3]`. +3. `test_no_match_returns_none`: stored + pending, `get_records_with_column_value({"g": 9})` and `get_records_by_ids([b"zz"])` are `None`. +4. `test_pending_row_with_stored_id_reads_as_flush_keeps_it[skip_duplicates]`: stored `a=1`; pending add of `a=10, d=4`; before flush each method returns `a=1` and `d=4` once (`get_all_records == [1, 4]`, `get_record_by_id(a) == [1]`, `get_records_by_ids([a, d]) == [1, 4]`, `get_records_with_column_value({"g": 0}) == [1, 4]`); after `flush()` the same values. +5. `test_record_id_column_is_returned_for_merged_rows`: stored + pending, `get_all_records(PATH, record_id_column="rid")` has column `rid` with all three ids and no `__record_id`; same for `get_records_by_ids` and `get_records_with_column_value`. + +- [ ] **Step 2: Run, verify the expected failures** + +Run: `uv run pytest tests/test_databases/test_delta_read_your_writes.py -v` +Expected: test 1 fails for every method after flush; test 2 fails for `get_records_by_ids` / `get_records_with_column_value`; test 3 fails (empty table, not `None`); test 4 fails for `skip_duplicates=False`. + +### Task 2: Failing PipelineJob test + +**Files:** Create `tests/test_pipeline/test_delta_node_records_after_run.py` + +- [ ] **Step 1: Write the test** + +```python +@pytest.mark.parametrize("orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +def test_node_returns_every_row_right_after_first_run(tmp_path, orchestrator): + source = op.sources.DictSource([{"id": i, "x": 10 * i} for i in range(3)], tag_columns=["id"]) + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + job = PipelineJob(store=DeltaTableDatabase(tmp_path)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orchestrator()) + + records = job.nodes["double"].get_all_records() + assert records is not None + assert sorted(records.column("result").to_pylist()) == [0, 20, 40] +``` + +- [ ] **Step 2: Run, verify it fails** (`1 != 3` rows) for both orchestrators. + +- [ ] **Step 3: Commit the failing tests** + +```bash +git add tests/test_databases/test_delta_read_your_writes.py tests/test_pipeline/test_delta_node_records_after_run.py +git commit -m "test(databases): regression tests for Delta reads missing own writes (ITL-864)" +``` + +### Task 3: Share the snapshot caches across the `at()` family + +**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (`__init__`, `at()`) + +- [ ] **Step 1:** Add keyword-only private constructor parameters `_shared_delta_table_cache`, `_shared_existing_ids_cache`, `_shared_cache_dirty`, used when given (same pattern as `_shared_pending_batches`); `at()` passes the root instance's dicts. Comment why: any family member's `flush()` writes every pending key and refreshes the cache entry for it, so the snapshot cache must be family-wide or a sibling keeps reading the old version. +- [ ] **Step 2:** Run Task 1 test 1 and the Task 2 test: expected PASS for test 1 and Task 2. + +### Task 4: Merge stored and pending rows the way flush does + +**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (read methods) + +- [ ] **Step 1:** Add helpers: + +```python +def _pending_not_stored(self, record_path): + """Pending rows whose record id is not stored; flush inserts only these.""" + pending = self._pending_batches.get(self._get_record_key(record_path)) + if pending is None or pending.num_rows == 0: + return None + stored_ids = self._get_existing_ids(record_path) + if stored_ids: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter(pc.invert(pc.is_in( + pending[self.RECORD_ID_COLUMN], value_set=pa.array(list(stored_ids), type=id_type)))) + return pending if pending.num_rows > 0 else None + +def _combine_stored_and_pending(self, stored, pending, record_id_column): + parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] + if not parts: + return None + table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) + return self._handle_record_id_column(table, record_id_column) +``` + +- [ ] **Step 2:** `get_all_records`: pending part uses `_pending_not_stored`. `get_record_by_id`: serve from pending only when the id is pending and not stored. `get_records_by_ids`: stored read (existing try/except unchanged) plus pending filtered by the ids. `get_records_with_column_value`: stored read (existing try/except unchanged) plus pending filtered by the expression; `None` when empty. +- [ ] **Step 3:** Run `uv run pytest tests/test_databases tests/test_pipeline -q`: all pass. +- [ ] **Step 4:** Commit `fix(databases): Delta reads return every row the handle family wrote (ITL-864)`. + +### Task 5: Design issue entry, full verification, PR + +- [ ] Append one `DESIGN_ISSUES.md` entry (resolved, with **Fix:**; note that writes from other processes are still not seen until a handle reloads the table — follow-up for the coordinator to file). +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (golden hashes unchanged). +- [ ] Merge `origin/main`, rerun if it moved, push, PR "Fixes ITL-864", `orca linear attach`, `orca worktree set --pr`. diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py new file mode 100644 index 000000000..9dade27f0 --- /dev/null +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -0,0 +1,183 @@ +"""Reads through a ``DeltaTableDatabase`` handle reflect every write of its family (ITL-864). + +Contract: a read through any handle of one ``at()`` family returns exactly the rows a +flush would leave, pending and stored together, with no duplicates and no missing +rows. Delta's flush inserts only rows whose record id is not stored yet, so on an id +overlap the stored row is the one returned. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.databases import DeltaTableDatabase + +PATH = ("table",) +SCOPE = "scope" + + +def _rows( + ids: list[str], values: list[int], groups: list[int] | None = None +) -> pa.Table: + """Build a record table keyed by ``__record_id`` with ``int64`` columns ``v`` and ``g``.""" + return pa.table( + { + "__record_id": pa.array([i.encode() for i in ids], type=pa.large_binary()), + "v": pa.array(values, type=pa.int64()), + "g": pa.array( + groups if groups is not None else [0] * len(ids), type=pa.int64() + ), + } + ) + + +def _values(table: pa.Table | None) -> list[int] | None: + """Return the sorted ``v`` values of ``table``, or ``None`` when there is no table.""" + if table is None: + return None + return sorted(table.column("v").to_pylist()) + + +Read = Callable[[DeltaTableDatabase, tuple[str, ...]], "pa.Table | None"] + +# Each read method, called for rows ``a=1`` and ``b=2`` (both in group 0). +READS: dict[str, Read] = { + "get_all_records": lambda db, path: db.get_all_records(path), + "get_record_by_id": lambda db, path: db.get_record_by_id(path, b"b"), + "get_records_by_ids": lambda db, path: db.get_records_by_ids(path, [b"a", b"b"]), + "get_records_with_column_value": lambda db, path: db.get_records_with_column_value( + path, {"g": 0} + ), +} +EXPECTED = { + "get_all_records": [1, 2], + "get_record_by_id": [2], + "get_records_by_ids": [1, 2], + "get_records_with_column_value": [1, 2], +} + + +@pytest.mark.parametrize("flusher", ["writer", "root", "reader"]) +@pytest.mark.parametrize("method", list(READS)) +def test_sibling_handle_sees_rows_flushed_by_another_handle( + tmp_path: Path, method: str, flusher: str +) -> None: + """A handle that already loaded the table sees rows a sibling handle wrote and flushed.""" + root = DeltaTableDatabase(tmp_path) + reader = root.at(SCOPE) + writer = root.at(SCOPE) + read = READS[method] + + reader.add_records(PATH, _rows(["a"], [1]), flush=True) + assert _values(reader.get_all_records(PATH)) == [1] # reader loads its snapshot + + writer.add_records(PATH, _rows(["b"], [2])) + assert _values(read(reader, PATH)) == EXPECTED[method] + assert _values(read(root, (SCOPE, *PATH))) == EXPECTED[method] + + {"writer": writer, "root": root, "reader": reader}[flusher].flush() + + assert _values(read(reader, PATH)) == EXPECTED[method] + assert _values(read(writer, PATH)) == EXPECTED[method] + assert _values(read(root, (SCOPE, *PATH))) == EXPECTED[method] + assert _values(read(root.at(SCOPE), PATH)) == EXPECTED[method] + + +@pytest.fixture +def stored_and_pending(tmp_path: Path) -> DeltaTableDatabase: + """A handle with stored rows ``a=1 (g=0)``, ``b=2 (g=1)`` and pending row ``c=3 (g=1)``.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["a", "b"], [1, 2], [0, 1]), flush=True) + db.add_records(PATH, _rows(["c"], [3], [1])) + return db + + +def test_get_all_records_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + assert _values(stored_and_pending.get_all_records(PATH)) == [1, 2, 3] + + +def test_get_record_by_id_finds_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + assert _values(stored_and_pending.get_record_by_id(PATH, b"a")) == [1] + assert _values(stored_and_pending.get_record_by_id(PATH, b"c")) == [3] + + +def test_get_records_by_ids_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + assert _values(stored_and_pending.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] + + +def test_get_records_with_column_value_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + db = stored_and_pending + assert _values(db.get_records_with_column_value(PATH, {"g": 1})) == [2, 3] + assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1] + + +def test_reads_with_no_match_return_none( + stored_and_pending: DeltaTableDatabase, +) -> None: + db = stored_and_pending + assert db.get_records_with_column_value(PATH, {"g": 9}) is None + assert db.get_records_by_ids(PATH, [b"zz"]) is None + assert db.get_record_by_id(PATH, b"zz") is None + + +@pytest.mark.parametrize( + ("read", "expected_ids"), + [ + ( + lambda db: db.get_all_records(PATH, record_id_column="rid"), + [b"a", b"b", b"c"], + ), + ( + lambda db: db.get_records_by_ids( + PATH, [b"a", b"c"], record_id_column="rid" + ), + [b"a", b"c"], + ), + ( + lambda db: db.get_records_with_column_value( + PATH, [("g", 1)], record_id_column="rid" + ), + [b"b", b"c"], + ), + ], + ids=["get_all_records", "get_records_by_ids", "get_records_with_column_value"], +) +def test_record_id_column_names_the_merged_ids( + stored_and_pending: DeltaTableDatabase, + read: Callable[[DeltaTableDatabase], pa.Table | None], + expected_ids: list[bytes], +) -> None: + """With ``record_id_column`` the ids of stored and pending rows come back under that name.""" + table = read(stored_and_pending) + assert table is not None + assert "__record_id" not in table.column_names + assert sorted(table.column("rid").to_pylist()) == expected_ids + + +@pytest.mark.parametrize("skip_duplicates", [False, True]) +def test_pending_row_with_stored_id_reads_as_flush_keeps_it( + tmp_path: Path, skip_duplicates: bool +) -> None: + """A pending row whose id is already stored is never returned; the stored row is.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["a"], [1]), flush=True) + db.add_records(PATH, _rows(["a", "d"], [10, 4]), skip_duplicates=skip_duplicates) + + for when in ("before flush", "after flush"): + assert _values(db.get_all_records(PATH)) == [1, 4], when + assert _values(db.get_record_by_id(PATH, b"a")) == [1], when + assert _values(db.get_records_by_ids(PATH, [b"a", b"d"])) == [1, 4], when + assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1, 4], when + db.flush() diff --git a/tests/test_pipeline/test_delta_node_records_after_run.py b/tests/test_pipeline/test_delta_node_records_after_run.py new file mode 100644 index 000000000..15c925f1f --- /dev/null +++ b/tests/test_pipeline/test_delta_node_records_after_run.py @@ -0,0 +1,46 @@ +"""A function node on a Delta store returns every row right after its first run (ITL-864). + +``PipelineJob.run`` flushes through handles other than the node's own, so a node +handle that cached the table during the run used to keep reading an old snapshot +and return only some of the rows it had just written. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +@pytest.mark.parametrize("n_rows", [2, 3]) +def test_node_returns_every_row_right_after_first_run( + tmp_path: Path, + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], + n_rows: int, +) -> None: + source = op.sources.DictSource( + [{"id": i, "x": 10 * i} for i in range(n_rows)], tag_columns=["id"] + ) + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + job = PipelineJob(store=DeltaTableDatabase(tmp_path)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orchestrator()) + + records = job.nodes["double"].get_all_records() + assert records is not None + assert sorted(records.column("result").to_pylist()) == [ + 20 * i for i in range(n_rows) + ] From b727e0c6f52787ec6a91ad050a7ec6329d0e2306 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:39:50 -0800 Subject: [PATCH 07/30] fix(databases): Delta reads return every row the handle family wrote (ITL-864) Every at() handle shares the root's pending batches and any handle's flush() writes every pending key, but each handle cached its own loaded DeltaTable, so the node's pipeline handle kept reading the version it loaded during the run: node.get_all_records() returned 1 of 3 rows right after a first PipelineJob run. The loaded-table, existing-ids and cache-dirty dicts are now shared across the at() family, like pending state. Every read method now returns what a flush would leave: stored rows plus the pending rows whose record id is not stored, None when empty. Before, get_records_by_ids ignored pending rows, get_records_with_column_value ignored stored rows whenever a pending batch existed, and a pending row with a stored id was returned twice or instead of the stored row. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 34 ++++ src/orcapod/databases/delta_lake_databases.py | 158 ++++++++++++------ 2 files changed, 143 insertions(+), 49 deletions(-) diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 5b167d8de..ee1f0d0c8 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2208,3 +2208,37 @@ subprocesses with different `PYTHONHASHSEED` values and were verified identical 3.11, 3.12 and 3.13. The generator's header says when regeneration is legitimate: a changed value needs a migration assessment and the developer's approval. Not pinned yet: the `StarfixArrowHasher` gaps tracked in ITL-848. + +--- + +## `src/orcapod/databases/delta_lake_databases.py` — reading back a handle family's own writes + +### DW1 — A read through one `at()` handle could miss rows its family had just written +**Status:** resolved +**Severity:** high +**Issue:** ITL-864 + +Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` +returned 1 of 2 or 3 rows (plain `int -> int` pod, sync and async orchestrators), while +`iter_data` and a fresh handle saw every row. Every `at()` handle shares the root's pending +batches, and any handle's `flush()` writes every pending key, so the result-store handle +flushed the pipeline table's rows. But each handle kept its own loaded-table cache, so the +node's pipeline handle kept reading the table version it had loaded during the run. Every read +method missed the rows a sibling flushed. Separately, the read methods merged pending and stored +rows inconsistently: `get_records_by_ids` ignored pending rows, `get_records_with_column_value` +ignored stored rows whenever a pending batch existed (and returned an empty table instead of +`None`), and a pending row whose record id was already stored was returned as well, although a +flush keeps the stored row and drops the pending one. + +**Fix:** the loaded-table, existing-ids and cache-dirty dicts are shared across the `at()` +family exactly as pending state is. Every read method returns what a flush would leave: the +stored rows plus the pending rows whose record id is not stored (`_pending_not_stored`), and +`None` when there are none. These dicts are touched only from the thread that drives the +pipeline; the async orchestrator runs only user functions in worker threads. Regression tests: +`tests/test_databases/test_delta_read_your_writes.py` and +`tests/test_pipeline/test_delta_node_records_after_run.py`. + +Still open: a handle does not see writes made by another process, or by a separate +`DeltaTableDatabase` root on the same path, until it reloads the table; this is unchanged. +Reading with a pending batch also refreshes the stored-id set when it is stale, which loads the +whole table (D3). diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index c7dec5dcf..f9b99bb09 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -57,6 +57,9 @@ def __init__( _scoped_path: tuple[str, ...] = (), _shared_pending_batches: "dict[str, pa.Table] | None" = None, _shared_pending_record_ids: "defaultdict[str, set[bytes]] | None" = None, + _shared_delta_table_cache: "dict[str, deltalake.DeltaTable] | None" = None, + _shared_existing_ids_cache: "defaultdict[str, set[str]] | None" = None, + _shared_cache_dirty: "defaultdict[str, bool] | None" = None, ): self._root_uri, self._storage_options = parse_base_path( base_path, storage_options @@ -81,9 +84,16 @@ def __init__( ) # For cloud paths: create_base_path is silently ignored (no directory needed). - self._delta_table_cache: dict[str, deltalake.DeltaTable] = {} - # Shared pending state — when constructed via at(), these dicts are shared - # with the root instance so that root.flush() can flush all scoped writes. + # Shared state — when constructed via at(), these dicts are shared with the + # root instance. Any handle's flush() writes every pending key of the family + # and refreshes the loaded table and existing-ids entries for it, so these + # caches must be family-wide too: a per-handle cache would keep serving the + # snapshot a sibling loaded before the flush (ITL-864). All of them are + # touched only from the thread that drives the pipeline (the async + # orchestrator runs only user functions in worker threads). + self._delta_table_cache: dict[str, deltalake.DeltaTable] = ( + _shared_delta_table_cache if _shared_delta_table_cache is not None else {} + ) if _shared_pending_batches is not None: self._pending_batches = _shared_pending_batches else: @@ -92,8 +102,16 @@ def __init__( self._pending_record_ids = _shared_pending_record_ids else: self._pending_record_ids: dict[str, set[bytes]] = defaultdict(set) - self._existing_ids_cache: dict[str, set[str]] = defaultdict(set) - self._cache_dirty: dict[str, bool] = defaultdict(lambda: True) + self._existing_ids_cache: dict[str, set[str]] = ( + _shared_existing_ids_cache + if _shared_existing_ids_cache is not None + else defaultdict(set) + ) + self._cache_dirty: dict[str, bool] = ( + _shared_cache_dirty + if _shared_cache_dirty is not None + else defaultdict(lambda: True) + ) def _clear_pending(self): """Clear all pending state.""" @@ -620,6 +638,57 @@ def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool: and self._pending_batches[record_key].num_rows >= self.batch_size ) + def _pending_not_stored(self, record_path: tuple[str, ...]) -> pa.Table | None: + """Return the pending rows a flush would insert at ``record_path``. + + A flush inserts only rows whose record id is not stored yet (the merge is + ``when_not_matched_insert_all``), so a pending row with a stored id is + never returned by a read: the stored row is. + + Args: + record_path: Path components relative to this database view. + + Returns: + The pending rows whose record id is not stored, or ``None`` if there + are none. + """ + pending = self._pending_batches.get(self._get_record_key(record_path)) + if pending is None or pending.num_rows == 0: + return None + stored_ids = self._get_existing_ids(record_path) + if stored_ids: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + is_stored = pc.is_in( + pending[self.RECORD_ID_COLUMN], + value_set=pa.array(list(stored_ids), type=id_type), + ) + pending = pending.filter(pc.invert(is_stored)) + return pending if pending.num_rows > 0 else None + + def _combine_stored_and_pending( + self, + stored: pa.Table | None, + pending: pa.Table | None, + record_id_column: str | None, + ) -> pa.Table | None: + """Combine stored rows with pending rows into one read result. + + Args: + stored: Rows read from the Delta table, or ``None``. + pending: Pending rows that are not stored (see ``_pending_not_stored``), + already filtered by the read's condition, or ``None``. + record_id_column: Name to give the record id column, or ``None`` to + drop it. + + Returns: + The combined rows, or ``None`` if there are none. + """ + parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] + if not parts: + return None + table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) + return self._handle_record_id_column(table, record_id_column) + def get_all_records( self, record_path: tuple[str, ...], @@ -629,11 +698,13 @@ def get_all_records( """ Get all records from both pending batch and Delta table. + Every read method returns the rows a flush would leave: the stored rows + plus the pending rows whose record id is not stored yet, as seen through + any handle of this ``at()`` family. + Returns: Combined Arrow table with all records, or None if no records exist """ - record_key = self._get_record_key(record_path) - tables_to_combine = [] # Add Delta table data @@ -647,24 +718,10 @@ def get_all_records( except Exception as e: logger.warning(f"Error reading Delta table: {e}") - # Add pending batch data - if ( - retrieve_pending - and (pending_batch := self._pending_batches.get(record_key)) is not None - ): - if pending_batch.num_rows > 0: - tables_to_combine.append(pending_batch) - - if not tables_to_combine: - return None - - if len(tables_to_combine) == 1: - table_to_return = tables_to_combine[0] - else: - table_to_return = pa.concat_tables(tables_to_combine) - - # Handle record_id_column if specified - return self._handle_record_id_column(table_to_return, record_id_column) + # Add the pending rows a flush would insert + pending = self._pending_not_stored(record_path) if retrieve_pending else None + stored = pa.concat_tables(tables_to_combine) if tables_to_combine else None + return self._combine_stored_and_pending(stored, pending, record_id_column) def get_records_with_column_value( self, @@ -675,9 +732,6 @@ def get_records_with_column_value( ) -> "pa.Table | None": if flush: self.flush_batch(record_path) - # check if record_id is found in pending batches - record_key = self._get_record_key(record_path) - pending_batch = self._pending_batches.get(record_key) if isinstance(column_values, Mapping): # Convert Mapping to list of tuples @@ -691,31 +745,26 @@ def get_records_with_column_value( for next_expression in expressions[1:]: combined_expression = combined_expression & next_expression - if pending_batch is not None: - filtered_table = pending_batch.filter(combined_expression) - return self._handle_record_id_column(filtered_table, record_id_column) + pending = self._pending_not_stored(record_path) + if pending is not None: + pending = pending.filter(combined_expression) # Now check the Delta table delta_table = self._get_delta_table(record_path) if delta_table is None: - return None + return self._combine_stored_and_pending(None, pending, record_id_column) try: # Use schema-preserving read result = self._read_delta_table(delta_table, expression=combined_expression) - - if len(result) == 0: - return None - - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) - except Exception as e: logger.error( f"Error getting record with {column_values} from {'/'.join(record_path)}: {e}" ) raise e + return self._combine_stored_and_pending(result, pending, record_id_column) + def get_record_by_id( self, record_path: tuple[str, ...], @@ -738,9 +787,12 @@ def get_record_by_id( self.flush_batch(record_path) record_id = coerce_record_id(record_id) - # check if record_id is found in pending batches + # Serve a pending record only if its id is not stored: a flush keeps the + # stored row and drops the pending one. record_key = self._get_record_key(record_path) - if record_id in self._pending_record_ids[record_key]: + if record_id in self._pending_record_ids[ + record_key + ] and record_id not in self._get_existing_ids(record_path): # Return the pending record after removing the entry id column pending_batch = self._pending_batches[record_key] assert pending_batch is not None, "Pending batch should not be None" @@ -812,26 +864,31 @@ def get_records_by_ids( return None # check inside the batch + pending = self._pending_not_stored(record_path) + if pending is not None: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter( + pc.is_in( + pending[self.RECORD_ID_COLUMN], + value_set=pa.array(record_ids_list, type=id_type), + ) + ) + delta_table = self._get_delta_table(record_path) if delta_table is None: - return None + return self._combine_stored_and_pending(None, pending, record_id_column) try: # Use schema-preserving read with filters filter_expr = self._create_record_ids_filter(record_ids_list) result = self._read_delta_table(delta_table, filters=filter_expr) - - if len(result) == 0: - return None - - # Handle record_id column based on parameter - return self._handle_record_id_column(result, record_id_column) - except Exception as e: logger.error( f"Error getting records by IDs from {'/'.join(record_path)}: {e}" ) return None + return self._combine_stored_and_pending(result, pending, record_id_column) + def _read_delta_table( self, delta_table: deltalake.DeltaTable, @@ -1098,6 +1155,9 @@ def at(self, *path_components: str) -> "DeltaTableDatabase": _scoped_path=new_scoped_path, _shared_pending_batches=root_instance._pending_batches, _shared_pending_record_ids=root_instance._pending_record_ids, + _shared_delta_table_cache=root_instance._delta_table_cache, + _shared_existing_ids_cache=root_instance._existing_ids_cache, + _shared_cache_dirty=root_instance._cache_dirty, ) def list_sources(self) -> list[tuple[str, ...]]: From ce32c9e6d7230da649ef1bbb0cb54431853723f2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:42:08 -0800 Subject: [PATCH 08/30] refactor(pipeline): give the async store-error selection #287's shape (ITL-884) Rename _single_store_read_error to _single_store_error(group, topo_order) and declare topo_order and store_error before the try, exactly as PR #287 does for StoreWriteError, so whichever PR merges second only has to widen the helper to both error types. Behaviour is unchanged: a StoreReadError names no node, so the first one in the group is still chosen. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/async_orchestrator.py | 31 ++++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 4fb3a0db9..36e710a86 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -138,6 +138,8 @@ async def _run_async( effective_observer = observer if observer is not None else NoOpObserver() effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) + topo_order: list[Any] = [] + store_error: StoreReadError | None = None try: topo_order = list(graph.topological_sort()) buf = self._buffer_size @@ -254,9 +256,7 @@ async def _run_async( except ExceptionGroup as group: # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. - # A plain except, not except*: raising inside except* can wrap the - # error in a new group again. - store_error = _single_store_read_error(group) + store_error = _single_store_error(group, topo_order) if store_error is None: raise raise store_error @@ -264,27 +264,36 @@ async def _run_async( effective_observer.on_run_end(run_id) -def _single_store_read_error(group: ExceptionGroup) -> StoreReadError | None: +def _single_store_error( + group: ExceptionGroup, topo_order: list[Any] +) -> StoreReadError | None: """Pick the one ``StoreReadError`` an async run raises from its exception group. Concurrent node tasks can fail together, so the ``TaskGroup``'s group may hold - several store read errors and other exceptions. The caller gets a single - ``StoreReadError``, as from the sync orchestrator: the first one in the group. - Every other exception stays attached for diagnosis: each is noted on the - chosen error, other store read errors are also logged at WARNING, and the - whole group is the chosen error's ``__context__``. Its ``__cause__`` stays the + several store errors and other exceptions. The caller gets a single + ``StoreReadError``, as from the sync orchestrator: the one whose node comes + first in topological order, with ties and node-less errors (a + ``StoreReadError`` names a table, not a node) broken by their order in the + group. Every other exception stays attached for diagnosis: each is noted on + the chosen error, other store errors are also logged at WARNING, and the whole + group is the chosen error's ``__context__``. Its ``__cause__`` stays the backend error. Args: group: The exception group the run's ``TaskGroup`` raised. + topo_order: The run's nodes in topological order. Returns: - The store read error to raise, or ``None`` when the group holds none. + The store error to raise, or ``None`` when the group holds none. """ store_errors = StoreReadError.in_group(group) if not store_errors: return None - chosen = store_errors[0] + rank = {getattr(node, "label", None): i for i, node in enumerate(topo_order)} + chosen = min( + store_errors, + key=lambda err: rank.get(getattr(err, "node_label", None), len(rank)), + ) for other in _leaves(group): if other is chosen: continue From 3785d71beec0f314badd6a61b86a046bedf5485e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:42:17 -0800 Subject: [PATCH 09/30] docs(plans): name the async helper as implemented (ITL-884) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-10-itl-884-coderabbit-flush-and-async-group.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md index caaebe4b1..394e968b6 100644 --- a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -14,7 +14,7 @@ each failure. When at least one batch failed with a store error, it raises the f `StoreReadError` (else the first `InputValidationError`) itself, with a note for every other failed batch. Only non-store failures keep the `RuntimeError` wrapper. The async orchestrator replaces `except* StoreReadError` with a plain `except ExceptionGroup` that selects one -`StoreReadError` (`_single_store_read_error`), notes and logs the rest, and raises it, in the same +`StoreReadError` (`_single_store_error`), notes and logs the rest, and raises it, in the same shape as PR #287's `_single_store_error` for `StoreWriteError` so the second PR to merge has a trivial conflict. @@ -42,7 +42,7 @@ No hash, path or stored-schema changes. No public API added. - `src/orcapod/databases/delta_lake_databases.py`: `flush()` and a module helper `_store_error_to_raise(errors)`. - `src/orcapod/pipeline/async_orchestrator.py`: plain `except ExceptionGroup`, helpers - `_single_store_read_error(group)` and `_leaves(group)` (identical to #287's `_leaves`). + `_single_store_error(group, topo_order)` and `_leaves(group)` (identical to #287's `_leaves`). - Tests: `tests/test_databases/test_delta_read_failures.py` (assert the bare error types, add a multi-batch test), `tests/test_pipeline/test_store_read_failure.py` (flush during per-row processing, both orchestrators; concurrent unrelated failure, async). @@ -134,7 +134,7 @@ docstring gets a Raises section. Run Task 1's tests: PASS. raise store_error ``` -`_single_store_read_error(group)`: `StoreReadError.in_group(group)`; none gives `None`; the first +`_single_store_error(group, topo_order)`: `StoreReadError.in_group(group)`; none gives `None`; the first is chosen; every other leaf (`_leaves`) is noted on it ("Another store read failed in the same run: ..." also logged at WARNING; "Another node failed in the same run: : "). Its `__cause__` stays the backend error; the group is its `__context__`. Run: PASS. From 7d96a01549681aec530f4aec257a6de41acdef0f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:42:37 -0800 Subject: [PATCH 10/30] docs(plans): fix the remaining helper name in the plan's code sample (ITL-884) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-10-itl-884-coderabbit-flush-and-async-group.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md index 394e968b6..89c03292c 100644 --- a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -128,7 +128,7 @@ docstring gets a Raises section. Run Task 1's tests: PASS. # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. # A plain except (not except*) so the raise is never wrapped again. - store_error = _single_store_read_error(group) + store_error = _single_store_error(group, topo_order) if store_error is None: raise raise store_error From e9c78b5c6e9f2b15725db8f33eb445d97a77dc1c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:43:53 -0800 Subject: [PATCH 11/30] fix(databases): type Delta stored-id sets as bytes and fix test typing (ITL-864) Record ids are bytes; the set[str] annotation made basedpyright report the new stored-id check as always true. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 10 +++++----- tests/test_databases/test_delta_read_your_writes.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index f9b99bb09..a50853084 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -58,8 +58,8 @@ def __init__( _shared_pending_batches: "dict[str, pa.Table] | None" = None, _shared_pending_record_ids: "defaultdict[str, set[bytes]] | None" = None, _shared_delta_table_cache: "dict[str, deltalake.DeltaTable] | None" = None, - _shared_existing_ids_cache: "defaultdict[str, set[str]] | None" = None, - _shared_cache_dirty: "defaultdict[str, bool] | None" = None, + _shared_existing_ids_cache: "dict[str, set[bytes]] | None" = None, + _shared_cache_dirty: "dict[str, bool] | None" = None, ): self._root_uri, self._storage_options = parse_base_path( base_path, storage_options @@ -102,7 +102,7 @@ def __init__( self._pending_record_ids = _shared_pending_record_ids else: self._pending_record_ids: dict[str, set[bytes]] = defaultdict(set) - self._existing_ids_cache: dict[str, set[str]] = ( + self._existing_ids_cache: dict[str, set[bytes]] = ( _shared_existing_ids_cache if _shared_existing_ids_cache is not None else defaultdict(set) @@ -343,7 +343,7 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: self._existing_ids_cache[record_key] = set() else: existing_ids = cast( - set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) + set[bytes], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) ) self._existing_ids_cache[record_key] = existing_ids @@ -358,7 +358,7 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: self._cache_dirty[record_key] = False raise - def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[str]: + def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[bytes]: """Get the set of existing IDs in the Delta table, using cache when possible.""" record_key = self._get_record_key(record_path) if ( diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index 9dade27f0..632b73a08 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import cast from pathlib import Path import pyarrow as pa @@ -39,7 +40,7 @@ def _values(table: pa.Table | None) -> list[int] | None: """Return the sorted ``v`` values of ``table``, or ``None`` when there is no table.""" if table is None: return None - return sorted(table.column("v").to_pylist()) + return sorted(cast(list[int], table.column("v").to_pylist())) Read = Callable[[DeltaTableDatabase, tuple[str, ...]], "pa.Table | None"] @@ -163,7 +164,7 @@ def test_record_id_column_names_the_merged_ids( table = read(stored_and_pending) assert table is not None assert "__record_id" not in table.column_names - assert sorted(table.column("rid").to_pylist()) == expected_ids + assert sorted(cast(list[bytes], table.column("rid").to_pylist())) == expected_ids @pytest.mark.parametrize("skip_duplicates", [False, True]) From 486db1d95865fd7bf5495d6e6fb61755073e144d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:18:30 -0800 Subject: [PATCH 12/30] docs(tests): docstrings for the store-read-failure tests (ITL-884) CodeRabbit's pre-merge Docstring Coverage check failed on the two test files this PR adds. Every test, fixture and helper in them now has a one-line Google-style docstring. Co-Authored-By: Claude Opus 5.5 --- .../test_delta_read_failures.py | 41 +++++++++++++++++++ .../test_pipeline/test_store_read_failure.py | 13 ++++++ 2 files changed, 54 insertions(+) diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index 94ae2d228..8cea2e3e7 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -37,6 +37,7 @@ def rows(ids: list[bytes], x: pa.Array[Any], **extra: pa.Array[Any]) -> pa.Table def table_dir(base: Path) -> Path: + """Return the directory of the table at ``PATH`` under ``base``.""" return base.joinpath(*PATH) @@ -53,24 +54,29 @@ class _Ext(pa.ExtensionType): """A minimal extension type standing in for a logical type's storage.""" def __init__(self, storage: pa.DataType) -> None: + """Wrap ``storage`` in the test extension type.""" super().__init__(storage, "orcapod.test.itl884") def __arrow_ext_serialize__(self) -> bytes: + """Return the (empty) serialized parameters of the type.""" return b"" @classmethod def __arrow_ext_deserialize__( cls, storage_type: pa.DataType, serialized: bytes ) -> "_Ext": + """Rebuild the type over ``storage_type``.""" return cls(storage_type) def ext_array(values: list[Any], storage: pa.DataType) -> pa.ExtensionArray[Any]: + """Build an extension array of ``values`` over the given storage type.""" return pa.ExtensionArray.from_storage(_Ext(storage), pa.array(values, storage)) @pytest.fixture def base(tmp_path: Path) -> Path: + """Return the store root, inside the test's temporary directory.""" return tmp_path / "store" @@ -113,6 +119,7 @@ def unsupported_reader_feature(base: Path) -> None: def assert_names_table(exc: StoreReadError, base: Path) -> None: + """Assert that ``exc`` names the table at ``PATH`` and chains its cause.""" assert Path(exc.table) == table_dir(base) assert exc.table in str(exc) assert exc.__cause__ is not None @@ -133,13 +140,17 @@ def assert_names_table(exc: StoreReadError, base: Path) -> None: class TestUnreadableTable: + """A table that exists but cannot be read raises on every read path.""" + @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) def test_read_raises_store_read_error(self, base, unreadable, read): + """Each read of the unreadable table raises ``StoreReadError`` naming it.""" with pytest.raises(StoreReadError) as info: read(unreadable) assert_names_table(info.value, base) def test_skip_duplicates_append_raises_store_read_error(self, base, unreadable): + """The existing-ids read for ``skip_duplicates`` raises, never assumes none.""" # skip_duplicates reads the stored record ids before buffering the rows. with pytest.raises(StoreReadError) as info: unreadable.add_records( @@ -151,18 +162,23 @@ def test_skip_duplicates_append_raises_store_read_error(self, base, unreadable): assert_names_table(info.value, base) def test_table_still_exists(self, unreadable): + """The unreadable table is still reported as existing.""" assert unreadable.table_exists(PATH) def test_bad_filter_is_not_reported_as_unreadable_table(stored): + """A bad filter column is the caller's error, not a ``StoreReadError``.""" # The table reads fine; the filter names a column it does not have. with pytest.raises(pa.ArrowInvalid): stored.get_records_with_column_value(PATH, {"no_such_column": 1}) class TestAbsentTable: + """A table that does not exist still reads as ``None``.""" + @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) def test_read_returns_none(self, base, read): + """Each read of a table that was never written returns ``None``.""" db = DeltaTableDatabase(base_path=base) assert read(db) is None assert not db.table_exists(PATH) @@ -174,7 +190,10 @@ def test_read_returns_none(self, base, read): class TestUnloadableTableIsNeverOverwritten: + """A table whose log cannot be loaded is never replaced by a flush.""" + def test_unsupported_reader_feature(self, base, stored): + """An unloadable log makes reads and the flush raise; nothing is written.""" unsupported_reader_feature(base) before = snapshot(table_dir(base)) db = DeltaTableDatabase(base_path=base) @@ -194,6 +213,7 @@ def test_unsupported_reader_feature(self, base, stored): def test_protocol_error_on_load_is_not_treated_as_absent( self, base, stored, monkeypatch ): + """A ``DeltaProtocolError`` on load makes the flush raise, not overwrite.""" # deltalake 1.0.2 does not raise DeltaProtocolError for a local table we can # build here, so the load is replaced by one that raises it for this table. # Before ITL-884 the flush treated this error as "no table" and wrote the @@ -202,6 +222,7 @@ def test_protocol_error_on_load_is_not_treated_as_absent( target = table_dir(base) def failing_load(table_uri, *args, **kwargs): + """Raise ``DeltaProtocolError`` for the target table only.""" if Path(str(table_uri)) == target: raise DeltaProtocolError("simulated") return real(table_uri, *args, **kwargs) @@ -226,9 +247,12 @@ def failing_load(table_uri, *args, **kwargs): class TestFlushRaisesStoreErrorsThemselves: + """``flush`` raises store errors as themselves, with the other failures noted.""" + def test_store_read_error_is_raised_with_a_note_per_other_failed_batch( self, base, stored ): + """The ``StoreReadError`` is raised and the refused batch is noted on it.""" other = ("results", "u") stored.add_records( other, @@ -260,7 +284,10 @@ def test_store_read_error_is_raised_with_a_note_per_other_failed_batch( db.flush() def test_other_flush_failures_still_raise_runtime_error(self, base, monkeypatch): + """A flush that fails for another reason still raises ``RuntimeError``.""" + def failing_write(*args, **kwargs): + """Raise ``OSError`` for every write.""" raise OSError("simulated disk failure") db = DeltaTableDatabase(base_path=base) @@ -280,10 +307,13 @@ def failing_write(*args, **kwargs): def append(db: DeltaTableDatabase, table: pa.Table) -> None: + """Add ``table`` at ``PATH`` and flush it.""" db.add_records(PATH, table, record_id_column="rid", flush=True) class TestChangedColumnTypeIsRefused: + """An append whose column type differs from the stored table is refused.""" + @pytest.mark.parametrize( ("new_type", "value", "named"), [ @@ -294,6 +324,7 @@ class TestChangedColumnTypeIsRefused: ids=["int64->float64", "int64->int32", "int64->large_string"], ) def test_plain_type_change(self, base, stored, new_type, value, named): + """A changed column type is refused, naming the column and both types.""" with pytest.raises(InputValidationError) as info: append(stored, rows([b"c"], pa.array([value], new_type))) message = str(info.value) @@ -308,6 +339,7 @@ def test_plain_type_change(self, base, stored, new_type, value, named): assert back.schema.field("x").type == pa.int64() def test_extension_storage_change(self, base): + """A changed extension storage type is refused and the table stays readable.""" db = DeltaTableDatabase(base_path=base) append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) @@ -327,12 +359,16 @@ def test_extension_storage_change(self, base): class TestLegitimateAppendsStillPass: + """Appends whose types match the stored table, as Delta stores them, succeed.""" + def read_back(self, base: Path) -> pa.Table: + """Read every row at ``PATH`` with a fresh database view.""" back = DeltaTableDatabase(base_path=base).get_all_records(PATH) assert back is not None return back def test_identical_schema(self, base, stored): + """An append with the stored schema succeeds.""" append(stored, rows([b"c"], pa.array([3], pa.int64()))) assert self.read_back(base).num_rows == 3 @@ -350,12 +386,14 @@ def test_identical_schema(self, base, stored): ids=["string->large", "large->string", "list->large", "binary->large"], ) def test_large_and_non_large_layouts(self, base, first, second): + """Large and non-large layouts of one type count as the same type.""" db = DeltaTableDatabase(base_path=base) append(db, rows([b"a"], first)) append(db, rows([b"b"], second)) assert self.read_back(base).num_rows == 2 def test_new_column(self, base, stored): + """An append that adds a column succeeds.""" append( stored, rows([b"c"], pa.array([3], pa.int64()), y=pa.array(["new"])), @@ -365,18 +403,21 @@ def test_new_column(self, base, stored): assert "y" in back.column_names def test_missing_column(self, base): + """An append that omits a stored column succeeds.""" db = DeltaTableDatabase(base_path=base) append(db, rows([b"a"], pa.array([1], pa.int64()), y=pa.array(["old"]))) append(db, rows([b"b"], pa.array([2], pa.int64()))) assert self.read_back(base).num_rows == 2 def test_extension_type_with_unchanged_storage(self, base): + """An extension type with unchanged storage appends.""" db = DeltaTableDatabase(base_path=base) append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) append(db, rows([b"b"], ext_array([b"y"], pa.large_binary()))) assert self.read_back(base).num_rows == 2 def test_nullability_only_difference_is_accepted(self, base): + """A nullability-only difference is accepted (pinned, ITL-842).""" # Pins today's behaviour: the append check ignores nullability (ITL-884, # decision D3). Whether a nullability change should be refused belongs to # the nullability audit, ITL-842. The read-back assertion records what diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index c1eaa2b86..6a3c52c8d 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -35,6 +35,7 @@ @function_pod(output_keys="y") def double(x: int) -> int: + """Record the call, damage any queued table, and return twice ``x``.""" _CALLS.append(x) while _DAMAGE_ON_NEXT_CALL: require_unknown_reader_feature(_DAMAGE_ON_NEXT_CALL.pop()) @@ -43,6 +44,7 @@ def double(x: int) -> int: @function_pod(output_keys="z") def triple(x: int) -> int: + """Return three times ``x``.""" return 3 * x @@ -54,6 +56,7 @@ def triple(x: int) -> int: @pytest.fixture(autouse=True) def _reset_calls(): + """Start and end every test with no recorded calls and no queued damage.""" _CALLS.clear() _DAMAGE_ON_NEXT_CALL.clear() yield @@ -63,10 +66,12 @@ def _reset_calls(): @pytest.fixture(params=list(ORCHESTRATORS)) def make_orchestrator(request) -> Callable[[], object]: + """Return a factory for the parametrized orchestrator.""" return ORCHESTRATORS[request.param] def build_job(store_path: Path, keys: str = "abc") -> PipelineJob: + """Build a job that doubles ``x`` for one row per key in ``keys``.""" source = ArrowTableSource( pa.table({"k": list(keys), "x": list(range(1, len(keys) + 1))}), tag_columns=["k"], @@ -88,6 +93,7 @@ def table_dirs(store_path: Path) -> tuple[Path, Path]: def delete_data_files(table: Path) -> None: + """Delete a table's data files, leaving its log intact.""" for f in table.glob("*.parquet"): f.unlink() @@ -120,6 +126,7 @@ def stored(tmp_path: Path, make_orchestrator) -> Path: def test_unreadable_result_table_fails_the_run(stored, make_orchestrator): + """An unreadable result table fails the run before any function call.""" result_table, _ = table_dirs(stored) delete_data_files(result_table) @@ -134,6 +141,7 @@ def test_unreadable_result_table_fails_the_run(stored, make_orchestrator): def test_unreadable_result_table_fails_the_run_under_continue( stored, make_orchestrator ): + """The per-row lookup's read failure fails the run under ``continue``.""" # With the pipeline table gone, the job has no stored tag rows to join, so the # first read of the result table is the per-row cache lookup, which runs inside # the error_policy="continue" handler. @@ -150,6 +158,7 @@ def test_unreadable_result_table_fails_the_run_under_continue( def test_store_read_failure_in_a_row_flush_fails_the_run(stored, make_orchestrator): + """A read failure in the flush that stores a row's result fails the run.""" # Rows d and e are new. The result table breaks while d is computed, after # d's cache lookup succeeded, so the failure surfaces in the flush that stores # d's result, inside the error_policy="continue" per-row handler. Before the @@ -174,6 +183,7 @@ def test_store_read_failure_in_a_row_flush_fails_the_run(stored, make_orchestrat def test_absent_tables_are_computed(stored, make_orchestrator): + """Tables that do not exist are computed and stored, not reported unreadable.""" result_table, pipeline_table = table_dirs(stored) shutil.rmtree(result_table) shutil.rmtree(pipeline_table) @@ -192,16 +202,19 @@ class _FailsToStartNode(NoOpObserver): """An observer whose ``on_node_start`` raises for one node label.""" def __init__(self, label: str) -> None: + """Fail ``on_node_start`` for the node labelled ``label``.""" self.label = label def on_node_start( self, node_label: str, node_hash: str, tag_schema: object = None ) -> None: + """Raise ``ValueError`` when the node labelled ``self.label`` starts.""" if node_label == self.label: raise ValueError(f"observer failed for {node_label}") def test_unrelated_concurrent_failure_does_not_hide_store_read_error(tmp_path): + """An unrelated concurrent failure does not hide the ``StoreReadError``.""" # Node "dbl" raises StoreReadError while loading its stored results and node # "tri" fails in the observer's on_node_start. Both fail before their first # await, so both land in the async TaskGroup's exception group. Before the fix, From d32c84d114eabbcea1ef84004aac6f2bbef644d9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:49:09 -0800 Subject: [PATCH 13/30] style(errors): unquote a local annotation ruff UP037 flags (ITL-884) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index d27e850fc..393870433 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -240,7 +240,7 @@ def in_group(cls, group: BaseExceptionGroup) -> "list[StoreReadError]": Returns: The ``StoreReadError`` leaves in the order the group holds them. """ - found: "list[StoreReadError]" = [] + found: list[StoreReadError] = [] for exc in group.exceptions: if isinstance(exc, BaseExceptionGroup): found.extend(cls.in_group(exc)) From 8f9adf8bf7d8ee4213973c0979fe123644d07a74 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 23:01:13 -0800 Subject: [PATCH 14/30] fix(databases): retry a failed Delta stored-id refresh on the next read (ITL-864) A failed refresh marked an empty stored-id cache clean while the loaded table stayed cached, so later reads served pending rows whose ids were already stored next to the stored rows. Keep the cache dirty on failure. In get_records_by_ids the refresh now runs inside the stored read's error handler, so both failures behave the same. Addresses CodeRabbit comments 4534785488 and 4534785497 on PR #293. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 5 +- src/orcapod/databases/delta_lake_databases.py | 36 +++++++------- ...1-10-pr-293-coderabbit-id-cache-refresh.md | 45 ++++++++++++++++++ .../test_delta_read_your_writes.py | 47 +++++++++++++++++++ 4 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index ee1f0d0c8..34473dd89 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2236,7 +2236,10 @@ stored rows plus the pending rows whose record id is not stored (`_pending_not_s `None` when there are none. These dicts are touched only from the thread that drives the pipeline; the async orchestrator runs only user functions in worker threads. Regression tests: `tests/test_databases/test_delta_read_your_writes.py` and -`tests/test_pipeline/test_delta_node_records_after_run.py`. +`tests/test_pipeline/test_delta_node_records_after_run.py`. A failed stored-id refresh leaves +the cache dirty, so the next read retries it instead of treating every stored id as unstored +and returning duplicates; in `get_records_by_ids` the refresh shares the stored read's error +handling. Still open: a handle does not see writes made by another process, or by a separate `DeltaTableDatabase` root on the same path, until it reloads the table; this is unchanged. diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index a50853084..6ce9d1199 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -354,8 +354,10 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: except Exception as e: logger.error(f"Failed to refresh existing IDs cache: {e}") + # Stay dirty: the loaded table is cached, so a clean empty cache would + # never be refreshed and stored ids would read as unstored. self._existing_ids_cache[record_key] = set() - self._cache_dirty[record_key] = False + self._cache_dirty[record_key] = True raise def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[bytes]: @@ -863,30 +865,30 @@ def get_records_by_ids( if len(record_ids) == 0: return None - # check inside the batch - pending = self._pending_not_stored(record_path) - if pending is not None: - id_type = pending.schema.field(self.RECORD_ID_COLUMN).type - pending = pending.filter( - pc.is_in( - pending[self.RECORD_ID_COLUMN], - value_set=pa.array(record_ids_list, type=id_type), - ) - ) - delta_table = self._get_delta_table(record_path) - if delta_table is None: - return self._combine_stored_and_pending(None, pending, record_id_column) try: - # Use schema-preserving read with filters - filter_expr = self._create_record_ids_filter(record_ids_list) - result = self._read_delta_table(delta_table, filters=filter_expr) + # Reading the stored ids (to drop pending rows a flush would not + # insert) is a stored read too, so it shares the read's handler. + pending = self._pending_not_stored(record_path) + result = None + if delta_table is not None: + # Use schema-preserving read with filters + filter_expr = self._create_record_ids_filter(record_ids_list) + result = self._read_delta_table(delta_table, filters=filter_expr) except Exception as e: logger.error( f"Error getting records by IDs from {'/'.join(record_path)}: {e}" ) return None + if pending is not None: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter( + pc.is_in( + pending[self.RECORD_ID_COLUMN], + value_set=pa.array(record_ids_list, type=id_type), + ) + ) return self._combine_stored_and_pending(result, pending, record_id_column) def _read_delta_table( diff --git a/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md b/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md new file mode 100644 index 000000000..ebdef0e62 --- /dev/null +++ b/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md @@ -0,0 +1,45 @@ +# PR #293 CodeRabbit follow-up: stored-ID cache refresh failures (ITL-864) + +Revision plan approved by the coordinator (task_355ce670f6bb), provided each finding +reproduces. Branch `eywalker/itl-864-deltatabledatabase-reading-through-the`. + +## Findings + +1. **Comment 4534785488 (Major), `delta_lake_databases.py:658`.** When + `_refresh_existing_ids_cache` fails after the Delta table loaded, it stores an empty ID + set and marks the cache **clean**. The loaded table is cached, so `_get_existing_ids` + never refreshes again: a pending row whose id is already stored is served next to the + stored row (a duplicate) by every read method. + *Fix:* the failure path marks the cache dirty, so the next read retries the refresh. + +2. **Comment 4534785497 (Minor), `delta_lake_databases.py:891`.** In + `get_records_by_ids`, `_pending_not_stored` refreshes the ID cache before the + method's stored-read `try`, so a refresh failure escapes while a stored-read failure is + logged and returns `None`. + *Fix:* run the refresh inside the same `try` as the stored read (one handler, no second + `return None`). After PR #290 (ITL-884) merges, that handler is gone and both failures + raise `StoreReadError` naming the table. + +## Steps (TDD) + +1. Regression tests in `tests/test_databases/test_delta_read_your_writes.py`, public + read methods only, with the backend's `DeltaTable.to_pyarrow_table` (used only by the + ID refresh) patched to fail once: + - stored `a=1`, flushed; pending `a=10`, `d=4`; one read fails in the refresh; every + read method then returns `a=1`, `d=4` and never `a=10`. + - stored `a=1`, flushed; pending `c=3`; `get_records_by_ids` with the refresh failing + follows the method's stored-read error behaviour (now `None`), and the retry returns + `[1, 3]`. +2. Run them, confirm both fail for the reasons above. +3. Apply the two fixes; rerun the file, then the full suite, ruff, basedpyright. +4. Commit, push, signed replies on both threads, wait for CI and CodeRabbit. + +## After PR #290 merges + +Merge `main`; in the read methods take #290's error handling (`StoreReadError`, no +`try`/`return None`) around this branch's merge helpers. #290's +`_refresh_existing_ids_cache` leaves `_cache_dirty` untouched on failure, which is still +clean when the refresh ran only because the table was not cached yet, so keep an explicit +`self._cache_dirty[record_key] = True` there. Change the test of finding 2 to expect +`StoreReadError` and extend it to every read method. Rerun the full suite (3.11, 3.12, +3.13), golden tests unchanged. diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index 632b73a08..8dbeee1a6 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -12,6 +12,7 @@ from typing import cast from pathlib import Path +import deltalake import pyarrow as pa import pytest @@ -182,3 +183,49 @@ def test_pending_row_with_stored_id_reads_as_flush_keeps_it( assert _values(db.get_records_by_ids(PATH, [b"a", b"d"])) == [1, 4], when assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1, 4], when db.flush() + + +@pytest.fixture +def fail_id_refresh_once(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the next stored-ID refresh fail once, as a transient storage error would. + + ``DeltaTable.to_pyarrow_table`` is the backend call the stored-ID refresh makes; the + read methods scan through ``to_pyarrow_dataset``, so only the refresh fails. + """ + original = deltalake.DeltaTable.to_pyarrow_table + calls = {"n": 0} + + def to_pyarrow_table(self: deltalake.DeltaTable) -> pa.Table: + calls["n"] += 1 + if calls["n"] == 1: + raise OSError("transient storage error") + return original(self) + + monkeypatch.setattr(deltalake.DeltaTable, "to_pyarrow_table", to_pyarrow_table) + + +@pytest.mark.usefixtures("fail_id_refresh_once") +@pytest.mark.parametrize("method", list(READS)) +def test_failed_id_refresh_is_retried_by_the_next_read( + tmp_path: Path, method: str +) -> None: + """After a failed stored-ID refresh, a pending row with a stored id is still not returned.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["b"], [2]), flush=True) + db.add_records(PATH, _rows(["a", "b"], [1, 20])) + + with pytest.raises(OSError, match="transient storage error"): + _ = db.get_all_records(PATH) + + assert _values(READS[method](db, PATH)) == EXPECTED[method] + + +@pytest.mark.usefixtures("fail_id_refresh_once") +def test_get_records_by_ids_handles_failed_id_refresh_like_a_failed_read( + stored_and_pending: DeltaTableDatabase, +) -> None: + """A failed stored-ID refresh follows the method's stored-read error behaviour.""" + db = stored_and_pending + + assert db.get_records_by_ids(PATH, [b"a", b"c"]) is None + assert _values(db.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] From b8bcc817f29bb2b314221bb62317c08a6ad16e0b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 23:41:42 -0800 Subject: [PATCH 15/30] fix(databases): Delta reads refresh missing stored ids and combine evolved schemas (ITL-864) A handle that loaded a stored table before reading its ids (a fresh root over an existing table) never refreshed them, so pending rows with stored ids were served next to the stored rows. Refresh when the id entry is missing as well as when it is dirty. Reads that combine stored and pending rows raised ArrowInvalid on any schema difference, including a pending string column next to the stored large_string. Large-normalize both parts (null kept) and concatenate with promote_options='default', so a read before flush returns what the read after flush does: missing columns null-filled, nullability and struct children merged as Delta's merge does. Differing types still raise. Addresses CodeRabbit review 5855237778 on PR #293. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 20 ++- src/orcapod/databases/delta_lake_databases.py | 42 ++++- ...1-10-pr-293-coderabbit-id-cache-refresh.md | 14 ++ .../test_delta_read_your_writes.py | 161 +++++++++++++++++- 4 files changed, 234 insertions(+), 3 deletions(-) diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 34473dd89..50d0f488b 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2239,7 +2239,25 @@ pipeline; the async orchestrator runs only user functions in worker threads. Reg `tests/test_pipeline/test_delta_node_records_after_run.py`. A failed stored-id refresh leaves the cache dirty, so the next read retries it instead of treating every stored id as unstored and returning duplicates; in `get_records_by_ids` the refresh shares the stored read's error -handling. +handling. A handle that loaded the table before it had any stored ids (a fresh root over an +existing table) refreshes them on its first read with pending rows. + +When stored and pending rows are read together, the result has the schema a flush would give: +both parts are large-normalized (`null` kept), then combined with +`pa.concat_tables(promote_options="default")`. A column missing on one side is null-filled, and a +`null`-typed column takes the other side's type. Columns whose types differ (`int64` vs `int32`, +a string vs an int, a timestamp unit, a decimal precision) raise `pyarrow.ArrowTypeError`; nothing +is widened. Before this, a read of stored rows next to pending rows with any schema difference, +including a pending `string` column next to the stored `large_string`, raised `ArrowInvalid`; +`get_all_records` already did so on `main`. + +Recorded for the nullability audit (ITL-842) and ITL-890: Delta's merge relaxes nullability (a +non-null pending column into a nullable stored one stays nullable) and evolves struct children +(`struct

` stored + `struct` pending becomes `struct`, stored rows `q = null`). The +read path now mirrors both. Whether Orcapod should allow either at all is for those issues. A +pending all-null `null`-typed column reads back as `null`, but Delta cannot store it: the flush +fails and the batch stays pending (unchanged, pinned by +`test_pending_null_typed_column_reads_back_but_does_not_flush`). Still open: a handle does not see writes made by another process, or by a separate `DeltaTableDatabase` root on the same path, until it reloads the table; this is unchanged. diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 6ce9d1199..131b67c31 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -363,9 +363,12 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[bytes]: """Get the set of existing IDs in the Delta table, using cache when possible.""" record_key = self._get_record_key(record_path) + # A table loaded by a read is cached before any ids are, so a missing + # id entry needs a refresh as much as a dirty one does. if ( self._cache_dirty.get(record_key) or record_key not in self._delta_table_cache + or record_key not in self._existing_ids_cache ): self._refresh_existing_ids_cache(record_path) return self._existing_ids_cache.get(record_key) or set() @@ -675,6 +678,14 @@ def _combine_stored_and_pending( ) -> pa.Table | None: """Combine stored rows with pending rows into one read result. + Combined rows have the schema a flush would give them: both parts take + their large layouts (a stored table reads back large), a column missing + on one side is null-filled, a ``null``-typed column takes the other + side's type, nullability is merged (nullable if either side is), and + struct children are merged, as Delta's merge evolves the stored schema. + A column whose types still differ raises ``pyarrow.ArrowTypeError``; + nothing is widened. A single part is returned as it is. + Args: stored: Rows read from the Delta table, or ``None``. pending: Pending rows that are not stored (see ``_pending_not_stored``), @@ -688,9 +699,38 @@ def _combine_stored_and_pending( parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] if not parts: return None - table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) + if len(parts) == 1: + table = parts[0] + else: + table = pa.concat_tables( + [self._large_layout(t) for t in parts], promote_options="default" + ) return self._handle_record_id_column(table, record_id_column) + @staticmethod + def _large_layout(table: pa.Table) -> pa.Table: + """Return ``table`` with every column in its large layout. + + Only layouts change (``string`` to ``large_string``, ``list`` to + ``large_list``, recursively); ``null`` stays ``null``, and names, + nullability and metadata are kept. + + Args: + table: The table to normalize. + + Returns: + The table with large-normalized column types. + """ + schema = table.schema + for index, field in enumerate(table.schema): + schema = schema.set( + index, + field.with_type( + arrow_utils.normalize_to_large_types(field.type, keep_null=True) + ), + ) + return table if schema.equals(table.schema) else table.cast(schema) + def get_all_records( self, record_path: tuple[str, ...], diff --git a/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md b/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md index ebdef0e62..6aea99fca 100644 --- a/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md +++ b/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md @@ -43,3 +43,17 @@ clean when the refresh ran only because the table was not cached yet, so keep an `self._cache_dirty[record_key] = True` there. Change the test of finding 2 to expect `StoreReadError` and extend it to every read method. Rerun the full suite (3.11, 3.12, 3.13), golden tests unchanged. + +## Round 2 (CodeRabbit review of 8f9adf8b; coordinator approved both behaviour fixes) + +- **Outside-diff (Major), `_get_existing_ids`:** a handle that loaded the table before any + stored ids were read (a fresh root over an existing table) never refreshed them, so pending + duplicates were served. Refresh also when the record key has no id entry. Test: + `test_fresh_handle_drops_pending_rows_with_stored_ids` (four read methods). +- **4535100450 (Major), `_combine_stored_and_pending`:** stored and pending rows with any + schema difference raised `ArrowInvalid`. Large-normalize both parts (`keep_null=True`), + then `pa.concat_tables(promote_options="default")`. Contract tests assert the read before + flush equals the read after flush (missing column either way, nullability merge, struct + child added, `string`/`list` layouts); type differences keep raising `ArrowTypeError`; a + `null`-typed new column is pinned (reads back, flush fails). +- **4535100459 (nit):** Google docstrings on the five tests and the wrapper. diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index 8dbeee1a6..3185ca82c 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -9,7 +9,7 @@ from __future__ import annotations from collections.abc import Callable -from typing import cast +from typing import Any, cast from pathlib import Path import deltalake @@ -101,12 +101,14 @@ def stored_and_pending(tmp_path: Path) -> DeltaTableDatabase: def test_get_all_records_returns_stored_and_pending( stored_and_pending: DeltaTableDatabase, ) -> None: + """``get_all_records`` returns the stored rows and the pending row together.""" assert _values(stored_and_pending.get_all_records(PATH)) == [1, 2, 3] def test_get_record_by_id_finds_stored_and_pending( stored_and_pending: DeltaTableDatabase, ) -> None: + """``get_record_by_id`` finds a stored row and a pending row alike.""" assert _values(stored_and_pending.get_record_by_id(PATH, b"a")) == [1] assert _values(stored_and_pending.get_record_by_id(PATH, b"c")) == [3] @@ -114,12 +116,14 @@ def test_get_record_by_id_finds_stored_and_pending( def test_get_records_by_ids_returns_stored_and_pending( stored_and_pending: DeltaTableDatabase, ) -> None: + """``get_records_by_ids`` returns the requested stored and pending rows together.""" assert _values(stored_and_pending.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] def test_get_records_with_column_value_returns_stored_and_pending( stored_and_pending: DeltaTableDatabase, ) -> None: + """``get_records_with_column_value`` matches stored and pending rows alike.""" db = stored_and_pending assert _values(db.get_records_with_column_value(PATH, {"g": 1})) == [2, 3] assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1] @@ -128,6 +132,7 @@ def test_get_records_with_column_value_returns_stored_and_pending( def test_reads_with_no_match_return_none( stored_and_pending: DeltaTableDatabase, ) -> None: + """A read that matches no stored or pending row returns ``None``.""" db = stored_and_pending assert db.get_records_with_column_value(PATH, {"g": 9}) is None assert db.get_records_by_ids(PATH, [b"zz"]) is None @@ -196,6 +201,7 @@ def fail_id_refresh_once(monkeypatch: pytest.MonkeyPatch) -> None: calls = {"n": 0} def to_pyarrow_table(self: deltalake.DeltaTable) -> pa.Table: + """Raise on the first call, then read the table as the backend does.""" calls["n"] += 1 if calls["n"] == 1: raise OSError("transient storage error") @@ -229,3 +235,156 @@ def test_get_records_by_ids_handles_failed_id_refresh_like_a_failed_read( assert db.get_records_by_ids(PATH, [b"a", b"c"]) is None assert _values(db.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] + + +@pytest.mark.parametrize("method", list(READS)) +def test_fresh_handle_drops_pending_rows_with_stored_ids( + tmp_path: Path, method: str +) -> None: + """A handle that loaded a table written by another root still drops pending rows with stored ids.""" + DeltaTableDatabase(tmp_path).add_records(PATH, _rows(["b"], [2]), flush=True) + db = DeltaTableDatabase(tmp_path) + assert _values(db.get_all_records(PATH)) == [2] # loads the table, no ids yet + + db.add_records(PATH, _rows(["a", "b"], [1, 20])) + + assert _values(READS[method](db, PATH)) == EXPECTED[method] + + +def _table(record_id: str, x: pa.Array[Any], x_nullable: bool = True) -> pa.Table: + """Build one row keyed by ``record_id`` with ``v=0`` and the column ``x``.""" + schema = pa.schema( + [ + pa.field("__record_id", pa.large_binary()), + pa.field("v", pa.int64()), + pa.field("x", x.type, nullable=x_nullable), + ] + ) + return pa.table( + [ + pa.array([record_id.encode()], type=pa.large_binary()), + pa.array([0], type=pa.int64()), + x, + ], + schema=schema, + ) + + +def _without_x(table: pa.Table) -> pa.Table: + """Return ``table`` without its ``x`` column.""" + return table.drop_columns(["x"]) + + +# (stored row, pending row) pairs whose schemas differ in a way a flush accepts. +SCHEMA_CHANGES: dict[str, tuple[pa.Table, pa.Table]] = { + "pending_adds_column": ( + _without_x(_table("a", pa.array([1], pa.int64()))), + _table("c", pa.array([3], pa.int64())), + ), + "pending_lacks_column": ( + _table("a", pa.array([1], pa.int64())), + _without_x(_table("c", pa.array([3], pa.int64()))), + ), + "non_null_pending_into_nullable": ( + _table("a", pa.array([1], pa.int64())), + _table("c", pa.array([3], pa.int64()), x_nullable=False), + ), + "struct_child_added": ( + _table("a", pa.array([{"p": 1}])), + _table("c", pa.array([{"p": 3, "q": 4}])), + ), + "non_large_string": ( + _table("a", pa.array(["s"], pa.string())), + _table("c", pa.array(["t"], pa.string())), + ), + "non_large_list": ( + _table("a", pa.array([[1]], pa.list_(pa.int64()))), + _table("c", pa.array([[3]], pa.list_(pa.int64()))), + ), +} + +ReadAc = Callable[[DeltaTableDatabase], "pa.Table | None"] + +# Each read method that can return rows ``a`` and ``c`` together, with the record id kept. +READS_AC: dict[str, ReadAc] = { + "get_all_records": lambda db: db.get_all_records(PATH, record_id_column="rid"), + "get_records_by_ids": lambda db: db.get_records_by_ids( + PATH, [b"a", b"c"], record_id_column="rid" + ), + "get_records_with_column_value": lambda db: db.get_records_with_column_value( + PATH, {"v": 0}, record_id_column="rid" + ), +} + + +def _read_sorted(db: DeltaTableDatabase, method: str) -> pa.Table: + """Read rows ``a`` and ``c`` with ``method`` and sort them by record id.""" + table = READS_AC[method](db) + assert table is not None + return table.sort_by("rid") + + +@pytest.mark.parametrize("method", list(READS_AC)) +@pytest.mark.parametrize("change", list(SCHEMA_CHANGES)) +def test_read_before_flush_matches_read_after_flush_across_schema_change( + tmp_path: Path, change: str, method: str +) -> None: + """Stored and pending rows whose schemas differ read back as the flushed table does.""" + stored, pending = SCHEMA_CHANGES[change] + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, stored, flush=True) + db.add_records(PATH, pending) + + before = _read_sorted(db, method) + db.flush() + after = _read_sorted(db, method) + + assert before.num_rows == 2 + assert before.schema == after.schema + assert before.to_pylist() == after.to_pylist() + + +@pytest.mark.parametrize( + ("stored_x", "pending_x"), + [ + (pa.array([1], pa.int64()), pa.array([3], pa.int32())), + (pa.array([1], pa.int64()), pa.array(["t"], pa.large_string())), + (pa.array([1], pa.timestamp("us")), pa.array([3], pa.timestamp("ms"))), + ( + pa.array([1], pa.decimal128(10, 2)), + pa.array([3], pa.decimal128(12, 2)), + ), + ], + ids=["int64_vs_int32", "int64_vs_string", "timestamp_unit", "decimal_precision"], +) +@pytest.mark.parametrize("method", list(READS_AC)) +def test_read_of_stored_and_pending_rows_with_different_types_raises( + tmp_path: Path, method: str, stored_x: pa.Array[Any], pending_x: pa.Array[Any] +) -> None: + """A column whose stored and pending types differ is never widened: the read raises.""" + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, _table("a", stored_x), flush=True) + db.add_records(PATH, _table("c", pending_x)) + + with pytest.raises(pa.ArrowTypeError, match="Field x has incompatible types"): + _ = READS_AC[method](db) + + +def test_pending_null_typed_column_reads_back_but_does_not_flush( + tmp_path: Path, +) -> None: + """A new all-null ``null``-typed column reads back as ``null``; Delta cannot store it. + + The flush fails and the batch stays pending, so the read is unchanged after it. + """ + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, _without_x(_table("a", pa.array([1]))), flush=True) + db.add_records(PATH, _table("c", pa.nulls(1))) + + before = _read_sorted(db, "get_all_records") + assert before.schema.field("x").type == pa.null() + assert before.column("x").to_pylist() == [None, None] + + with pytest.raises(RuntimeError, match="Failed to flush"): + db.flush() + assert _read_sorted(db, "get_all_records").equals(before) From 65c2bd909b959d0ef193c847bad5df780b06a1fb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:04:08 -0800 Subject: [PATCH 16/30] docs(plans): resume notes after merging main (ITL-884) Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-10-itl-884-delta-read-failures.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md index 71ef7fa94..d084ae08b 100644 --- a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md +++ b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md @@ -187,3 +187,12 @@ Run the Task 2 tests and expect PASS. `uv run --with basedpyright basedpyright` (exit code 0) and `uv run pytest tests/`. - Commit in Conventional Commits style: `test(databases): ...` for the failing tests first, then `fix(databases): ...`, then `docs: ...`. + +## Resume notes (2026-11-11, after machine restart) + +- Arrived to a clean worktree at d32c84d1, in sync with origin; PR #290 had 0 open threads + but conflicted with main after #291 (ITL-847). +- Merged origin/main (b197544e) as b4387c00: only DESIGN_ISSUES.md conflicted; kept SJ1 (main) + then DR1 (this PR) at the end, additions-only against both sides. CHANGELOG auto-merged + (#291 did not touch it). Pushed. +- Next: full suite on 3.11/3.12/3.13, CI green, one CodeRabbit request, worker_done. From ff90d33890437dccb77f134a22f42633533e5d44 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:07:38 -0800 Subject: [PATCH 17/30] fix(databases): Delta column-value query skips the side that lacks the column (ITL-864) A query on a column only the stored or only the pending rows have raised 'No match for FieldRef' before flush but returned the matching rows after flush. The side without the column now contributes no rows (after a flush it holds null there, which never matches); a column neither side has still raises. Also type pending record ids as bytes in _add_to_pending_batch. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 22 ++++++-- ...26-11-10-itl-864-delta-read-your-writes.md | 16 ++++++ .../test_delta_read_your_writes.py | 53 +++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 131b67c31..1e1ec8557 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -632,7 +632,7 @@ def _add_to_pending_batch(self, record_path: tuple[str, ...], table: pa.Table): else: self._pending_batches[record_key] = pa.concat_tables([pending_batch, table]) - pending_ids = cast(list[str], table[self.RECORD_ID_COLUMN].to_pylist()) + pending_ids = cast(list[bytes], table[self.RECORD_ID_COLUMN].to_pylist()) self._pending_record_ids[record_key].update(pending_ids) def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool: @@ -788,11 +788,27 @@ def get_records_with_column_value( combined_expression = combined_expression & next_expression pending = self._pending_not_stored(record_path) + delta_table = self._get_delta_table(record_path) + + # A side without a queried column holds null there once flushed, and + # ``column == value`` never matches null, so it contributes no rows. A + # column neither side has is unknown: the filter raises as before. + queried = {c for c, _ in pair_list} + stored_columns = ( + set() + if delta_table is None + else {field.name for field in delta_table.schema().fields} + ) + pending_columns = set() if pending is None else set(pending.column_names) + if queried <= stored_columns | pending_columns: + if not queried <= pending_columns: + pending = None + if not queried <= stored_columns: + delta_table = None + if pending is not None: pending = pending.filter(combined_expression) - # Now check the Delta table - delta_table = self._get_delta_table(record_path) if delta_table is None: return self._combine_stored_and_pending(None, pending, record_id_column) diff --git a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md index 993c7eaa2..546c5e252 100644 --- a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md +++ b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md @@ -156,3 +156,19 @@ def _combine_stored_and_pending(self, stored, pending, record_id_column): - [ ] Append one `DESIGN_ISSUES.md` entry (resolved, with **Fix:**; note that writes from other processes are still not seen until a handle reloads the table — follow-up for the coordinator to file). - [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (golden hashes unchanged). - [ ] Merge `origin/main`, rerun if it moved, push, PR "Fixes ITL-864", `orca linear attach`, `orca worktree set --pr`. + +--- + +## Review rounds and continuation notes (successor-readable) + +Done and pushed (verify, do not redo): +- Round 1 (e9c78b5c): a failed ID refresh keeps the cache dirty; `get_records_by_ids` refresh shares the stored-read handler (8f9adf8b). +- Round 2 (8f9adf8b): refresh when the record key has no ID-cache entry; stored + pending combined with `normalize_table_to_large_types` + `pa.concat_tables(promote_options="default")` (b8bcc817). Tests: `test_fresh_handle_drops_pending_rows_with_stored_ids`, evolved-schema tests. + +Round 3 (CodeRabbit review of b8bcc817, outside-diff, `get_records_with_column_value`), approved fix: +- A side (stored schema, read without loading data; pending column names) that lacks a queried column contributes no rows; after a flush its rows hold null there and `col == value` never matches null. A column on neither side keeps today's `ArrowInvalid` ("No match for FieldRef"). +- Tests: `test_query_on_column_one_side_lacks_matches_read_after_flush[column_only_pending|column_only_stored]`, `test_query_on_column_neither_side_has_raises[flushed]`. +- Also the round-1 outside-diff trivial: `_add_to_pending_batch` cast `list[str]` -> `list[bytes]`. +- Reply: signed top-level PR comment (no thread for outside-diff findings), and one for the round-1 trivial. + +Then: merge origin/main (b197544e); wait for PR #290 (ITL-884, StoreReadError) to merge; merge main again, take #290's error handling around `_pending_not_stored` / `_combine_stored_and_pending`; a failed ID refresh must raise `StoreReadError` on every read method; full suite; worker_done only after the post-#290 merge is green. diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index 3185ca82c..0549cb639 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -388,3 +388,56 @@ def test_pending_null_typed_column_reads_back_but_does_not_flush( with pytest.raises(RuntimeError, match="Failed to flush"): db.flush() assert _read_sorted(db, "get_all_records").equals(before) + + +@pytest.mark.parametrize( + ("stored", "pending", "query", "expected_ids"), + [ + ( + _without_x(_table("a", pa.array([1], pa.int64()))), + _table("c", pa.array([3], pa.int64())), + {"x": 3}, + [b"c"], + ), + ( + _table("a", pa.array([1], pa.int64())), + _without_x(_table("c", pa.array([3], pa.int64()))), + {"x": 1}, + [b"a"], + ), + ], + ids=["column_only_pending", "column_only_stored"], +) +def test_query_on_column_one_side_lacks_matches_read_after_flush( + tmp_path: Path, + stored: pa.Table, + pending: pa.Table, + query: dict[str, int], + expected_ids: list[bytes], +) -> None: + """A query on a column only one side has returns before flush what it returns after.""" + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, stored, flush=True) + db.add_records(PATH, pending) + + before = db.get_records_with_column_value(PATH, query, record_id_column="rid") + db.flush() + after = db.get_records_with_column_value(PATH, query, record_id_column="rid") + + assert before is not None and after is not None + assert before.column("rid").to_pylist() == expected_ids + assert before.schema == after.schema + assert before.to_pylist() == after.to_pylist() + + +@pytest.mark.parametrize("flushed", [False, True]) +def test_query_on_column_neither_side_has_raises(tmp_path: Path, flushed: bool) -> None: + """A query on a column neither the stored nor the pending rows have raises, flushed or not.""" + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, _rows(["a"], [1]), flush=True) + db.add_records(PATH, _rows(["c"], [3])) + if flushed: + db.flush() + + with pytest.raises(pa.ArrowInvalid, match="No match for FieldRef"): + _ = db.get_records_with_column_value(PATH, {"y": 1}) From f6a3c3adc1de1780c112f103021dde748300859c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:24:06 -0800 Subject: [PATCH 18/30] docs(errors): StoreReadError no longer claims nothing was recomputed (ITL-884) A flush can fail after the function already ran for a row, so the message, docstring and docs note now say only that the stored rows are not treated as missing and the table is not overwritten. The plan's test and orchestrator notes now match what was implemented. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 5 +++-- src/orcapod/errors.py | 8 ++++---- .../2026-11-10-itl-884-delta-read-failures.md | 20 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2d8bdd384..becb5a6f0 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -222,8 +222,9 @@ print("function calls:", len(calls)) A table that does not exist yet reads as empty, and its rows are computed. A table that exists but cannot be read (for example, a data file was deleted or damaged) is never treated as empty: the run stops with `orcapod.errors.StoreReadError`, which names the - table and the underlying error, whatever the orchestrator's `error_policy`. Nothing is - recomputed or overwritten, so you can repair the table or restore the file and run again. + table and the underlying error, whatever the orchestrator's `error_policy`. Its stored rows + are not recomputed and the table is not overwritten, so you can repair the table or restore + the file and run again. In a `DeltaTableDatabase`, rows whose column types differ from the stored table's (`float64` values for a column stored as `int64`, say) are refused with diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 393870433..25e96ddd6 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -203,8 +203,8 @@ class StoreReadError(RuntimeError): A table that does not exist reads as empty. A table that exists but cannot be read (a missing or damaged data file, a log the backend cannot load, columns whose stored data no longer matches the table schema) raises this error - instead, so stored results never look dropped. Nothing is recomputed or - overwritten. A failed store read always fails the run, whatever the + instead, so stored results never look dropped. Its rows are not recomputed + and the table is not overwritten. A failed store read always fails the run, whatever the orchestrator's ``error_policy``. The backend's own exception is chained as ``__cause__``. @@ -225,8 +225,8 @@ def __str__(self) -> str: reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" return ( f"Could not read the stored table at {self.table!r}{reason}. The table " - "exists, so its rows are not treated as missing: nothing was recomputed " - "or overwritten." + "exists, so its rows are not treated as missing and the table was not " + "overwritten." ) @classmethod diff --git a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md index d084ae08b..8eca242f5 100644 --- a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md +++ b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md @@ -39,8 +39,9 @@ No hash, path or stored-schema changes. - `src/orcapod/core/nodes/function_node.py` (`execute` per-row handler) and `src/orcapod/core/function_pod.py` (async `process_one`) both get `except StoreReadError: ...; raise`. -- `src/orcapod/pipeline/async_orchestrator.py` gets `except* StoreReadError` and raises the - first leaf. +- `src/orcapod/pipeline/async_orchestrator.py` catches the `TaskGroup`'s `ExceptionGroup` + with a plain `except`, selects one `StoreReadError` with `_single_store_error`, and raises + it on its own, noting the other failures on it. - Tests: `tests/test_databases/test_delta_read_failures.py` and `tests/test_pipeline/test_store_read_failure.py`. - Docs: `docs/concepts/function-pods.md` gets an admonition, `CHANGELOG.md` gets Fixed and @@ -69,8 +70,10 @@ No hash, path or stored-schema changes. 1.0.2 does not raise naturally from a local table). Flush raises and the stored files are untouched. The comment explains the monkeypatch. - Append refusal: - - `test_castable_type_change_is_refused`: int64 stored, float64 appended. Raises (flush - wraps it in `RuntimeError`, so assert that `InputValidationError` is the `__cause__`). + - `test_castable_type_change_is_refused` (implemented as the `int64->float64` case of + `test_plain_type_change`): int64 stored, float64 appended. `flush` raises + `InputValidationError` itself (Task 2 of `2026-11-10-itl-884-coderabbit-flush-and-async-group.md`); assert it + directly. The message names the column `x`, `int64` and `double`. A re-read gives the original row only, with type int64. - `test_narrowing_change_is_refused`: int64 stored, int32 appended. @@ -123,7 +126,7 @@ class StoreReadError(RuntimeError): reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" return ( f"Could not read the stored table at {self.table!r}{reason}. The table exists, " - "so its rows are not treated as missing: nothing was recomputed or overwritten." + "so its rows are not treated as missing and the table was not overwritten." ) @classmethod @@ -169,8 +172,11 @@ Run the Task 1 and Task 2 tests and expect PASS. then `raise`. - `FunctionPod` async `process_one`: put `except StoreReadError as exc:` before `except Exception`. It calls `obs.on_data_crash(...)` and then `raise`. -- `AsyncPipelineOrchestrator._run_async`: add `except* StoreReadError as group: raise - StoreReadError.in_group(group)[0]`. +- `AsyncPipelineOrchestrator._run_async`: add `except ExceptionGroup as group:` and pick + the error with `_single_store_error(group, topo_order)` (the `StoreReadError` of the + earliest node in topological order, the other failures noted on it); raise it on its own, + or re-raise the group unchanged when it holds no `StoreReadError`. A leaf raised inside + `except*` would be wrapped in a new group whenever another task failed concurrently. Run the Task 2 tests and expect PASS. From 0369cfcc0d76ed5babf956f45afccfb4c5e1c94d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 02:49:39 -0800 Subject: [PATCH 19/30] refactor(databases): callers wrap store errors; no message-only parameters (ITL-884) Per the developer's review of PR #290, no helper receives a value it uses only to build an error message; the caller that owns the context wraps the error. - _read_delta_table no longer takes record_path. A failed scan raises the module-private _DeltaTableReadError (backend error as __cause__); its three callers raise StoreReadError with the path. - _require_compatible_append(table_uri, ...) becomes _incompatible_append_column(...), which returns the mismatching column; flush_batch raises the InputValidationError naming the table. - _store_error_to_raise only selects the error; flush() adds the notes naming the other failed batches. - _single_store_error drops topo_order, which ranked by a node_label that StoreReadError does not have; it picks the first StoreReadError in the group, as before. Messages and behaviour are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 109 ++++++++++-------- src/orcapod/pipeline/async_orchestrator.py | 21 +--- ...tl-884-coderabbit-flush-and-async-group.md | 6 + .../2026-11-10-itl-884-delta-read-failures.md | 6 + ...1-11-itl-884-no-message-only-parameters.md | 46 ++++++++ .../test_delta_table_database.py | 2 +- 6 files changed, 127 insertions(+), 63 deletions(-) create mode 100644 superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index afb121aa5..e6483d225 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -71,36 +71,34 @@ def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: return arrow_type -def _store_error_to_raise(errors: list[tuple[str, Exception]]) -> Exception | None: +def _store_error_to_raise(errors: list[Exception]) -> Exception | None: """Pick the store error a failed flush raises as itself. A ``StoreReadError`` is preferred, because it is the error that fails a run whatever the orchestrator's ``error_policy``; failing that, an - ``InputValidationError`` from a refused append. Every other failed batch is - added to the chosen error as a note, so no failure is hidden. + ``InputValidationError`` from a refused append. Args: - errors: The ``(record_key, error)`` pair of every batch that failed, in - flush order. + errors: The error of every batch that failed, in flush order. Returns: - The chosen error with its notes added, or ``None`` when no batch failed - with a store error. + The first ``StoreReadError``, else the first ``InputValidationError``, or + ``None`` when no batch failed with a store error. """ - chosen: Exception | None = None for kind in (StoreReadError, InputValidationError): - chosen = next((e for _, e in errors if isinstance(e, kind)), None) + chosen = next((e for e in errors if isinstance(e, kind)), None) if chosen is not None: - break - if chosen is None: - return None - for record_key, other in errors: - if other is not chosen: - chosen.add_note( - f"Another batch also failed to flush ({record_key}): " - f"{type(other).__name__}: {other}" - ) - return chosen + return chosen + return None + + +class _DeltaTableReadError(Exception): + """A Delta table's data could not be read; the backend error is the ``__cause__``. + + Raised by ``DeltaTableDatabase._read_delta_table`` so that its callers, which + know the table's path, can tell a failed read from a bad filter and raise + ``StoreReadError``. + """ class DeltaTableDatabase: @@ -790,9 +788,10 @@ def get_records_with_column_value( return None # Use schema-preserving read - result = self._read_delta_table( - delta_table, record_path, expression=combined_expression - ) + try: + result = self._read_delta_table(delta_table, expression=combined_expression) + except _DeltaTableReadError as e: + raise self._read_failed(record_path) from e.__cause__ if len(result) == 0: return None @@ -844,7 +843,10 @@ def get_record_by_id( # Use schema-preserving read filter_expr = self._create_record_id_filter(record_id) - result = self._read_delta_table(delta_table, record_path, filters=filter_expr) + try: + result = self._read_delta_table(delta_table, filters=filter_expr) + except _DeltaTableReadError as e: + raise self._read_failed(record_path) from e.__cause__ if len(result) == 0: return None @@ -894,7 +896,10 @@ def get_records_by_ids( return None # Use schema-preserving read with filters filter_expr = self._create_record_ids_filter(record_ids_list) - result = self._read_delta_table(delta_table, record_path, filters=filter_expr) + try: + result = self._read_delta_table(delta_table, filters=filter_expr) + except _DeltaTableReadError as e: + raise self._read_failed(record_path) from e.__cause__ if len(result) == 0: return None @@ -905,7 +910,6 @@ def get_records_by_ids( def _read_delta_table( self, delta_table: deltalake.DeltaTable, - record_path: tuple[str, ...], filters: list | None = None, expression: pc.Expression | None = None, ) -> pa.Table: @@ -914,7 +918,6 @@ def _read_delta_table( Args: delta_table: The Delta table to read from - record_path: Path of the table relative to this view, for errors filters: Optional filters to apply expression: Optional filter expression, used instead of ``filters`` @@ -922,15 +925,16 @@ def _read_delta_table( Arrow table with preserved schema Raises: - StoreReadError: If the table's data could not be read. An error in - the filter itself (an unknown column, say) is raised as is. + _DeltaTableReadError: If the table's data could not be read, with the + backend's error as its ``__cause__``. An error in the filter itself + (an unknown column, say) is raised as is. """ filter_expr = None # Use to_pyarrow_dataset with as_large_types for Polars compatible arrow table loading try: dataset = delta_table.to_pyarrow_dataset(as_large_types=True) except Exception as e: - raise self._read_failed(record_path) from e + raise _DeltaTableReadError from e if filters and expression is None: for filt in filters: if len(filt) == 3: @@ -959,7 +963,7 @@ def _read_delta_table( try: scanned = dataset.to_table() except Exception as e: - raise self._read_failed(record_path) from e + raise _DeltaTableReadError from e table = arrow_utils.normalize_table_view_types(scanned) if filter_expr is not None: return table.filter(filter_expr) @@ -1032,8 +1036,15 @@ def flush(self) -> None: logger.error(f"Error flushing batch for {record_key}: {e}") errors.append((record_key, e)) if errors: - store_error = _store_error_to_raise(errors) + store_error = _store_error_to_raise([e for _, e in errors]) if store_error is not None: + # Keep every other failure visible on the error that is raised. + for record_key, other in errors: + if other is not store_error: + store_error.add_note( + f"Another batch also failed to flush ({record_key}): " + f"{type(other).__name__}: {other}" + ) raise store_error failed_keys = [k for k, _ in errors] raise RuntimeError( @@ -1119,7 +1130,16 @@ def _flush_by_key(self, record_key: str) -> None: f"Created new Delta table for {record_key} with {len(combined_table)} records" ) else: - self._require_compatible_append(table_uri, delta_table, combined_table) + mismatch = self._incompatible_append_column(delta_table, combined_table) + if mismatch is not None: + column, stored_type, incoming_type = mismatch + raise InputValidationError( + f"Cannot append to the stored table {table_uri!r}: column " + f"{column!r} is stored as {stored_type} but the new rows have " + f"{incoming_type} (input type " + f"{combined_table.schema.field(column).type}). " + "The stored rows are unchanged and nothing was written." + ) delta_table.merge( source=combined_table, predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}", @@ -1150,28 +1170,27 @@ def _flush_by_key(self, record_key: str) -> None: raise @staticmethod - def _require_compatible_append( - table_uri: str, delta_table: deltalake.DeltaTable, batch: pa.Table - ) -> None: - """Refuse a batch whose column types differ from the stored table's. + def _incompatible_append_column( + delta_table: deltalake.DeltaTable, batch: pa.Table + ) -> tuple[str, pa.DataType, pa.DataType] | None: + """Find a column whose type in ``batch`` differs from the stored table's. Delta's merge either casts such a batch to the stored type (``int32`` or ``float64`` rows into an ``int64`` column) or, when an extension type is involved, writes data files the table can no longer read. Both sides are compared through Delta's own type mapping, with extension types reduced to storage, large-normalized and nullability ignored, so a type Delta stores - differently (``uint64`` as ``int64``) is not refused. Columns present on + differently (``uint64`` as ``int64``) is not reported. Columns present on only one side are allowed (schema evolution). The stored schema comes from the loaded log; no data is read. Args: - table_uri: Location of the stored table, for the error message. delta_table: The loaded stored table. batch: The rows about to be merged into it. - Raises: - InputValidationError: If a column's type in ``batch`` differs from its - stored type. Nothing is written. + Returns: + ``(column, stored_type, incoming_type)`` for the first column whose + compared types differ, or ``None`` when the batch can be appended. """ stored = pa.schema(delta_table.schema().to_arrow()) # pyright: ignore[reportArgumentType] # arro3 schema; pyarrow reads it through the C data interface incoming = pa.schema( @@ -1197,12 +1216,8 @@ def _require_compatible_append( arrow_utils.normalize_to_large_types(field.type, keep_null=True) ) if stored_type != incoming_type: - raise InputValidationError( - f"Cannot append to the stored table {table_uri!r}: column " - f"{field.name!r} is stored as {stored_type} but the new rows have " - f"{incoming_type} (input type {batch.schema.field(field.name).type}). " - "The stored rows are unchanged and nothing was written." - ) + return field.name, stored_type, incoming_type + return None @property def base_path(self) -> tuple[str, ...]: diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 36e710a86..313b749cd 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -138,7 +138,6 @@ async def _run_async( effective_observer = observer if observer is not None else NoOpObserver() effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) - topo_order: list[Any] = [] store_error: StoreReadError | None = None try: topo_order = list(graph.topological_sort()) @@ -256,7 +255,7 @@ async def _run_async( except ExceptionGroup as group: # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. - store_error = _single_store_error(group, topo_order) + store_error = _single_store_error(group) if store_error is None: raise raise store_error @@ -264,24 +263,20 @@ async def _run_async( effective_observer.on_run_end(run_id) -def _single_store_error( - group: ExceptionGroup, topo_order: list[Any] -) -> StoreReadError | None: +def _single_store_error(group: ExceptionGroup) -> StoreReadError | None: """Pick the one ``StoreReadError`` an async run raises from its exception group. Concurrent node tasks can fail together, so the ``TaskGroup``'s group may hold several store errors and other exceptions. The caller gets a single - ``StoreReadError``, as from the sync orchestrator: the one whose node comes - first in topological order, with ties and node-less errors (a - ``StoreReadError`` names a table, not a node) broken by their order in the - group. Every other exception stays attached for diagnosis: each is noted on + ``StoreReadError``, as from the sync orchestrator: the first one in the group + (a ``StoreReadError`` names a table, not a node, so there is no node order to + rank by). Every other exception stays attached for diagnosis: each is noted on the chosen error, other store errors are also logged at WARNING, and the whole group is the chosen error's ``__context__``. Its ``__cause__`` stays the backend error. Args: group: The exception group the run's ``TaskGroup`` raised. - topo_order: The run's nodes in topological order. Returns: The store error to raise, or ``None`` when the group holds none. @@ -289,11 +284,7 @@ def _single_store_error( store_errors = StoreReadError.in_group(group) if not store_errors: return None - rank = {getattr(node, "label", None): i for i, node in enumerate(topo_order)} - chosen = min( - store_errors, - key=lambda err: rank.get(getattr(err, "node_label", None), len(rank)), - ) + chosen = store_errors[0] for other in _leaves(group): if other is chosen: continue diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md index 89c03292c..89c7a0f42 100644 --- a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -3,6 +3,12 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan > task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **Superseded in part (November 11, 2026):** the developer's review of PR #290 removed every +> parameter used only for an error message. `_read_delta_table` no longer takes `record_path`, +> `_require_compatible_append(table_uri, ...)` became `_incompatible_append_column(...)`, +> `_store_error_to_raise` takes only the errors, and `_single_store_error` no longer takes +> `topo_order`. See `2026-11-11-itl-884-no-message-only-parameters.md`. + **Goal:** Answer the two open CodeRabbit threads on PR #290 (ids 4534230764 and 4534230775). A `StoreReadError` or an append-refusal `InputValidationError` raised while flushing reaches the caller as itself, so a store read failure in a per-row result-cache flush fails the run under diff --git a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md index 8eca242f5..e2c5a9fe5 100644 --- a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md +++ b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md @@ -3,6 +3,12 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan > task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **Superseded in part (November 11, 2026):** the developer's review of PR #290 removed every +> parameter used only for an error message. `_read_delta_table` no longer takes `record_path`, +> `_require_compatible_append(table_uri, ...)` became `_incompatible_append_column(...)`, +> `_store_error_to_raise` takes only the errors, and `_single_store_error` no longer takes +> `topo_order`. See `2026-11-11-itl-884-no-message-only-parameters.md`. + **Goal:** A Delta table that exists but cannot be read raises `StoreReadError`, never an empty result. A flush that would make an existing table unreadable, or silently cast the stored column type, is refused before anything is written. diff --git a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md new file mode 100644 index 000000000..cfe260872 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md @@ -0,0 +1,46 @@ +# ITL-884 — remove message-only parameters (developer review of PR #290) + +**Review:** comment 4536812218 on `src/orcapod/databases/delta_lake_databases.py` +(`_read_delta_table`'s new `record_path`): a function must not receive a value used only to +build an error message; the caller, which has the context, wraps the error. + +**Goal:** no helper in the PR diff takes a parameter it uses only for a message. User-visible +behaviour and every `StoreReadError` / `InputValidationError` message stay exactly as they are. + +## Audit of the PR diff (`git diff origin/main...HEAD`) + +| Function | Parameter | Used for | Action | +|---|---|---|---| +| `DeltaTableDatabase._read_delta_table` | `record_path` | message only (`_read_failed`) | Remove; restore main's signature. The function raises a module-private `_DeltaTableReadError` (backend error as `__cause__`) for a failed scan; its three callers catch it and raise `self._read_failed(record_path) from exc.__cause__`. A bad filter still propagates as is. | +| `DeltaTableDatabase._require_compatible_append` | `table_uri` | message only | Becomes `_incompatible_append_column(delta_table, batch)`, returning `(column, stored_type, incoming_type)` or `None`; `flush_batch` (owns `table_uri`) raises the `InputValidationError` with the same message. | +| `_store_error_to_raise` | `record_key` in the `(record_key, error)` pairs | message only (the notes) | Takes `list[Exception]` and only selects; `flush` adds the notes naming the other batches' keys. | +| `async_orchestrator._single_store_error` | `topo_order` | nothing: ranks by `node_label`, which `StoreReadError` lacks, so the first error in the group always wins | Remove the parameter and the ranking (and the `topo_order` pre-init in `run`); select the first store error in group order. Same behaviour. | +| `DeltaTableDatabase._read_failed` | `record_path` | builds the error (its job); called only by owners of `record_path` | Keep. | +| `_get_delta_table`, `_refresh_existing_ids_cache`, `get_all_records` | `record_path` | locating the table (work) | Keep. | +| `flush_batch` | `table_uri`, `abs_path_components` | write location (work) | Keep. | +| `StoreReadError.__init__` | `record_path` | public attribute; not in the message, not read in src/tests | Keep as error payload (coordinator approved). | +| `FunctionJobNode` / `_FunctionPodBase` `StoreReadError` handlers | `node_label`, `pod_label` | observer callbacks (work) | Keep. | + +## Steps + +1. Ask the coordinator to approve the table (repo review process). **Status: approved** (item 8 kept: + payload for a caller that catches the error, same shape as `StoreWriteError` on PR #287). +2. Edit `delta_lake_databases.py`, `async_orchestrator.py`; restore + `tests/test_databases/test_delta_table_database.py:564` to main's call. +3. Tests unchanged in behaviour: `tests/test_databases/test_delta_read_failures.py`, + `tests/test_pipeline/test_store_read_failure.py`, full suite on 3.11–3.13, ruff, + basedpyright. +4. Update the earlier ITL-884 plans' descriptions where they name the removed parameters. +5. One commit, push, reply on thread 4536812218 (do not resolve), CodeRabbit round, CI. + +## Notes for successors + +- PR #293 calls `_read_delta_table`; after this it no longer passes `record_path`. + +## Status + +- Steps 2–4 done: suite 6926 passed / 96 skipped / 15 xfailed on 3.11, 3.12 and 3.13; ruff clean; + basedpyright 0 errors (warnings 1595 → 1587). +- `_single_store_error` here is now `(group) -> StoreReadError | None`, picking + `StoreReadError.in_group(group)[0]`. PR #287's version takes `topo_order` and ranks + `StoreWriteError` by `node_label`; whichever PR merges second reconciles the two. diff --git a/tests/test_databases/test_delta_table_database.py b/tests/test_databases/test_delta_table_database.py index 27142c3fc..4a5296ac1 100644 --- a/tests/test_databases/test_delta_table_database.py +++ b/tests/test_databases/test_delta_table_database.py @@ -561,7 +561,7 @@ def test_read_delta_table_no_filter_normalizes_views(self, db): tbl = make_string_view_table(__record_id=["p", "q"], category=["A", "B"]) deltalake.write_deltalake(uri, tbl, mode="overwrite") - result = db._read_delta_table(db._get_delta_table(self.PATH), self.PATH) + result = db._read_delta_table(db._get_delta_table(self.PATH)) assert result.num_rows == 2 assert not any(pa.types.is_string_view(f.type) for f in result.schema) From b55a3e642b5c7416e6b38b4894752ec6e53c9abc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 03:43:30 -0800 Subject: [PATCH 20/30] fix(databases): the append check accepts new nested struct fields (ITL-884) The append check compared whole struct types, so appending rows whose struct column gained a field was refused with InputValidationError, although main accepted it and Delta's merge adds the field (CodeRabbit, PR #290). Struct fields are now matched by name at every depth, through list and map types: a field present on only one side is schema evolution, as a top-level column is, and a changed type of an existing field is still refused. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +- docs/concepts/function-pods.md | 1 + src/orcapod/databases/delta_lake_databases.py | 48 ++++++++++++++-- ...1-11-itl-884-no-message-only-parameters.md | 10 ++++ .../test_delta_read_failures.py | 57 +++++++++++++++++++ 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 067c140cf..e325a723e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ into an `int64` column, `large_binary` into `large_string`), or, when an extensi was involved, accepted rows the table could then no longer read. The comparison treats types the way Delta stores them: `string` and `large_string` (and the other large and non-large layouts) are the same type, an extension type is compared by its storage type, and nullability is not -compared. New columns and missing columns are still accepted. +compared. Struct fields are compared by name at any depth, including inside lists and maps. +New and missing columns, and new and missing struct fields, are still accepted. `DeltaTableDatabase.flush()` (and `add_records(..., flush=True)`) now raises a `StoreReadError` or this `InputValidationError` as itself. Before, it wrapped every failure in a diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index becb5a6f0..cdd82edfa 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -231,6 +231,7 @@ print("function calls:", len(calls)) `InputValidationError`, which names the table, the column and both types. They are not converted to the stored type. `string` and `large_string`, and the other large and non-large layouts of one type, count as the same type. + New columns and new struct fields are still accepted. Both errors reach you as themselves, also from `flush()` and from `add_records(..., flush=True)`. When several pending tables fail in one flush, the diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index e6483d225..4fb6f9310 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -71,6 +71,45 @@ def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: return arrow_type +def _types_conflict(stored: pa.DataType, incoming: pa.DataType) -> bool: + """Return ``True`` if ``incoming`` changes a type that ``stored`` already has. + + Both types must already be reduced by ``_comparable_storage_type``. Struct + fields are matched by name at every depth, through list and map types. A + field present on only one side is not a conflict, just as a top-level column + present on only one side is not: adding or omitting it is schema evolution, + which Delta's merge accepts or rejects itself. Any other difference is a + conflict. + + Args: + stored: The type the stored table has. + incoming: The type the new rows have. + + Returns: + Whether a type the stored table already has would change. + """ + if pa.types.is_struct(stored) and pa.types.is_struct(incoming): + stored_fields = {f.name: f.type for f in stored} + return any( + _types_conflict(stored_fields[f.name], f.type) + for f in incoming + if f.name in stored_fields + ) + if (pa.types.is_large_list(stored) and pa.types.is_large_list(incoming)) or ( + pa.types.is_list(stored) and pa.types.is_list(incoming) + ): + return _types_conflict(stored.value_type, incoming.value_type) + if pa.types.is_fixed_size_list(stored) and pa.types.is_fixed_size_list(incoming): + return stored.list_size != incoming.list_size or _types_conflict( + stored.value_type, incoming.value_type + ) + if pa.types.is_map(stored) and pa.types.is_map(incoming): + return _types_conflict(stored.key_type, incoming.key_type) or _types_conflict( + stored.item_type, incoming.item_type + ) + return stored != incoming + + def _store_error_to_raise(errors: list[Exception]) -> Exception | None: """Pick the store error a failed flush raises as itself. @@ -1180,9 +1219,10 @@ def _incompatible_append_column( involved, writes data files the table can no longer read. Both sides are compared through Delta's own type mapping, with extension types reduced to storage, large-normalized and nullability ignored, so a type Delta stores - differently (``uint64`` as ``int64``) is not reported. Columns present on - only one side are allowed (schema evolution). The stored schema comes from - the loaded log; no data is read. + differently (``uint64`` as ``int64``) is not reported. Columns and nested + struct fields present on only one side are allowed (schema evolution); + see ``_types_conflict``. The stored schema comes from the loaded log; no + data is read. Args: delta_table: The loaded stored table. @@ -1215,7 +1255,7 @@ def _incompatible_append_column( incoming_type = _comparable_storage_type( arrow_utils.normalize_to_large_types(field.type, keep_null=True) ) - if stored_type != incoming_type: + if _types_conflict(stored_type, incoming_type): return field.name, stored_type, incoming_type return None diff --git a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md index cfe260872..cdc4a379a 100644 --- a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md +++ b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md @@ -44,3 +44,13 @@ behaviour and every `StoreReadError` / `InputValidationError` message stay exact - `_single_store_error` here is now `(group) -> StoreReadError | None`, picking `StoreReadError.in_group(group)[0]`. PR #287's version takes `topo_order` and ranks `StoreWriteError` by `node_label`; whichever PR merges second reconciles the two. + +## CodeRabbit round on 0369cfcc (thread 4537325251, Major) + +The append check compared whole struct types, so adding a nested struct field (which main accepts +and Delta's merge adds) was refused: a regression from this PR. Fix (coordinator approved): the +module function `_types_conflict(stored, incoming)` matches struct fields by name at every depth, +through list and map types; a field on one side only is not a conflict, like a top-level column. +`_incompatible_append_column` still returns what differs; `flush_batch` raises. New tests: +`test_nested_field_type_change`, `test_new_struct_field`, `test_new_struct_field_inside_a_list` +(stored schema and null back-fill asserted). Suite 6929 passed / 96 skipped / 15 xfailed on 3.11–3.13. diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index 8cea2e3e7..f27e0c499 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -357,6 +357,26 @@ def test_extension_storage_change(self, base): assert back is not None assert back.num_rows == 1 + def test_nested_field_type_change(self, base: Path) -> None: + """A changed type of an existing struct field is refused.""" + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], pa.array([{"a": 1}], pa.struct([("a", pa.int64())])))) + + with pytest.raises(InputValidationError) as info: + append( + db, + rows([b"b"], pa.array([{"a": 2.0}], pa.struct([("a", pa.float64())]))), + ) + message = str(info.value) + assert "'x'" in message + assert "struct" in message + assert "struct" in message + + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + assert back.num_rows == 1 + assert back.schema.field("x").type == pa.struct([("a", pa.int64())]) + class TestLegitimateAppendsStillPass: """Appends whose types match the stored table, as Delta stores them, succeed.""" @@ -402,6 +422,43 @@ def test_new_column(self, base, stored): assert back.num_rows == 3 assert "y" in back.column_names + def test_new_struct_field(self, base: Path) -> None: + """An append that adds a field to a stored struct column succeeds.""" + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], pa.array([{"a": 1}], pa.struct([("a", pa.int64())])))) + evolved = pa.struct([("a", pa.int64()), ("b", pa.int64())]) + append(db, rows([b"b"], pa.array([{"a": 2, "b": 3}], evolved))) + + back = DeltaTableDatabase(base_path=base).get_all_records( + PATH, record_id_column="rid" + ) + assert back is not None + back = back.sort_by("rid") + assert back.schema.field("x").type == evolved + assert back.column("x").to_pylist() == [ + {"a": 1, "b": None}, + {"a": 2, "b": 3}, + ] + + def test_new_struct_field_inside_a_list(self, base: Path) -> None: + """An append that adds a field to a struct inside a list column succeeds.""" + db = DeltaTableDatabase(base_path=base) + stored_type = pa.large_list(pa.struct([("a", pa.int64())])) + append(db, rows([b"a"], pa.array([[{"a": 1}]], stored_type))) + evolved = pa.large_list(pa.struct([("a", pa.int64()), ("b", pa.int64())])) + append(db, rows([b"b"], pa.array([[{"a": 2, "b": 3}]], evolved))) + + back = DeltaTableDatabase(base_path=base).get_all_records( + PATH, record_id_column="rid" + ) + assert back is not None + back = back.sort_by("rid") + assert back.schema.field("x").type == evolved + assert back.column("x").to_pylist() == [ + [{"a": 1, "b": None}], + [{"a": 2, "b": 3}], + ] + def test_missing_column(self, base): """An append that omits a stored column succeeds.""" db = DeltaTableDatabase(base_path=base) From 488e2a2b763c55a563974af05fe96a2104ec1616 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 03:44:09 -0800 Subject: [PATCH 21/30] docs(agents): add the no message-only parameters rule (ITL-884) Developer, November 11, 2026, from the review of PR #290: a function must not receive a value it uses only for an error or log message; the caller wraps or annotates the error. Added to both CLAUDE.md and .zed/rules. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 9 +++++++++ CLAUDE.md | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/.zed/rules b/.zed/rules index f6e613504..0c226b8ac 100644 --- a/.zed/rules +++ b/.zed/rules @@ -46,6 +46,15 @@ Just change the code and update all references directly. Never manipulate sys.modules directly (e.g. sys.modules.setdefault). If a subpackage import path doesn't work, create a proper re-export package with an __init__.py instead. +## No message-only parameters + +Never pass a value into a function only so that the function can put it in an error or log +message (developer, November 11, 2026). If the function does not use the value to do its job, +it must not receive it: the function raises what it knows, and the caller, which has the +context, catches and wraps or annotates the error (path, row, column, node label). A helper +whose job is to report or record (log a suppressed failure, attach a note) legitimately takes +the label and context it records. + ## Docstrings Use Google style (https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) diff --git a/CLAUDE.md b/CLAUDE.md index bc810d382..1908ec144 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,6 +50,15 @@ Just change the code and update all references directly. Never manipulate `sys.modules` directly (e.g. `sys.modules.setdefault`). If a subpackage import path doesn't work, create a proper re-export package with an `__init__.py` instead. +## No message-only parameters + +Never pass a value into a function only so that the function can put it in an error or log +message (developer, November 11, 2026). If the function does not use the value to do its job, +it must not receive it: the function raises what it knows, and the caller, which has the +context, catches and wraps or annotates the error (path, row, column, node label). A helper +whose job is to report or record (log a suppressed failure, attach a note) legitimately takes +the label and context it records. + ## Docstrings Use [Google style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) From 65c6330adfe4a529970985af24f014e531a92048 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 05:32:14 -0800 Subject: [PATCH 22/30] fix(pipeline): name node and row on async pipeline-record store errors (ITL-884) CodeRabbit round on 4f0b4f1c: - The async FunctionJobNode records pipeline entries outside the pod's per-row handler; record_and_forward now names the node and row on a StoreError raised there. - The async orchestrator's per-node wrapper also labels store errors that a node raises inside its own exception group. - CHANGELOG and the function-pods guide: a refused append writes nothing to that table (other pending tables may commit); new columns and struct fields pass the check and are added per allow_schema_evolution; the read guarantee is stated for DeltaTableDatabase; in a run, a refused append is a StoreWriteError caused by the InputValidationError. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- docs/concepts/function-pods.md | 23 ++++---- src/orcapod/core/nodes/function_node.py | 24 +++++--- src/orcapod/pipeline/async_orchestrator.py | 11 +++- ...1-11-itl-884-no-message-only-parameters.md | 9 +++ .../test_store_errors_together.py | 58 +++++++++++++++++++ 6 files changed, 110 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28e0bf7c..bc6879e8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,14 +8,16 @@ Before writing to an existing `DeltaTableDatabase` table, the flush now compares the new rows' column types with the stored ones. If a column's type differs, the flush raises -`InputValidationError` naming the table, the column and both types, and writes nothing. Before, +`InputValidationError` naming the table, the column and both types, and writes nothing to that +table (other tables' pending rows in the same flush may still be committed). Before, Delta's merge silently cast the new rows to the stored type (`int32` or `float64` rows written into an `int64` column, `large_binary` into `large_string`), or, when an extension (logical) type was involved, accepted rows the table could then no longer read. The comparison treats types the way Delta stores them: `string` and `large_string` (and the other large and non-large layouts) are the same type, an extension type is compared by its storage type, and nullability is not compared. Struct fields are compared by name at any depth, including inside lists and maps. -New and missing columns, and new and missing struct fields, are still accepted. +New and missing columns, and new and missing struct fields, pass the check; whether Delta's +merge adds a new one follows the database's `allow_schema_evolution` setting, as before. `DeltaTableDatabase.flush()` (and `add_records(..., flush=True)`) now raises a `StoreReadError` or this `InputValidationError` as itself. Before, it wrapped every failure in a diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 659e242d3..387ae2176 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -220,9 +220,10 @@ print("function calls:", len(calls)) `"v0.0"`; a minor bump such as `"v0.1"` is not enough), or rename the function. !!! note "When the store cannot be read" - A table that does not exist yet reads as empty, and its rows are computed. A table that - exists but cannot be read (for example, a data file was deleted or damaged) is never - treated as empty: the run stops with `orcapod.errors.StoreReadError`, which names the + A table that does not exist yet reads as empty, and its rows are computed. In a + `DeltaTableDatabase`, a table that exists but cannot be read (for example, a data file was + deleted or damaged) is never treated as empty: the run stops with + `orcapod.errors.StoreReadError`, which names the table, the node and row being processed when they are known, and the underlying error, whatever the orchestrator's `error_policy`. Its stored rows are not recomputed and the table is not overwritten, so you can repair the table or restore the file and run again. @@ -231,13 +232,15 @@ print("function calls:", len(calls)) In a `DeltaTableDatabase`, rows whose column types differ from the stored table's (`float64` values for a column stored as `int64`, say) are refused with - `InputValidationError`, which names the table, the column and both types. They are not - converted to the stored type. `string` and `large_string`, and the other large and - non-large layouts of one type, count as the same type. - New columns and new struct fields are still accepted. - - Both errors reach you as themselves, also from `flush()` and from - `add_records(..., flush=True)`. When several pending tables fail in one flush, the + `InputValidationError`, which names the table, the column and both types, and nothing is + written to that table. They are not converted to the stored type. `string` and + `large_string`, and the other large and non-large layouts of one type, count as the same + type. New columns and new struct fields pass this check; whether they are added follows + the database's `allow_schema_evolution` setting. In a job run, a refused append fails the + run as a `StoreWriteError` whose cause is the `InputValidationError`. + + When you use the database directly, both errors reach you as themselves, also from + `flush()` and from `add_records(..., flush=True)`. When several pending tables fail in one flush, the `StoreReadError` (or else the first `InputValidationError`) is raised, and each other failure is attached to it as a note and logged. Failed rows stay queued, so a later flush tries them again. A flush that fails for any other reason raises `RuntimeError`, diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index c302d7a99..3454c8856 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -2453,14 +2453,22 @@ async def record_and_forward() -> None: execution_pod.RESULT_COMPUTED_FLAG, False ) ) - self.add_pipeline_record( - original_tag, - input_data, - data_record_id=output_data.datagram_uuid, - computed=result_computed, - is_ephemeral=is_ephemeral, - output_data_hash=output_data.content_hash(), - ) + try: + self.add_pipeline_record( + original_tag, + input_data, + data_record_id=output_data.datagram_uuid, + computed=result_computed, + is_ephemeral=is_ephemeral, + output_data_hash=output_data.content_hash(), + ) + except StoreError as exc: + # This runs outside the pod's per-row handler: name the + # node and row here. + exc.add_context( + node_label=node_label, tag=original_tag.as_dict() + ) + raise # Update in-memory cache so iter_data() sees the result. base_entry_id = self.compute_base_entry_id( original_tag, input_data diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 9049f1429..87ffed444 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -280,7 +280,10 @@ async def _run_async( def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: - """Start a node's coroutine in ``tg``, naming the node on a store error it raises. + """Start a node's coroutine in ``tg``, naming the node on the store errors it raises. + + A store error raised on its own or inside an exception group (a node that runs + its own task group) gets the node's label when it has none. Args: tg: The run's task group. @@ -296,6 +299,12 @@ async def run() -> None: # so _single_store_error can order it like the sync orchestrator. exc.add_context(node_label=node.label) raise + except BaseExceptionGroup as group: + # A node that runs its own task group raises its store errors inside + # a group; name the node on each and re-raise the group unchanged. + for store_error in StoreError.in_group(group): + store_error.add_context(node_label=node.label) + raise tg.create_task(run()) diff --git a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md index d5c9e3737..43da3a4a0 100644 --- a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md +++ b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md @@ -70,3 +70,12 @@ through list and map types; a field on one side only is not a conflict, like a t - D7 CHANGELOG / DESIGN_ISSUES / function-pods concept page. - D8 `tests/test_pipeline/test_store_errors_together.py` (sync/async × both orders, async order independence). + +## CodeRabbit round on 4f0b4f1c (coordinator approved) + +Accepted: CHANGELOG wording (refused append writes nothing *to that table*; new fields pass the +check, Delta adds them per `allow_schema_evolution`), concept page scoped to `DeltaTableDatabase` +and says a refused append in a run is a `StoreWriteError`; async `record_and_forward` names node +and row on a `StoreError`; `_start_node` labels store errors inside a node's exception group. +Declined: pending-rows read skip (fixed by #293). Deferred: table data without a `_delta_log` +and connector-DB read errors -> ITL-916. diff --git a/tests/test_pipeline/test_store_errors_together.py b/tests/test_pipeline/test_store_errors_together.py index 56fb8f57f..9b9863eb0 100644 --- a/tests/test_pipeline/test_store_errors_together.py +++ b/tests/test_pipeline/test_store_errors_together.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import json from collections.abc import Callable, Iterator from pathlib import Path @@ -32,6 +33,7 @@ from orcapod.pipeline.async_orchestrator import ( AsyncPipelineOrchestrator, _single_store_error, # pyright: ignore[reportPrivateUsage] + _start_node, # pyright: ignore[reportPrivateUsage] ) from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator @@ -273,3 +275,59 @@ def test_refused_append_is_a_store_write_error(tmp_path: Path) -> None: assert err.node_label == "rd" assert isinstance(err.__cause__, InputValidationError) assert "'y'" in str(err.__cause__) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_pipeline_record_read_failure_names_node_and_row( + tmp_path: Path, orchestrator: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A read failure while recording a row's pipeline entry names the node and row.""" + # The async node records pipeline entries outside the pod's per-row handler, + # so this checks that the error still gets its node and row. + store_path = tmp_path / "store" + build_job(store_path, "abc", reader_first=True).run( + orchestrator=ORCHESTRATORS[orchestrator]() + ) + original = DeltaTableDatabase.add_record + + def add_record(self: DeltaTableDatabase, record_path, record_id, record, **kwargs): + # Stand in for a pipeline table that cannot be read while row d's entry + # is recorded (the skip_duplicates check reads the stored record ids). + if "_result" not in self.base_path and record.column("k")[0].as_py() == "d": + raise StoreReadError("pipeline-table", tuple(record_path)) from OSError( + "data file missing" + ) + return original(self, record_path, record_id, record, **kwargs) + + monkeypatch.setattr(DeltaTableDatabase, "add_record", add_record) + with pytest.raises(StoreReadError) as info: + build_job(store_path, "abcd", reader_first=True).run( + orchestrator=ORCHESTRATORS[orchestrator]() + ) + + err = info.value + assert err.table == "pipeline-table" + assert err.node_label == "rd" + assert err.tag == {"k": "d"} + + +def test_store_error_inside_a_node_group_gets_the_node_label() -> None: + """A store error a node raises inside its own exception group is labelled.""" + # Calls the private helper directly: no public pipeline raises a store error + # from a node's inner task group without an annotation on the way. + + class Node: + label = "inner" + + read = StoreReadError("table", ("t",)) + + async def node_work() -> None: + raise ExceptionGroup("node", [read]) + + async def main() -> None: + async with asyncio.TaskGroup() as tg: + _start_node(tg, Node(), node_work()) + + with pytest.raises(ExceptionGroup): + asyncio.run(main()) + assert read.node_label == "inner" From ffe44481566bc17786c70529346750c08a77eb51 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 06:25:03 -0800 Subject: [PATCH 23/30] fix(databases): the append check compares the types the Delta writer stores (ITL-884) The append check converted the batch with deltalake.Schema.from_arrow, which rejects non-microsecond timestamps, so appending timestamp[s|ms|ns] rows to an existing table failed although main appended them (CodeRabbit, PR #290). A differential guard over every storable type also found that a logical type whose Delta storage differs (list[Path]: large_list written, list read back) broke the check when the stored schema's extension metadata was rebuilt. - _as_delta_writer_stores converts the batch with the Delta writer's own conversion (deltalake.writer._conversion, used by write_deltalake and merge), then Delta's type mapping, instead of a hand-written mapping. - _stored_arrow_schema reads the stored schema without field metadata. - tests/test_databases/test_delta_append_types.py appends an identical schema for every Arrow and registered logical type Delta can store (nullable and not, same view and reopened); 40 cases failed before the fix. Types Delta cannot store are listed and must still fail on the first write; fixed_size_list, whose merge Delta rejects on main too, is a strict xfail (ITL-916). A timestamp onto int64 is still refused. - The function-pods guide and CHANGELOG say the check compares the types as Delta stores them. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- DESIGN_ISSUES.md | 8 +- docs/concepts/function-pods.md | 8 +- src/orcapod/databases/delta_lake_databases.py | 114 +++++++- ...1-11-itl-884-no-message-only-parameters.md | 9 + .../test_databases/test_delta_append_types.py | 260 ++++++++++++++++++ .../test_delta_read_failures.py | 9 +- 7 files changed, 392 insertions(+), 21 deletions(-) create mode 100644 tests/test_databases/test_delta_append_types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6879e8d..2044dd015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,9 @@ Delta's merge silently cast the new rows to the stored type (`int32` or `float64 into an `int64` column, `large_binary` into `large_string`), or, when an extension (logical) type was involved, accepted rows the table could then no longer read. The comparison treats types the way Delta stores them: `string` and `large_string` (and the other large and non-large layouts) -are the same type, an extension type is compared by its storage type, and nullability is not -compared. Struct fields are compared by name at any depth, including inside lists and maps. +are the same type, types the Delta writer converts on the way in are compared as converted +(unsigned integers as signed, every timestamp in microseconds), an extension type is compared by +its storage type, and nullability is not compared. Struct fields are compared by name at any depth, including inside lists and maps. New and missing columns, and new and missing struct fields, pass the check; whether Delta's merge adds a new one follows the database's `allow_schema_evolution` setting, as before. diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index cccca70c8..24f977a51 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2322,7 +2322,13 @@ async `FunctionPod` row handler and the async orchestrator let it fail the run w the stored Delta schema through Delta's own type mapping (extension types by storage, large-normalized, nullability ignored). On a difference it raises `InputValidationError` naming the table, column and both types, and nothing is written; struct fields are compared by name at -every depth, so a new nested field is schema evolution, not a difference. `flush()` raises a `StoreReadError` or +every depth, so a new nested field is schema evolution, not a difference. The batch's types +are converted with the Delta writer's own conversion (`deltalake.writer._conversion`, the one +`write_deltalake` and `merge` use) before the comparison, and the stored schema is read without +field metadata, so a type the writer stores differently (timestamps in microseconds, unsigned as +signed) or a logical type whose Delta storage differs is never refused; the differential guard +`tests/test_databases/test_delta_append_types.py` appends an identical schema for every Arrow and +logical type Delta can store. `flush()` raises a `StoreReadError` or that `InputValidationError` as itself, not wrapped in `RuntimeError`, with every other failed batch noted on it, so a read failure in a per-row result-cache flush still fails the run. The async orchestrator selects the `StoreReadError` from the `TaskGroup`'s exception group with a diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 387ae2176..4a6a5aa77 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -230,12 +230,14 @@ print("function calls:", len(calls)) A write that fails because the stored table cannot be read raises `StoreReadError` too, not `StoreWriteError` (see *When the store cannot be written* below). - In a `DeltaTableDatabase`, rows whose column types differ from the stored table's - (`float64` values for a column stored as `int64`, say) are refused with + In a `DeltaTableDatabase`, rows whose column types differ from the stored table's, as + Delta stores them (`float64` values for a column stored as `int64`, say), are refused with `InputValidationError`, which names the table, the column and both types, and nothing is written to that table. They are not converted to the stored type. `string` and `large_string`, and the other large and non-large layouts of one type, count as the same - type. New columns and new struct fields pass this check; whether they are added follows + type, and so do types the Delta writer converts on the way in (`uint64` is stored as + `int64`, every timestamp in microseconds). New columns and new struct fields pass this + check; whether they are added follows the database's `allow_schema_evolution` setting. In a job run, a refused append fails the run as a `StoreWriteError` whose cause is the `InputValidationError`. diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 4fb6f9310..de89a49e3 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -71,6 +71,92 @@ def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: return arrow_type +def _as_delta_writer_stores(schema: pa.Schema) -> pa.Schema: + """Return ``schema`` with the types the Delta writer would store it as. + + Applies the conversion ``deltalake.write_deltalake`` and ``DeltaTable.merge`` + apply to incoming data (unsigned to signed integers, every timestamp to + microseconds, fixed-size binary to binary), then Delta's own type mapping. + Calling the writer's conversion instead of copying it keeps the append check + in step with what the writer accepts; ``test_identical_schema_appends`` + guards it across ``deltalake`` versions. + + Args: + schema: The schema of the rows about to be written. + + Returns: + The schema as the Delta table would store it. + """ + from arro3.core import Schema as Arro3Schema + from deltalake.writer._conversion import ( # pyright: ignore[reportPrivateUsage] # the writer's own conversion, see the docstring + _convert_arro3_schema_to_delta, # pyright: ignore[reportPrivateUsage] + ) + + converted = _convert_arro3_schema_to_delta(Arro3Schema.from_arrow(schema)) + return pa.schema( + deltalake.Schema.from_arrow(converted).to_arrow() # pyright: ignore[reportArgumentType] # arro3 schema; pyarrow reads it through the C data interface + ) + + +def _without_field_metadata(delta_type: Any) -> Any: + """Return a Delta type with the metadata of every nested field dropped. + + Args: + delta_type: A ``deltalake.schema`` type. + + Returns: + The same type with no field metadata at any depth. + """ + from deltalake.schema import ArrayType, Field, MapType, StructType + + if isinstance(delta_type, StructType): + return StructType( + [ + Field(f.name, _without_field_metadata(f.type), nullable=f.nullable) + for f in delta_type.fields + ] + ) + if isinstance(delta_type, ArrayType): + return ArrayType( + _without_field_metadata(delta_type.element_type), + contains_null=delta_type.contains_null, + ) + if isinstance(delta_type, MapType): + return MapType( + _without_field_metadata(delta_type.key_type), + _without_field_metadata(delta_type.value_type), + value_contains_null=delta_type.value_contains_null, + ) + return delta_type + + +def _stored_arrow_schema(delta_table: deltalake.DeltaTable) -> pa.Schema: + """Return the Arrow schema of the table's stored Delta schema, without metadata. + + Delta keeps the Arrow field metadata it was written with, including the + extension type name of a logical type. Reading that back into pyarrow would + rebuild the registered extension type over Delta's own storage type (``list`` + where Orcapod wrote ``large_list``), which the extension type rejects. The + append check compares storage types only, so the metadata is dropped first. + + Args: + delta_table: The loaded stored table. + + Returns: + The stored schema as plain Arrow types. + """ + from deltalake.schema import Field, Schema + + stored = delta_table.schema() + plain = Schema( + [ + Field(f.name, _without_field_metadata(f.type), nullable=f.nullable) + for f in stored.fields + ] + ) + return pa.schema(plain.to_arrow()) # pyright: ignore[reportArgumentType] # arro3 schema; pyarrow reads it through the C data interface + + def _types_conflict(stored: pa.DataType, incoming: pa.DataType) -> bool: """Return ``True`` if ``incoming`` changes a type that ``stored`` already has. @@ -1216,10 +1302,12 @@ def _incompatible_append_column( Delta's merge either casts such a batch to the stored type (``int32`` or ``float64`` rows into an ``int64`` column) or, when an extension type is - involved, writes data files the table can no longer read. Both sides are - compared through Delta's own type mapping, with extension types reduced to - storage, large-normalized and nullability ignored, so a type Delta stores - differently (``uint64`` as ``int64``) is not reported. Columns and nested + involved, writes data files the table can no longer read. The batch's types + are first converted exactly as the Delta writer converts them (see + ``_as_delta_writer_stores``), then both sides are compared with extension + types reduced to storage, large-normalized and nullability ignored, so a + type Delta stores differently (``uint64`` as ``int64``, ``timestamp[ns]`` + as ``timestamp[us]``) is not reported. Columns and nested struct fields present on only one side are allowed (schema evolution); see ``_types_conflict``. The stored schema comes from the loaded log; no data is read. @@ -1232,16 +1320,14 @@ def _incompatible_append_column( ``(column, stored_type, incoming_type)`` for the first column whose compared types differ, or ``None`` when the batch can be appended. """ - stored = pa.schema(delta_table.schema().to_arrow()) # pyright: ignore[reportArgumentType] # arro3 schema; pyarrow reads it through the C data interface - incoming = pa.schema( - deltalake.Schema.from_arrow( - pa.schema( - [ - pa.field(f.name, _comparable_storage_type(f.type)) - for f in batch.schema - ] - ) - ).to_arrow() # pyright: ignore[reportArgumentType] # arro3 schema, as above + stored = _stored_arrow_schema(delta_table) + incoming = _as_delta_writer_stores( + pa.schema( + [ + pa.field(f.name, _comparable_storage_type(f.type)) + for f in batch.schema + ] + ) ) for field in incoming: index = stored.get_field_index(field.name) diff --git a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md index 43da3a4a0..557b74e3d 100644 --- a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md +++ b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md @@ -79,3 +79,12 @@ and says a refused append in a run is a `StoreWriteError`; async `record_and_for and row on a `StoreError`; `_start_node` labels store errors inside a node's exception group. Declined: pending-rows read skip (fixed by #293). Deferred: table data without a `_delta_log` and connector-DB read errors -> ITL-916. + +## CodeRabbit round on cf029bbf (coordinator approved) + +Regression: non-microsecond timestamps (and `list[Path]`, found by the new differential guard) +could not be appended to an existing Delta table. Fix: `_as_delta_writer_stores` applies the +writer's own conversion, `_stored_arrow_schema` reads the stored schema without field metadata. +Guard: `tests/test_databases/test_delta_append_types.py` (every Arrow/logical type, identical +schema, same view and reopened; 40 cases failed before the fix). Deferred: ExceptionGroup-wrapped +store errors -> ITL-918; fixed_size_list merge failure (also on main) -> ITL-916 (strict xfail). diff --git a/tests/test_databases/test_delta_append_types.py b/tests/test_databases/test_delta_append_types.py new file mode 100644 index 000000000..f0a242d12 --- /dev/null +++ b/tests/test_databases/test_delta_append_types.py @@ -0,0 +1,260 @@ +"""Appending rows with the stored schema succeeds for every type Delta can store. + +ITL-884 added a check that refuses an append whose column types differ from the +stored table's. It twice refused appends that ``main`` accepted (a new nested +struct field, then non-microsecond timestamps), so this differential guard writes +one row of each Arrow type, appends a second row with the IDENTICAL schema, and +reads both back: once on the same database view and once after reopening the +store. A type the Delta writer cannot store at all is listed in +``NOT_STORABLE`` and must still fail on the first write, so the list stays true +as ``deltalake`` changes. +""" + +from __future__ import annotations + +import datetime +import decimal +import pathlib +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest +from upath import UPath + +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase +from orcapod.logical_types.directory_type import Directory +from orcapod.logical_types.file_type import File + +PATH = ("types", "t") + + +@dataclass +class Point: + """A dataclass output type, stored as a logical (extension) type.""" + + x: int + y: float + + +def _converter() -> Any: + """The default data context's type converter.""" + return get_default_context().type_converter + + +def _logical(python_type: Any) -> pa.DataType: + """The Arrow type Orcapod stores ``python_type`` as.""" + converter = _converter() + if python_type is Point: + # Dataclasses are logical types once registered with the converter. + converter.register_python_class(Point) + return converter.python_type_to_arrow_type(python_type) + + +def sample(arrow_type: pa.DataType) -> Any: + """Return one valid Python value of ``arrow_type`` (its storage, for extensions).""" + if isinstance(arrow_type, pa.BaseExtensionType): + return sample(arrow_type.storage_type) + if pa.types.is_boolean(arrow_type): + return True + if pa.types.is_integer(arrow_type): + return 1 + if pa.types.is_floating(arrow_type): + return 1.5 + if pa.types.is_decimal(arrow_type): + return decimal.Decimal("1.5") + if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type): + return "a" + if pa.types.is_fixed_size_binary(arrow_type): + return b"a" * arrow_type.byte_width + if pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type): + return b"a" + if pa.types.is_date(arrow_type): + return datetime.date(2020, 1, 2) + if pa.types.is_time(arrow_type): + return datetime.time(1, 2, 3) + if pa.types.is_timestamp(arrow_type): + return datetime.datetime(2020, 1, 2, 3, 4, 5) + if pa.types.is_duration(arrow_type): + return datetime.timedelta(seconds=3) + if pa.types.is_dictionary(arrow_type): + return sample(arrow_type.value_type) + if pa.types.is_fixed_size_list(arrow_type): + return [sample(arrow_type.value_type)] * arrow_type.list_size + if pa.types.is_list(arrow_type) or pa.types.is_large_list(arrow_type): + return [sample(arrow_type.value_type)] + if pa.types.is_map(arrow_type): + return [(sample(arrow_type.key_type), sample(arrow_type.item_type))] + if pa.types.is_struct(arrow_type): + return {f.name: sample(f.type) for f in arrow_type} + raise NotImplementedError(arrow_type) + + +def column(arrow_type: pa.DataType) -> pa.Array[Any]: + """A one-row array of ``arrow_type``.""" + if isinstance(arrow_type, pa.BaseExtensionType): + return pa.ExtensionArray.from_storage( + arrow_type, column(arrow_type.storage_type) + ) + if pa.types.is_dictionary(arrow_type): + return pa.array([sample(arrow_type)], arrow_type.value_type).dictionary_encode() + return pa.array([sample(arrow_type)], arrow_type) + + +PLAIN_TYPES: dict[str, pa.DataType] = { + "int8": pa.int8(), + "int16": pa.int16(), + "int32": pa.int32(), + "int64": pa.int64(), + "uint8": pa.uint8(), + "uint16": pa.uint16(), + "uint32": pa.uint32(), + "uint64": pa.uint64(), + "float16": pa.float16(), + "float32": pa.float32(), + "float64": pa.float64(), + "bool": pa.bool_(), + "string": pa.string(), + "large_string": pa.large_string(), + "binary": pa.binary(), + "large_binary": pa.large_binary(), + "fixed_size_binary": pa.binary(4), + "date32": pa.date32(), + "date64": pa.date64(), + "time32[s]": pa.time32("s"), + "time32[ms]": pa.time32("ms"), + "time64[us]": pa.time64("us"), + "time64[ns]": pa.time64("ns"), + **{f"timestamp[{u}]": pa.timestamp(u) for u in ("s", "ms", "us", "ns")}, + **{f"timestamp[{u}, UTC]": pa.timestamp(u, "UTC") for u in ("s", "ms", "us", "ns")}, + "timestamp[us, Europe/Paris]": pa.timestamp("us", "Europe/Paris"), + **{f"duration[{u}]": pa.duration(u) for u in ("s", "ms", "us", "ns")}, + "decimal128(10, 2)": pa.decimal128(10, 2), + "decimal128(38, 10)": pa.decimal128(38, 10), + "decimal256(40, 2)": pa.decimal256(40, 2), + "list": pa.list_(pa.int64()), + "large_list": pa.large_list(pa.int64()), + "fixed_size_list": pa.list_(pa.int64(), 2), + "list": pa.list_(pa.field("item", pa.int64(), nullable=False)), + "struct": pa.struct([("a", pa.int64()), ("b", pa.string())]), + "struct": pa.struct([pa.field("a", pa.int64(), nullable=False)]), + "map": pa.map_(pa.string(), pa.int64()), + "dictionary": pa.dictionary(pa.int32(), pa.string()), + # Nested combinations, with the units the Delta writer converts. + "struct": pa.struct( + [("t", pa.timestamp("ns")), ("u", pa.uint32())] + ), + "large_list>": pa.large_list( + pa.struct([("t", pa.timestamp("ms", "UTC"))]) + ), + "list>": pa.list_(pa.large_list(pa.uint64())), + "struct>>": pa.struct( + [("l", pa.list_(pa.struct([("a", pa.binary(3))])))] + ), +} + +LOGICAL_TYPES: dict[str, Any] = { + "logical Path": pathlib.Path, + "logical UPath": UPath, + "logical UUID": uuid.UUID, + "logical File": File, + "logical Directory": Directory, + "logical numpy.ndarray": np.ndarray, + "logical pandas.DataFrame": pd.DataFrame, + "logical pandas.Series": pd.Series, + "logical dataclass": Point, + "logical list[Path]": list[pathlib.Path], +} + +# Types the Delta writer cannot store at all (deltalake 1.0.2): the first write +# fails, before the append check is ever reached, on main as on this branch. Kept +# explicit so a deltalake upgrade that starts storing one of them makes the test +# fail and the type join the guarded list. +NOT_STORABLE: set[str] = { + "float16", + "date64", + "time32[s]", + "time32[ms]", + "time64[us]", + "time64[ns]", + "duration[s]", + "duration[ms]", + "duration[us]", + "duration[ns]", + "decimal256(40, 2)", +} + +# Types Delta stores but whose append Delta's own merge rejects, on main as on +# this branch (not the append check): tracked in ITL-916. +MERGE_REJECTS: set[str] = {"fixed_size_list"} + + +def table(arrow_type: pa.DataType, rid: bytes, *, nullable: bool) -> pa.Table: + """A one-row table keyed by ``rid`` with an ``x`` column of ``arrow_type``.""" + schema = pa.schema( + [ + pa.field("rid", pa.large_binary(), nullable=False), + pa.field("x", arrow_type, nullable=nullable), + ] + ) + return pa.Table.from_arrays( + [pa.array([rid], pa.large_binary()), column(arrow_type)], schema=schema + ) + + +def all_types() -> list[Any]: + """Every (id, Arrow type) pair, nullable and non-nullable.""" + types = dict(PLAIN_TYPES) + types.update({name: _logical(t) for name, t in LOGICAL_TYPES.items()}) + merge_rejects = pytest.mark.xfail( + raises=RuntimeError, + strict=True, + reason="Delta's merge rejects this type, also on main (ITL-916)", + ) + return [ + pytest.param( + name, + arrow_type, + nullable, + id=f"{name}-{'nullable' if nullable else 'non-null'}", + marks=[merge_rejects] if name in MERGE_REJECTS else [], + ) + for name, arrow_type in types.items() + for nullable in (True, False) + ] + + +def write(db: DeltaTableDatabase, rows: pa.Table) -> None: + """Add ``rows`` at ``PATH`` and flush them.""" + db.add_records(PATH, rows, record_id_column="rid", flush=True) + + +@pytest.mark.parametrize(("name", "arrow_type", "nullable"), all_types()) +@pytest.mark.parametrize("reopen", [False, True], ids=["same-view", "reopened"]) +def test_identical_schema_appends( + tmp_path: Path, name: str, arrow_type: pa.DataType, nullable: bool, reopen: bool +) -> None: + """A second batch with the stored batch's exact schema appends and reads back.""" + db = DeltaTableDatabase(base_path=tmp_path) + if name in NOT_STORABLE: + # Delta cannot store the type: the first write already fails. + with pytest.raises(RuntimeError): + write(db, table(arrow_type, b"a", nullable=nullable)) + return + write(db, table(arrow_type, b"a", nullable=nullable)) + + if reopen: + db = DeltaTableDatabase(base_path=tmp_path) + write(db, table(arrow_type, b"b", nullable=nullable)) + + back = DeltaTableDatabase(base_path=tmp_path).get_all_records( + PATH, record_id_column="rid" + ) + assert back is not None + assert back.num_rows == 2 + assert set(back.column("rid").to_pylist()) == {b"a", b"b"} diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index f27e0c499..e40ca3514 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -15,6 +15,7 @@ from __future__ import annotations +import datetime import json from collections.abc import Callable from pathlib import Path @@ -320,8 +321,14 @@ class TestChangedColumnTypeIsRefused: (pa.float64(), 2.0, "double"), (pa.int32(), 2, "int32"), (pa.large_string(), "2", "string"), + (pa.timestamp("ns"), datetime.datetime(2020, 1, 2), "timestamp"), + ], + ids=[ + "int64->float64", + "int64->int32", + "int64->large_string", + "int64->timestamp[ns]", ], - ids=["int64->float64", "int64->int32", "int64->large_string"], ) def test_plain_type_change(self, base, stored, new_type, value, named): """A changed column type is refused, naming the column and both types.""" From 51dc9050859ba29096a8a82ba4a3acf68c909b52 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 07:37:25 -0800 Subject: [PATCH 24/30] fix(databases): null-typed values append to a typed Delta column (ITL-884) The append check passed the batch's types to Delta's type mapping, which has no null type, so rows whose column, struct field or list items were typed null could not be appended to a typed column, although main accepts them (CodeRabbit, PR #290). A null-typed part fits any stored type, so the comparison now leaves it out: a null struct field is dropped, and a column holding null list items or map entries is not compared. Tests: the three null cases (all pass on main), and a null field does not hide a changed sibling field. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 56 ++++++++++++++++--- ...1-11-itl-884-no-message-only-parameters.md | 8 +++ .../test_databases/test_delta_append_types.py | 50 +++++++++++++++++ 3 files changed, 107 insertions(+), 7 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index de89a49e3..48ac855a4 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -71,6 +71,47 @@ def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: return arrow_type +def _without_null_types(arrow_type: pa.DataType) -> pa.DataType | None: + """Return ``arrow_type`` without the parts typed ``null``, or ``None`` if none is left. + + A ``null``-typed value fits a column of any type, and Delta has no ``null`` + type to compare it with, so the append check leaves it out: a ``null`` struct + field is dropped, and a type that is ``null`` or holds ``null`` list items or + map keys or values is not compared at all. + + Args: + arrow_type: A type already reduced by ``_comparable_storage_type``. + + Returns: + The type to compare, or ``None`` when it should not be compared. + """ + if pa.types.is_null(arrow_type): + return None + if pa.types.is_struct(arrow_type): + kept = [ + pa.field(f.name, t) + for f in arrow_type + if (t := _without_null_types(f.type)) is not None + ] + return pa.struct(kept) if kept else None + if ( + pa.types.is_list(arrow_type) + or pa.types.is_large_list(arrow_type) + or pa.types.is_fixed_size_list(arrow_type) + ): + if _without_null_types(arrow_type.value_type) != arrow_type.value_type: + return None + return arrow_type + if pa.types.is_map(arrow_type): + if ( + _without_null_types(arrow_type.key_type) != arrow_type.key_type + or _without_null_types(arrow_type.item_type) != arrow_type.item_type + ): + return None + return arrow_type + return arrow_type + + def _as_delta_writer_stores(schema: pa.Schema) -> pa.Schema: """Return ``schema`` with the types the Delta writer would store it as. @@ -1302,7 +1343,9 @@ def _incompatible_append_column( Delta's merge either casts such a batch to the stored type (``int32`` or ``float64`` rows into an ``int64`` column) or, when an extension type is - involved, writes data files the table can no longer read. The batch's types + involved, writes data files the table can no longer read. ``null``-typed + parts of the batch fit any stored type and are not compared (see + ``_without_null_types``). The batch's types are first converted exactly as the Delta writer converts them (see ``_as_delta_writer_stores``), then both sides are compared with extension types reduced to storage, large-normalized and nullability ignored, so a @@ -1321,13 +1364,12 @@ def _incompatible_append_column( compared types differ, or ``None`` when the batch can be appended. """ stored = _stored_arrow_schema(delta_table) + compared = [ + (f.name, _without_null_types(_comparable_storage_type(f.type))) + for f in batch.schema + ] incoming = _as_delta_writer_stores( - pa.schema( - [ - pa.field(f.name, _comparable_storage_type(f.type)) - for f in batch.schema - ] - ) + pa.schema([pa.field(name, t) for name, t in compared if t is not None]) ) for field in incoming: index = stored.get_field_index(field.name) diff --git a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md index 557b74e3d..44e2689f4 100644 --- a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md +++ b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md @@ -88,3 +88,11 @@ writer's own conversion, `_stored_arrow_schema` reads the stored schema without Guard: `tests/test_databases/test_delta_append_types.py` (every Arrow/logical type, identical schema, same view and reopened; 40 cases failed before the fix). Deferred: ExceptionGroup-wrapped store errors -> ITL-918; fixed_size_list merge failure (also on main) -> ITL-916 (strict xfail). + +## CodeRabbit round on ffe44481 + +Regression: rows whose column (or nested field / list item) is `null`-typed could not be appended +to a typed column; `main` accepts them. Fix: `_without_null_types` leaves `null` parts out of the +comparison (a `null` struct field is dropped; a column holding `null` list items or map entries is +not compared). Tests in `test_delta_append_types.py` (three null cases, all passing on `main`; +a `null` field does not hide a changed sibling). diff --git a/tests/test_databases/test_delta_append_types.py b/tests/test_databases/test_delta_append_types.py index f0a242d12..31778811b 100644 --- a/tests/test_databases/test_delta_append_types.py +++ b/tests/test_databases/test_delta_append_types.py @@ -28,6 +28,7 @@ from orcapod.contexts import get_default_context from orcapod.databases import DeltaTableDatabase +from orcapod.errors import InputValidationError from orcapod.logical_types.directory_type import Directory from orcapod.logical_types.file_type import File @@ -258,3 +259,52 @@ def test_identical_schema_appends( assert back is not None assert back.num_rows == 2 assert set(back.column("rid").to_pylist()) == {b"a", b"b"} + + +@pytest.mark.parametrize( + ("stored_type", "incoming_type"), + [ + (pa.int64(), pa.null()), + (pa.struct([("a", pa.int64())]), pa.struct([("a", pa.null())])), + (pa.large_list(pa.int64()), pa.large_list(pa.null())), + ], + ids=["int64<-null", "struct-field<-null", "list-element<-null"], +) +def test_null_typed_values_append_to_a_typed_column( + tmp_path: Path, stored_type: pa.DataType, incoming_type: pa.DataType +) -> None: + """Rows whose column (or nested field) is ``null``-typed append to a typed column.""" + db = DeltaTableDatabase(base_path=tmp_path) + write(db, table(stored_type, b"a", nullable=True)) + nulls = pa.Table.from_arrays( + [pa.array([b"b"], pa.large_binary()), pa.array([None], incoming_type)], + names=["rid", "x"], + ) + write(db, nulls) + + back = DeltaTableDatabase(base_path=tmp_path).get_all_records( + PATH, record_id_column="rid" + ) + assert back is not None + assert back.num_rows == 2 + + +def test_null_field_does_not_hide_a_changed_sibling(tmp_path: Path) -> None: + """A ``null`` struct field is skipped, but a changed sibling field is refused.""" + db = DeltaTableDatabase(base_path=tmp_path) + write( + db, + table(pa.struct([("a", pa.int64()), ("b", pa.int64())]), b"a", nullable=True), + ) + changed = pa.Table.from_arrays( + [ + pa.array([b"b"], pa.large_binary()), + pa.array( + [{"a": None, "b": 2.5}], + pa.struct([("a", pa.null()), ("b", pa.float64())]), + ), + ], + names=["rid", "x"], + ) + with pytest.raises(InputValidationError, match="'x'"): + write(db, changed) From 11efe638997ba281ec22bdf8787cae80be90a0d5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 07:56:37 -0800 Subject: [PATCH 25/30] fix(databases): a null struct field inside a list no longer hides its siblings (ITL-884) CodeRabbit round on 51dc9050: _without_null_types dropped a whole list or map when a struct item held a null field, so a changed sibling field there was never compared. The reduced container is kept when comparable fields remain, so such a change is refused with InputValidationError naming both types. The nested-null append tests now build {"a": None} and [None] (not an outer null) and assert the stored values. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 44 +++++++++----- ...1-11-itl-884-no-message-only-parameters.md | 7 +++ .../test_databases/test_delta_append_types.py | 59 ++++++++++++------- 3 files changed, 75 insertions(+), 35 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 48ac855a4..8c34c1f04 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -75,9 +75,10 @@ def _without_null_types(arrow_type: pa.DataType) -> pa.DataType | None: """Return ``arrow_type`` without the parts typed ``null``, or ``None`` if none is left. A ``null``-typed value fits a column of any type, and Delta has no ``null`` - type to compare it with, so the append check leaves it out: a ``null`` struct - field is dropped, and a type that is ``null`` or holds ``null`` list items or - map keys or values is not compared at all. + type to compare it with, so the append check leaves it out at every depth: a + ``null`` struct field is dropped (its siblings are still compared, inside lists + and maps too), and a type that is ``null``, or whose list items or map keys or + values are entirely ``null``, is not compared. Args: arrow_type: A type already reduced by ``_comparable_storage_type``. @@ -94,21 +95,34 @@ def _without_null_types(arrow_type: pa.DataType) -> pa.DataType | None: if (t := _without_null_types(f.type)) is not None ] return pa.struct(kept) if kept else None - if ( - pa.types.is_list(arrow_type) - or pa.types.is_large_list(arrow_type) - or pa.types.is_fixed_size_list(arrow_type) - ): - if _without_null_types(arrow_type.value_type) != arrow_type.value_type: + if pa.types.is_large_list(arrow_type) or pa.types.is_list(arrow_type): + item = _without_null_types(arrow_type.value_type) + if item is None: return None - return arrow_type + field = pa.field(arrow_type.value_field.name, item) + return ( + pa.large_list(field) + if pa.types.is_large_list(arrow_type) + else pa.list_(field) + ) + if pa.types.is_fixed_size_list(arrow_type): + item = _without_null_types(arrow_type.value_type) + if item is None: + return None + return pa.list_( + pa.field(arrow_type.value_field.name, item), arrow_type.list_size + ) if pa.types.is_map(arrow_type): - if ( - _without_null_types(arrow_type.key_type) != arrow_type.key_type - or _without_null_types(arrow_type.item_type) != arrow_type.item_type - ): + key = _without_null_types(arrow_type.key_type) + value = _without_null_types(arrow_type.item_type) + if key is None or value is None: return None - return arrow_type + # pyarrow accepts key and item Fields here; its stubs only list types. + return pa.map_( # pyright: ignore[reportCallIssue] + pa.field(arrow_type.key_field.name, key, nullable=False), + pa.field(arrow_type.item_field.name, value), + keys_sorted=arrow_type.keys_sorted, + ) return arrow_type diff --git a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md index 44e2689f4..a0def4030 100644 --- a/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md +++ b/superpowers/plans/2026-11-11-itl-884-no-message-only-parameters.md @@ -96,3 +96,10 @@ to a typed column; `main` accepts them. Fix: `_without_null_types` leaves `null` comparison (a `null` struct field is dropped; a column holding `null` list items or map entries is not compared). Tests in `test_delta_append_types.py` (three null cases, all passing on `main`; a `null` field does not hide a changed sibling). +- Round on 51dc9050: `_without_null_types` keeps a list or map whose items still have comparable + fields (a changed sibling of a `null` field inside `list` is refused); the nested-null + tests now build `{"a": None}` / `[None]` and assert the stored value. A `null` field inside + `list` cannot be appended by Delta's merge itself (also on main), so it is not a case + of the append test. CI's "Run Tests" / "Objective" / "PostgreSQL" / license-check workflows fail + on every branch since ~14:25Z: the runners cannot resolve security.ubuntu.com (apt install of + graphviz); "Python — Required CI" passes. diff --git a/tests/test_databases/test_delta_append_types.py b/tests/test_databases/test_delta_append_types.py index 31778811b..489388596 100644 --- a/tests/test_databases/test_delta_append_types.py +++ b/tests/test_databases/test_delta_append_types.py @@ -262,22 +262,28 @@ def test_identical_schema_appends( @pytest.mark.parametrize( - ("stored_type", "incoming_type"), + ("stored_type", "incoming_type", "value"), [ - (pa.int64(), pa.null()), - (pa.struct([("a", pa.int64())]), pa.struct([("a", pa.null())])), - (pa.large_list(pa.int64()), pa.large_list(pa.null())), + (pa.int64(), pa.null(), None), + (pa.struct([("a", pa.int64())]), pa.struct([("a", pa.null())]), {"a": None}), + (pa.large_list(pa.int64()), pa.large_list(pa.null()), [None]), + # A null field inside list is not here: Delta's own merge cannot + # coerce it (also on main), whatever the append check decides. + ], + ids=[ + "int64<-null", + "struct-field<-null", + "list-item<-null", ], - ids=["int64<-null", "struct-field<-null", "list-element<-null"], ) def test_null_typed_values_append_to_a_typed_column( - tmp_path: Path, stored_type: pa.DataType, incoming_type: pa.DataType + tmp_path: Path, stored_type: pa.DataType, incoming_type: pa.DataType, value: Any ) -> None: """Rows whose column (or nested field) is ``null``-typed append to a typed column.""" db = DeltaTableDatabase(base_path=tmp_path) write(db, table(stored_type, b"a", nullable=True)) nulls = pa.Table.from_arrays( - [pa.array([b"b"], pa.large_binary()), pa.array([None], incoming_type)], + [pa.array([b"b"], pa.large_binary()), pa.array([value], incoming_type)], names=["rid", "x"], ) write(db, nulls) @@ -287,24 +293,37 @@ def test_null_typed_values_append_to_a_typed_column( ) assert back is not None assert back.num_rows == 2 + rows = dict(zip(back.column("rid").to_pylist(), back.column("x").to_pylist())) + assert rows[b"b"] == value -def test_null_field_does_not_hide_a_changed_sibling(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("stored_type", "incoming_type", "value"), + [ + ( + pa.struct([("a", pa.int64()), ("b", pa.int64())]), + pa.struct([("a", pa.null()), ("b", pa.float64())]), + {"a": None, "b": 2.5}, + ), + ( + pa.large_list(pa.struct([("a", pa.int64()), ("b", pa.int64())])), + pa.large_list(pa.struct([("a", pa.null()), ("b", pa.float64())])), + [{"a": None, "b": 2.5}], + ), + ], + ids=["struct", "list"], +) +def test_null_field_does_not_hide_a_changed_sibling( + tmp_path: Path, stored_type: pa.DataType, incoming_type: pa.DataType, value: Any +) -> None: """A ``null`` struct field is skipped, but a changed sibling field is refused.""" db = DeltaTableDatabase(base_path=tmp_path) - write( - db, - table(pa.struct([("a", pa.int64()), ("b", pa.int64())]), b"a", nullable=True), - ) + write(db, table(stored_type, b"a", nullable=True)) changed = pa.Table.from_arrays( - [ - pa.array([b"b"], pa.large_binary()), - pa.array( - [{"a": None, "b": 2.5}], - pa.struct([("a", pa.null()), ("b", pa.float64())]), - ), - ], + [pa.array([b"b"], pa.large_binary()), pa.array([value], incoming_type)], names=["rid", "x"], ) - with pytest.raises(InputValidationError, match="'x'"): + with pytest.raises(InputValidationError, match="'x'") as info: write(db, changed) + assert "int64" in str(info.value) + assert "double" in str(info.value) From f9b2c6c3ca8e00953814ae3650e5d0770c6b0385 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 11:45:22 -0800 Subject: [PATCH 26/30] docs(plans): plan for stacking ITL-864 on PR #290 Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-11-itl-864-stack-on-290.md | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 superpowers/plans/2026-11-11-itl-864-stack-on-290.md diff --git a/superpowers/plans/2026-11-11-itl-864-stack-on-290.md b/superpowers/plans/2026-11-11-itl-864-stack-on-290.md new file mode 100644 index 000000000..42a09e133 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-864-stack-on-290.md @@ -0,0 +1,48 @@ +# ITL-864 / PR #293: rebuild on top of PR #290 (ITL-884) and main + +Date: 2026-11-11. Coordinator task task_0f94cf0766e2. Stack approved by the developer: +#290 -> #293 -> #295 -> Delta half of ITL-919. Previous plan: +`superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md`. + +**Goal:** the branch contains main (c8efc092) and #290 (head 11efe638) by merge commits, +with both behaviours intact: +- #290: an existing table that cannot be loaded or read raises `StoreReadError`, never + "no rows"; `_read_delta_table(delta_table, filters=None, expression=None)` raises the + private `_DeltaTableReadError` and the caller wraps it with `self._read_failed(record_path)`. +- #293: every read method returns stored rows plus pending rows whose id is not stored, + through any handle of the `at()` family (shared table / ids / dirty caches). + +No public API, hash, stored-schema or dependency change. + +## Tasks + +1. `git merge origin/main` (conflicts expected only where #293 touches the same code as + main; resolve, commit, push WIP). +2. `git merge origin/eywalker/itl-884-delta-read-failures-are-returned-as-no-rows`. + Resolution rules in `delta_lake_databases.py`: + - `_refresh_existing_ids_cache`: #290's shape (try only around the load/scan, raise + `self._read_failed(record_path) from e`), with #293's `set[bytes]` typing and the + cache left dirty on failure. + - `get_all_records`: #290's raise on a failed scan + #293's `_pending_not_stored` / + `_combine_stored_and_pending`. + - `get_records_with_column_value`: #293's one-side-lacks-the-column logic; the stored + read uses #290's `except _DeltaTableReadError -> StoreReadError`; filter errors + propagate as is. + - `get_record_by_id`: #293's "pending and not stored" check + #290's wrapping. + - `get_records_by_ids`: drop #293's `except Exception: return None` (that is exactly + what #290 forbids); `_pending_not_stored` propagates `StoreReadError`; the read uses + #290's wrapping. + - CHANGELOG: one `[0.3.0] — unreleased` section; DESIGN_ISSUES: all entries. +3. Tests: run #290's and #293's tests together. Add interaction tests (public interface, + in `tests/test_databases/test_delta_read_your_writes.py`): + - unreadable stored table with pending rows queued through a sibling handle: every read + method raises `StoreReadError` (pending rows are not returned alone as if the table + were empty), and the pending rows stay queued (nothing dropped); + - a failed stored-id refresh raises `StoreReadError` on every read method, and a later + read after the table is readable again returns stored + pending. + Show each new test's result before any code change it needs. +4. Full suite on 3.11/3.12/3.13, ruff check, ruff format --check, basedpyright (exit code), + golden hashes unchanged. +5. Push; PR description gets the "Stacked on #290" note; CI + CodeRabbit; worker_done. + +## Progress notes (successor-readable) From ba396d419fc0dd4ec8e1d544cce6e06ac6244621 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 11:48:22 -0800 Subject: [PATCH 27/30] test(databases): Delta reads raise StoreReadError with pending rows queued (ITL-864) After merging #290 (ITL-884), a failed stored-id refresh is a failed stored read: every read method raises StoreReadError instead of the backend error or None. Update #293's two refresh tests to that rule and add an interaction test: with a stored table that cannot be read and pending rows queued through a sibling handle, each read raises StoreReadError (never answers from the pending rows alone) and the pending rows are still returned and flushed once the table is readable again. Co-Authored-By: Claude Opus 5.5 --- .../test_delta_read_your_writes.py | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index 0549cb639..c6da7ca2f 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -17,6 +17,7 @@ import pytest from orcapod.databases import DeltaTableDatabase +from orcapod.errors import StoreReadError PATH = ("table",) SCOPE = "scope" @@ -220,23 +221,66 @@ def test_failed_id_refresh_is_retried_by_the_next_read( db.add_records(PATH, _rows(["b"], [2]), flush=True) db.add_records(PATH, _rows(["a", "b"], [1, 20])) - with pytest.raises(OSError, match="transient storage error"): + # The failed refresh is a failed stored read (ITL-884): never "no stored ids". + with pytest.raises(StoreReadError) as info: _ = db.get_all_records(PATH) + assert isinstance(info.value.__cause__, OSError) assert _values(READS[method](db, PATH)) == EXPECTED[method] @pytest.mark.usefixtures("fail_id_refresh_once") -def test_get_records_by_ids_handles_failed_id_refresh_like_a_failed_read( +def test_get_records_by_ids_raises_on_failed_id_refresh( stored_and_pending: DeltaTableDatabase, ) -> None: - """A failed stored-ID refresh follows the method's stored-read error behaviour.""" + """A failed stored-ID refresh raises ``StoreReadError``, never "no records" (ITL-884).""" db = stored_and_pending - assert db.get_records_by_ids(PATH, [b"a", b"c"]) is None + with pytest.raises(StoreReadError) as info: + _ = db.get_records_by_ids(PATH, [b"a", b"c"]) + assert isinstance(info.value.__cause__, OSError) assert _values(db.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] +@pytest.mark.parametrize("cache", ["cold", "warm"]) +@pytest.mark.parametrize("method", list(READS)) +def test_unreadable_table_with_sibling_pending_rows_raises_and_drops_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: str, cache: str +) -> None: + """A read never answers from pending rows alone when the stored table is unreadable. + + ``b`` is stored and ``a`` is pending through a sibling handle. While the stored + table cannot be read, every read method raises ``StoreReadError`` (ITL-884) + rather than returning only the pending rows (ITL-864); once it is readable again + the same read returns both rows, so nothing was dropped from the pending batch. + """ + DeltaTableDatabase(tmp_path).at(SCOPE).add_records( + PATH, _rows(["b"], [2]), flush=True + ) + root = DeltaTableDatabase(tmp_path) + reader, writer = root.at(SCOPE), root.at(SCOPE) + if cache == "warm": + # The family has loaded the table and its stored ids before the failure. + assert _values(reader.get_all_records(PATH)) == [2] + assert reader.get_record_by_id(PATH, b"a") is None + writer.add_records(PATH, _rows(["a"], [1])) + + def unreadable(*_args: object, **_kwargs: object) -> pa.Table: + """Fail as a storage outage would.""" + raise OSError("storage unavailable") + + with monkeypatch.context() as patch: + patch.setattr(deltalake.DeltaTable, "to_pyarrow_dataset", unreadable) + patch.setattr(deltalake.DeltaTable, "to_pyarrow_table", unreadable) + with pytest.raises(StoreReadError) as info: + _ = READS[method](reader, PATH) + assert isinstance(info.value.__cause__, OSError) + + assert _values(READS[method](reader, PATH)) == EXPECTED[method] + root.flush() + assert _values(READS[method](reader, PATH)) == EXPECTED[method] + + @pytest.mark.parametrize("method", list(READS)) def test_fresh_handle_drops_pending_rows_with_stored_ids( tmp_path: Path, method: str From c35bdf855a472af64290086637390f855f89ea32 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 11:49:40 -0800 Subject: [PATCH 28/30] docs(plans): progress notes for stacking ITL-864 on #290 Co-Authored-By: Claude Opus 5.5 --- superpowers/plans/2026-11-11-itl-864-stack-on-290.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/superpowers/plans/2026-11-11-itl-864-stack-on-290.md b/superpowers/plans/2026-11-11-itl-864-stack-on-290.md index 42a09e133..0e6a703b8 100644 --- a/superpowers/plans/2026-11-11-itl-864-stack-on-290.md +++ b/superpowers/plans/2026-11-11-itl-864-stack-on-290.md @@ -46,3 +46,15 @@ No public API, hash, stored-schema or dependency change. 5. Push; PR description gets the "Stacked on #290" note; CI + CodeRabbit; worker_done. ## Progress notes (successor-readable) +- Done: merged origin/main c8efc092 (7ba60eb4; conflict DESIGN_ISSUES.md only: kept main's + "Newcomer papercuts" entry and the ITL-864 entry, ours last). Merged #290 11efe638 + (a2ac8d9e; conflicts in `_refresh_existing_ids_cache`, `get_records_with_column_value`, + `get_records_by_ids`, resolved per the rules above; CHANGELOG/DESIGN_ISSUES auto-merged; + ITL-864 Fixed entry added to `[0.3.0] — unreleased` in the merge commit). +- Done: ba396d41 updates #293's two refresh tests to StoreReadError and adds + `test_unreadable_table_with_sibling_pending_rows_raises_and_drops_nothing` (verified to fail + when #293's old `except Exception: return None` is put back in `get_records_by_ids`). +- ITL-923 ephemeral re-run repro (scratchpad script): main and #290 lose records on both + orchestrators; this branch fixes sync (10 stored, 5 read) but async still stores 5 and reads + None. Out of scope; reported to the coordinator. +- Next: full suite 3.11/3.12/3.13, PR description "Stacked on #290", CI, CodeRabbit, worker_done. From 06ecbd5c576dd5e1534deb3862a4d7231f5769f2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 12:09:04 -0800 Subject: [PATCH 29/30] fix(databases): pending-row filter costs O(pending) and keeps null ids (ITL-864) _pending_not_stored built an Arrow value set from the whole stored-id set on every read. Build the keep-mask from the pending ids against the cached set instead. A pending row whose record id is null is now kept: Delta's merge matches ids with `=`, so a flush inserts it even when a stored row's id is null too, and pc.is_in matched null to null. Addresses CodeRabbit review comments on PR #293. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 15 +++++++----- .../test_delta_read_your_writes.py | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 8c701812b..f783dab9f 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -926,12 +926,15 @@ def _pending_not_stored(self, record_path: tuple[str, ...]) -> pa.Table | None: return None stored_ids = self._get_existing_ids(record_path) if stored_ids: - id_type = pending.schema.field(self.RECORD_ID_COLUMN).type - is_stored = pc.is_in( - pending[self.RECORD_ID_COLUMN], - value_set=pa.array(list(stored_ids), type=id_type), - ) - pending = pending.filter(pc.invert(is_stored)) + # The mask is built from the pending ids against the cached set, so its + # cost follows the pending batch, not the stored table. A null id is + # never matched by the merge (``null = null`` is not true), so its row + # is kept. + keep = [ + record_id is None or record_id not in stored_ids + for record_id in pending[self.RECORD_ID_COLUMN].to_pylist() + ] + pending = pending.filter(pa.array(keep, type=pa.bool_())) return pending if pending.num_rows > 0 else None def _combine_stored_and_pending( diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index c6da7ca2f..59db96a0b 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -485,3 +485,26 @@ def test_query_on_column_neither_side_has_raises(tmp_path: Path, flushed: bool) with pytest.raises(pa.ArrowInvalid, match="No match for FieldRef"): _ = db.get_records_with_column_value(PATH, {"y": 1}) + + +def test_pending_row_with_null_id_reads_back_like_after_flush(tmp_path: Path) -> None: + """A pending row with a null record id is returned when a stored row also has one. + + Delta's merge matches ids with ``=``, and ``null = null`` is not true, so a flush + inserts a pending row whose id is null even when a stored row's id is null too. + """ + null_id = pa.table( + { + "__record_id": pa.array([None], type=pa.large_binary()), + "v": pa.array([5], type=pa.int64()), + "g": pa.array([0], type=pa.int64()), + } + ) + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, pa.concat_tables([_rows(["a"], [1]), null_id]), flush=True) + db.add_records(PATH, null_id.set_column(1, "v", pa.array([6], type=pa.int64()))) + + before = _values(db.get_all_records(PATH)) + db.flush() + + assert before == _values(db.get_all_records(PATH)) == [1, 5, 6] From 97aa5a0959d0c80074307225b637ea110725c554 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 22:07:20 -0800 Subject: [PATCH 30/30] docs(changelog): move the ITL-864 entry to a changelog fragment (ITL-961) Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 15 --------------- changelog.d/ITL-864.md | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 changelog.d/ITL-864.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 964e588bf..b9688f8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,21 +61,6 @@ A write that fails because the stored table cannot be read raises `StoreReadErro `StoreWriteError`. When read and write failures happen in the same run, both orchestrators raise the error of the node that comes first in the pipeline (ITL-884 #290). -#### Delta reads return every row the database's handles wrote (ITL-864) - -Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` could -return only some of the rows (1 of 2 or 3) on both orchestrators. Handles made with `at()` share -their pending writes, and any handle's flush writes all of them, but each handle kept its own -loaded copy of the table, so a sibling kept reading the version it had loaded before the flush. -The loaded tables and the stored-id sets are now shared by every handle of one root. Every read -method (`get_all_records`, `get_record_by_id`, `get_records_by_ids`, -`get_records_with_column_value`) returns what a flush would leave: the stored rows plus the -pending rows whose record id is not stored, or `None` when there are none. Before, -`get_records_by_ids` ignored pending rows, `get_records_with_column_value` ignored stored rows -while a pending batch existed, and a pending row whose id was already stored could be returned -twice or instead of the stored row. Writes made by another process or another root on the same -path are still seen only after the table is reloaded. (ITL-864 #293) - #### A failed store write fails the run (ITL-878) A failed store write fails the run. `PipelineJob.run()` used to return normally when a node's diff --git a/changelog.d/ITL-864.md b/changelog.d/ITL-864.md new file mode 100644 index 000000000..cfc4bc3dc --- /dev/null +++ b/changelog.d/ITL-864.md @@ -0,0 +1,16 @@ +### Fixed + +#### Delta reads return every row the database's handles wrote (ITL-864) + +Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` could +return only some of the rows (1 of 2 or 3) on both orchestrators. Handles made with `at()` share +their pending writes, and any handle's flush writes all of them, but each handle kept its own +loaded copy of the table, so a sibling kept reading the version it had loaded before the flush. +The loaded tables and the stored-id sets are now shared by every handle of one root. Every read +method (`get_all_records`, `get_record_by_id`, `get_records_by_ids`, +`get_records_with_column_value`) returns what a flush would leave: the stored rows plus the +pending rows whose record id is not stored, or `None` when there are none. Before, +`get_records_by_ids` ignored pending rows, `get_records_with_column_value` ignored stored rows +while a pending batch existed, and a pending row whose id was already stored could be returned +twice or instead of the stored row. Writes made by another process or another root on the same +path are still seen only after the table is reloaded. (ITL-864 #293)