From b2a3a69801d3bff2bcc07ff6d1402cde7e092ea8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 20:49:13 -0800 Subject: [PATCH 01/73] test(databases): failing regression tests for Delta read failures reported as no rows (ITL-884) Adds StoreReadError and tests showing that an unreadable Delta table reads as None, that a table which fails to load can be overwritten, that a changed column type is appended (silently cast or left unreadable), and that both orchestrators finish a run over an unreadable result table without raising. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/errors.py | 52 +++ .../2026-11-10-itl-884-delta-read-failures.md | 189 ++++++++++ .../test_delta_read_failures.py | 340 ++++++++++++++++++ .../test_pipeline/test_store_read_failure.py | 133 +++++++ 4 files changed, 714 insertions(+) create mode 100644 superpowers/plans/2026-11-10-itl-884-delta-read-failures.md create mode 100644 tests/test_databases/test_delta_read_failures.py create mode 100644 tests/test_pipeline/test_store_read_failure.py diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 81c653627..d27e850fc 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -195,3 +195,55 @@ def __init__( f"node={node_identity_path!r} " f"cached_content_hash={cached_content_hash!r}" ) + + +class StoreReadError(RuntimeError): + """Raised when a stored table exists but could not be read. + + A table that does not exist reads as empty. A table that exists but cannot be + read (a missing or damaged data file, a log the backend cannot load, columns + whose stored data no longer matches the table schema) raises this error + instead, so stored results never look dropped. Nothing is recomputed or + overwritten. A failed store read always fails the run, whatever the + orchestrator's ``error_policy``. + + The backend's own exception is chained as ``__cause__``. + + Attributes: + table: Location of the table that could not be read (a directory or a + cloud URI). + record_path: Path of the table, including the database view's prefix. + """ + + def __init__(self, table: str, record_path: tuple[str, ...]) -> None: + self.table = table + self.record_path = tuple(record_path) + super().__init__(table, self.record_path) + + def __str__(self) -> str: + cause = self.__cause__ + reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" + return ( + f"Could not read the stored table at {self.table!r}{reason}. The table " + "exists, so its rows are not treated as missing: nothing was recomputed " + "or overwritten." + ) + + @classmethod + def in_group(cls, group: BaseExceptionGroup) -> "list[StoreReadError]": + """Return every ``StoreReadError`` in an exception group, nested ones included. + + Args: + group: An exception group, such as the one an ``asyncio.TaskGroup`` + raises. + + Returns: + The ``StoreReadError`` leaves in the order the group holds them. + """ + found: "list[StoreReadError]" = [] + for exc in group.exceptions: + if isinstance(exc, BaseExceptionGroup): + found.extend(cls.in_group(exc)) + elif isinstance(exc, cls): + found.append(exc) + return found diff --git a/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md new file mode 100644 index 000000000..71ef7fa94 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-884-delta-read-failures.md @@ -0,0 +1,189 @@ +# ITL-884: Delta read failures are never reported as "no rows" — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A Delta table that exists but cannot be read raises `StoreReadError`, never an +empty result. A flush that would make an existing table unreadable, or silently cast the +stored column type, is refused before anything is written. + +**Architecture:** Every read path in `DeltaTableDatabase` separates "table absent" +(`TableNotFoundError` → `None`, unchanged) from "table present but unreadable" (raise +`StoreReadError` naming the table URI, with the backend exception chained). `_flush_by_key` +creates a table only on `TableNotFoundError`. Before it merges into an existing table, it compares +the incoming batch's column types with the stored Delta schema. Both sides go through Delta's +own type mapping, with extension types stripped to storage, large-normalized and +nullability-blind. A difference raises `InputValidationError` naming the table, column and both +types. A store read failure fails the run whatever `error_policy` is (same shape as PR #287's +`StoreWriteError`). + +**Tech Stack:** deltalake 1.0.2, pyarrow, pytest. + +Approved by the coordinator (2026-11-10): contract as proposed. D1 adds a sibling +`StoreReadError(RuntimeError)`, and the append refusal raises `InputValidationError`. D2 is +option A: re-raise in the two per-row handlers and unwrap in the async orchestrator, with the +same shape as #287. D3 makes the comparison nullability-blind, plus a test that pins +nullability-only behaviour. D4 fixes the `DeltaProtocolError` → overwrite path. + +No hash, path or stored-schema changes. + +--- + +## File map + +- `src/orcapod/errors.py` gets `StoreReadError` (`table`, `record_path`, `__str__` from + `__cause__`, `in_group`). +- `src/orcapod/databases/delta_lake_databases.py` changes the read paths and + `_flush_by_key`, and adds `_require_compatible_append` and the module helper + `_comparable_storage_type`. +- `src/orcapod/core/nodes/function_node.py` (`execute` per-row handler) and + `src/orcapod/core/function_pod.py` (async `process_one`) both get + `except StoreReadError: ...; raise`. +- `src/orcapod/pipeline/async_orchestrator.py` gets `except* StoreReadError` and raises the + first leaf. +- Tests: `tests/test_databases/test_delta_read_failures.py` and + `tests/test_pipeline/test_store_read_failure.py`. +- Docs: `docs/concepts/function-pods.md` gets an admonition, `CHANGELOG.md` gets Fixed and + Breaking entries, and `DESIGN_ISSUES.md` gets a new entry at the end. + +## Task 1: Failing regression tests (database level) + +`tests/test_databases/test_delta_read_failures.py`, public `DeltaTableDatabase` API only: + +- `unreadable_table` fixture: write 2 rows, flush, delete the table's `*.parquet` files (the + log stays intact). Use a fresh `DeltaTableDatabase` instance for the reads. +- `test_get_all_records_raises` checks `StoreReadError`: `exc.table` ends with the table + directory, `str(exc)` contains the URI, and `__cause__` is set. +- `test_get_records_by_ids_raises`, `test_get_record_by_id_raises`, + `test_get_records_with_column_value_raises`, and + `test_add_records_skip_duplicates_raises` (existing-ids refresh). Each raises + `StoreReadError`. +- `test_absent_table_reads_return_none` covers all four reads on a path never written: + `None`, `table_exists` is False. +- `test_unloadable_log_raises_on_read_and_flush_never_overwrites`: the table's log names an + unsupported reader feature (the load fails). Reads raise `StoreReadError`. `add_records(..., + flush=True)` raises, the log files and parquet files are byte-identical afterwards, and no + new log version exists. +- `test_protocol_error_on_load_is_not_treated_as_absent`: monkeypatch + `deltalake.DeltaTable` to raise `DeltaProtocolError` for this table (the error deltalake + 1.0.2 does not raise naturally from a local table). Flush raises and the stored files are + untouched. The comment explains the monkeypatch. +- Append refusal: + - `test_castable_type_change_is_refused`: int64 stored, float64 appended. Raises (flush + wraps it in `RuntimeError`, so assert that `InputValidationError` is the `__cause__`). + The message names the column `x`, `int64` and `double`. A re-read gives the original row + only, with type int64. + - `test_narrowing_change_is_refused`: int64 stored, int32 appended. + - `test_extension_storage_change_is_refused`: `ext` stored, `ext>` + appended. Refused, and the table is still readable afterwards (1 row). + - Legitimate cases still pass: identical schema; `string` vs `large_string` (and + `list` vs `large_list`); a new column; a missing column; an extension type + with unchanged storage (2 rows read back). + - `test_nullability_only_difference_is_accepted`: a nested non-null struct field vs a + nullable one is accepted. This pins today's nullability-blind behaviour (ITL-842 audit). + It also records what Delta stores for nullability after the merge, as an assertion on + the read-back schema. + +Run `uv run pytest tests/test_databases/test_delta_read_failures.py -v`. Expected: the read +tests FAIL (`None` / no raise / raw errors), and the refusal tests FAIL (the write is +accepted). + +## Task 2: Failing regression tests (pipeline level, both orchestrators) + +`tests/test_pipeline/test_store_read_failure.py`, parametrized over +`SyncPipelineOrchestrator()` and `AsyncPipelineOrchestrator()`: + +- Run `source -> FunctionPod(double) label="dbl"` on a `DeltaTableDatabase`, then delete + the parquet files of the `_result` table. +- `test_unreadable_result_table_fails_the_run`: a fresh store and job, `job.run(orchestrator=...)` + raises `StoreReadError`, and the function is not called. +- `test_unreadable_result_table_fails_the_run_under_continue`: also delete the pipeline + (tag) table, so that Phase 1 sees no tag rows and the per-row lookup is the read that fails. + With `error_policy="continue"` it still raises `StoreReadError` and makes 0 calls. +- `test_absent_tables_still_compute`: control case. Delete both tables entirely, then the + run computes all 3 rows. + +Run the tests and expect FAIL. + +## Task 3: `StoreReadError` + +Add the class to `src/orcapod/errors.py`: + +```python +class StoreReadError(RuntimeError): + """Raised when a stored table exists but could not be read. ...""" + + def __init__(self, table: str, record_path: tuple[str, ...]) -> None: + self.table = table + self.record_path = tuple(record_path) + super().__init__(table, self.record_path) + + def __str__(self) -> str: + cause = self.__cause__ + reason = f": {type(cause).__name__}: {cause}" if cause is not None else "" + return ( + f"Could not read the stored table at {self.table!r}{reason}. The table exists, " + "so its rows are not treated as missing: nothing was recomputed or overwritten." + ) + + @classmethod + def in_group(cls, group: BaseExceptionGroup) -> "list[StoreReadError]": ... +``` + +## Task 4: Delta read paths + +- `_get_delta_table`: on `TableNotFoundError` return `None`. On any other exception, drop + the cache entry and `raise StoreReadError(table_uri, full_path) from exc`. +- Add the helper `_read_failed(record_path, exc) -> StoreReadError`, built from + `_get_table_uri(record_path)` and `self._path_prefix + record_path`. +- `get_all_records`: `except Exception as exc: raise self._read_failed(...) from exc` + (replaces the warning-and-skip). +- `get_records_by_ids`, `get_record_by_id`, `get_records_with_column_value`: narrow each + `try` to the read itself, then `raise self._read_failed(...) from exc` (replaces + `return None` / the raw re-raise). +- `_refresh_existing_ids_cache`: wrap the `to_pyarrow_table()` failure the same way. + +## Task 5: Delta flush path + +- In `_flush_by_key`, load only on `TableNotFoundError`, where `delta_table = None`. Any + other exception does `raise StoreReadError(table_uri, abs_path_components) from exc`. The + existing outer handler restores the pending batch. +- Before `merge`: `self._require_compatible_append(table_uri, delta_table, combined_table)`. +- `_comparable_storage_type(t)` is a module-level function. It strips extension types to + storage at every depth, rebuilds struct/list/fixed-size-list/map with nullable children + (comparison only, never stored), and recurses. +- `_require_compatible_append` builds the stored types from + `pa.schema(delta_table.schema().to_arrow())`. The incoming types go through + `deltalake.Schema.from_arrow(pa.schema([pa.field(name, _comparable_storage_type(type))]))` + and `.to_arrow()`. Both sides are passed through `normalize_to_large_types(..., + keep_null=True)` and `_comparable_storage_type`. Only columns present on both sides are + compared. On a mismatch it raises `InputValidationError(f"Cannot append to the stored table + {table_uri!r}: column {name!r} is stored as {stored} but the new rows have {incoming}. ...")`. + +Run the Task 1 and Task 2 tests and expect PASS. + +## Task 6: Run-level handling (same shape as #287) + +- `FunctionNode.execute` per-row handler: put `except StoreReadError as exc:` before + `except Exception`. It calls `ctx_obs.on_data_crash(...)` and `ctx_obs.on_node_end(...)`, + then `raise`. +- `FunctionPod` async `process_one`: put `except StoreReadError as exc:` before + `except Exception`. It calls `obs.on_data_crash(...)` and then `raise`. +- `AsyncPipelineOrchestrator._run_async`: add `except* StoreReadError as group: raise + StoreReadError.in_group(group)[0]`. + +Run the Task 2 tests and expect PASS. + +## Task 7: Docs, changelog, design issues, full suite + +- `docs/concepts/function-pods.md` gets a `!!! note "When the store cannot be read"` + admonition after "Re-runs compute only what is new". It has no python block. +- `CHANGELOG.md`: a Fixed entry (ITL-884: read failures raise `StoreReadError`; an unloadable + table is never overwritten). Breaking Changes: a castable column type change appended to an + existing Delta table is now refused with `InputValidationError` instead of being silently + cast to the stored type. +- `DESIGN_ISSUES.md`: one self-contained entry at the end with status resolved and a Fix note. +- Then run `uv run ruff format .`, `uv run ruff check .`, + `uv run --with basedpyright basedpyright` (exit code 0) and `uv run pytest tests/`. +- Commit in Conventional Commits style: `test(databases): ...` for the failing tests first, + then `fix(databases): ...`, then `docs: ...`. diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py new file mode 100644 index 000000000..597312934 --- /dev/null +++ b/tests/test_databases/test_delta_read_failures.py @@ -0,0 +1,340 @@ +"""Regression tests for ITL-884: a Delta table that cannot be read is never "no rows". + +Covers: +- A table that exists but cannot be read raises ``StoreReadError`` on every read + path, naming the table; a table that does not exist still reads as ``None``. +- A table whose log cannot be loaded is never overwritten by a flush. +- An append whose column types differ from the stored table is refused before + anything is written, naming the column and both types; legitimate appends + (same types in another layout, new or missing columns, unchanged extension + storage) still succeed. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path + +import deltalake +import pyarrow as pa +import pytest + +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import InputValidationError, StoreReadError + +PATH = ("results", "t") + + +def rows(ids: list[bytes], x: pa.Array, **extra: pa.Array) -> pa.Table: + """Build a table keyed by ``rid`` with an ``x`` column and optional extras.""" + return pa.table({"rid": pa.array(ids, pa.large_binary()), "x": x, **extra}) + + +def table_dir(base: Path) -> Path: + return base.joinpath(*PATH) + + +def snapshot(directory: Path) -> dict[str, bytes]: + """Every file under ``directory`` with its bytes, keyed by relative path.""" + return { + str(p.relative_to(directory)): p.read_bytes() + for p in sorted(directory.rglob("*")) + if p.is_file() + } + + +class _Ext(pa.ExtensionType): + """A minimal extension type standing in for a logical type's storage.""" + + def __init__(self, storage: pa.DataType) -> None: + super().__init__(storage, "orcapod.test.itl884") + + def __arrow_ext_serialize__(self) -> bytes: + return b"" + + @classmethod + def __arrow_ext_deserialize__( + cls, storage_type: pa.DataType, serialized: bytes + ) -> "_Ext": + return cls(storage_type) + + +def ext_array(values: list, storage: pa.DataType) -> pa.ExtensionArray: + return pa.ExtensionArray.from_storage(_Ext(storage), pa.array(values, storage)) + + +@pytest.fixture +def base(tmp_path: Path) -> Path: + return tmp_path / "store" + + +@pytest.fixture +def stored(base: Path) -> DeltaTableDatabase: + """A store holding two rows (``a``, ``b``) of ``x: int64`` at ``PATH``.""" + db = DeltaTableDatabase(base_path=base) + db.add_records( + PATH, + rows([b"a", b"b"], pa.array([1, 2], pa.int64())), + record_id_column="rid", + flush=True, + ) + return db + + +@pytest.fixture +def unreadable(base: Path, stored: DeltaTableDatabase) -> DeltaTableDatabase: + """A fresh view of a table whose log is intact but whose data files are gone.""" + data_files = list(table_dir(base).glob("*.parquet")) + assert data_files + for f in data_files: + f.unlink() + return DeltaTableDatabase(base_path=base) + + +def unsupported_reader_feature(base: Path) -> None: + """Make the table's log require a reader feature deltalake does not know.""" + log = table_dir(base) / "_delta_log" / "00000000000000000000.json" + actions = [json.loads(line) for line in log.read_text().splitlines()] + for action in actions: + if "protocol" in action: + action["protocol"] = { + "minReaderVersion": 3, + "minWriterVersion": 7, + "readerFeatures": ["orcapodMadeUpFeature"], + "writerFeatures": ["orcapodMadeUpFeature"], + } + log.write_text("\n".join(json.dumps(a) for a in actions)) + + +def assert_names_table(exc: StoreReadError, base: Path) -> None: + assert Path(exc.table) == table_dir(base) + assert exc.table in str(exc) + assert exc.__cause__ is not None + + +# --------------------------------------------------------------------------- +# Reads: unreadable table raises, absent table is None +# --------------------------------------------------------------------------- + +READS: dict[str, Callable[[DeltaTableDatabase], object]] = { + "get_all_records": lambda db: db.get_all_records(PATH), + "get_records_by_ids": lambda db: db.get_records_by_ids(PATH, [b"a"]), + "get_record_by_id": lambda db: db.get_record_by_id(PATH, b"a"), + "get_records_with_column_value": lambda db: db.get_records_with_column_value( + PATH, {"x": 1} + ), +} + + +class TestUnreadableTable: + @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) + def test_read_raises_store_read_error(self, base, unreadable, read): + with pytest.raises(StoreReadError) as info: + read(unreadable) + assert_names_table(info.value, base) + + def test_skip_duplicates_append_raises_store_read_error(self, base, unreadable): + # skip_duplicates reads the stored record ids before buffering the rows. + with pytest.raises(StoreReadError) as info: + unreadable.add_records( + PATH, + rows([b"c"], pa.array([3], pa.int64())), + record_id_column="rid", + skip_duplicates=True, + ) + assert_names_table(info.value, base) + + def test_table_still_exists(self, unreadable): + assert unreadable.table_exists(PATH) + + +class TestAbsentTable: + @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) + def test_read_returns_none(self, base, read): + db = DeltaTableDatabase(base_path=base) + assert read(db) is None + assert not db.table_exists(PATH) + + +# --------------------------------------------------------------------------- +# Flush never overwrites a table it could not load +# --------------------------------------------------------------------------- + + +class TestUnloadableTableIsNeverOverwritten: + def test_unsupported_reader_feature(self, base, stored): + unsupported_reader_feature(base) + before = snapshot(table_dir(base)) + db = DeltaTableDatabase(base_path=base) + + with pytest.raises(StoreReadError): + db.get_all_records(PATH) + with pytest.raises(RuntimeError) as info: + db.add_records( + PATH, + rows([b"c"], pa.array([3], pa.int64())), + record_id_column="rid", + flush=True, + ) + assert isinstance(info.value.__cause__, StoreReadError) + assert snapshot(table_dir(base)) == before + + def test_protocol_error_on_load_is_not_treated_as_absent( + self, base, stored, monkeypatch + ): + # deltalake 1.0.2 does not raise DeltaProtocolError for a local table we can + # build here, so the load is replaced by one that raises it for this table. + # Before ITL-884 the flush treated this error as "no table" and wrote the + # batch with mode="overwrite", replacing the stored rows. + real = deltalake.DeltaTable + target = table_dir(base) + + def failing_load(table_uri, *args, **kwargs): + if Path(str(table_uri)) == target: + raise deltalake.exceptions.DeltaProtocolError("simulated") + return real(table_uri, *args, **kwargs) + + before = snapshot(target) + db = DeltaTableDatabase(base_path=base) + monkeypatch.setattr(deltalake, "DeltaTable", failing_load) + with pytest.raises(RuntimeError) as info: + db.add_records( + PATH, + rows([b"c"], pa.array([3], pa.int64())), + record_id_column="rid", + flush=True, + ) + monkeypatch.undo() + + assert isinstance(info.value.__cause__, StoreReadError) + assert snapshot(target) == before + stored_rows = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert stored_rows is not None + assert stored_rows.num_rows == 2 + + +# --------------------------------------------------------------------------- +# Append refusal: a column's type differs from the stored table +# --------------------------------------------------------------------------- + + +def append(db: DeltaTableDatabase, table: pa.Table) -> None: + db.add_records(PATH, table, record_id_column="rid", flush=True) + + +def refusal(info: pytest.ExceptionInfo[RuntimeError]) -> InputValidationError: + """The ``InputValidationError`` behind a refused flush.""" + cause = info.value.__cause__ + assert isinstance(cause, InputValidationError), repr(info.value) + return cause + + +class TestChangedColumnTypeIsRefused: + @pytest.mark.parametrize( + ("new_type", "value", "named"), + [ + (pa.float64(), 2.0, "double"), + (pa.int32(), 2, "int32"), + (pa.large_string(), "2", "string"), + ], + ids=["int64->float64", "int64->int32", "int64->large_string"], + ) + def test_plain_type_change(self, base, stored, new_type, value, named): + with pytest.raises(RuntimeError) as info: + append(stored, rows([b"c"], pa.array([value], new_type))) + message = str(refusal(info)) + assert "'x'" in message + assert "int64" in message + assert named in message + assert str(table_dir(base)) in message + + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + assert back.num_rows == 2 + assert back.schema.field("x").type == pa.int64() + + def test_extension_storage_change(self, base): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) + + with pytest.raises(RuntimeError) as info: + append( + db, + rows([b"b"], ext_array([[1.0]], pa.large_list(pa.float32()))), + ) + message = str(refusal(info)) + assert "'x'" in message + assert "binary" in message + assert "list" in message + + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + assert back.num_rows == 1 + + +class TestLegitimateAppendsStillPass: + def read_back(self, base: Path) -> pa.Table: + back = DeltaTableDatabase(base_path=base).get_all_records(PATH) + assert back is not None + return back + + def test_identical_schema(self, base, stored): + append(stored, rows([b"c"], pa.array([3], pa.int64()))) + assert self.read_back(base).num_rows == 3 + + @pytest.mark.parametrize( + ("first", "second"), + [ + (pa.array(["a"], pa.string()), pa.array(["b"], pa.large_string())), + (pa.array(["a"], pa.large_string()), pa.array(["b"], pa.string())), + ( + pa.array([[1]], pa.list_(pa.int64())), + pa.array([[2]], pa.large_list(pa.int64())), + ), + (pa.array([b"a"], pa.binary()), pa.array([b"b"], pa.large_binary())), + ], + ids=["string->large", "large->string", "list->large", "binary->large"], + ) + def test_large_and_non_large_layouts(self, base, first, second): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], first)) + append(db, rows([b"b"], second)) + assert self.read_back(base).num_rows == 2 + + def test_new_column(self, base, stored): + append( + stored, + rows([b"c"], pa.array([3], pa.int64()), y=pa.array(["new"])), + ) + back = self.read_back(base) + assert back.num_rows == 3 + assert "y" in back.column_names + + def test_missing_column(self, base): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], pa.array([1], pa.int64()), y=pa.array(["old"]))) + append(db, rows([b"b"], pa.array([2], pa.int64()))) + assert self.read_back(base).num_rows == 2 + + def test_extension_type_with_unchanged_storage(self, base): + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) + append(db, rows([b"b"], ext_array([b"y"], pa.large_binary()))) + assert self.read_back(base).num_rows == 2 + + def test_nullability_only_difference_is_accepted(self, base): + # Pins today's behaviour: the append check ignores nullability (ITL-884, + # decision D3). Whether a nullability change should be refused belongs to + # the nullability audit, ITL-842. The read-back assertion records what + # Delta does on merge: the stored non-null child becomes nullable, so the + # rows written first now read back with a different (looser) type. + strict = pa.struct([pa.field("a", pa.int64(), nullable=False)]) + loose = pa.struct([pa.field("a", pa.int64(), nullable=True)]) + db = DeltaTableDatabase(base_path=base) + append(db, rows([b"a"], pa.array([{"a": 1}], strict))) + append(db, rows([b"b"], pa.array([{"a": 2}], loose))) + + back = self.read_back(base) + assert back.num_rows == 2 + assert back.schema.field("x").type == loose diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py new file mode 100644 index 000000000..4010ae2cb --- /dev/null +++ b/tests/test_pipeline/test_store_read_failure.py @@ -0,0 +1,133 @@ +"""Regression tests for ITL-884 at the job level: an unreadable store fails the run. + +Before ITL-884 a Delta result table that existed but could not be read was +returned as "no rows": the job warned that the data "may have been deleted +externally", swallowed the per-row read errors under ``error_policy="continue"``, +finished without raising, and the node showed no records. Both orchestrators +must now raise ``StoreReadError`` instead, and a table that does not exist must +still be computed. +""" + +from __future__ import annotations + +import shutil +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.function_pod import FunctionPod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import StoreReadError +from orcapod.pipeline import PipelineJob +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +_CALLS: list[int] = [] + + +def double(x: int) -> int: + _CALLS.append(x) + return 2 * x + + +ORCHESTRATORS: dict[str, Callable[[], object]] = { + "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), + "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), +} + + +@pytest.fixture(autouse=True) +def _reset_calls(): + _CALLS.clear() + yield + _CALLS.clear() + + +@pytest.fixture(params=list(ORCHESTRATORS)) +def make_orchestrator(request) -> Callable[[], object]: + return ORCHESTRATORS[request.param] + + +def build_job(store_path: Path) -> PipelineJob: + source = ArrowTableSource( + pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + tag_columns=["k"], + infer_nullable=True, + ) + job = PipelineJob(name="itl884", store=DeltaTableDatabase(base_path=store_path)) + with job: + FunctionPod(PythonDataFunction(double, output_keys="y"))(source, label="dbl") + return job + + +def table_dirs(store_path: Path) -> tuple[Path, Path]: + """The (result table, pipeline table) directories the job wrote.""" + dirs = {p.parent for p in store_path.rglob("*.parquet")} + result = [d for d in dirs if "_result" in d.parts] + pipeline = [d for d in dirs if "_result" not in d.parts] + assert len(result) == 1 and len(pipeline) == 1, sorted(dirs) + return result[0], pipeline[0] + + +def delete_data_files(table: Path) -> None: + for f in table.glob("*.parquet"): + f.unlink() + + +@pytest.fixture +def stored(tmp_path: Path, make_orchestrator) -> Path: + """A store in which the job has computed and stored all three rows.""" + store_path = tmp_path / "store" + build_job(store_path).run(orchestrator=make_orchestrator()) + assert len(_CALLS) == 3 + _CALLS.clear() + return store_path + + +def test_unreadable_result_table_fails_the_run(stored, make_orchestrator): + result_table, _ = table_dirs(stored) + delete_data_files(result_table) + + job = build_job(stored) + with pytest.raises(StoreReadError) as info: + job.run(orchestrator=make_orchestrator()) + + assert Path(info.value.table) == result_table + assert _CALLS == [] + + +def test_unreadable_result_table_fails_the_run_under_continue( + stored, make_orchestrator +): + # With the pipeline table gone, the job has no stored tag rows to join, so the + # first read of the result table is the per-row cache lookup, which runs inside + # the error_policy="continue" handler. + result_table, pipeline_table = table_dirs(stored) + delete_data_files(result_table) + shutil.rmtree(pipeline_table) + + job = build_job(stored) + with pytest.raises(StoreReadError) as info: + job.run(orchestrator=make_orchestrator()) + + assert Path(info.value.table) == result_table + assert _CALLS == [] + + +def test_absent_tables_are_computed(stored, make_orchestrator): + result_table, pipeline_table = table_dirs(stored) + shutil.rmtree(result_table) + shutil.rmtree(pipeline_table) + + build_job(stored).run(orchestrator=make_orchestrator()) + assert sorted(_CALLS) == [1, 2, 3] + + # The recomputed rows were stored: a fresh job over the same store computes + # nothing. (Reading the node right after its own run can miss rows, ITL-864.) + _CALLS.clear() + build_job(stored).run(orchestrator=make_orchestrator()) + assert _CALLS == [] From 79141a595832013d5c85db14a49e659b172bb1cb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 20:55:08 -0800 Subject: [PATCH 02/73] fix(databases): a Delta table that cannot be read raises StoreReadError, never no rows (ITL-884) Only TableNotFoundError means a table is absent. Every other load or read error in DeltaTableDatabase raises StoreReadError naming the table, and a flush that cannot load an existing table raises instead of overwriting it. Before merging into an existing table the flush compares column types with the stored schema through Delta's own type mapping (extension types by storage, large-normalized, nullability ignored) and refuses a mismatch with InputValidationError naming the table, column and both types. A store read failure fails the run under both orchestrators whatever the error_policy. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 6 + src/orcapod/core/nodes/function_node.py | 7 + src/orcapod/databases/delta_lake_databases.py | 265 ++++++++++++------ src/orcapod/pipeline/async_orchestrator.py | 5 + .../test_delta_read_failures.py | 14 +- .../test_delta_table_database.py | 2 +- .../test_pipeline/test_store_read_failure.py | 6 +- 7 files changed, 220 insertions(+), 85 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index ba7e3b1d4..4db5ed7bb 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -14,6 +14,7 @@ from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.streams.base import StreamBase from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.errors import StoreReadError from orcapod.protocols.core_protocols import ( ArgumentGroup, FunctionPodProtocol, @@ -587,6 +588,11 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: out_tag, result_data = await self._async_invoke_with_hooks( tag, data, logger=pkt_logger, run_id=run_id ) + except StoreReadError as exc: + # A failed store read fails the run; only rows whose + # function raised are skipped. + obs.on_data_crash(pod_label, tag, data, exc) + raise except Exception as exc: logger.debug( "Data processing failed, skipping: %s", exc, exc_info=True diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 7af980049..ff6fed40f 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -35,6 +35,7 @@ EphemeralResultMissingError, PipelineJobRequiredError, SchemaVersionError, + StoreReadError, ) from orcapod.protocols.core_protocols import ( FunctionPodProtocol, @@ -1294,6 +1295,12 @@ def execute( tag_out, result = self._process_data_internal( tag, data, logger=pkt_logger, run_id=run_id ) + except StoreReadError as exc: + # A failed store read fails the run whatever the error policy: + # error_policy only decides about rows whose function raised. + ctx_obs.on_data_crash(node_label, tag, data, exc) + ctx_obs.on_node_end(node_label, node_hash) + raise except Exception as exc: logger.warning( "Data execution failed in %s: %s", diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index c7dec5dcf..e767854cd 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from orcapod.databases.utils import coerce_record_id +from orcapod.errors import InputValidationError, StoreReadError from orcapod.databases.storage_utils import is_cloud_uri, parse_base_path from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -27,6 +28,49 @@ logger = logging.getLogger(__name__) +def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: + """Return ``arrow_type`` reduced to what the append check compares. + + Extension types are replaced by their storage type and every nested field is + made nullable, at every depth. The result is used only to compare a batch + with a stored table (ITL-884); it is never stored, so dropping nested + non-null flags here changes no schema or hash. + + Args: + arrow_type: The Arrow type to reduce. + + Returns: + The storage type with all nested fields nullable. + """ + if isinstance(arrow_type, pa.ExtensionType): + return _comparable_storage_type(arrow_type.storage_type) + if pa.types.is_struct(arrow_type): + return pa.struct( + [pa.field(f.name, _comparable_storage_type(f.type)) for f in arrow_type] + ) + if pa.types.is_large_list(arrow_type): + field = arrow_type.value_field + return pa.large_list(pa.field(field.name, _comparable_storage_type(field.type))) + if pa.types.is_list(arrow_type): + field = arrow_type.value_field + return pa.list_(pa.field(field.name, _comparable_storage_type(field.type))) + if pa.types.is_fixed_size_list(arrow_type): + field = arrow_type.value_field + return pa.list_( + pa.field(field.name, _comparable_storage_type(field.type)), + arrow_type.list_size, + ) + if pa.types.is_map(arrow_type): + key, item = arrow_type.key_field, arrow_type.item_field + # pyarrow accepts key and item Fields here; its stubs only list types. + return pa.map_( # pyright: ignore[reportCallIssue] + pa.field(key.name, _comparable_storage_type(key.type), nullable=False), + pa.field(item.name, _comparable_storage_type(item.type)), + keys_sorted=arrow_type.keys_sorted, + ) + return arrow_type + + class DeltaTableDatabase: """ A Delta table database with clear insert vs update semantics. @@ -213,11 +257,23 @@ def _get_delta_table( # Table doesn't exist return None except Exception as e: - logger.error(f"Error loading Delta table for {record_key}: {e}") - # Try to clear any corrupted cache - if record_key in self._delta_table_cache: - self._delta_table_cache.pop(record_key) - raise + # The table exists but could not be loaded: never report it as absent. + self._delta_table_cache.pop(record_key, None) + raise self._read_failed(record_path) from e + + def _read_failed(self, record_path: tuple[str, ...]) -> StoreReadError: + """Build the error for a table at ``record_path`` that exists but cannot be read. + + Args: + record_path: Path components relative to this database view. + + Returns: + A ``StoreReadError`` naming the table; the caller raises it from the + backend's exception. + """ + return StoreReadError( + self._get_table_uri(record_path), self._path_prefix + record_path + ) def table_exists(self, record_path: tuple[str, ...]) -> bool: """Return ``True`` if a Delta table exists at ``record_path``. @@ -317,28 +373,26 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: # Get all existing IDs from Delta table using standard RECORD_ID_COLUMN # TODO: replace this with more targetted loading of only the target column and in batches arrow_table = delta_table.to_pyarrow_table() - if arrow_table.num_rows == 0: - self._existing_ids_cache[record_key] = set() - elif self.RECORD_ID_COLUMN not in arrow_table.column_names: - # TODO: replace this with proper checking of the table schema first! - logger.warning(f"Delta table missing {self.RECORD_ID_COLUMN} column") - self._existing_ids_cache[record_key] = set() - else: - existing_ids = cast( - set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) - ) - self._existing_ids_cache[record_key] = existing_ids + except Exception as e: + # Leave the cache dirty so the next call reads the table again. + raise self._read_failed(record_path) from e - self._cache_dirty[record_key] = False - logger.debug( - f"Refreshed existing IDs cache: {len(self._existing_ids_cache)} IDs" + if arrow_table.num_rows == 0: + self._existing_ids_cache[record_key] = set() + elif self.RECORD_ID_COLUMN not in arrow_table.column_names: + # TODO: replace this with proper checking of the table schema first! + logger.warning(f"Delta table missing {self.RECORD_ID_COLUMN} column") + self._existing_ids_cache[record_key] = set() + else: + existing_ids = cast( + set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) ) + self._existing_ids_cache[record_key] = existing_ids - except Exception as e: - logger.error(f"Failed to refresh existing IDs cache: {e}") - self._existing_ids_cache[record_key] = set() - self._cache_dirty[record_key] = False - raise + self._cache_dirty[record_key] = False + logger.debug( + f"Refreshed existing IDs cache: {len(self._existing_ids_cache)} IDs" + ) def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[str]: """Get the set of existing IDs in the Delta table, using cache when possible.""" @@ -631,6 +685,9 @@ def get_all_records( Returns: Combined Arrow table with all records, or None if no records exist + + Raises: + StoreReadError: If the Delta table exists but could not be read. """ record_key = self._get_record_key(record_path) @@ -642,10 +699,10 @@ def get_all_records( delta_table_data = delta_table.to_pyarrow_dataset( as_large_types=True ).to_table() - if delta_table_data.num_rows > 0: - tables_to_combine.append(delta_table_data) except Exception as e: - logger.warning(f"Error reading Delta table: {e}") + raise self._read_failed(record_path) from e + if delta_table_data.num_rows > 0: + tables_to_combine.append(delta_table_data) # Add pending batch data if ( @@ -700,21 +757,16 @@ def get_records_with_column_value( if delta_table is None: return None - try: - # Use schema-preserving read - result = self._read_delta_table(delta_table, expression=combined_expression) - - if len(result) == 0: - return None + # Use schema-preserving read + result = self._read_delta_table( + delta_table, record_path, expression=combined_expression + ) - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) + if len(result) == 0: + return None - except Exception as e: - logger.error( - f"Error getting record with {column_values} from {'/'.join(record_path)}: {e}" - ) - raise e + # Handle (remove/rename) the record id column before returning + return self._handle_record_id_column(result, record_id_column) def get_record_by_id( self, @@ -758,22 +810,15 @@ def get_record_by_id( if delta_table is None: return None - try: - # Use schema-preserving read - filter_expr = self._create_record_id_filter(record_id) - result = self._read_delta_table(delta_table, filters=filter_expr) - - if len(result) == 0: - return None + # Use schema-preserving read + filter_expr = self._create_record_id_filter(record_id) + result = self._read_delta_table(delta_table, record_path, filters=filter_expr) - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) + if len(result) == 0: + return None - except Exception as e: - logger.error( - f"Error getting record {record_id} from {'/'.join(record_path)}: {e}" - ) - raise e + # Handle (remove/rename) the record id column before returning + return self._handle_record_id_column(result, record_id_column) def get_records_by_ids( self, @@ -815,26 +860,20 @@ def get_records_by_ids( delta_table = self._get_delta_table(record_path) if delta_table is None: return None - try: - # Use schema-preserving read with filters - filter_expr = self._create_record_ids_filter(record_ids_list) - result = self._read_delta_table(delta_table, filters=filter_expr) - - if len(result) == 0: - return None - - # Handle record_id column based on parameter - return self._handle_record_id_column(result, record_id_column) + # Use schema-preserving read with filters + filter_expr = self._create_record_ids_filter(record_ids_list) + result = self._read_delta_table(delta_table, record_path, filters=filter_expr) - except Exception as e: - logger.error( - f"Error getting records by IDs from {'/'.join(record_path)}: {e}" - ) + if len(result) == 0: return None + # Handle record_id column based on parameter + return self._handle_record_id_column(result, record_id_column) + def _read_delta_table( self, delta_table: deltalake.DeltaTable, + record_path: tuple[str, ...], filters: list | None = None, expression: pc.Expression | None = None, ) -> pa.Table: @@ -843,14 +882,23 @@ def _read_delta_table( Args: delta_table: The Delta table to read from + record_path: Path of the table relative to this view, for errors filters: Optional filters to apply + expression: Optional filter expression, used instead of ``filters`` Returns: Arrow table with preserved schema + + Raises: + StoreReadError: If the table's data could not be read. An error in + the filter itself (an unknown column, say) is raised as is. """ filter_expr = None # Use to_pyarrow_dataset with as_large_types for Polars compatible arrow table loading - dataset = delta_table.to_pyarrow_dataset(as_large_types=True) + try: + dataset = delta_table.to_pyarrow_dataset(as_large_types=True) + except Exception as e: + raise self._read_failed(record_path) from e if filters and expression is None: for filt in filters: if len(filt) == 3: @@ -876,7 +924,11 @@ def _read_delta_table( # Convert view types before filtering: PyArrow has no comparison/sort # kernels for string_view, so filtering the scan directly crashes # (ENG-601). Trades predicate pushdown for correctness. - table = arrow_utils.normalize_table_view_types(dataset.to_table()) + try: + scanned = dataset.to_table() + except Exception as e: + raise self._read_failed(record_path) from e + table = arrow_utils.normalize_table_view_types(scanned) if filter_expr is not None: return table.filter(filter_expr) return table @@ -991,18 +1043,19 @@ def _flush_by_key(self, record_key: str) -> None: os.makedirs(table_uri, exist_ok=True) - # Check if Delta table exists at this URI - try: - import deltalake as _dl + # Check if Delta table exists at this URI. Only TableNotFoundError + # means "absent": a table that exists but cannot be loaded must never + # be replaced by the mode="overwrite" write below (ITL-884). + from deltalake.exceptions import TableNotFoundError - delta_table = _dl.DeltaTable( + try: + delta_table = deltalake.DeltaTable( table_uri, storage_options=self._storage_options or None ) - except ( - _dl.exceptions.TableNotFoundError, - _dl.exceptions.DeltaProtocolError, - ): + except TableNotFoundError: delta_table = None + except Exception as load_error: + raise StoreReadError(table_uri, abs_path_components) from load_error if delta_table is None: # TODO: reconsider mode="overwrite" here @@ -1017,6 +1070,7 @@ def _flush_by_key(self, record_key: str) -> None: f"Created new Delta table for {record_key} with {len(combined_table)} records" ) else: + self._require_compatible_append(table_uri, delta_table, combined_table) delta_table.merge( source=combined_table, predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}", @@ -1046,6 +1100,61 @@ def _flush_by_key(self, record_key: str) -> None: self._pending_record_ids[record_key] = pending_ids raise + @staticmethod + def _require_compatible_append( + table_uri: str, delta_table: deltalake.DeltaTable, batch: pa.Table + ) -> None: + """Refuse a batch whose column types differ from the stored table's. + + Delta's merge either casts such a batch to the stored type (``int32`` or + ``float64`` rows into an ``int64`` column) or, when an extension type is + involved, writes data files the table can no longer read. Both sides are + compared through Delta's own type mapping, with extension types reduced to + storage, large-normalized and nullability ignored, so a type Delta stores + differently (``uint64`` as ``int64``) is not refused. Columns present on + only one side are allowed (schema evolution). The stored schema comes from + the loaded log; no data is read. + + Args: + table_uri: Location of the stored table, for the error message. + delta_table: The loaded stored table. + batch: The rows about to be merged into it. + + Raises: + InputValidationError: If a column's type in ``batch`` differs from its + stored type. Nothing is written. + """ + stored = pa.schema(delta_table.schema().to_arrow()) # pyright: ignore[reportArgumentType] # arro3 schema; pyarrow reads it through the C data interface + incoming = pa.schema( + deltalake.Schema.from_arrow( + pa.schema( + [ + pa.field(f.name, _comparable_storage_type(f.type)) + for f in batch.schema + ] + ) + ).to_arrow() # pyright: ignore[reportArgumentType] # arro3 schema, as above + ) + for field in incoming: + index = stored.get_field_index(field.name) + if index < 0: + continue + stored_type = _comparable_storage_type( + arrow_utils.normalize_to_large_types( + stored.field(index).type, keep_null=True + ) + ) + incoming_type = _comparable_storage_type( + arrow_utils.normalize_to_large_types(field.type, keep_null=True) + ) + if stored_type != incoming_type: + raise InputValidationError( + f"Cannot append to the stored table {table_uri!r}: column " + f"{field.name!r} is stored as {stored_type} but the new rows have " + f"{incoming_type} (input type {batch.schema.field(field.name).type}). " + "The stored rows are unchanged and nothing was written." + ) + @property def base_path(self) -> tuple[str, ...]: """The current relative root of this database view (always () for root instances).""" diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 4a75564c5..ecb9b1b42 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Literal from orcapod.channels import BroadcastChannel, Channel +from orcapod.errors import StoreReadError from orcapod.pipeline.result import OrchestratorResult from orcapod.protocols.node_protocols import ( is_function_node, @@ -250,6 +251,10 @@ async def _run_async( return OrchestratorResult( node_outputs=collectors if materialize_results else {} ) + except* StoreReadError as group: + # A failed store read stops the run. Raise it on its own, as the sync + # orchestrator does, rather than inside the TaskGroup's exception group. + raise StoreReadError.in_group(group)[0] finally: effective_observer.on_run_end(run_id) diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index 597312934..aaf1f77ca 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -15,10 +15,12 @@ import json from collections.abc import Callable from pathlib import Path +from typing import Any import deltalake import pyarrow as pa import pytest +from deltalake.exceptions import DeltaProtocolError from orcapod.databases import DeltaTableDatabase from orcapod.errors import InputValidationError, StoreReadError @@ -26,7 +28,7 @@ PATH = ("results", "t") -def rows(ids: list[bytes], x: pa.Array, **extra: pa.Array) -> pa.Table: +def rows(ids: list[bytes], x: pa.Array[Any], **extra: pa.Array[Any]) -> pa.Table: """Build a table keyed by ``rid`` with an ``x`` column and optional extras.""" return pa.table({"rid": pa.array(ids, pa.large_binary()), "x": x, **extra}) @@ -60,7 +62,7 @@ def __arrow_ext_deserialize__( return cls(storage_type) -def ext_array(values: list, storage: pa.DataType) -> pa.ExtensionArray: +def ext_array(values: list[Any], storage: pa.DataType) -> pa.ExtensionArray[Any]: return pa.ExtensionArray.from_storage(_Ext(storage), pa.array(values, storage)) @@ -149,6 +151,12 @@ def test_table_still_exists(self, unreadable): assert unreadable.table_exists(PATH) +def test_bad_filter_is_not_reported_as_unreadable_table(stored): + # The table reads fine; the filter names a column it does not have. + with pytest.raises(pa.ArrowInvalid): + stored.get_records_with_column_value(PATH, {"no_such_column": 1}) + + class TestAbsentTable: @pytest.mark.parametrize("read", READS.values(), ids=list(READS)) def test_read_returns_none(self, base, read): @@ -192,7 +200,7 @@ def test_protocol_error_on_load_is_not_treated_as_absent( def failing_load(table_uri, *args, **kwargs): if Path(str(table_uri)) == target: - raise deltalake.exceptions.DeltaProtocolError("simulated") + raise DeltaProtocolError("simulated") return real(table_uri, *args, **kwargs) before = snapshot(target) diff --git a/tests/test_databases/test_delta_table_database.py b/tests/test_databases/test_delta_table_database.py index 4a5296ac1..27142c3fc 100644 --- a/tests/test_databases/test_delta_table_database.py +++ b/tests/test_databases/test_delta_table_database.py @@ -561,7 +561,7 @@ def test_read_delta_table_no_filter_normalizes_views(self, db): tbl = make_string_view_table(__record_id=["p", "q"], category=["A", "B"]) deltalake.write_deltalake(uri, tbl, mode="overwrite") - result = db._read_delta_table(db._get_delta_table(self.PATH)) + result = db._read_delta_table(db._get_delta_table(self.PATH), self.PATH) assert result.num_rows == 2 assert not any(pa.types.is_string_view(f.type) for f in result.schema) diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index 4010ae2cb..f36b97d1c 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -17,8 +17,7 @@ import pyarrow as pa import pytest -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod +from orcapod import function_pod from orcapod.core.sources import ArrowTableSource from orcapod.databases import DeltaTableDatabase from orcapod.errors import StoreReadError @@ -29,6 +28,7 @@ _CALLS: list[int] = [] +@function_pod(output_keys="y") def double(x: int) -> int: _CALLS.append(x) return 2 * x @@ -60,7 +60,7 @@ def build_job(store_path: Path) -> PipelineJob: ) job = PipelineJob(name="itl884", store=DeltaTableDatabase(base_path=store_path)) with job: - FunctionPod(PythonDataFunction(double, output_keys="y"))(source, label="dbl") + _ = double.pod(source, label="dbl") return job From c7ffc7dc6c3c3324138b220026bb2a710bfcbf08 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 20:55:08 -0800 Subject: [PATCH 03/73] docs: state the Delta read-failure and append type-check contract (ITL-884) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ DESIGN_ISSUES.md | 34 ++++++++++++++++++++++++++++++++++ docs/concepts/function-pods.md | 13 +++++++++++++ 3 files changed, 75 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a71b50c0..ecd675777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,22 @@ info. Hashes of pipelines that already worked are unchanged. ### Fixed +#### A Delta table that cannot be read is no longer reported as having no rows (ITL-884) + +When a `DeltaTableDatabase` table existed but could not be read (a missing or damaged data +file, a log the backend cannot load, data files whose column type no longer matches the +table), `get_all_records` and `get_records_by_ids` returned `None`, the same answer as for a +table that does not exist. A `PipelineJob` then warned that the stored results "may have been +deleted externally", swallowed the per-row read errors under `error_policy="continue"`, finished +without raising, and its nodes showed no records. Every read now raises +`orcapod.errors.StoreReadError`, which names the table and chains the backend's error, and a +store read failure fails the run under both orchestrators whatever the `error_policy`. A table +that does not exist still reads as empty. + +A flush that could not load an existing table no longer treats it as absent. Before this fix a +`DeltaProtocolError` while loading the table made the flush write the batch with +`mode="overwrite"`, replacing the stored rows. The flush now raises and writes nothing. + #### Materialized rows keep their own source info (ITL-845) When rows were collected and turned back into a stream, every row got the source-info @@ -115,6 +131,18 @@ replaced by `_`. Correcting stored values is tracked in ITL-853. ### Breaking Changes +#### A changed column type is refused when appending to a Delta table (ITL-884) + +Before writing to an existing `DeltaTableDatabase` table, the flush now compares the new rows' +column types with the stored ones. If a column's type differs, the flush raises +`InputValidationError` naming the table, the column and both types, and writes nothing. Before, +Delta's merge silently cast the new rows to the stored type (`int32` or `float64` rows written +into an `int64` column, `large_binary` into `large_string`), or, when an extension (logical) type +was involved, accepted rows the table could then no longer read. The comparison treats types the +way Delta stores them: `string` and `large_string` (and the other large and non-large layouts) +are the same type, an extension type is compared by its storage type, and nullability is not +compared. New columns and missing columns are still accepted. + #### Operator outputs keep their inputs' Arrow types, normalized to large variants (ITL-832, ITL-813) `MergeJoin` no longer infers Arrow types from Python values, and it no longer fails on a diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index a6f9b59ed..7bc13f001 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2171,3 +2171,37 @@ equals the materialized one. Hashes of pipelines that already worked are unchang Regressions: `tests/test_core/operators/test_chained_aggregation.py`. Open: on the async barrier path, per-row source info is still row 0's (ITL-845); that assertion is a strict xfail. + +--- + +## `src/orcapod/databases/delta_lake_databases.py` — read failures + +### DR1 — A Delta table that could not be read was reported as having no rows +**Status:** resolved +**Severity:** critical +**Issue:** ITL-884 + +`DeltaTableDatabase.get_all_records` logged a read error at WARNING and returned `None`, and +`get_records_by_ids` returned `None` on any error. Both conflated "the table does not exist" +with "the table exists but cannot be read". Callers treat `None` as no rows. A function node +recomputed with a "deleted externally" warning, its per-row lookups failed inside the +`error_policy="continue"` handler, and the run finished without raising, so stored results +looked dropped. `_flush_by_key` also treated a `DeltaProtocolError` on load as "no table" and +wrote the batch with `mode="overwrite"`. The case that exposed it: an append whose column type +differs from the stored table, which Delta's merge either casts silently to the stored type or, +with an extension type involved, accepts and then cannot read. Other backends were not +affected: the in-memory and connector databases (SQLite, PostgreSQL, SpiralDB) return `None` +only for a table that does not exist and let read errors propagate. + +**Fix:** only `TableNotFoundError` means absent. Every other load or read error raises +`orcapod.errors.StoreReadError`, which names the table and chains the cause. `FunctionNode.execute`, the +async `FunctionPod` row handler and the async orchestrator let it fail the run whatever the +`error_policy`. Before merging into an existing table, `_flush_by_key` compares the batch with +the stored Delta schema through Delta's own type mapping (extension types by storage, +large-normalized, nullability ignored). On a difference it raises `InputValidationError` naming the +table, column and both types, and nothing is written. Regressions: +`tests/test_databases/test_delta_read_failures.py` and +`tests/test_pipeline/test_store_read_failure.py`. Open: whether a nullability-only change +should be refused (Delta's merge relaxes a non-null nested field to nullable) belongs to the +nullability audit, ITL-842. + diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8618052c..602056047 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -218,6 +218,19 @@ print("function calls:", len(calls)) version, e.g. `@function_pod(output_keys="bmi", version="v1.0")` (the default is `"v0.0"`; a minor bump such as `"v0.1"` is not enough), or rename the function. +!!! note "When the store cannot be read" + A table that does not exist yet reads as empty, and its rows are computed. A table that + exists but cannot be read (for example, a data file was deleted or damaged) is never + treated as empty: the run stops with `orcapod.errors.StoreReadError`, which names the + table and the underlying error, whatever the orchestrator's `error_policy`. Nothing is + recomputed or overwritten, so you can repair the table or restore the file and run again. + + In a `DeltaTableDatabase`, rows whose column types differ from the stored table's + (`float64` values for a column stored as `int64`, say) are refused with + `InputValidationError`, which names the table, the column and both types. They are not + converted to the stored type. `string` and `large_string`, and the other large and + non-large layouts of one type, count as the same type. + A node from a job that has run also provides: - **`get_all_records()`** -- every record this step of the job has stored, as a PyArrow From 5a067645d6bc8053a1cedc66965024619b528836 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:29:14 -0800 Subject: [PATCH 04/73] fix(databases): flush raises StoreReadError and the append refusal as themselves (ITL-884) DeltaTableDatabase.flush() wrapped every failed batch in a RuntimeError. A StoreReadError raised by a result-cache flush during per-row processing was therefore skipped as a row failure under error_policy="continue": the next row was computed and the run ended with RuntimeError. Callers catching InputValidationError also missed refused appends. flush() now raises the first StoreReadError, or else the first InputValidationError, as itself, with a note for every other batch that failed. Every failure is still logged and every failed batch stays queued. A flush that fails only for other reasons still raises RuntimeError. Addresses CodeRabbit comment 4534230764 on PR #290. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 +- docs/concepts/function-pods.md | 7 + src/orcapod/databases/delta_lake_databases.py | 51 +++++- ...tl-884-coderabbit-flush-and-async-group.md | 159 ++++++++++++++++++ .../test_delta_read_failures.py | 75 +++++++-- .../test_pipeline/test_store_read_failure.py | 53 +++++- 6 files changed, 337 insertions(+), 20 deletions(-) create mode 100644 superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd675777..306ea1d00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,8 +92,9 @@ table that does not exist. A `PipelineJob` then warned that the stored results " deleted externally", swallowed the per-row read errors under `error_policy="continue"`, finished without raising, and its nodes showed no records. Every read now raises `orcapod.errors.StoreReadError`, which names the table and chains the backend's error, and a -store read failure fails the run under both orchestrators whatever the `error_policy`. A table -that does not exist still reads as empty. +store read failure fails the run under both orchestrators whatever the `error_policy`, also +when it happens while a row's result is being stored. A table that does not exist still reads as +empty. A flush that could not load an existing table no longer treats it as absent. Before this fix a `DeltaProtocolError` while loading the table made the flush write the batch with @@ -143,6 +144,13 @@ way Delta stores them: `string` and `large_string` (and the other large and non- are the same type, an extension type is compared by its storage type, and nullability is not compared. New columns and missing columns are still accepted. +`DeltaTableDatabase.flush()` (and `add_records(..., flush=True)`) now raises a +`StoreReadError` or this `InputValidationError` as itself. Before, it wrapped every failure in a +`RuntimeError`. When several tables fail in one flush, the `StoreReadError` is raised, or else the +first `InputValidationError`, with each other failure attached as a note and logged. Failed rows +stay queued for the next flush. A flush that fails only for other reasons still raises +`RuntimeError`, with the first failure as its cause. + #### Operator outputs keep their inputs' Arrow types, normalized to large variants (ITL-832, ITL-813) `MergeJoin` no longer infers Arrow types from Python values, and it no longer fails on a diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 602056047..2d8bdd384 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -231,6 +231,13 @@ print("function calls:", len(calls)) converted to the stored type. `string` and `large_string`, and the other large and non-large layouts of one type, count as the same type. + Both errors reach you as themselves, also from `flush()` and from + `add_records(..., flush=True)`. When several pending tables fail in one flush, the + `StoreReadError` (or else the first `InputValidationError`) is raised, and each other + failure is attached to it as a note and logged. Failed rows stay queued, so a later + flush tries them again. A flush that fails for any other reason raises `RuntimeError`, + with the first failure as its cause. + A node from a job that has run also provides: - **`get_all_records()`** -- every record this step of the job has stored, as a PyArrow diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index e767854cd..afb121aa5 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -71,6 +71,38 @@ def _comparable_storage_type(arrow_type: pa.DataType) -> pa.DataType: return arrow_type +def _store_error_to_raise(errors: list[tuple[str, Exception]]) -> Exception | None: + """Pick the store error a failed flush raises as itself. + + A ``StoreReadError`` is preferred, because it is the error that fails a run + whatever the orchestrator's ``error_policy``; failing that, an + ``InputValidationError`` from a refused append. Every other failed batch is + added to the chosen error as a note, so no failure is hidden. + + Args: + errors: The ``(record_key, error)`` pair of every batch that failed, in + flush order. + + Returns: + The chosen error with its notes added, or ``None`` when no batch failed + with a store error. + """ + chosen: Exception | None = None + for kind in (StoreReadError, InputValidationError): + chosen = next((e for _, e in errors if isinstance(e, kind)), None) + if chosen is not None: + break + if chosen is None: + return None + for record_key, other in errors: + if other is not chosen: + chosen.add_note( + f"Another batch also failed to flush ({record_key}): " + f"{type(other).__name__}: {other}" + ) + return chosen + + class DeltaTableDatabase: """ A Delta table database with clear insert vs update semantics. @@ -977,7 +1009,21 @@ def from_config(cls, config: dict[str, Any]) -> "DeltaTableDatabase": ) def flush(self) -> None: - """Flush all pending batches.""" + """Flush all pending batches. + + Every pending batch is attempted, even after one fails. A batch that fails + stays queued, so a later flush tries it again, and each failure is logged. + + Raises: + StoreReadError: If a batch's existing table could not be loaded. It is + raised as itself; every other batch that failed in this flush is + added to it as a note. + InputValidationError: If no batch failed with ``StoreReadError`` but a + batch's column types differ from its stored table's. Other failed + batches are added to it as notes. + RuntimeError: If batches failed with other errors only, naming the + failed batches, with the first failure as its ``__cause__``. + """ errors: list[tuple[str, Exception]] = [] for record_key in list(self._pending_batches.keys()): try: @@ -986,6 +1032,9 @@ def flush(self) -> None: logger.error(f"Error flushing batch for {record_key}: {e}") errors.append((record_key, e)) if errors: + store_error = _store_error_to_raise(errors) + if store_error is not None: + raise store_error failed_keys = [k for k, _ in errors] raise RuntimeError( f"Failed to flush {len(errors)} batch(es): {failed_keys}" diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md new file mode 100644 index 000000000..caaebe4b1 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -0,0 +1,159 @@ +# ITL-884 follow-up: store errors keep their type through `flush()` and out of the async TaskGroup + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Answer the two open CodeRabbit threads on PR #290 (ids 4534230764 and 4534230775). +A `StoreReadError` or an append-refusal `InputValidationError` raised while flushing reaches the +caller as itself, so a store read failure in a per-row result-cache flush fails the run under +`error_policy="continue"`. The async orchestrator raises a lone `StoreReadError` even when an +unrelated node failed in the same `TaskGroup`. + +**Architecture:** `DeltaTableDatabase.flush()` keeps attempting every pending batch and logging +each failure. When at least one batch failed with a store error, it raises the first +`StoreReadError` (else the first `InputValidationError`) itself, with a note for every other +failed batch. Only non-store failures keep the `RuntimeError` wrapper. The async orchestrator +replaces `except* StoreReadError` with a plain `except ExceptionGroup` that selects one +`StoreReadError` (`_single_store_read_error`), notes and logs the rest, and raises it, in the same +shape as PR #287's `_single_store_error` for `StoreWriteError` so the second PR to merge has a +trivial conflict. + +**Tech Stack:** deltalake 1.0.2, pyarrow, asyncio `TaskGroup`, pytest. + +Approved by the coordinator (2026-11-10) on condition that each finding reproduces. Both do: + +- (1b) Run a job, then run it again over two new rows, with the function corrupting the result + table's log (a new log version that requires an unknown reader feature) during its first call. + The per-row lookup still succeeds, the result-cache flush raises `StoreReadError` wrapped in + `RuntimeError`, the sync per-row handler under `continue` skips the row, the next row is + computed, and the run ends with `RuntimeError` from the job's end-of-run flush. +- (2) CI's 3.11 is 3.11.18, so the 3.11.0-3.11.3 single-leaf wrapping does not occur there. The + concurrent case reproduces deterministically on 3.11.18 and 3.13.18: node `dbl` raises + `StoreReadError` in `_load_cached_entries` and node `tri` fails in the observer's + `on_node_start`, both before their first `await`, so both land in one group and `job.run` + raises `ExceptionGroup([StoreReadError, ExceptionGroup([ValueError])])`. + +No hash, path or stored-schema changes. No public API added. + +--- + +## File map + +- `src/orcapod/databases/delta_lake_databases.py`: `flush()` and a module helper + `_store_error_to_raise(errors)`. +- `src/orcapod/pipeline/async_orchestrator.py`: plain `except ExceptionGroup`, helpers + `_single_store_read_error(group)` and `_leaves(group)` (identical to #287's `_leaves`). +- Tests: `tests/test_databases/test_delta_read_failures.py` (assert the bare error types, add a + multi-batch test), `tests/test_pipeline/test_store_read_failure.py` (flush during per-row + processing, both orchestrators; concurrent unrelated failure, async). +- Docs: `docs/concepts/function-pods.md` admonition, `CHANGELOG.md` ITL-884 entries, + `DESIGN_ISSUES.md` DR1 fix note. + +## Task 1: Failing tests for finding (1) + +- [ ] `tests/test_databases/test_delta_read_failures.py`: the `refusal` helper is replaced by + `pytest.raises(InputValidationError)` at each refusal site; the two unloadable-table flush tests + use `pytest.raises(StoreReadError)`; the absent `__cause__` assertions go. +- [ ] New `test_flush_raises_the_store_error_and_notes_every_other_failed_batch`: two tables + with pending batches (one whose stored `x` is int64 and receives float64, one whose log requires + an unknown reader feature); `flush()` raises the `StoreReadError`, one note names the other + table's key and `InputValidationError`, and both batches stay queued (a second `flush()` raises + again). +- [ ] New `test_flush_keeps_runtime_error_for_other_failures`: control case. Monkeypatch + `deltalake.write_deltalake` to raise `OSError` for a new table; `flush()` raises `RuntimeError` + with the `OSError` as `__cause__` (unchanged behaviour). +- [ ] `tests/test_pipeline/test_store_read_failure.py`: + `test_store_read_failure_in_a_row_flush_fails_the_run`, both orchestrators with + `error_policy="continue"`. Rerun over `a..e` with the function corrupting the result table on + its first call; `job.run` raises `StoreReadError` naming the result table, and the function ran + for exactly one new row. +- [ ] Run `uv run pytest tests/test_databases/test_delta_read_failures.py + tests/test_pipeline/test_store_read_failure.py -v`. Expected: the new and changed tests FAIL with + `RuntimeError`, and the per-row test also shows two calls. + +## Task 2: Fix `flush()` + +```python +def flush(self) -> None: + errors: list[tuple[str, Exception]] = [] + for record_key in list(self._pending_batches.keys()): + try: + self._flush_by_key(record_key) + except Exception as e: + logger.error(f"Error flushing batch for {record_key}: {e}") + errors.append((record_key, e)) + if errors: + store_error = _store_error_to_raise(errors) + if store_error is not None: + raise store_error + failed_keys = [k for k, _ in errors] + raise RuntimeError( + f"Failed to flush {len(errors)} batch(es): {failed_keys}" + ) from errors[0][1] + + +def _store_error_to_raise(errors): + """First StoreReadError, else first InputValidationError, with notes; else None.""" + for kind in (StoreReadError, InputValidationError): + chosen = next((e for _, e in errors if isinstance(e, kind)), None) + if chosen is not None: + break + else: + return None + for key, other in errors: + if other is not chosen: + chosen.add_note( + f"Another batch also failed to flush ({key}): " + f"{type(other).__name__}: {other}" + ) + return chosen +``` + +`StoreReadError` is preferred because it is the one the per-row handlers treat as fatal. The +docstring gets a Raises section. Run Task 1's tests: PASS. + +## Task 3: Failing test for finding (2) + +- [ ] `test_concurrent_unrelated_failure_still_raises_store_read_error` (async only): an + observer (a `NoOpObserver` subclass whose `contextualize` returns itself and whose + `on_node_start` raises `ValueError` for `tri`), two function nodes over one source, the `dbl` + result table's data files deleted. `job.run` raises `StoreReadError` (not a group) naming the + result table, `__cause__` is set, and a note names `ValueError: observer failed for tri`. +- [ ] Run it: FAIL with `ExceptionGroup`. + +## Task 4: Fix the async orchestrator + +```python + except ExceptionGroup as group: + # A failed store read stops the run. Raise it on its own, as the sync + # orchestrator does, rather than inside the TaskGroup's exception group. + # A plain except (not except*) so the raise is never wrapped again. + store_error = _single_store_read_error(group) + if store_error is None: + raise + raise store_error +``` + +`_single_store_read_error(group)`: `StoreReadError.in_group(group)`; none gives `None`; the first +is chosen; every other leaf (`_leaves`) is noted on it ("Another store read failed in the same +run: ..." also logged at WARNING; "Another node failed in the same run: : "). Its +`__cause__` stays the backend error; the group is its `__context__`. Run: PASS. + +## Task 5: Docs, design log, verification, commit, replies + +- [ ] Concept page and CHANGELOG: `flush()` / `add_records(..., flush=True)` raise + `StoreReadError` and `InputValidationError` themselves; when several batches fail, the first + store error is raised with a note per other batch, every failed batch stays queued and each + failure is logged; other flush failures still raise `RuntimeError`. +- [ ] `DESIGN_ISSUES.md` DR1: one sentence on the flush and async-group fix. +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, basedpyright (exit code), the full + suite on 3.11, 3.12 and 3.13, `tests/test_hashing/golden` unchanged. +- [ ] One commit per finding, push, signed replies on both threads, wait for CI and CodeRabbit. + +## Interaction with PR #287 (report, do not fix here) + +#287's `ResultCache.store` wraps every `add_record`/`flush` failure in `StoreWriteError`, and its +`PipelineJob.run` wraps the end-of-run flush the same way. Once both PRs are in, a +`StoreReadError` from a flush reaches the caller as the `__cause__` of a `StoreWriteError`. It +is still fatal, but `except StoreReadError` misses it. Whichever PR merges second should decide +whether those wrappers let `StoreReadError` through. diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index aaf1f77ca..94ae2d228 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -4,6 +4,9 @@ - A table that exists but cannot be read raises ``StoreReadError`` on every read path, naming the table; a table that does not exist still reads as ``None``. - A table whose log cannot be loaded is never overwritten by a flush. +- ``flush`` raises ``StoreReadError`` and the append refusal's + ``InputValidationError`` themselves, not wrapped in ``RuntimeError``, and + notes every other batch that failed in the same flush. - An append whose column types differ from the stored table is refused before anything is written, naming the column and both types; legitimate appends (same types in another layout, new or missing columns, unchanged extension @@ -178,14 +181,14 @@ def test_unsupported_reader_feature(self, base, stored): with pytest.raises(StoreReadError): db.get_all_records(PATH) - with pytest.raises(RuntimeError) as info: + with pytest.raises(StoreReadError) as info: db.add_records( PATH, rows([b"c"], pa.array([3], pa.int64())), record_id_column="rid", flush=True, ) - assert isinstance(info.value.__cause__, StoreReadError) + assert_names_table(info.value, base) assert snapshot(table_dir(base)) == before def test_protocol_error_on_load_is_not_treated_as_absent( @@ -206,7 +209,7 @@ def failing_load(table_uri, *args, **kwargs): before = snapshot(target) db = DeltaTableDatabase(base_path=base) monkeypatch.setattr(deltalake, "DeltaTable", failing_load) - with pytest.raises(RuntimeError) as info: + with pytest.raises(StoreReadError) as info: db.add_records( PATH, rows([b"c"], pa.array([3], pa.int64())), @@ -215,13 +218,62 @@ def failing_load(table_uri, *args, **kwargs): ) monkeypatch.undo() - assert isinstance(info.value.__cause__, StoreReadError) + assert isinstance(info.value.__cause__, DeltaProtocolError) assert snapshot(target) == before stored_rows = DeltaTableDatabase(base_path=base).get_all_records(PATH) assert stored_rows is not None assert stored_rows.num_rows == 2 +class TestFlushRaisesStoreErrorsThemselves: + def test_store_read_error_is_raised_with_a_note_per_other_failed_batch( + self, base, stored + ): + other = ("results", "u") + stored.add_records( + other, + rows([b"a"], pa.array([1], pa.int64())), + record_id_column="rid", + flush=True, + ) + unsupported_reader_feature(base) + db = DeltaTableDatabase(base_path=base) + # Queued first: refused at flush, because x is stored as int64. + db.add_records(other, rows([b"b"], pa.array([2.0])), record_id_column="rid") + # Queued second: its table's log cannot be loaded. + db.add_records( + PATH, rows([b"c"], pa.array([3], pa.int64())), record_id_column="rid" + ) + + with pytest.raises(StoreReadError) as info: + db.flush() + + # The StoreReadError wins over the earlier refusal, because it is the one + # that fails a run whatever the error policy; the refusal is noted on it. + assert_names_table(info.value, base) + notes = getattr(info.value, "__notes__", []) + assert len(notes) == 1 + assert "results/u" in notes[0] + assert "InputValidationError" in notes[0] + # Both failed batches stay queued, so the next flush fails the same way. + with pytest.raises(StoreReadError): + db.flush() + + def test_other_flush_failures_still_raise_runtime_error(self, base, monkeypatch): + def failing_write(*args, **kwargs): + raise OSError("simulated disk failure") + + db = DeltaTableDatabase(base_path=base) + db.add_records( + PATH, rows([b"a"], pa.array([1], pa.int64())), record_id_column="rid" + ) + monkeypatch.setattr(deltalake, "write_deltalake", failing_write) + + with pytest.raises(RuntimeError, match="Failed to flush 1 batch") as info: + db.flush() + assert isinstance(info.value.__cause__, OSError) + + # --------------------------------------------------------------------------- # Append refusal: a column's type differs from the stored table # --------------------------------------------------------------------------- @@ -231,13 +283,6 @@ def append(db: DeltaTableDatabase, table: pa.Table) -> None: db.add_records(PATH, table, record_id_column="rid", flush=True) -def refusal(info: pytest.ExceptionInfo[RuntimeError]) -> InputValidationError: - """The ``InputValidationError`` behind a refused flush.""" - cause = info.value.__cause__ - assert isinstance(cause, InputValidationError), repr(info.value) - return cause - - class TestChangedColumnTypeIsRefused: @pytest.mark.parametrize( ("new_type", "value", "named"), @@ -249,9 +294,9 @@ class TestChangedColumnTypeIsRefused: ids=["int64->float64", "int64->int32", "int64->large_string"], ) def test_plain_type_change(self, base, stored, new_type, value, named): - with pytest.raises(RuntimeError) as info: + with pytest.raises(InputValidationError) as info: append(stored, rows([b"c"], pa.array([value], new_type))) - message = str(refusal(info)) + message = str(info.value) assert "'x'" in message assert "int64" in message assert named in message @@ -266,12 +311,12 @@ def test_extension_storage_change(self, base): db = DeltaTableDatabase(base_path=base) append(db, rows([b"a"], ext_array([b"x"], pa.large_binary()))) - with pytest.raises(RuntimeError) as info: + with pytest.raises(InputValidationError) as info: append( db, rows([b"b"], ext_array([[1.0]], pa.large_list(pa.float32()))), ) - message = str(refusal(info)) + message = str(info.value) assert "'x'" in message assert "binary" in message assert "list" in message diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index f36b97d1c..3311033df 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -10,6 +10,7 @@ from __future__ import annotations +import json import shutil from collections.abc import Callable from pathlib import Path @@ -26,11 +27,16 @@ from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator _CALLS: list[int] = [] +# Tables the function damages on its next call, standing in for another process +# breaking a table while the job runs. +_DAMAGE_ON_NEXT_CALL: list[Path] = [] @function_pod(output_keys="y") def double(x: int) -> int: _CALLS.append(x) + while _DAMAGE_ON_NEXT_CALL: + require_unknown_reader_feature(_DAMAGE_ON_NEXT_CALL.pop()) return 2 * x @@ -43,8 +49,10 @@ def double(x: int) -> int: @pytest.fixture(autouse=True) def _reset_calls(): _CALLS.clear() + _DAMAGE_ON_NEXT_CALL.clear() yield _CALLS.clear() + _DAMAGE_ON_NEXT_CALL.clear() @pytest.fixture(params=list(ORCHESTRATORS)) @@ -52,9 +60,9 @@ def make_orchestrator(request) -> Callable[[], object]: return ORCHESTRATORS[request.param] -def build_job(store_path: Path) -> PipelineJob: +def build_job(store_path: Path, keys: str = "abc") -> PipelineJob: source = ArrowTableSource( - pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + pa.table({"k": list(keys), "x": list(range(1, len(keys) + 1))}), tag_columns=["k"], infer_nullable=True, ) @@ -78,6 +86,23 @@ def delete_data_files(table: Path) -> None: f.unlink() +def require_unknown_reader_feature(table: Path) -> None: + """Commit a log version that requires a reader feature deltalake does not know. + + Data files already loaded stay readable through the job's open table handle, + but loading the table again, as every flush does, fails. + """ + log = table / "_delta_log" + version = max(int(p.stem) for p in log.glob("*.json")) + 1 + protocol = { + "minReaderVersion": 3, + "minWriterVersion": 7, + "readerFeatures": ["orcapodMadeUpFeature"], + "writerFeatures": ["orcapodMadeUpFeature"], + } + (log / f"{version:020d}.json").write_text(json.dumps({"protocol": protocol})) + + @pytest.fixture def stored(tmp_path: Path, make_orchestrator) -> Path: """A store in which the job has computed and stored all three rows.""" @@ -118,6 +143,30 @@ def test_unreadable_result_table_fails_the_run_under_continue( assert _CALLS == [] +def test_store_read_failure_in_a_row_flush_fails_the_run(stored, make_orchestrator): + # Rows d and e are new. The result table breaks while d is computed, after + # d's cache lookup succeeded, so the failure surfaces in the flush that stores + # d's result, inside the error_policy="continue" per-row handler. Before the + # fix the flush wrapped the StoreReadError in a RuntimeError, the handler + # skipped d as a failed row, computed e, and the run ended with RuntimeError. + result_table, _ = table_dirs(stored) + _DAMAGE_ON_NEXT_CALL.append(result_table) + + orchestrator = make_orchestrator() + job = build_job(stored, keys="abcde") + with pytest.raises(StoreReadError) as info: + job.run(orchestrator=orchestrator) + + assert Path(info.value.table) == result_table + if isinstance(orchestrator, SyncPipelineOrchestrator): + # The run stops at d: e is never computed. + assert _CALLS == [4] + else: + # The async pod computes rows concurrently, so e may already be in flight + # when d's flush fails. + assert 4 in _CALLS and len(_CALLS) <= 2 + + def test_absent_tables_are_computed(stored, make_orchestrator): result_table, pipeline_table = table_dirs(stored) shutil.rmtree(result_table) From 3c75c095804a9eff1c594565627af1109984ffa3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:32:28 -0800 Subject: [PATCH 05/73] fix(pipeline): raise StoreReadError on its own when another node also failed (ITL-884) The async orchestrator re-raised the StoreReadError inside an "except* StoreReadError" handler. When an unrelated node failed in the same TaskGroup, the unmatched exception stayed in the group and the caller got an ExceptionGroup instead of the StoreReadError (on any Python version; on 3.11.0-3.11.3 even a lone error was re-wrapped). A plain "except ExceptionGroup" now selects the first StoreReadError and raises it, noting every other failure on it (other store read errors are also logged), with the group as its __context__ and the backend error kept as __cause__. Same shape as PR #287's handling of StoreWriteError. Addresses CodeRabbit comment 4534230775 on PR #290. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 7 ++- src/orcapod/pipeline/async_orchestrator.py | 58 ++++++++++++++++++- .../test_pipeline/test_store_read_failure.py | 53 +++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 7ce70ad6b..9de98f4fc 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2236,7 +2236,12 @@ async `FunctionPod` row handler and the async orchestrator let it fail the run w `error_policy`. Before merging into an existing table, `_flush_by_key` compares the batch with the stored Delta schema through Delta's own type mapping (extension types by storage, large-normalized, nullability ignored). On a difference it raises `InputValidationError` naming the -table, column and both types, and nothing is written. Regressions: +table, column and both types, and nothing is written. `flush()` raises a `StoreReadError` or +that `InputValidationError` as itself, not wrapped in `RuntimeError`, with every other failed +batch noted on it, so a read failure in a per-row result-cache flush still fails the run. The +async orchestrator selects the `StoreReadError` from the `TaskGroup`'s exception group with a +plain `except`, noting the other failures, so a concurrent unrelated failure cannot turn it into +an `ExceptionGroup`. Regressions: `tests/test_databases/test_delta_read_failures.py` and `tests/test_pipeline/test_store_read_failure.py`. Open: whether a nullability-only change should be refused (Delta's merge relaxes a non-null nested field to nullable) belongs to the diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index ecb9b1b42..4fb3a0db9 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -251,14 +251,68 @@ async def _run_async( return OrchestratorResult( node_outputs=collectors if materialize_results else {} ) - except* StoreReadError as group: + except ExceptionGroup as group: # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. - raise StoreReadError.in_group(group)[0] + # A plain except, not except*: raising inside except* can wrap the + # error in a new group again. + store_error = _single_store_read_error(group) + if store_error is None: + raise + raise store_error finally: effective_observer.on_run_end(run_id) +def _single_store_read_error(group: ExceptionGroup) -> StoreReadError | None: + """Pick the one ``StoreReadError`` an async run raises from its exception group. + + Concurrent node tasks can fail together, so the ``TaskGroup``'s group may hold + several store read errors and other exceptions. The caller gets a single + ``StoreReadError``, as from the sync orchestrator: the first one in the group. + Every other exception stays attached for diagnosis: each is noted on the + chosen error, other store read errors are also logged at WARNING, and the + whole group is the chosen error's ``__context__``. Its ``__cause__`` stays the + backend error. + + Args: + group: The exception group the run's ``TaskGroup`` raised. + + Returns: + The store read error to raise, or ``None`` when the group holds none. + """ + store_errors = StoreReadError.in_group(group) + if not store_errors: + return None + chosen = store_errors[0] + for other in _leaves(group): + if other is chosen: + continue + if isinstance(other, StoreReadError): + logger.warning( + "Another store read failed in the same run: %s", + other, + exc_info=other, + ) + chosen.add_note(f"Another store read failed in the same run: {other}") + else: + chosen.add_note( + f"Another node failed in the same run: {type(other).__name__}: {other}" + ) + return chosen + + +def _leaves(group: BaseExceptionGroup) -> list[BaseException]: + """Return every non-group exception in ``group``, nested groups flattened.""" + found: list[BaseException] = [] + for exc in group.exceptions: + if isinstance(exc, BaseExceptionGroup): + found.extend(_leaves(exc)) + else: + found.append(exc) + return found + + class _CollectingWriter: """Wrapper that collects items while forwarding to real writer.""" diff --git a/tests/test_pipeline/test_store_read_failure.py b/tests/test_pipeline/test_store_read_failure.py index 3311033df..c1eaa2b86 100644 --- a/tests/test_pipeline/test_store_read_failure.py +++ b/tests/test_pipeline/test_store_read_failure.py @@ -24,6 +24,7 @@ from orcapod.errors import StoreReadError from orcapod.pipeline import PipelineJob from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator _CALLS: list[int] = [] @@ -40,6 +41,11 @@ def double(x: int) -> int: return 2 * x +@function_pod(output_keys="z") +def triple(x: int) -> int: + return 3 * x + + ORCHESTRATORS: dict[str, Callable[[], object]] = { "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), @@ -180,3 +186,50 @@ def test_absent_tables_are_computed(stored, make_orchestrator): _CALLS.clear() build_job(stored).run(orchestrator=make_orchestrator()) assert _CALLS == [] + + +class _FailsToStartNode(NoOpObserver): + """An observer whose ``on_node_start`` raises for one node label.""" + + def __init__(self, label: str) -> None: + self.label = label + + def on_node_start( + self, node_label: str, node_hash: str, tag_schema: object = None + ) -> None: + if node_label == self.label: + raise ValueError(f"observer failed for {node_label}") + + +def test_unrelated_concurrent_failure_does_not_hide_store_read_error(tmp_path): + # Node "dbl" raises StoreReadError while loading its stored results and node + # "tri" fails in the observer's on_node_start. Both fail before their first + # await, so both land in the async TaskGroup's exception group. Before the fix, + # re-raising inside "except* StoreReadError" left the unmatched ValueError in + # the group and the caller got an ExceptionGroup, whatever the Python version. + stored = tmp_path / "store" + _ = build_job(stored).run() + result_table, _ = table_dirs(stored) + delete_data_files(result_table) + + source = ArrowTableSource( + pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + tag_columns=["k"], + infer_nullable=True, + ) + job = PipelineJob(name="itl884", store=DeltaTableDatabase(base_path=stored)) + with job: + _ = double.pod(source, label="dbl") + _ = triple.pod(source, label="tri") + + with pytest.raises(StoreReadError) as info: + job.run( + orchestrator=AsyncPipelineOrchestrator(), + observer=_FailsToStartNode("tri"), + ) + + assert type(info.value) is StoreReadError + assert Path(info.value.table) == result_table + assert info.value.__cause__ is not None + notes = getattr(info.value, "__notes__", []) + assert any("ValueError: observer failed for tri" in note for note in notes), notes From 2e238c695e612d4e43c883baa8bcb515edc25815 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:37:53 -0800 Subject: [PATCH 06/73] test(databases): regression tests for Delta reads missing their own writes (ITL-864) Database-level tests for each read method (sibling at() handles, stored plus pending rows, id overlap under skip_duplicates True and False, record_id_column) and a PipelineJob test on both orchestrators. They fail before the fix. Co-Authored-By: Claude Opus 5.5 --- ...26-11-10-itl-864-delta-read-your-writes.md | 158 +++++++++++++++ .../test_delta_read_your_writes.py | 183 ++++++++++++++++++ .../test_delta_node_records_after_run.py | 46 +++++ 3 files changed, 387 insertions(+) create mode 100644 superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md create mode 100644 tests/test_databases/test_delta_read_your_writes.py create mode 100644 tests/test_pipeline/test_delta_node_records_after_run.py diff --git a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md new file mode 100644 index 000000000..993c7eaa2 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md @@ -0,0 +1,158 @@ +# ITL-864 DeltaTableDatabase read-your-writes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every read method of `DeltaTableDatabase` returns exactly the rows a flush would leave, through every handle of an `at()` family, so `node.get_all_records()` returns every row right after the first run on a Delta store. + +**Architecture:** Two defects, one contract. (1) `at()` handles share the root's pending batches, and any handle's `flush()` writes every pending key, but each handle keeps its own `_delta_table_cache`, so a handle that loaded a table earlier keeps serving a stale `DeltaTable` snapshot after a sibling flushed. The fix shares `_delta_table_cache`, `_existing_ids_cache` and `_cache_dirty` across the family, exactly as pending state is already shared. (2) The read methods merge pending and stored rows inconsistently (`get_records_by_ids` ignores pending; `get_records_with_column_value` ignores stored whenever a pending batch exists; overlapping ids show twice). The fix: every read returns stored rows plus pending rows whose record id is not stored (Delta's flush is `when_not_matched_insert_all`, so the stored row wins), `None` when empty. + +**Tech Stack:** Python, pyarrow, deltalake, pytest, `uv run`. + +Approved by the coordinator (option A, Nov 10, 2026). No public API, hash, stored schema or dependency change. Error handling is untouched (PR #290 / ITL-884 rewrites it in the same file; merge main after it lands). + +--- + +## Reproduction (Step 1 of the brief) + +`PipelineJob`, `DictSource` of 2 or 3 ints, `int -> int` pod, `DeltaTableDatabase`: + +| store | orchestrator | n | `get_all_records` | `iter_data` | +|---|---|---|---|---| +| delta | sync | 2 / 3 | 1 / 1 | 2 / 3 | +| delta | async | 2 / 3 | 1 / 2 | 2 / 3 | +| memory | both | 2 / 3 | 2 / 3 | 2 / 3 | + +Trace: the result-store handle (`at("pipeline", "_result")`) flushes the pipeline table's rows; the node's pipeline handle (`at("pipeline")`) cached the table at version 0 and still reads version 0 while the table is at version 2. Its pending batch is empty; a fresh handle reads all rows. + +## Contract and per-method findings (database level) + +Contract: a read through a handle reflects all writes of its `at()` family, pending and flushed together, and equals what a flush would leave: no duplicates, no missing rows. + +| case | `get_all_records` | `get_record_by_id` | `get_records_by_ids` | `get_records_with_column_value` | +|---|---|---|---|---| +| A sibling handle flushed | stale | stale | stale | stale | +| B stored + pending, one handle | ok | ok | misses pending | misses stored when pending exists; empty table instead of `None` | +| C pending id already stored, `skip_duplicates=False` | duplicate | returns pending row, flush keeps stored | (dup) | (dup) | +| D same, `skip_duplicates=True` | ok | ok | ok | ok | +| `record_id_column=` | returns (no hang) | | | | + +Other backends: `InMemoryArrowDatabase` shares `_tables` across `at()` and `ConnectorArrowDatabase` re-reads committed rows on every call, so neither goes stale; both merge pending + committed in every method. Both flush with the NEW row winning on id overlap (Delta keeps the stored row) and show both rows before flush under `skip_duplicates=False`. Report only. + +Thread safety: the async orchestrator runs only the user function in `run_in_executor` (`core/executors/local.py`); every database read and write happens on the event-loop thread, as for the already-shared pending dicts, so sharing the caches adds no cross-thread access. + +## File structure + +- Modify: `src/orcapod/databases/delta_lake_databases.py` — `__init__` / `at()` share caches; new helpers `_pending_not_stored` and `_combine_stored_and_pending`; the four read methods use them. +- Create: `tests/test_databases/test_delta_read_your_writes.py` — database-level regression tests. +- Create: `tests/test_pipeline/test_delta_node_records_after_run.py` — `PipelineJob` regression, both orchestrators. +- Modify: `DESIGN_ISSUES.md` — one entry appended at the end. + +--- + +### Task 1: Failing database-level tests + +**Files:** Create `tests/test_databases/test_delta_read_your_writes.py` + +- [ ] **Step 1: Write the tests** + +Helpers: `_rows(db, ids, vals, groups)` builds a table with `__record_id` (`large_binary`), `v` (`int64`), `g` (`int64`); `_values(table)` returns sorted `v` or `None`. A `READS` parametrization maps each read method to a call and its expected values: + +```python +READS = { + "get_all_records": (lambda db: db.get_all_records(PATH), [1, 2]), + "get_record_by_id": (lambda db: db.get_record_by_id(PATH, b"b"), [2]), + "get_records_by_ids": (lambda db: db.get_records_by_ids(PATH, [b"a", b"b"]), [1, 2]), + "get_records_with_column_value": ( + lambda db: db.get_records_with_column_value(PATH, {"g": 0}), [1, 2]), +} +``` + +Tests: +1. `test_sibling_handle_sees_rows_flushed_by_another_handle[method][flusher]`: `root = DeltaTableDatabase(tmp_path)`, `h1, h2 = root.at("s"), root.at("s")`; `h1` writes `a=1` with `flush=True` and reads once (loads its snapshot); `h2` writes `b=2`; read through `h1` and through `root` (path `("s",) + PATH`) before flush; flush through `flusher` in (`h2`, `root`, `h1`); read through `h1`, `root` and a new `root.at("s")` after flush. All equal the expected values. +2. `test_pending_and_stored_rows_are_both_returned[method]`: one handle, stored `a=1 (g=0), b=2 (g=1)`, pending `c=3 (g=1)`: `get_records_by_ids([a, c]) == [1, 3]`, `get_records_with_column_value({"g": 1}) == [2, 3]`, `{"g": 0} == [1]`, `get_all_records == [1, 2, 3]`, `get_record_by_id(c) == [3]`. +3. `test_no_match_returns_none`: stored + pending, `get_records_with_column_value({"g": 9})` and `get_records_by_ids([b"zz"])` are `None`. +4. `test_pending_row_with_stored_id_reads_as_flush_keeps_it[skip_duplicates]`: stored `a=1`; pending add of `a=10, d=4`; before flush each method returns `a=1` and `d=4` once (`get_all_records == [1, 4]`, `get_record_by_id(a) == [1]`, `get_records_by_ids([a, d]) == [1, 4]`, `get_records_with_column_value({"g": 0}) == [1, 4]`); after `flush()` the same values. +5. `test_record_id_column_is_returned_for_merged_rows`: stored + pending, `get_all_records(PATH, record_id_column="rid")` has column `rid` with all three ids and no `__record_id`; same for `get_records_by_ids` and `get_records_with_column_value`. + +- [ ] **Step 2: Run, verify the expected failures** + +Run: `uv run pytest tests/test_databases/test_delta_read_your_writes.py -v` +Expected: test 1 fails for every method after flush; test 2 fails for `get_records_by_ids` / `get_records_with_column_value`; test 3 fails (empty table, not `None`); test 4 fails for `skip_duplicates=False`. + +### Task 2: Failing PipelineJob test + +**Files:** Create `tests/test_pipeline/test_delta_node_records_after_run.py` + +- [ ] **Step 1: Write the test** + +```python +@pytest.mark.parametrize("orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +def test_node_returns_every_row_right_after_first_run(tmp_path, orchestrator): + source = op.sources.DictSource([{"id": i, "x": 10 * i} for i in range(3)], tag_columns=["id"]) + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + job = PipelineJob(store=DeltaTableDatabase(tmp_path)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orchestrator()) + + records = job.nodes["double"].get_all_records() + assert records is not None + assert sorted(records.column("result").to_pylist()) == [0, 20, 40] +``` + +- [ ] **Step 2: Run, verify it fails** (`1 != 3` rows) for both orchestrators. + +- [ ] **Step 3: Commit the failing tests** + +```bash +git add tests/test_databases/test_delta_read_your_writes.py tests/test_pipeline/test_delta_node_records_after_run.py +git commit -m "test(databases): regression tests for Delta reads missing own writes (ITL-864)" +``` + +### Task 3: Share the snapshot caches across the `at()` family + +**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (`__init__`, `at()`) + +- [ ] **Step 1:** Add keyword-only private constructor parameters `_shared_delta_table_cache`, `_shared_existing_ids_cache`, `_shared_cache_dirty`, used when given (same pattern as `_shared_pending_batches`); `at()` passes the root instance's dicts. Comment why: any family member's `flush()` writes every pending key and refreshes the cache entry for it, so the snapshot cache must be family-wide or a sibling keeps reading the old version. +- [ ] **Step 2:** Run Task 1 test 1 and the Task 2 test: expected PASS for test 1 and Task 2. + +### Task 4: Merge stored and pending rows the way flush does + +**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (read methods) + +- [ ] **Step 1:** Add helpers: + +```python +def _pending_not_stored(self, record_path): + """Pending rows whose record id is not stored; flush inserts only these.""" + pending = self._pending_batches.get(self._get_record_key(record_path)) + if pending is None or pending.num_rows == 0: + return None + stored_ids = self._get_existing_ids(record_path) + if stored_ids: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter(pc.invert(pc.is_in( + pending[self.RECORD_ID_COLUMN], value_set=pa.array(list(stored_ids), type=id_type)))) + return pending if pending.num_rows > 0 else None + +def _combine_stored_and_pending(self, stored, pending, record_id_column): + parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] + if not parts: + return None + table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) + return self._handle_record_id_column(table, record_id_column) +``` + +- [ ] **Step 2:** `get_all_records`: pending part uses `_pending_not_stored`. `get_record_by_id`: serve from pending only when the id is pending and not stored. `get_records_by_ids`: stored read (existing try/except unchanged) plus pending filtered by the ids. `get_records_with_column_value`: stored read (existing try/except unchanged) plus pending filtered by the expression; `None` when empty. +- [ ] **Step 3:** Run `uv run pytest tests/test_databases tests/test_pipeline -q`: all pass. +- [ ] **Step 4:** Commit `fix(databases): Delta reads return every row the handle family wrote (ITL-864)`. + +### Task 5: Design issue entry, full verification, PR + +- [ ] Append one `DESIGN_ISSUES.md` entry (resolved, with **Fix:**; note that writes from other processes are still not seen until a handle reloads the table — follow-up for the coordinator to file). +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (golden hashes unchanged). +- [ ] Merge `origin/main`, rerun if it moved, push, PR "Fixes ITL-864", `orca linear attach`, `orca worktree set --pr`. diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py new file mode 100644 index 000000000..9dade27f0 --- /dev/null +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -0,0 +1,183 @@ +"""Reads through a ``DeltaTableDatabase`` handle reflect every write of its family (ITL-864). + +Contract: a read through any handle of one ``at()`` family returns exactly the rows a +flush would leave, pending and stored together, with no duplicates and no missing +rows. Delta's flush inserts only rows whose record id is not stored yet, so on an id +overlap the stored row is the one returned. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.databases import DeltaTableDatabase + +PATH = ("table",) +SCOPE = "scope" + + +def _rows( + ids: list[str], values: list[int], groups: list[int] | None = None +) -> pa.Table: + """Build a record table keyed by ``__record_id`` with ``int64`` columns ``v`` and ``g``.""" + return pa.table( + { + "__record_id": pa.array([i.encode() for i in ids], type=pa.large_binary()), + "v": pa.array(values, type=pa.int64()), + "g": pa.array( + groups if groups is not None else [0] * len(ids), type=pa.int64() + ), + } + ) + + +def _values(table: pa.Table | None) -> list[int] | None: + """Return the sorted ``v`` values of ``table``, or ``None`` when there is no table.""" + if table is None: + return None + return sorted(table.column("v").to_pylist()) + + +Read = Callable[[DeltaTableDatabase, tuple[str, ...]], "pa.Table | None"] + +# Each read method, called for rows ``a=1`` and ``b=2`` (both in group 0). +READS: dict[str, Read] = { + "get_all_records": lambda db, path: db.get_all_records(path), + "get_record_by_id": lambda db, path: db.get_record_by_id(path, b"b"), + "get_records_by_ids": lambda db, path: db.get_records_by_ids(path, [b"a", b"b"]), + "get_records_with_column_value": lambda db, path: db.get_records_with_column_value( + path, {"g": 0} + ), +} +EXPECTED = { + "get_all_records": [1, 2], + "get_record_by_id": [2], + "get_records_by_ids": [1, 2], + "get_records_with_column_value": [1, 2], +} + + +@pytest.mark.parametrize("flusher", ["writer", "root", "reader"]) +@pytest.mark.parametrize("method", list(READS)) +def test_sibling_handle_sees_rows_flushed_by_another_handle( + tmp_path: Path, method: str, flusher: str +) -> None: + """A handle that already loaded the table sees rows a sibling handle wrote and flushed.""" + root = DeltaTableDatabase(tmp_path) + reader = root.at(SCOPE) + writer = root.at(SCOPE) + read = READS[method] + + reader.add_records(PATH, _rows(["a"], [1]), flush=True) + assert _values(reader.get_all_records(PATH)) == [1] # reader loads its snapshot + + writer.add_records(PATH, _rows(["b"], [2])) + assert _values(read(reader, PATH)) == EXPECTED[method] + assert _values(read(root, (SCOPE, *PATH))) == EXPECTED[method] + + {"writer": writer, "root": root, "reader": reader}[flusher].flush() + + assert _values(read(reader, PATH)) == EXPECTED[method] + assert _values(read(writer, PATH)) == EXPECTED[method] + assert _values(read(root, (SCOPE, *PATH))) == EXPECTED[method] + assert _values(read(root.at(SCOPE), PATH)) == EXPECTED[method] + + +@pytest.fixture +def stored_and_pending(tmp_path: Path) -> DeltaTableDatabase: + """A handle with stored rows ``a=1 (g=0)``, ``b=2 (g=1)`` and pending row ``c=3 (g=1)``.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["a", "b"], [1, 2], [0, 1]), flush=True) + db.add_records(PATH, _rows(["c"], [3], [1])) + return db + + +def test_get_all_records_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + assert _values(stored_and_pending.get_all_records(PATH)) == [1, 2, 3] + + +def test_get_record_by_id_finds_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + assert _values(stored_and_pending.get_record_by_id(PATH, b"a")) == [1] + assert _values(stored_and_pending.get_record_by_id(PATH, b"c")) == [3] + + +def test_get_records_by_ids_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + assert _values(stored_and_pending.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] + + +def test_get_records_with_column_value_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + db = stored_and_pending + assert _values(db.get_records_with_column_value(PATH, {"g": 1})) == [2, 3] + assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1] + + +def test_reads_with_no_match_return_none( + stored_and_pending: DeltaTableDatabase, +) -> None: + db = stored_and_pending + assert db.get_records_with_column_value(PATH, {"g": 9}) is None + assert db.get_records_by_ids(PATH, [b"zz"]) is None + assert db.get_record_by_id(PATH, b"zz") is None + + +@pytest.mark.parametrize( + ("read", "expected_ids"), + [ + ( + lambda db: db.get_all_records(PATH, record_id_column="rid"), + [b"a", b"b", b"c"], + ), + ( + lambda db: db.get_records_by_ids( + PATH, [b"a", b"c"], record_id_column="rid" + ), + [b"a", b"c"], + ), + ( + lambda db: db.get_records_with_column_value( + PATH, [("g", 1)], record_id_column="rid" + ), + [b"b", b"c"], + ), + ], + ids=["get_all_records", "get_records_by_ids", "get_records_with_column_value"], +) +def test_record_id_column_names_the_merged_ids( + stored_and_pending: DeltaTableDatabase, + read: Callable[[DeltaTableDatabase], pa.Table | None], + expected_ids: list[bytes], +) -> None: + """With ``record_id_column`` the ids of stored and pending rows come back under that name.""" + table = read(stored_and_pending) + assert table is not None + assert "__record_id" not in table.column_names + assert sorted(table.column("rid").to_pylist()) == expected_ids + + +@pytest.mark.parametrize("skip_duplicates", [False, True]) +def test_pending_row_with_stored_id_reads_as_flush_keeps_it( + tmp_path: Path, skip_duplicates: bool +) -> None: + """A pending row whose id is already stored is never returned; the stored row is.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["a"], [1]), flush=True) + db.add_records(PATH, _rows(["a", "d"], [10, 4]), skip_duplicates=skip_duplicates) + + for when in ("before flush", "after flush"): + assert _values(db.get_all_records(PATH)) == [1, 4], when + assert _values(db.get_record_by_id(PATH, b"a")) == [1], when + assert _values(db.get_records_by_ids(PATH, [b"a", b"d"])) == [1, 4], when + assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1, 4], when + db.flush() diff --git a/tests/test_pipeline/test_delta_node_records_after_run.py b/tests/test_pipeline/test_delta_node_records_after_run.py new file mode 100644 index 000000000..15c925f1f --- /dev/null +++ b/tests/test_pipeline/test_delta_node_records_after_run.py @@ -0,0 +1,46 @@ +"""A function node on a Delta store returns every row right after its first run (ITL-864). + +``PipelineJob.run`` flushes through handles other than the node's own, so a node +handle that cached the table during the run used to keep reading an old snapshot +and return only some of the rows it had just written. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +@pytest.mark.parametrize("n_rows", [2, 3]) +def test_node_returns_every_row_right_after_first_run( + tmp_path: Path, + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], + n_rows: int, +) -> None: + source = op.sources.DictSource( + [{"id": i, "x": 10 * i} for i in range(n_rows)], tag_columns=["id"] + ) + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + job = PipelineJob(store=DeltaTableDatabase(tmp_path)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orchestrator()) + + records = job.nodes["double"].get_all_records() + assert records is not None + assert sorted(records.column("result").to_pylist()) == [ + 20 * i for i in range(n_rows) + ] From b727e0c6f52787ec6a91ad050a7ec6329d0e2306 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:39:50 -0800 Subject: [PATCH 07/73] fix(databases): Delta reads return every row the handle family wrote (ITL-864) Every at() handle shares the root's pending batches and any handle's flush() writes every pending key, but each handle cached its own loaded DeltaTable, so the node's pipeline handle kept reading the version it loaded during the run: node.get_all_records() returned 1 of 3 rows right after a first PipelineJob run. The loaded-table, existing-ids and cache-dirty dicts are now shared across the at() family, like pending state. Every read method now returns what a flush would leave: stored rows plus the pending rows whose record id is not stored, None when empty. Before, get_records_by_ids ignored pending rows, get_records_with_column_value ignored stored rows whenever a pending batch existed, and a pending row with a stored id was returned twice or instead of the stored row. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 34 ++++ src/orcapod/databases/delta_lake_databases.py | 158 ++++++++++++------ 2 files changed, 143 insertions(+), 49 deletions(-) diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 5b167d8de..ee1f0d0c8 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2208,3 +2208,37 @@ subprocesses with different `PYTHONHASHSEED` values and were verified identical 3.11, 3.12 and 3.13. The generator's header says when regeneration is legitimate: a changed value needs a migration assessment and the developer's approval. Not pinned yet: the `StarfixArrowHasher` gaps tracked in ITL-848. + +--- + +## `src/orcapod/databases/delta_lake_databases.py` — reading back a handle family's own writes + +### DW1 — A read through one `at()` handle could miss rows its family had just written +**Status:** resolved +**Severity:** high +**Issue:** ITL-864 + +Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` +returned 1 of 2 or 3 rows (plain `int -> int` pod, sync and async orchestrators), while +`iter_data` and a fresh handle saw every row. Every `at()` handle shares the root's pending +batches, and any handle's `flush()` writes every pending key, so the result-store handle +flushed the pipeline table's rows. But each handle kept its own loaded-table cache, so the +node's pipeline handle kept reading the table version it had loaded during the run. Every read +method missed the rows a sibling flushed. Separately, the read methods merged pending and stored +rows inconsistently: `get_records_by_ids` ignored pending rows, `get_records_with_column_value` +ignored stored rows whenever a pending batch existed (and returned an empty table instead of +`None`), and a pending row whose record id was already stored was returned as well, although a +flush keeps the stored row and drops the pending one. + +**Fix:** the loaded-table, existing-ids and cache-dirty dicts are shared across the `at()` +family exactly as pending state is. Every read method returns what a flush would leave: the +stored rows plus the pending rows whose record id is not stored (`_pending_not_stored`), and +`None` when there are none. These dicts are touched only from the thread that drives the +pipeline; the async orchestrator runs only user functions in worker threads. Regression tests: +`tests/test_databases/test_delta_read_your_writes.py` and +`tests/test_pipeline/test_delta_node_records_after_run.py`. + +Still open: a handle does not see writes made by another process, or by a separate +`DeltaTableDatabase` root on the same path, until it reloads the table; this is unchanged. +Reading with a pending batch also refreshes the stored-id set when it is stale, which loads the +whole table (D3). diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index c7dec5dcf..f9b99bb09 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -57,6 +57,9 @@ def __init__( _scoped_path: tuple[str, ...] = (), _shared_pending_batches: "dict[str, pa.Table] | None" = None, _shared_pending_record_ids: "defaultdict[str, set[bytes]] | None" = None, + _shared_delta_table_cache: "dict[str, deltalake.DeltaTable] | None" = None, + _shared_existing_ids_cache: "defaultdict[str, set[str]] | None" = None, + _shared_cache_dirty: "defaultdict[str, bool] | None" = None, ): self._root_uri, self._storage_options = parse_base_path( base_path, storage_options @@ -81,9 +84,16 @@ def __init__( ) # For cloud paths: create_base_path is silently ignored (no directory needed). - self._delta_table_cache: dict[str, deltalake.DeltaTable] = {} - # Shared pending state — when constructed via at(), these dicts are shared - # with the root instance so that root.flush() can flush all scoped writes. + # Shared state — when constructed via at(), these dicts are shared with the + # root instance. Any handle's flush() writes every pending key of the family + # and refreshes the loaded table and existing-ids entries for it, so these + # caches must be family-wide too: a per-handle cache would keep serving the + # snapshot a sibling loaded before the flush (ITL-864). All of them are + # touched only from the thread that drives the pipeline (the async + # orchestrator runs only user functions in worker threads). + self._delta_table_cache: dict[str, deltalake.DeltaTable] = ( + _shared_delta_table_cache if _shared_delta_table_cache is not None else {} + ) if _shared_pending_batches is not None: self._pending_batches = _shared_pending_batches else: @@ -92,8 +102,16 @@ def __init__( self._pending_record_ids = _shared_pending_record_ids else: self._pending_record_ids: dict[str, set[bytes]] = defaultdict(set) - self._existing_ids_cache: dict[str, set[str]] = defaultdict(set) - self._cache_dirty: dict[str, bool] = defaultdict(lambda: True) + self._existing_ids_cache: dict[str, set[str]] = ( + _shared_existing_ids_cache + if _shared_existing_ids_cache is not None + else defaultdict(set) + ) + self._cache_dirty: dict[str, bool] = ( + _shared_cache_dirty + if _shared_cache_dirty is not None + else defaultdict(lambda: True) + ) def _clear_pending(self): """Clear all pending state.""" @@ -620,6 +638,57 @@ def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool: and self._pending_batches[record_key].num_rows >= self.batch_size ) + def _pending_not_stored(self, record_path: tuple[str, ...]) -> pa.Table | None: + """Return the pending rows a flush would insert at ``record_path``. + + A flush inserts only rows whose record id is not stored yet (the merge is + ``when_not_matched_insert_all``), so a pending row with a stored id is + never returned by a read: the stored row is. + + Args: + record_path: Path components relative to this database view. + + Returns: + The pending rows whose record id is not stored, or ``None`` if there + are none. + """ + pending = self._pending_batches.get(self._get_record_key(record_path)) + if pending is None or pending.num_rows == 0: + return None + stored_ids = self._get_existing_ids(record_path) + if stored_ids: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + is_stored = pc.is_in( + pending[self.RECORD_ID_COLUMN], + value_set=pa.array(list(stored_ids), type=id_type), + ) + pending = pending.filter(pc.invert(is_stored)) + return pending if pending.num_rows > 0 else None + + def _combine_stored_and_pending( + self, + stored: pa.Table | None, + pending: pa.Table | None, + record_id_column: str | None, + ) -> pa.Table | None: + """Combine stored rows with pending rows into one read result. + + Args: + stored: Rows read from the Delta table, or ``None``. + pending: Pending rows that are not stored (see ``_pending_not_stored``), + already filtered by the read's condition, or ``None``. + record_id_column: Name to give the record id column, or ``None`` to + drop it. + + Returns: + The combined rows, or ``None`` if there are none. + """ + parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] + if not parts: + return None + table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) + return self._handle_record_id_column(table, record_id_column) + def get_all_records( self, record_path: tuple[str, ...], @@ -629,11 +698,13 @@ def get_all_records( """ Get all records from both pending batch and Delta table. + Every read method returns the rows a flush would leave: the stored rows + plus the pending rows whose record id is not stored yet, as seen through + any handle of this ``at()`` family. + Returns: Combined Arrow table with all records, or None if no records exist """ - record_key = self._get_record_key(record_path) - tables_to_combine = [] # Add Delta table data @@ -647,24 +718,10 @@ def get_all_records( except Exception as e: logger.warning(f"Error reading Delta table: {e}") - # Add pending batch data - if ( - retrieve_pending - and (pending_batch := self._pending_batches.get(record_key)) is not None - ): - if pending_batch.num_rows > 0: - tables_to_combine.append(pending_batch) - - if not tables_to_combine: - return None - - if len(tables_to_combine) == 1: - table_to_return = tables_to_combine[0] - else: - table_to_return = pa.concat_tables(tables_to_combine) - - # Handle record_id_column if specified - return self._handle_record_id_column(table_to_return, record_id_column) + # Add the pending rows a flush would insert + pending = self._pending_not_stored(record_path) if retrieve_pending else None + stored = pa.concat_tables(tables_to_combine) if tables_to_combine else None + return self._combine_stored_and_pending(stored, pending, record_id_column) def get_records_with_column_value( self, @@ -675,9 +732,6 @@ def get_records_with_column_value( ) -> "pa.Table | None": if flush: self.flush_batch(record_path) - # check if record_id is found in pending batches - record_key = self._get_record_key(record_path) - pending_batch = self._pending_batches.get(record_key) if isinstance(column_values, Mapping): # Convert Mapping to list of tuples @@ -691,31 +745,26 @@ def get_records_with_column_value( for next_expression in expressions[1:]: combined_expression = combined_expression & next_expression - if pending_batch is not None: - filtered_table = pending_batch.filter(combined_expression) - return self._handle_record_id_column(filtered_table, record_id_column) + pending = self._pending_not_stored(record_path) + if pending is not None: + pending = pending.filter(combined_expression) # Now check the Delta table delta_table = self._get_delta_table(record_path) if delta_table is None: - return None + return self._combine_stored_and_pending(None, pending, record_id_column) try: # Use schema-preserving read result = self._read_delta_table(delta_table, expression=combined_expression) - - if len(result) == 0: - return None - - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) - except Exception as e: logger.error( f"Error getting record with {column_values} from {'/'.join(record_path)}: {e}" ) raise e + return self._combine_stored_and_pending(result, pending, record_id_column) + def get_record_by_id( self, record_path: tuple[str, ...], @@ -738,9 +787,12 @@ def get_record_by_id( self.flush_batch(record_path) record_id = coerce_record_id(record_id) - # check if record_id is found in pending batches + # Serve a pending record only if its id is not stored: a flush keeps the + # stored row and drops the pending one. record_key = self._get_record_key(record_path) - if record_id in self._pending_record_ids[record_key]: + if record_id in self._pending_record_ids[ + record_key + ] and record_id not in self._get_existing_ids(record_path): # Return the pending record after removing the entry id column pending_batch = self._pending_batches[record_key] assert pending_batch is not None, "Pending batch should not be None" @@ -812,26 +864,31 @@ def get_records_by_ids( return None # check inside the batch + pending = self._pending_not_stored(record_path) + if pending is not None: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter( + pc.is_in( + pending[self.RECORD_ID_COLUMN], + value_set=pa.array(record_ids_list, type=id_type), + ) + ) + delta_table = self._get_delta_table(record_path) if delta_table is None: - return None + return self._combine_stored_and_pending(None, pending, record_id_column) try: # Use schema-preserving read with filters filter_expr = self._create_record_ids_filter(record_ids_list) result = self._read_delta_table(delta_table, filters=filter_expr) - - if len(result) == 0: - return None - - # Handle record_id column based on parameter - return self._handle_record_id_column(result, record_id_column) - except Exception as e: logger.error( f"Error getting records by IDs from {'/'.join(record_path)}: {e}" ) return None + return self._combine_stored_and_pending(result, pending, record_id_column) + def _read_delta_table( self, delta_table: deltalake.DeltaTable, @@ -1098,6 +1155,9 @@ def at(self, *path_components: str) -> "DeltaTableDatabase": _scoped_path=new_scoped_path, _shared_pending_batches=root_instance._pending_batches, _shared_pending_record_ids=root_instance._pending_record_ids, + _shared_delta_table_cache=root_instance._delta_table_cache, + _shared_existing_ids_cache=root_instance._existing_ids_cache, + _shared_cache_dirty=root_instance._cache_dirty, ) def list_sources(self) -> list[tuple[str, ...]]: From ce32c9e6d7230da649ef1bbb0cb54431853723f2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:42:08 -0800 Subject: [PATCH 08/73] refactor(pipeline): give the async store-error selection #287's shape (ITL-884) Rename _single_store_read_error to _single_store_error(group, topo_order) and declare topo_order and store_error before the try, exactly as PR #287 does for StoreWriteError, so whichever PR merges second only has to widen the helper to both error types. Behaviour is unchanged: a StoreReadError names no node, so the first one in the group is still chosen. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/async_orchestrator.py | 31 ++++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 4fb3a0db9..36e710a86 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -138,6 +138,8 @@ async def _run_async( effective_observer = observer if observer is not None else NoOpObserver() effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) + topo_order: list[Any] = [] + store_error: StoreReadError | None = None try: topo_order = list(graph.topological_sort()) buf = self._buffer_size @@ -254,9 +256,7 @@ async def _run_async( except ExceptionGroup as group: # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. - # A plain except, not except*: raising inside except* can wrap the - # error in a new group again. - store_error = _single_store_read_error(group) + store_error = _single_store_error(group, topo_order) if store_error is None: raise raise store_error @@ -264,27 +264,36 @@ async def _run_async( effective_observer.on_run_end(run_id) -def _single_store_read_error(group: ExceptionGroup) -> StoreReadError | None: +def _single_store_error( + group: ExceptionGroup, topo_order: list[Any] +) -> StoreReadError | None: """Pick the one ``StoreReadError`` an async run raises from its exception group. Concurrent node tasks can fail together, so the ``TaskGroup``'s group may hold - several store read errors and other exceptions. The caller gets a single - ``StoreReadError``, as from the sync orchestrator: the first one in the group. - Every other exception stays attached for diagnosis: each is noted on the - chosen error, other store read errors are also logged at WARNING, and the - whole group is the chosen error's ``__context__``. Its ``__cause__`` stays the + several store errors and other exceptions. The caller gets a single + ``StoreReadError``, as from the sync orchestrator: the one whose node comes + first in topological order, with ties and node-less errors (a + ``StoreReadError`` names a table, not a node) broken by their order in the + group. Every other exception stays attached for diagnosis: each is noted on + the chosen error, other store errors are also logged at WARNING, and the whole + group is the chosen error's ``__context__``. Its ``__cause__`` stays the backend error. Args: group: The exception group the run's ``TaskGroup`` raised. + topo_order: The run's nodes in topological order. Returns: - The store read error to raise, or ``None`` when the group holds none. + The store error to raise, or ``None`` when the group holds none. """ store_errors = StoreReadError.in_group(group) if not store_errors: return None - chosen = store_errors[0] + rank = {getattr(node, "label", None): i for i, node in enumerate(topo_order)} + chosen = min( + store_errors, + key=lambda err: rank.get(getattr(err, "node_label", None), len(rank)), + ) for other in _leaves(group): if other is chosen: continue From 3785d71beec0f314badd6a61b86a046bedf5485e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:42:17 -0800 Subject: [PATCH 09/73] docs(plans): name the async helper as implemented (ITL-884) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-10-itl-884-coderabbit-flush-and-async-group.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md index caaebe4b1..394e968b6 100644 --- a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -14,7 +14,7 @@ each failure. When at least one batch failed with a store error, it raises the f `StoreReadError` (else the first `InputValidationError`) itself, with a note for every other failed batch. Only non-store failures keep the `RuntimeError` wrapper. The async orchestrator replaces `except* StoreReadError` with a plain `except ExceptionGroup` that selects one -`StoreReadError` (`_single_store_read_error`), notes and logs the rest, and raises it, in the same +`StoreReadError` (`_single_store_error`), notes and logs the rest, and raises it, in the same shape as PR #287's `_single_store_error` for `StoreWriteError` so the second PR to merge has a trivial conflict. @@ -42,7 +42,7 @@ No hash, path or stored-schema changes. No public API added. - `src/orcapod/databases/delta_lake_databases.py`: `flush()` and a module helper `_store_error_to_raise(errors)`. - `src/orcapod/pipeline/async_orchestrator.py`: plain `except ExceptionGroup`, helpers - `_single_store_read_error(group)` and `_leaves(group)` (identical to #287's `_leaves`). + `_single_store_error(group, topo_order)` and `_leaves(group)` (identical to #287's `_leaves`). - Tests: `tests/test_databases/test_delta_read_failures.py` (assert the bare error types, add a multi-batch test), `tests/test_pipeline/test_store_read_failure.py` (flush during per-row processing, both orchestrators; concurrent unrelated failure, async). @@ -134,7 +134,7 @@ docstring gets a Raises section. Run Task 1's tests: PASS. raise store_error ``` -`_single_store_read_error(group)`: `StoreReadError.in_group(group)`; none gives `None`; the first +`_single_store_error(group, topo_order)`: `StoreReadError.in_group(group)`; none gives `None`; the first is chosen; every other leaf (`_leaves`) is noted on it ("Another store read failed in the same run: ..." also logged at WARNING; "Another node failed in the same run: : "). Its `__cause__` stays the backend error; the group is its `__context__`. Run: PASS. From 7d96a01549681aec530f4aec257a6de41acdef0f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:42:37 -0800 Subject: [PATCH 10/73] docs(plans): fix the remaining helper name in the plan's code sample (ITL-884) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-10-itl-884-coderabbit-flush-and-async-group.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md index 394e968b6..89c03292c 100644 --- a/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md +++ b/superpowers/plans/2026-11-10-itl-884-coderabbit-flush-and-async-group.md @@ -128,7 +128,7 @@ docstring gets a Raises section. Run Task 1's tests: PASS. # A failed store read stops the run. Raise it on its own, as the sync # orchestrator does, rather than inside the TaskGroup's exception group. # A plain except (not except*) so the raise is never wrapped again. - store_error = _single_store_read_error(group) + store_error = _single_store_error(group, topo_order) if store_error is None: raise raise store_error From e9c78b5c6e9f2b15725db8f33eb445d97a77dc1c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:43:53 -0800 Subject: [PATCH 11/73] fix(databases): type Delta stored-id sets as bytes and fix test typing (ITL-864) Record ids are bytes; the set[str] annotation made basedpyright report the new stored-id check as always true. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/databases/delta_lake_databases.py | 10 +++++----- tests/test_databases/test_delta_read_your_writes.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index f9b99bb09..a50853084 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -58,8 +58,8 @@ def __init__( _shared_pending_batches: "dict[str, pa.Table] | None" = None, _shared_pending_record_ids: "defaultdict[str, set[bytes]] | None" = None, _shared_delta_table_cache: "dict[str, deltalake.DeltaTable] | None" = None, - _shared_existing_ids_cache: "defaultdict[str, set[str]] | None" = None, - _shared_cache_dirty: "defaultdict[str, bool] | None" = None, + _shared_existing_ids_cache: "dict[str, set[bytes]] | None" = None, + _shared_cache_dirty: "dict[str, bool] | None" = None, ): self._root_uri, self._storage_options = parse_base_path( base_path, storage_options @@ -102,7 +102,7 @@ def __init__( self._pending_record_ids = _shared_pending_record_ids else: self._pending_record_ids: dict[str, set[bytes]] = defaultdict(set) - self._existing_ids_cache: dict[str, set[str]] = ( + self._existing_ids_cache: dict[str, set[bytes]] = ( _shared_existing_ids_cache if _shared_existing_ids_cache is not None else defaultdict(set) @@ -343,7 +343,7 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: self._existing_ids_cache[record_key] = set() else: existing_ids = cast( - set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) + set[bytes], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) ) self._existing_ids_cache[record_key] = existing_ids @@ -358,7 +358,7 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: self._cache_dirty[record_key] = False raise - def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[str]: + def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[bytes]: """Get the set of existing IDs in the Delta table, using cache when possible.""" record_key = self._get_record_key(record_path) if ( diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py index 9dade27f0..632b73a08 100644 --- a/tests/test_databases/test_delta_read_your_writes.py +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import cast from pathlib import Path import pyarrow as pa @@ -39,7 +40,7 @@ def _values(table: pa.Table | None) -> list[int] | None: """Return the sorted ``v`` values of ``table``, or ``None`` when there is no table.""" if table is None: return None - return sorted(table.column("v").to_pylist()) + return sorted(cast(list[int], table.column("v").to_pylist())) Read = Callable[[DeltaTableDatabase, tuple[str, ...]], "pa.Table | None"] @@ -163,7 +164,7 @@ def test_record_id_column_names_the_merged_ids( table = read(stored_and_pending) assert table is not None assert "__record_id" not in table.column_names - assert sorted(table.column("rid").to_pylist()) == expected_ids + assert sorted(cast(list[bytes], table.column("rid").to_pylist())) == expected_ids @pytest.mark.parametrize("skip_duplicates", [False, True]) From a87cfd5bb7e042e4a255bae8d5ef81e54400c1cf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:05:07 -0800 Subject: [PATCH 12/73] test(itl-892): exit-hang reproduction harness, matrix and temporary Linux probe Refs ITL-892 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/itl-892-exit-hang-probe.yml | 78 ++++++++ .../plans/2026-11-10-itl-892-exit-hang.md | 98 +++++++++ .../specs/itl-892-exit-hang/harness.py | 129 ++++++++++++ .../specs/itl-892-exit-hang/repro_orcapod.py | 109 ++++++++++ .../specs/itl-892-exit-hang/repro_pure.py | 187 ++++++++++++++++++ .../specs/itl-892-exit-hang/run_matrix.sh | 67 +++++++ 6 files changed, 668 insertions(+) create mode 100644 .github/workflows/itl-892-exit-hang-probe.yml create mode 100644 superpowers/plans/2026-11-10-itl-892-exit-hang.md create mode 100644 superpowers/specs/itl-892-exit-hang/harness.py create mode 100644 superpowers/specs/itl-892-exit-hang/repro_orcapod.py create mode 100644 superpowers/specs/itl-892-exit-hang/repro_pure.py create mode 100644 superpowers/specs/itl-892-exit-hang/run_matrix.sh diff --git a/.github/workflows/itl-892-exit-hang-probe.yml b/.github/workflows/itl-892-exit-hang-probe.yml new file mode 100644 index 000000000..4911039c5 --- /dev/null +++ b/.github/workflows/itl-892-exit-hang-probe.yml @@ -0,0 +1,78 @@ +# TEMPORARY (ITL-892): measures the interpreter-exit hang on Linux. Not a required +# check; runs only on the investigation branch. Delete before the PR leaves draft. +name: ITL-892 exit-hang probe + +on: + push: + branches: [eywalker/itl-892-process-hangs-at-interpreter-exit-after] + workflow_dispatch: + +permissions: + contents: read + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + env: + J: "4" + T: "8" + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Allow gdb to attach to hung children + run: | + sudo apt-get update && sudo apt-get install -y gdb + sudo sysctl -w kernel.yama.ptrace_scope=0 + + - name: Install dependencies (locked) + run: uv sync --locked --dev --python ${{ matrix.python-version }} + + - name: Probe + env: + UV_PYTHON: ${{ matrix.python-version }} + run: | + D=superpowers/specs/itl-892-exit-hang + rm -f $D/results.jsonl + H() { uv run python $D/harness.py -n 100 -t "$T" -j "$J" "$@"; } + py=${{ matrix.python-version }} + H --label "linux op=dataset_scan py$py locked" -- python $D/repro_pure.py --op dataset_scan + H --label "linux op=to_table py$py locked" -- python $D/repro_pure.py --op to_table + H --label "linux op=itl864 py$py locked" -- python $D/repro_pure.py --op itl864 + H --label "linux dataset_scan native_fs py$py locked" -- python $D/repro_pure.py --op dataset_scan --read-via native_fs + H --label "linux dataset_scan no_threads py$py locked" -- python $D/repro_pure.py --op dataset_scan --read-via no_threads + H --label "linux dataset_scan io_threads=1 py$py locked" -- python $D/repro_pure.py --op dataset_scan --io-threads 1 + H --label "linux dataset_scan mitigation=del py$py locked" -- python $D/repro_pure.py --op dataset_scan --mitigation del + for m in plain-read source-read pipeline-sync pipeline-async; do + H --label "linux orcapod $m py$py locked" -- python $D/repro_orcapod.py "$m" + done + for v in "22.0.0 1.0.2" "26.0.1 1.6.6"; do + set -- $v + uv run --no-project --with "pyarrow==$1" --with "deltalake==$2" python $D/harness.py \ + -n 100 -t "$T" -j "$J" --label "linux op=dataset_scan py$py pa$1 dl$2" -- \ + python $D/repro_pure.py --op dataset_scan + done + cat $D/results.jsonl >> "$GITHUB_STEP_SUMMARY" + + - name: Upload results and stacks + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: itl-892-linux-py${{ matrix.python-version }} + path: | + superpowers/specs/itl-892-exit-hang/results.jsonl + superpowers/specs/itl-892-exit-hang/samples/ diff --git a/superpowers/plans/2026-11-10-itl-892-exit-hang.md b/superpowers/plans/2026-11-10-itl-892-exit-hang.md new file mode 100644 index 000000000..743e36d8f --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-892-exit-hang.md @@ -0,0 +1,98 @@ +# ITL-892 Exit Hang Investigation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reproduce and characterise the interpreter-exit hang after a Delta read (ITL-892), evaluate +mitigations with hang rates over at least 40 runs each, and report to the coordinator before any +`src/` or `uv.lock` change. + +**Architecture:** One timeout harness (`harness.py`) runs a target script N times as child processes, +counts exits that miss the deadline, and samples the native stack of the first hung child (`sample` +on macOS, `py-spy`/`gdb` not required). Target scripts are small and parameterised by argv so one +harness drives every matrix cell. Version cells run in throwaway environments +(`uv run --no-project --with pyarrow==X --with deltalake==Y`) so `uv.lock` never changes. + +**Tech Stack:** Python 3.11/3.12/3.13 (uv-managed), pyarrow, deltalake, polars, Orcapod +`PipelineJob` on `DeltaTableDatabase`, GitHub Actions (temporary, non-required job) for Linux. + +Constraints (from the dispatch brief): no change to `uv.lock` or `src/` before the coordinator +approves a mitigation. All experiments live in `superpowers/specs/itl-892-exit-hang/`. Draft PR body +says `Refs ITL-892`. + +--- + +### Task 1: Harness and minimal pure reproduction + +**Files:** +- Create: `superpowers/specs/itl-892-exit-hang/harness.py` +- Create: `superpowers/specs/itl-892-exit-hang/repro_pure.py` + +- [ ] Write `repro_pure.py`: builds a small Delta table in a temp dir and performs one operation + chosen by `--op` (`write`, `merge`, `to_table`, `to_dataset`, `dataset_scan`, `write_and_read` + — the last is the ITL-864 worker's script: 1 write, 4 merges, a `to_pyarrow_dataset().to_table()` + after each). Flags: `--import-polars`, `--cpu-threads N` (`pa.set_cpu_count`), `--io-threads N` + (`pa.set_io_thread_count`), `--mitigation {none,del,gc,os_exit,pool_shutdown}`, `--mp {none,fork,spawn}`. + Prints `done` with flush then returns. +- [ ] Write `harness.py`: `harness.py N TIMEOUT -- `; runs N children with up to `-j` in + parallel, each in its own session; a child that misses TIMEOUT is sampled once (first hang only, + `sample 2`) then killed; prints `hung/N`, exit-code histogram, whether `done` was printed + before the hang, and writes a JSON line to `results.jsonl` with the cell label. +- [ ] Run the ITL-864 script 40 times on Python 3.12 with the locked versions. Expected: a hang rate + near 50% and a stack in `ThreadPool::~ThreadPool`. If 0/40, vary parallelism (-j1 vs -j8) and + table size before concluding. + +### Task 2: Characterise the trigger + +- [ ] Run each `--op` alone, 40 runs each: which operation is necessary. +- [ ] With the triggering op: `--cpu-threads 1`, `--io-threads 1`, env `OMP_NUM_THREADS=1`, + `ARROW_IO_THREADS=1`, `RAYON_NUM_THREADS=1`; `--import-polars`; deltalake without pyarrow + imported where the API allows (`DeltaTable.to_pyarrow_*` needs pyarrow; use `write_deltalake` + with an arro3 table / `QueryBuilder` to separate the runtimes); `--mp fork/spawn`. +- [ ] Python 3.11 and 3.13 at the triggering cell. + +### Task 3: Orcapod paths + +**Files:** +- Create: `superpowers/specs/itl-892-exit-hang/repro_orcapod.py` + +- [ ] Modes: `pipeline-sync`, `pipeline-async` (a two-source `PipelineJob` with one function pod on + `DeltaTableDatabase`, then `get_all_records`), `plain-read` (`DeltaTableDatabase` write, flush, + read). 40 runs each on 3.12 via the project environment; 3.11/3.13 via `uv run --isolated --python`. + +### Task 4: Version bisect and upstream search + +- [ ] Grid over pyarrow {20.0.0, 21, 22, 23.0.1, 24, 25.0.1, 26.0.1} × deltalake {1.0.2, 1.1.x, + 1.2.x, 1.3.x, 1.4.2, 1.5.1, 1.6.6} at the triggering cell, 40 runs each (narrow by halving once a + boundary is visible); record newest non-hanging and oldest hanging combinations. +- [ ] Search apache/arrow and delta-io/delta-rs issues (WebSearch, `gh search issues`) for + `ThreadPool` destructor / exit hang / `__cxa_finalize`; link matches. + +### Task 5: Mitigations + +- [ ] (a) Best version combination from Task 4: copy the worktree to a scratch dir, `uv lock + --upgrade-package`, run golden hash tests and the full suite there (never in this worktree). +- [ ] (b) Orcapod-side shutdown, prototyped in the repro scripts only: drop `DeltaTable`/dataset + references + `gc.collect()`; an `atexit` hook; pyarrow pool shutdown if exposed; `os._exit` after + flush (for comparison only). +- [ ] (c) Alternative read path: `DeltaTable.to_pyarrow_table`, `QueryBuilder`/arro3, polars + `scan_delta`/`read_delta`, reading the parquet files listed by `DeltaTable.file_uris()` with + pyarrow directly. +- [ ] (d) Documentation workaround text. +- [ ] 40 runs per option; record hang rate. + +### Task 6: Linux + +- [ ] Docker is not installed on this machine. Push a temporary, non-required workflow + `.github/workflows/itl-892-exit-hang-probe.yml` (`workflow_dispatch` + push on this branch only) + running the harness on ubuntu-latest for 3.11/3.12/3.13. Remove it before the PR leaves draft. + +### Task 7: Report + +**Files:** +- Create: `superpowers/specs/itl-892-exit-hang/FINDINGS.md` +- Modify: `DESIGN_ISSUES.md` (one appended entry for ITL-892) + +- [ ] Findings: hang-rate tables, stack, trigger, versions, upstream links, options (a)-(d) with + evidence and cost, recommendation, proposed regression check and its reliability. +- [ ] Commit (Conventional Commits, Co-Authored-By trailer), push, open draft PR "Refs ITL-892". +- [ ] `ask` the coordinator with results and options; do not implement a mitigation before approval. diff --git a/superpowers/specs/itl-892-exit-hang/harness.py b/superpowers/specs/itl-892-exit-hang/harness.py new file mode 100644 index 000000000..2443352de --- /dev/null +++ b/superpowers/specs/itl-892-exit-hang/harness.py @@ -0,0 +1,129 @@ +"""Run a command many times and count the runs that do not exit before a deadline. + +Usage: + harness.py --label LABEL -n 40 -t 20 -j 4 -- + +Each run is a child process in its own session. A run that misses the deadline is +counted as hung; the first hung child of a cell has its native stack sampled with +macOS ``sample`` (or ``eu-stack``/``gdb`` on Linux when present) before it is killed. +A summary line is printed and appended as JSON to ``results.jsonl`` next to this file. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import shutil +import signal +import subprocess +import sys +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +HERE = Path(__file__).resolve().parent +SAMPLES = HERE / "samples" + + +def _sample_stack(pid: int, label: str) -> str | None: + """Capture the native stack of a hung process, returning the file path.""" + SAMPLES.mkdir(exist_ok=True) + out = SAMPLES / f"{label.replace('/', '_').replace(' ', '_')}.txt" + if sys.platform == "darwin" and shutil.which("sample"): + cmd = ["sample", str(pid), "1", "-file", str(out)] + elif shutil.which("gdb"): + cmd = ["gdb", "-p", str(pid), "-batch", "-ex", "thread apply all bt"] + elif shutil.which("eu-stack"): + cmd = ["eu-stack", "-p", str(pid)] + else: + return None + try: + res = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + except Exception: # noqa: BLE001 - diagnostics only + return None + if not out.exists(): + out.write_text(res.stdout + res.stderr) + return str(out.relative_to(HERE)) + + +def _run_one(cmd: list[str], timeout: float, env: dict[str, str]) -> dict: + t0 = time.monotonic() + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + env=env, + ) + try: + out, err = proc.communicate(timeout=timeout) + return { + "hung": False, + "rc": proc.returncode, + "done": "done" in out, + "secs": round(time.monotonic() - t0, 2), + "err": err[-400:] if proc.returncode else "", + } + except subprocess.TimeoutExpired: + return {"hung": True, "proc": proc, "secs": timeout} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--label", required=True) + ap.add_argument("-n", type=int, default=40) + ap.add_argument("-t", type=float, default=20.0) + ap.add_argument("-j", type=int, default=4) + ap.add_argument("--no-sample", action="store_true") + ap.add_argument("cmd", nargs=argparse.REMAINDER) + args = ap.parse_args() + cmd = args.cmd[1:] if args.cmd and args.cmd[0] == "--" else args.cmd + env = dict(os.environ) + + results: list[dict] = [] + sampled: str | None = None + with ThreadPoolExecutor(max_workers=args.j) as ex: + for r in ex.map(lambda _: _run_one(cmd, args.t, env), range(args.n)): + if r["hung"]: + proc = r.pop("proc") + if sampled is None and not args.no_sample: + sampled = _sample_stack(proc.pid, args.label) or "unavailable" + os.killpg(proc.pid, signal.SIGKILL) + out, _ = proc.communicate() + r["done"] = "done" in (out or "") + results.append(r) + + hung = sum(r["hung"] for r in results) + rcs = Counter("HUNG" if r["hung"] else r["rc"] for r in results) + hung_after_done = sum(r["hung"] and r["done"] for r in results) + errors = sorted({r["err"] for r in results if r.get("err")})[:2] + summary = { + "label": args.label, + "n": args.n, + "hung": hung, + "hung_after_done": hung_after_done, + "rc": {str(k): v for k, v in rcs.items()}, + "timeout_s": args.t, + "jobs": args.j, + "platform": f"{platform.system()} {platform.machine()}", + "stack": sampled, + "errors": errors, + "cmd": " ".join(cmd), + } + print( + f"{args.label}: hung {hung}/{args.n} (after 'done': {hung_after_done}) rc={dict(rcs)}" + + (f" stack={sampled}" if sampled else "") + + (f" ERR={errors[0][-200:]!r}" if errors else ""), + flush=True, + ) + with open(HERE / "results.jsonl", "a") as fh: + fh.write(json.dumps(summary) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/superpowers/specs/itl-892-exit-hang/repro_orcapod.py b/superpowers/specs/itl-892-exit-hang/repro_orcapod.py new file mode 100644 index 000000000..4a230dda5 --- /dev/null +++ b/superpowers/specs/itl-892-exit-hang/repro_orcapod.py @@ -0,0 +1,109 @@ +"""ITL-892 through Orcapod: does a process that used a Delta store exit? + +Modes: + pipeline-sync ``PipelineJob`` on ``DeltaTableDatabase``, sync orchestrator, then + ``get_all_records`` on the function node. + pipeline-async the same with the async orchestrator. + plain-read ``DeltaTableDatabase`` write, flush, ``get_all_records``. + source-read ``DeltaTableSource`` over a table written by ``write_deltalake``. + rerun ``pipeline-sync`` twice on the same store (second run is all cache hits). + +Prints ``done`` and returns normally so the interpreter runs its shutdown. +""" + +from __future__ import annotations + +import argparse +import tempfile +from pathlib import Path + + +def pipeline(store_dir: Path, orchestrator_name: str, runs: int = 1) -> None: + import orcapod as op + from orcapod.databases import DeltaTableDatabase + from orcapod.pipeline import AsyncPipelineOrchestrator, PipelineJob + from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + orch = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, + }[orchestrator_name] + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + for _ in range(runs): + source = op.sources.DictSource( + [{"id": i, "x": 10 * i} for i in range(3)], tag_columns=["id"] + ) + job = PipelineJob(store=DeltaTableDatabase(store_dir)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orch()) + # Row counts are not checked: main still has the ITL-864 partial read. + job.nodes["double"].get_all_records() + + +def plain_read(store_dir: Path) -> None: + import pyarrow as pa + + from orcapod.databases import DeltaTableDatabase + + db = DeltaTableDatabase(store_dir) + table = pa.table( + { + "__record_id": pa.array(["a", "b", "c"], pa.large_string()), + "v": pa.array([1, 2, 3], pa.int64()), + } + ) + db.add_records(("t",), table, record_id_column="__record_id") + db.flush() + fresh = DeltaTableDatabase(store_dir) + fresh.get_all_records(("t",), record_id_column="__record_id") + + +def source_read(store_dir: Path) -> None: + import deltalake + import pyarrow as pa + + import orcapod as op + + path = store_dir / "src" + deltalake.write_deltalake( + str(path), + pa.table({"id": pa.array([1, 2, 3]), "x": pa.array([1.0, 2.0, 3.0])}), + ) + src = op.sources.DeltaTableSource(path, tag_columns=["id"]) + src.as_table() + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument( + "mode", + choices=[ + "pipeline-sync", + "pipeline-async", + "plain-read", + "source-read", + "rerun", + ], + ) + args = ap.parse_args() + store = Path(tempfile.mkdtemp()) + if args.mode == "pipeline-sync": + pipeline(store, "sync") + elif args.mode == "pipeline-async": + pipeline(store, "async") + elif args.mode == "rerun": + pipeline(store, "sync", runs=2) + elif args.mode == "plain-read": + plain_read(store) + else: + source_read(store) + print("done", flush=True) + + +if __name__ == "__main__": + main() diff --git a/superpowers/specs/itl-892-exit-hang/repro_pure.py b/superpowers/specs/itl-892-exit-hang/repro_pure.py new file mode 100644 index 000000000..d1674ca8a --- /dev/null +++ b/superpowers/specs/itl-892-exit-hang/repro_pure.py @@ -0,0 +1,187 @@ +"""Minimal ITL-892 reproduction: pyarrow + deltalake only, no Orcapod. + +Builds a small Delta table in a temporary directory, performs the operations chosen by +``--op``, prints ``done`` and returns normally so the interpreter runs its shutdown. + +Ops: + write one ``write_deltalake`` (pyarrow input), no read. + merge one write, then four ``DeltaTable.merge`` calls, no read. + to_table one write, then ``DeltaTable.to_pyarrow_table()``. + to_dataset one write, then ``DeltaTable.to_pyarrow_dataset()`` (no scan). + dataset_scan one write, then ``to_pyarrow_dataset().to_table()``. + itl864 the ITL-864 worker's script: write, four merges, and a + ``to_pyarrow_dataset(as_large_types=True).to_table()`` after each. + pyarrow_only pyarrow parquet write + dataset scan, deltalake imported but unused. +""" + +from __future__ import annotations + +import argparse +import atexit +import gc +import os +import sys +import tempfile + + +def build(path: str, i: int): + import pyarrow as pa + + return pa.table( + {"k": pa.array([i], pa.int64()), "v": pa.array([str(i)], pa.large_string())} + ) + + +def run(op: str, read_via: str, rounds: int = 4) -> list[object]: + """Perform ``op``; return the objects created so mitigations can drop them.""" + import deltalake + import pyarrow as pa + + keep: list[object] = [] + path = tempfile.mkdtemp() + + def read() -> None: + dt = deltalake.DeltaTable(path) + keep.append(dt) + if read_via == "dataset": + ds = dt.to_pyarrow_dataset(as_large_types=True) + keep.append(ds) + keep.append(ds.to_table()) + elif read_via == "table": + keep.append(dt.to_pyarrow_table()) + elif read_via == "native_fs": + # Same dataset path, but a native pyarrow filesystem instead of the default + # PyFileSystem(DeltaStorageHandler), so no Python file object is involved. + import pyarrow.fs as pa_fs + + fs = pa_fs.SubTreeFileSystem(path, pa_fs.LocalFileSystem()) + ds = dt.to_pyarrow_dataset(as_large_types=True, filesystem=fs) + keep.append(ds) + keep.append(ds.to_table()) + elif read_via == "no_threads": + ds = dt.to_pyarrow_dataset(as_large_types=True) + keep.append(ds) + keep.append(ds.to_table(use_threads=False)) + elif read_via == "querybuilder": + ctx = deltalake.QueryBuilder().register("t", dt) + keep.append(pa.table(ctx.execute("select * from t").read_all())) + elif read_via == "files": + import pyarrow.parquet as pq + + keep.append(pq.read_table(dt.file_uris())) + elif read_via == "polars": + import polars as pl + + keep.append(pl.read_delta(path)) + else: + raise ValueError(read_via) + + if op == "pyarrow_only": + import pyarrow.dataset as pads + import pyarrow.parquet as pq + + pq.write_table(build(path, 0), os.path.join(path, "a.parquet")) + keep.append(pads.dataset(path).to_table()) + return keep + + deltalake.write_deltalake(path, build(path, 0), mode="overwrite") + if op == "write": + return keep + if op == "merge": + for i in range(1, 5): + deltalake.DeltaTable(path).merge( + source=build(path, i), + predicate="target.k = source.k", + source_alias="source", + target_alias="target", + ).when_not_matched_insert_all().execute() + return keep + if op == "to_table": + read_via = "table" if read_via == "dataset" else read_via + read() + return keep + if op == "to_dataset": + dt = deltalake.DeltaTable(path) + keep += [dt, dt.to_pyarrow_dataset(as_large_types=True)] + return keep + if op == "dataset_scan": + read() + return keep + if op == "itl864": + read() + for i in range(1, rounds + 1): + deltalake.DeltaTable(path).merge( + source=build(path, i), + predicate="target.k = source.k", + source_alias="source", + target_alias="target", + ).when_not_matched_insert_all().execute() + read() + return keep + raise ValueError(op) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--op", default="itl864") + ap.add_argument( + "--read-via", + default="dataset", + choices=[ + "dataset", + "table", + "native_fs", + "no_threads", + "querybuilder", + "files", + "polars", + ], + ) + ap.add_argument("--rounds", type=int, default=4, help="merge rounds for itl864") + ap.add_argument("--import-polars", action="store_true") + ap.add_argument("--polars-first", action="store_true") + ap.add_argument("--cpu-threads", type=int) + ap.add_argument("--io-threads", type=int) + ap.add_argument( + "--mitigation", + default="none", + choices=["none", "del", "atexit_del", "os_exit"], + ) + ap.add_argument("--mp", default="none", choices=["none", "fork", "spawn"]) + args = ap.parse_args() + + if args.polars_first: + import polars # noqa: F401 + import pyarrow as pa + + if args.import_polars: + import polars # noqa: F401 + if args.cpu_threads is not None: + pa.set_cpu_count(args.cpu_threads) + if args.io_threads is not None: + pa.set_io_thread_count(args.io_threads) + + if args.mp != "none": + import multiprocessing as mp + + ctx = mp.get_context(args.mp) + p = ctx.Process(target=run, args=(args.op, args.read_via, args.rounds)) + p.start() + p.join() + keep: list[object] = [] + else: + keep = run(args.op, args.read_via, args.rounds) + + print("done", flush=True) + if args.mitigation == "del": + keep.clear() + gc.collect() + elif args.mitigation == "atexit_del": + atexit.register(lambda: (keep.clear(), gc.collect())) + elif args.mitigation == "os_exit": + sys.stdout.flush() + os._exit(0) + + +if __name__ == "__main__": + main() diff --git a/superpowers/specs/itl-892-exit-hang/run_matrix.sh b/superpowers/specs/itl-892-exit-hang/run_matrix.sh new file mode 100644 index 000000000..a1f004a0b --- /dev/null +++ b/superpowers/specs/itl-892-exit-hang/run_matrix.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# ITL-892 macOS matrix. Run from the repository root: +# bash superpowers/specs/itl-892-exit-hang/run_matrix.sh [N] +# Groups: ops, readpath, threads, mitigations, python, versions, orcapod. +# Each cell appends one JSON line to superpowers/specs/itl-892-exit-hang/results.jsonl. +set -u +D=superpowers/specs/itl-892-exit-hang +N=${2:-100} +J=${J:-10} +T=${T:-8} +OP=${OP:-dataset_scan} +H() { uv run python $D/harness.py -n "$N" -t "$T" -j "$J" "$@"; } +# Throwaway environment with chosen versions; uv.lock is untouched. +V() { # V