diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 616fc31fe..da94045cb 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2380,6 +2380,67 @@ newcomer. Re-checked on `b197544e` (v0.2.0 + #291): column ... because it's a binary field` lines on every Delta write still appear. **Open:** awaiting the developer's choice of how to silence them. +--- + +## `src/orcapod/databases/delta_lake_databases.py` — reading back a handle family's own writes + +### DW1 — A read through one `at()` handle could miss rows its family had just written +**Status:** resolved +**Severity:** high +**Issue:** ITL-864 + +Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` +returned 1 of 2 or 3 rows (plain `int -> int` pod, sync and async orchestrators), while +`iter_data` and a fresh handle saw every row. Every `at()` handle shares the root's pending +batches, and any handle's `flush()` writes every pending key, so the result-store handle +flushed the pipeline table's rows. But each handle kept its own loaded-table cache, so the +node's pipeline handle kept reading the table version it had loaded during the run. Every read +method missed the rows a sibling flushed. Separately, the read methods merged pending and stored +rows inconsistently: `get_records_by_ids` ignored pending rows, `get_records_with_column_value` +ignored stored rows whenever a pending batch existed (and returned an empty table instead of +`None`), and a pending row whose record id was already stored was returned as well, although a +flush keeps the stored row and drops the pending one. + +**Fix:** the loaded-table, existing-ids and cache-dirty dicts are shared across the `at()` +family exactly as pending state is. Every read method returns what a flush would leave: the +stored rows plus the pending rows whose record id is not stored (`_pending_not_stored`), and +`None` when there are none. These dicts are touched only from the thread that drives the +pipeline; the async orchestrator runs only user functions in worker threads. Regression tests: +`tests/test_databases/test_delta_read_your_writes.py` and +`tests/test_pipeline/test_delta_node_records_after_run.py`. A failed stored-id refresh leaves +the cache dirty, so the next read retries it instead of treating every stored id as unstored +and returning duplicates; in `get_records_by_ids` the refresh shares the stored read's error +handling. A handle that loaded the table before it had any stored ids (a fresh root over an +existing table) refreshes them on its first read with pending rows. + +When stored and pending rows are read together, the result has the schema a flush would give: +both parts are large-normalized (`null` kept), then combined with +`pa.concat_tables(promote_options="default")`. A column missing on one side is null-filled, and a +`null`-typed column takes the other side's type. Columns whose types differ (`int64` vs `int32`, +a string vs an int, a timestamp unit, a decimal precision) raise `pyarrow.ArrowTypeError`; nothing +is widened. Before this, a read of stored rows next to pending rows with any schema difference, +including a pending `string` column next to the stored `large_string`, raised `ArrowInvalid`; +`get_all_records` already did so on `main`. +A `get_records_with_column_value` query on a column that only the stored or only the pending +rows have skips the side without it (after a flush that side holds null there, and +`column == value` never matches null), so it returns before a flush what it returns after; a +column neither side has still raises `ArrowInvalid`. + +Recorded for the nullability audit (ITL-842) and ITL-890: Delta's merge relaxes nullability (a +non-null pending column into a nullable stored one stays nullable) and evolves struct children +(`struct
` stored + `struct
` pending becomes `struct
`, stored rows `q = null`). The +read path now mirrors both. Whether Orcapod should allow either at all is for those issues. A +pending all-null `null`-typed column reads back as `null`, but Delta cannot store it: the flush +fails and the batch stays pending (unchanged, pinned by +`test_pending_null_typed_column_reads_back_but_does_not_flush`). + +Still open: a handle does not see writes made by another process, or by a separate +`DeltaTableDatabase` root on the same path, until it reloads the table; this is unchanged. +Reading with a pending batch also refreshes the stored-id set when it is stale, which loads the +whole table (D3). + +--- + ## `src/orcapod/core/operators/filters.py` — `PolarsFilter` identity and nullability (ITL-828) ### PF1 — An expression predicate could not be hashed, and a filter broke a join inside a job diff --git a/changelog.d/ITL-864.md b/changelog.d/ITL-864.md new file mode 100644 index 000000000..cfc4bc3dc --- /dev/null +++ b/changelog.d/ITL-864.md @@ -0,0 +1,16 @@ +### Fixed + +#### Delta reads return every row the database's handles wrote (ITL-864) + +Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` could +return only some of the rows (1 of 2 or 3) on both orchestrators. Handles made with `at()` share +their pending writes, and any handle's flush writes all of them, but each handle kept its own +loaded copy of the table, so a sibling kept reading the version it had loaded before the flush. +The loaded tables and the stored-id sets are now shared by every handle of one root. Every read +method (`get_all_records`, `get_record_by_id`, `get_records_by_ids`, +`get_records_with_column_value`) returns what a flush would leave: the stored rows plus the +pending rows whose record id is not stored, or `None` when there are none. Before, +`get_records_by_ids` ignored pending rows, `get_records_with_column_value` ignored stored rows +while a pending batch existed, and a pending row whose id was already stored could be returned +twice or instead of the stored row. Writes made by another process or another root on the same +path are still seen only after the table is reloaded. (ITL-864 #293) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 8c34c1f04..f783dab9f 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -311,6 +311,9 @@ def __init__( _scoped_path: tuple[str, ...] = (), _shared_pending_batches: "dict[str, pa.Table] | None" = None, _shared_pending_record_ids: "defaultdict[str, set[bytes]] | None" = None, + _shared_delta_table_cache: "dict[str, deltalake.DeltaTable] | None" = None, + _shared_existing_ids_cache: "dict[str, set[bytes]] | None" = None, + _shared_cache_dirty: "dict[str, bool] | None" = None, ): self._root_uri, self._storage_options = parse_base_path( base_path, storage_options @@ -335,9 +338,16 @@ def __init__( ) # For cloud paths: create_base_path is silently ignored (no directory needed). - self._delta_table_cache: dict[str, deltalake.DeltaTable] = {} - # Shared pending state — when constructed via at(), these dicts are shared - # with the root instance so that root.flush() can flush all scoped writes. + # Shared state — when constructed via at(), these dicts are shared with the + # root instance. Any handle's flush() writes every pending key of the family + # and refreshes the loaded table and existing-ids entries for it, so these + # caches must be family-wide too: a per-handle cache would keep serving the + # snapshot a sibling loaded before the flush (ITL-864). All of them are + # touched only from the thread that drives the pipeline (the async + # orchestrator runs only user functions in worker threads). + self._delta_table_cache: dict[str, deltalake.DeltaTable] = ( + _shared_delta_table_cache if _shared_delta_table_cache is not None else {} + ) if _shared_pending_batches is not None: self._pending_batches = _shared_pending_batches else: @@ -346,8 +356,16 @@ def __init__( self._pending_record_ids = _shared_pending_record_ids else: self._pending_record_ids: dict[str, set[bytes]] = defaultdict(set) - self._existing_ids_cache: dict[str, set[str]] = defaultdict(set) - self._cache_dirty: dict[str, bool] = defaultdict(lambda: True) + self._existing_ids_cache: dict[str, set[bytes]] = ( + _shared_existing_ids_cache + if _shared_existing_ids_cache is not None + else defaultdict(set) + ) + self._cache_dirty: dict[str, bool] = ( + _shared_cache_dirty + if _shared_cache_dirty is not None + else defaultdict(lambda: True) + ) def _clear_pending(self): """Clear all pending state.""" @@ -584,7 +602,9 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: # TODO: replace this with more targetted loading of only the target column and in batches arrow_table = delta_table.to_pyarrow_table() except Exception as e: - # Leave the cache dirty so the next call reads the table again. + # Stay dirty: the loaded table is cached, so a clean cache would never + # be refreshed and stored ids would read as unstored (ITL-864). + self._cache_dirty[record_key] = True raise self._read_failed(record_path) from e if arrow_table.num_rows == 0: @@ -595,7 +615,7 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: self._existing_ids_cache[record_key] = set() else: existing_ids = cast( - set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) + set[bytes], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist()) ) self._existing_ids_cache[record_key] = existing_ids @@ -604,12 +624,15 @@ def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None: f"Refreshed existing IDs cache: {len(self._existing_ids_cache)} IDs" ) - def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[str]: + def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[bytes]: """Get the set of existing IDs in the Delta table, using cache when possible.""" record_key = self._get_record_key(record_path) + # A table loaded by a read is cached before any ids are, so a missing + # id entry needs a refresh as much as a dirty one does. if ( self._cache_dirty.get(record_key) or record_key not in self._delta_table_cache + or record_key not in self._existing_ids_cache ): self._refresh_existing_ids_cache(record_path) return self._existing_ids_cache.get(record_key) or set() @@ -873,7 +896,7 @@ def _add_to_pending_batch(self, record_path: tuple[str, ...], table: pa.Table): else: self._pending_batches[record_key] = pa.concat_tables([pending_batch, table]) - pending_ids = cast(list[str], table[self.RECORD_ID_COLUMN].to_pylist()) + pending_ids = cast(list[bytes], table[self.RECORD_ID_COLUMN].to_pylist()) self._pending_record_ids[record_key].update(pending_ids) def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool: @@ -884,6 +907,97 @@ def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool: and self._pending_batches[record_key].num_rows >= self.batch_size ) + def _pending_not_stored(self, record_path: tuple[str, ...]) -> pa.Table | None: + """Return the pending rows a flush would insert at ``record_path``. + + A flush inserts only rows whose record id is not stored yet (the merge is + ``when_not_matched_insert_all``), so a pending row with a stored id is + never returned by a read: the stored row is. + + Args: + record_path: Path components relative to this database view. + + Returns: + The pending rows whose record id is not stored, or ``None`` if there + are none. + """ + pending = self._pending_batches.get(self._get_record_key(record_path)) + if pending is None or pending.num_rows == 0: + return None + stored_ids = self._get_existing_ids(record_path) + if stored_ids: + # The mask is built from the pending ids against the cached set, so its + # cost follows the pending batch, not the stored table. A null id is + # never matched by the merge (``null = null`` is not true), so its row + # is kept. + keep = [ + record_id is None or record_id not in stored_ids + for record_id in pending[self.RECORD_ID_COLUMN].to_pylist() + ] + pending = pending.filter(pa.array(keep, type=pa.bool_())) + return pending if pending.num_rows > 0 else None + + def _combine_stored_and_pending( + self, + stored: pa.Table | None, + pending: pa.Table | None, + record_id_column: str | None, + ) -> pa.Table | None: + """Combine stored rows with pending rows into one read result. + + Combined rows have the schema a flush would give them: both parts take + their large layouts (a stored table reads back large), a column missing + on one side is null-filled, a ``null``-typed column takes the other + side's type, nullability is merged (nullable if either side is), and + struct children are merged, as Delta's merge evolves the stored schema. + A column whose types still differ raises ``pyarrow.ArrowTypeError``; + nothing is widened. A single part is returned as it is. + + Args: + stored: Rows read from the Delta table, or ``None``. + pending: Pending rows that are not stored (see ``_pending_not_stored``), + already filtered by the read's condition, or ``None``. + record_id_column: Name to give the record id column, or ``None`` to + drop it. + + Returns: + The combined rows, or ``None`` if there are none. + """ + parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] + if not parts: + return None + if len(parts) == 1: + table = parts[0] + else: + table = pa.concat_tables( + [self._large_layout(t) for t in parts], promote_options="default" + ) + return self._handle_record_id_column(table, record_id_column) + + @staticmethod + def _large_layout(table: pa.Table) -> pa.Table: + """Return ``table`` with every column in its large layout. + + Only layouts change (``string`` to ``large_string``, ``list`` to + ``large_list``, recursively); ``null`` stays ``null``, and names, + nullability and metadata are kept. + + Args: + table: The table to normalize. + + Returns: + The table with large-normalized column types. + """ + schema = table.schema + for index, field in enumerate(table.schema): + schema = schema.set( + index, + field.with_type( + arrow_utils.normalize_to_large_types(field.type, keep_null=True) + ), + ) + return table if schema.equals(table.schema) else table.cast(schema) + def get_all_records( self, record_path: tuple[str, ...], @@ -893,14 +1007,16 @@ def get_all_records( """ Get all records from both pending batch and Delta table. + Every read method returns the rows a flush would leave: the stored rows + plus the pending rows whose record id is not stored yet, as seen through + any handle of this ``at()`` family. + Returns: Combined Arrow table with all records, or None if no records exist Raises: StoreReadError: If the Delta table exists but could not be read. """ - record_key = self._get_record_key(record_path) - tables_to_combine = [] # Add Delta table data @@ -914,24 +1030,10 @@ def get_all_records( if delta_table_data.num_rows > 0: tables_to_combine.append(delta_table_data) - # Add pending batch data - if ( - retrieve_pending - and (pending_batch := self._pending_batches.get(record_key)) is not None - ): - if pending_batch.num_rows > 0: - tables_to_combine.append(pending_batch) - - if not tables_to_combine: - return None - - if len(tables_to_combine) == 1: - table_to_return = tables_to_combine[0] - else: - table_to_return = pa.concat_tables(tables_to_combine) - - # Handle record_id_column if specified - return self._handle_record_id_column(table_to_return, record_id_column) + # Add the pending rows a flush would insert + pending = self._pending_not_stored(record_path) if retrieve_pending else None + stored = pa.concat_tables(tables_to_combine) if tables_to_combine else None + return self._combine_stored_and_pending(stored, pending, record_id_column) def get_records_with_column_value( self, @@ -942,9 +1044,6 @@ def get_records_with_column_value( ) -> "pa.Table | None": if flush: self.flush_batch(record_path) - # check if record_id is found in pending batches - record_key = self._get_record_key(record_path) - pending_batch = self._pending_batches.get(record_key) if isinstance(column_values, Mapping): # Convert Mapping to list of tuples @@ -958,14 +1057,30 @@ def get_records_with_column_value( for next_expression in expressions[1:]: combined_expression = combined_expression & next_expression - if pending_batch is not None: - filtered_table = pending_batch.filter(combined_expression) - return self._handle_record_id_column(filtered_table, record_id_column) - - # Now check the Delta table + pending = self._pending_not_stored(record_path) delta_table = self._get_delta_table(record_path) + + # A side without a queried column holds null there once flushed, and + # ``column == value`` never matches null, so it contributes no rows. A + # column neither side has is unknown: the filter raises as before. + queried = {c for c, _ in pair_list} + stored_columns = ( + set() + if delta_table is None + else {field.name for field in delta_table.schema().fields} + ) + pending_columns = set() if pending is None else set(pending.column_names) + if queried <= stored_columns | pending_columns: + if not queried <= pending_columns: + pending = None + if not queried <= stored_columns: + delta_table = None + + if pending is not None: + pending = pending.filter(combined_expression) + if delta_table is None: - return None + return self._combine_stored_and_pending(None, pending, record_id_column) # Use schema-preserving read try: @@ -973,11 +1088,7 @@ def get_records_with_column_value( except _DeltaTableReadError as e: raise self._read_failed(record_path) from e.__cause__ - if len(result) == 0: - return None - - # Handle (remove/rename) the record id column before returning - return self._handle_record_id_column(result, record_id_column) + return self._combine_stored_and_pending(result, pending, record_id_column) def get_record_by_id( self, @@ -1001,9 +1112,12 @@ def get_record_by_id( self.flush_batch(record_path) record_id = coerce_record_id(record_id) - # check if record_id is found in pending batches + # Serve a pending record only if its id is not stored: a flush keeps the + # stored row and drops the pending one. record_key = self._get_record_key(record_path) - if record_id in self._pending_record_ids[record_key]: + if record_id in self._pending_record_ids[ + record_key + ] and record_id not in self._get_existing_ids(record_path): # Return the pending record after removing the entry id column pending_batch = self._pending_batches[record_key] assert pending_batch is not None, "Pending batch should not be None" @@ -1070,22 +1184,28 @@ def get_records_by_ids( if len(record_ids) == 0: return None - # check inside the batch delta_table = self._get_delta_table(record_path) - if delta_table is None: - return None - # Use schema-preserving read with filters - filter_expr = self._create_record_ids_filter(record_ids_list) - try: - result = self._read_delta_table(delta_table, filters=filter_expr) - except _DeltaTableReadError as e: - raise self._read_failed(record_path) from e.__cause__ - - if len(result) == 0: - return None - - # Handle record_id column based on parameter - return self._handle_record_id_column(result, record_id_column) + # Reading the stored ids drops the pending rows a flush would not insert; + # it raises StoreReadError when the stored table cannot be read. + pending = self._pending_not_stored(record_path) + result = None + if delta_table is not None: + # Use schema-preserving read with filters + filter_expr = self._create_record_ids_filter(record_ids_list) + try: + result = self._read_delta_table(delta_table, filters=filter_expr) + except _DeltaTableReadError as e: + raise self._read_failed(record_path) from e.__cause__ + + if pending is not None: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter( + pc.is_in( + pending[self.RECORD_ID_COLUMN], + value_set=pa.array(record_ids_list, type=id_type), + ) + ) + return self._combine_stored_and_pending(result, pending, record_id_column) def _read_delta_table( self, @@ -1453,6 +1573,9 @@ def at(self, *path_components: str) -> "DeltaTableDatabase": _scoped_path=new_scoped_path, _shared_pending_batches=root_instance._pending_batches, _shared_pending_record_ids=root_instance._pending_record_ids, + _shared_delta_table_cache=root_instance._delta_table_cache, + _shared_existing_ids_cache=root_instance._existing_ids_cache, + _shared_cache_dirty=root_instance._cache_dirty, ) def list_sources(self) -> list[tuple[str, ...]]: diff --git a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md new file mode 100644 index 000000000..546c5e252 --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md @@ -0,0 +1,174 @@ +# ITL-864 DeltaTableDatabase read-your-writes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every read method of `DeltaTableDatabase` returns exactly the rows a flush would leave, through every handle of an `at()` family, so `node.get_all_records()` returns every row right after the first run on a Delta store. + +**Architecture:** Two defects, one contract. (1) `at()` handles share the root's pending batches, and any handle's `flush()` writes every pending key, but each handle keeps its own `_delta_table_cache`, so a handle that loaded a table earlier keeps serving a stale `DeltaTable` snapshot after a sibling flushed. The fix shares `_delta_table_cache`, `_existing_ids_cache` and `_cache_dirty` across the family, exactly as pending state is already shared. (2) The read methods merge pending and stored rows inconsistently (`get_records_by_ids` ignores pending; `get_records_with_column_value` ignores stored whenever a pending batch exists; overlapping ids show twice). The fix: every read returns stored rows plus pending rows whose record id is not stored (Delta's flush is `when_not_matched_insert_all`, so the stored row wins), `None` when empty. + +**Tech Stack:** Python, pyarrow, deltalake, pytest, `uv run`. + +Approved by the coordinator (option A, Nov 10, 2026). No public API, hash, stored schema or dependency change. Error handling is untouched (PR #290 / ITL-884 rewrites it in the same file; merge main after it lands). + +--- + +## Reproduction (Step 1 of the brief) + +`PipelineJob`, `DictSource` of 2 or 3 ints, `int -> int` pod, `DeltaTableDatabase`: + +| store | orchestrator | n | `get_all_records` | `iter_data` | +|---|---|---|---|---| +| delta | sync | 2 / 3 | 1 / 1 | 2 / 3 | +| delta | async | 2 / 3 | 1 / 2 | 2 / 3 | +| memory | both | 2 / 3 | 2 / 3 | 2 / 3 | + +Trace: the result-store handle (`at("pipeline", "_result")`) flushes the pipeline table's rows; the node's pipeline handle (`at("pipeline")`) cached the table at version 0 and still reads version 0 while the table is at version 2. Its pending batch is empty; a fresh handle reads all rows. + +## Contract and per-method findings (database level) + +Contract: a read through a handle reflects all writes of its `at()` family, pending and flushed together, and equals what a flush would leave: no duplicates, no missing rows. + +| case | `get_all_records` | `get_record_by_id` | `get_records_by_ids` | `get_records_with_column_value` | +|---|---|---|---|---| +| A sibling handle flushed | stale | stale | stale | stale | +| B stored + pending, one handle | ok | ok | misses pending | misses stored when pending exists; empty table instead of `None` | +| C pending id already stored, `skip_duplicates=False` | duplicate | returns pending row, flush keeps stored | (dup) | (dup) | +| D same, `skip_duplicates=True` | ok | ok | ok | ok | +| `record_id_column=` | returns (no hang) | | | | + +Other backends: `InMemoryArrowDatabase` shares `_tables` across `at()` and `ConnectorArrowDatabase` re-reads committed rows on every call, so neither goes stale; both merge pending + committed in every method. Both flush with the NEW row winning on id overlap (Delta keeps the stored row) and show both rows before flush under `skip_duplicates=False`. Report only. + +Thread safety: the async orchestrator runs only the user function in `run_in_executor` (`core/executors/local.py`); every database read and write happens on the event-loop thread, as for the already-shared pending dicts, so sharing the caches adds no cross-thread access. + +## File structure + +- Modify: `src/orcapod/databases/delta_lake_databases.py` — `__init__` / `at()` share caches; new helpers `_pending_not_stored` and `_combine_stored_and_pending`; the four read methods use them. +- Create: `tests/test_databases/test_delta_read_your_writes.py` — database-level regression tests. +- Create: `tests/test_pipeline/test_delta_node_records_after_run.py` — `PipelineJob` regression, both orchestrators. +- Modify: `DESIGN_ISSUES.md` — one entry appended at the end. + +--- + +### Task 1: Failing database-level tests + +**Files:** Create `tests/test_databases/test_delta_read_your_writes.py` + +- [ ] **Step 1: Write the tests** + +Helpers: `_rows(db, ids, vals, groups)` builds a table with `__record_id` (`large_binary`), `v` (`int64`), `g` (`int64`); `_values(table)` returns sorted `v` or `None`. A `READS` parametrization maps each read method to a call and its expected values: + +```python +READS = { + "get_all_records": (lambda db: db.get_all_records(PATH), [1, 2]), + "get_record_by_id": (lambda db: db.get_record_by_id(PATH, b"b"), [2]), + "get_records_by_ids": (lambda db: db.get_records_by_ids(PATH, [b"a", b"b"]), [1, 2]), + "get_records_with_column_value": ( + lambda db: db.get_records_with_column_value(PATH, {"g": 0}), [1, 2]), +} +``` + +Tests: +1. `test_sibling_handle_sees_rows_flushed_by_another_handle[method][flusher]`: `root = DeltaTableDatabase(tmp_path)`, `h1, h2 = root.at("s"), root.at("s")`; `h1` writes `a=1` with `flush=True` and reads once (loads its snapshot); `h2` writes `b=2`; read through `h1` and through `root` (path `("s",) + PATH`) before flush; flush through `flusher` in (`h2`, `root`, `h1`); read through `h1`, `root` and a new `root.at("s")` after flush. All equal the expected values. +2. `test_pending_and_stored_rows_are_both_returned[method]`: one handle, stored `a=1 (g=0), b=2 (g=1)`, pending `c=3 (g=1)`: `get_records_by_ids([a, c]) == [1, 3]`, `get_records_with_column_value({"g": 1}) == [2, 3]`, `{"g": 0} == [1]`, `get_all_records == [1, 2, 3]`, `get_record_by_id(c) == [3]`. +3. `test_no_match_returns_none`: stored + pending, `get_records_with_column_value({"g": 9})` and `get_records_by_ids([b"zz"])` are `None`. +4. `test_pending_row_with_stored_id_reads_as_flush_keeps_it[skip_duplicates]`: stored `a=1`; pending add of `a=10, d=4`; before flush each method returns `a=1` and `d=4` once (`get_all_records == [1, 4]`, `get_record_by_id(a) == [1]`, `get_records_by_ids([a, d]) == [1, 4]`, `get_records_with_column_value({"g": 0}) == [1, 4]`); after `flush()` the same values. +5. `test_record_id_column_is_returned_for_merged_rows`: stored + pending, `get_all_records(PATH, record_id_column="rid")` has column `rid` with all three ids and no `__record_id`; same for `get_records_by_ids` and `get_records_with_column_value`. + +- [ ] **Step 2: Run, verify the expected failures** + +Run: `uv run pytest tests/test_databases/test_delta_read_your_writes.py -v` +Expected: test 1 fails for every method after flush; test 2 fails for `get_records_by_ids` / `get_records_with_column_value`; test 3 fails (empty table, not `None`); test 4 fails for `skip_duplicates=False`. + +### Task 2: Failing PipelineJob test + +**Files:** Create `tests/test_pipeline/test_delta_node_records_after_run.py` + +- [ ] **Step 1: Write the test** + +```python +@pytest.mark.parametrize("orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +def test_node_returns_every_row_right_after_first_run(tmp_path, orchestrator): + source = op.sources.DictSource([{"id": i, "x": 10 * i} for i in range(3)], tag_columns=["id"]) + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + job = PipelineJob(store=DeltaTableDatabase(tmp_path)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orchestrator()) + + records = job.nodes["double"].get_all_records() + assert records is not None + assert sorted(records.column("result").to_pylist()) == [0, 20, 40] +``` + +- [ ] **Step 2: Run, verify it fails** (`1 != 3` rows) for both orchestrators. + +- [ ] **Step 3: Commit the failing tests** + +```bash +git add tests/test_databases/test_delta_read_your_writes.py tests/test_pipeline/test_delta_node_records_after_run.py +git commit -m "test(databases): regression tests for Delta reads missing own writes (ITL-864)" +``` + +### Task 3: Share the snapshot caches across the `at()` family + +**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (`__init__`, `at()`) + +- [ ] **Step 1:** Add keyword-only private constructor parameters `_shared_delta_table_cache`, `_shared_existing_ids_cache`, `_shared_cache_dirty`, used when given (same pattern as `_shared_pending_batches`); `at()` passes the root instance's dicts. Comment why: any family member's `flush()` writes every pending key and refreshes the cache entry for it, so the snapshot cache must be family-wide or a sibling keeps reading the old version. +- [ ] **Step 2:** Run Task 1 test 1 and the Task 2 test: expected PASS for test 1 and Task 2. + +### Task 4: Merge stored and pending rows the way flush does + +**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (read methods) + +- [ ] **Step 1:** Add helpers: + +```python +def _pending_not_stored(self, record_path): + """Pending rows whose record id is not stored; flush inserts only these.""" + pending = self._pending_batches.get(self._get_record_key(record_path)) + if pending is None or pending.num_rows == 0: + return None + stored_ids = self._get_existing_ids(record_path) + if stored_ids: + id_type = pending.schema.field(self.RECORD_ID_COLUMN).type + pending = pending.filter(pc.invert(pc.is_in( + pending[self.RECORD_ID_COLUMN], value_set=pa.array(list(stored_ids), type=id_type)))) + return pending if pending.num_rows > 0 else None + +def _combine_stored_and_pending(self, stored, pending, record_id_column): + parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0] + if not parts: + return None + table = parts[0] if len(parts) == 1 else pa.concat_tables(parts) + return self._handle_record_id_column(table, record_id_column) +``` + +- [ ] **Step 2:** `get_all_records`: pending part uses `_pending_not_stored`. `get_record_by_id`: serve from pending only when the id is pending and not stored. `get_records_by_ids`: stored read (existing try/except unchanged) plus pending filtered by the ids. `get_records_with_column_value`: stored read (existing try/except unchanged) plus pending filtered by the expression; `None` when empty. +- [ ] **Step 3:** Run `uv run pytest tests/test_databases tests/test_pipeline -q`: all pass. +- [ ] **Step 4:** Commit `fix(databases): Delta reads return every row the handle family wrote (ITL-864)`. + +### Task 5: Design issue entry, full verification, PR + +- [ ] Append one `DESIGN_ISSUES.md` entry (resolved, with **Fix:**; note that writes from other processes are still not seen until a handle reloads the table — follow-up for the coordinator to file). +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (golden hashes unchanged). +- [ ] Merge `origin/main`, rerun if it moved, push, PR "Fixes ITL-864", `orca linear attach`, `orca worktree set --pr`. + +--- + +## Review rounds and continuation notes (successor-readable) + +Done and pushed (verify, do not redo): +- Round 1 (e9c78b5c): a failed ID refresh keeps the cache dirty; `get_records_by_ids` refresh shares the stored-read handler (8f9adf8b). +- Round 2 (8f9adf8b): refresh when the record key has no ID-cache entry; stored + pending combined with `normalize_table_to_large_types` + `pa.concat_tables(promote_options="default")` (b8bcc817). Tests: `test_fresh_handle_drops_pending_rows_with_stored_ids`, evolved-schema tests. + +Round 3 (CodeRabbit review of b8bcc817, outside-diff, `get_records_with_column_value`), approved fix: +- A side (stored schema, read without loading data; pending column names) that lacks a queried column contributes no rows; after a flush its rows hold null there and `col == value` never matches null. A column on neither side keeps today's `ArrowInvalid` ("No match for FieldRef"). +- Tests: `test_query_on_column_one_side_lacks_matches_read_after_flush[column_only_pending|column_only_stored]`, `test_query_on_column_neither_side_has_raises[flushed]`. +- Also the round-1 outside-diff trivial: `_add_to_pending_batch` cast `list[str]` -> `list[bytes]`. +- Reply: signed top-level PR comment (no thread for outside-diff findings), and one for the round-1 trivial. + +Then: merge origin/main (b197544e); wait for PR #290 (ITL-884, StoreReadError) to merge; merge main again, take #290's error handling around `_pending_not_stored` / `_combine_stored_and_pending`; a failed ID refresh must raise `StoreReadError` on every read method; full suite; worker_done only after the post-#290 merge is green. diff --git a/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md b/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md new file mode 100644 index 000000000..6aea99fca --- /dev/null +++ b/superpowers/plans/2026-11-10-pr-293-coderabbit-id-cache-refresh.md @@ -0,0 +1,59 @@ +# PR #293 CodeRabbit follow-up: stored-ID cache refresh failures (ITL-864) + +Revision plan approved by the coordinator (task_355ce670f6bb), provided each finding +reproduces. Branch `eywalker/itl-864-deltatabledatabase-reading-through-the`. + +## Findings + +1. **Comment 4534785488 (Major), `delta_lake_databases.py:658`.** When + `_refresh_existing_ids_cache` fails after the Delta table loaded, it stores an empty ID + set and marks the cache **clean**. The loaded table is cached, so `_get_existing_ids` + never refreshes again: a pending row whose id is already stored is served next to the + stored row (a duplicate) by every read method. + *Fix:* the failure path marks the cache dirty, so the next read retries the refresh. + +2. **Comment 4534785497 (Minor), `delta_lake_databases.py:891`.** In + `get_records_by_ids`, `_pending_not_stored` refreshes the ID cache before the + method's stored-read `try`, so a refresh failure escapes while a stored-read failure is + logged and returns `None`. + *Fix:* run the refresh inside the same `try` as the stored read (one handler, no second + `return None`). After PR #290 (ITL-884) merges, that handler is gone and both failures + raise `StoreReadError` naming the table. + +## Steps (TDD) + +1. Regression tests in `tests/test_databases/test_delta_read_your_writes.py`, public + read methods only, with the backend's `DeltaTable.to_pyarrow_table` (used only by the + ID refresh) patched to fail once: + - stored `a=1`, flushed; pending `a=10`, `d=4`; one read fails in the refresh; every + read method then returns `a=1`, `d=4` and never `a=10`. + - stored `a=1`, flushed; pending `c=3`; `get_records_by_ids` with the refresh failing + follows the method's stored-read error behaviour (now `None`), and the retry returns + `[1, 3]`. +2. Run them, confirm both fail for the reasons above. +3. Apply the two fixes; rerun the file, then the full suite, ruff, basedpyright. +4. Commit, push, signed replies on both threads, wait for CI and CodeRabbit. + +## After PR #290 merges + +Merge `main`; in the read methods take #290's error handling (`StoreReadError`, no +`try`/`return None`) around this branch's merge helpers. #290's +`_refresh_existing_ids_cache` leaves `_cache_dirty` untouched on failure, which is still +clean when the refresh ran only because the table was not cached yet, so keep an explicit +`self._cache_dirty[record_key] = True` there. Change the test of finding 2 to expect +`StoreReadError` and extend it to every read method. Rerun the full suite (3.11, 3.12, +3.13), golden tests unchanged. + +## Round 2 (CodeRabbit review of 8f9adf8b; coordinator approved both behaviour fixes) + +- **Outside-diff (Major), `_get_existing_ids`:** a handle that loaded the table before any + stored ids were read (a fresh root over an existing table) never refreshed them, so pending + duplicates were served. Refresh also when the record key has no id entry. Test: + `test_fresh_handle_drops_pending_rows_with_stored_ids` (four read methods). +- **4535100450 (Major), `_combine_stored_and_pending`:** stored and pending rows with any + schema difference raised `ArrowInvalid`. Large-normalize both parts (`keep_null=True`), + then `pa.concat_tables(promote_options="default")`. Contract tests assert the read before + flush equals the read after flush (missing column either way, nullability merge, struct + child added, `string`/`list` layouts); type differences keep raising `ArrowTypeError`; a + `null`-typed new column is pinned (reads back, flush fails). +- **4535100459 (nit):** Google docstrings on the five tests and the wrapper. diff --git a/superpowers/plans/2026-11-11-itl-864-stack-on-290.md b/superpowers/plans/2026-11-11-itl-864-stack-on-290.md new file mode 100644 index 000000000..0e6a703b8 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-864-stack-on-290.md @@ -0,0 +1,60 @@ +# ITL-864 / PR #293: rebuild on top of PR #290 (ITL-884) and main + +Date: 2026-11-11. Coordinator task task_0f94cf0766e2. Stack approved by the developer: +#290 -> #293 -> #295 -> Delta half of ITL-919. Previous plan: +`superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md`. + +**Goal:** the branch contains main (c8efc092) and #290 (head 11efe638) by merge commits, +with both behaviours intact: +- #290: an existing table that cannot be loaded or read raises `StoreReadError`, never + "no rows"; `_read_delta_table(delta_table, filters=None, expression=None)` raises the + private `_DeltaTableReadError` and the caller wraps it with `self._read_failed(record_path)`. +- #293: every read method returns stored rows plus pending rows whose id is not stored, + through any handle of the `at()` family (shared table / ids / dirty caches). + +No public API, hash, stored-schema or dependency change. + +## Tasks + +1. `git merge origin/main` (conflicts expected only where #293 touches the same code as + main; resolve, commit, push WIP). +2. `git merge origin/eywalker/itl-884-delta-read-failures-are-returned-as-no-rows`. + Resolution rules in `delta_lake_databases.py`: + - `_refresh_existing_ids_cache`: #290's shape (try only around the load/scan, raise + `self._read_failed(record_path) from e`), with #293's `set[bytes]` typing and the + cache left dirty on failure. + - `get_all_records`: #290's raise on a failed scan + #293's `_pending_not_stored` / + `_combine_stored_and_pending`. + - `get_records_with_column_value`: #293's one-side-lacks-the-column logic; the stored + read uses #290's `except _DeltaTableReadError -> StoreReadError`; filter errors + propagate as is. + - `get_record_by_id`: #293's "pending and not stored" check + #290's wrapping. + - `get_records_by_ids`: drop #293's `except Exception: return None` (that is exactly + what #290 forbids); `_pending_not_stored` propagates `StoreReadError`; the read uses + #290's wrapping. + - CHANGELOG: one `[0.3.0] — unreleased` section; DESIGN_ISSUES: all entries. +3. Tests: run #290's and #293's tests together. Add interaction tests (public interface, + in `tests/test_databases/test_delta_read_your_writes.py`): + - unreadable stored table with pending rows queued through a sibling handle: every read + method raises `StoreReadError` (pending rows are not returned alone as if the table + were empty), and the pending rows stay queued (nothing dropped); + - a failed stored-id refresh raises `StoreReadError` on every read method, and a later + read after the table is readable again returns stored + pending. + Show each new test's result before any code change it needs. +4. Full suite on 3.11/3.12/3.13, ruff check, ruff format --check, basedpyright (exit code), + golden hashes unchanged. +5. Push; PR description gets the "Stacked on #290" note; CI + CodeRabbit; worker_done. + +## Progress notes (successor-readable) +- Done: merged origin/main c8efc092 (7ba60eb4; conflict DESIGN_ISSUES.md only: kept main's + "Newcomer papercuts" entry and the ITL-864 entry, ours last). Merged #290 11efe638 + (a2ac8d9e; conflicts in `_refresh_existing_ids_cache`, `get_records_with_column_value`, + `get_records_by_ids`, resolved per the rules above; CHANGELOG/DESIGN_ISSUES auto-merged; + ITL-864 Fixed entry added to `[0.3.0] — unreleased` in the merge commit). +- Done: ba396d41 updates #293's two refresh tests to StoreReadError and adds + `test_unreadable_table_with_sibling_pending_rows_raises_and_drops_nothing` (verified to fail + when #293's old `except Exception: return None` is put back in `get_records_by_ids`). +- ITL-923 ephemeral re-run repro (scratchpad script): main and #290 lose records on both + orchestrators; this branch fixes sync (10 stored, 5 read) but async still stores 5 and reads + None. Out of scope; reported to the coordinator. +- Next: full suite 3.11/3.12/3.13, PR description "Stacked on #290", CI, CodeRabbit, worker_done. diff --git a/tests/test_databases/test_delta_read_your_writes.py b/tests/test_databases/test_delta_read_your_writes.py new file mode 100644 index 000000000..59db96a0b --- /dev/null +++ b/tests/test_databases/test_delta_read_your_writes.py @@ -0,0 +1,510 @@ +"""Reads through a ``DeltaTableDatabase`` handle reflect every write of its family (ITL-864). + +Contract: a read through any handle of one ``at()`` family returns exactly the rows a +flush would leave, pending and stored together, with no duplicates and no missing +rows. Delta's flush inserts only rows whose record id is not stored yet, so on an id +overlap the stored row is the one returned. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, cast +from pathlib import Path + +import deltalake +import pyarrow as pa +import pytest + +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import StoreReadError + +PATH = ("table",) +SCOPE = "scope" + + +def _rows( + ids: list[str], values: list[int], groups: list[int] | None = None +) -> pa.Table: + """Build a record table keyed by ``__record_id`` with ``int64`` columns ``v`` and ``g``.""" + return pa.table( + { + "__record_id": pa.array([i.encode() for i in ids], type=pa.large_binary()), + "v": pa.array(values, type=pa.int64()), + "g": pa.array( + groups if groups is not None else [0] * len(ids), type=pa.int64() + ), + } + ) + + +def _values(table: pa.Table | None) -> list[int] | None: + """Return the sorted ``v`` values of ``table``, or ``None`` when there is no table.""" + if table is None: + return None + return sorted(cast(list[int], table.column("v").to_pylist())) + + +Read = Callable[[DeltaTableDatabase, tuple[str, ...]], "pa.Table | None"] + +# Each read method, called for rows ``a=1`` and ``b=2`` (both in group 0). +READS: dict[str, Read] = { + "get_all_records": lambda db, path: db.get_all_records(path), + "get_record_by_id": lambda db, path: db.get_record_by_id(path, b"b"), + "get_records_by_ids": lambda db, path: db.get_records_by_ids(path, [b"a", b"b"]), + "get_records_with_column_value": lambda db, path: db.get_records_with_column_value( + path, {"g": 0} + ), +} +EXPECTED = { + "get_all_records": [1, 2], + "get_record_by_id": [2], + "get_records_by_ids": [1, 2], + "get_records_with_column_value": [1, 2], +} + + +@pytest.mark.parametrize("flusher", ["writer", "root", "reader"]) +@pytest.mark.parametrize("method", list(READS)) +def test_sibling_handle_sees_rows_flushed_by_another_handle( + tmp_path: Path, method: str, flusher: str +) -> None: + """A handle that already loaded the table sees rows a sibling handle wrote and flushed.""" + root = DeltaTableDatabase(tmp_path) + reader = root.at(SCOPE) + writer = root.at(SCOPE) + read = READS[method] + + reader.add_records(PATH, _rows(["a"], [1]), flush=True) + assert _values(reader.get_all_records(PATH)) == [1] # reader loads its snapshot + + writer.add_records(PATH, _rows(["b"], [2])) + assert _values(read(reader, PATH)) == EXPECTED[method] + assert _values(read(root, (SCOPE, *PATH))) == EXPECTED[method] + + {"writer": writer, "root": root, "reader": reader}[flusher].flush() + + assert _values(read(reader, PATH)) == EXPECTED[method] + assert _values(read(writer, PATH)) == EXPECTED[method] + assert _values(read(root, (SCOPE, *PATH))) == EXPECTED[method] + assert _values(read(root.at(SCOPE), PATH)) == EXPECTED[method] + + +@pytest.fixture +def stored_and_pending(tmp_path: Path) -> DeltaTableDatabase: + """A handle with stored rows ``a=1 (g=0)``, ``b=2 (g=1)`` and pending row ``c=3 (g=1)``.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["a", "b"], [1, 2], [0, 1]), flush=True) + db.add_records(PATH, _rows(["c"], [3], [1])) + return db + + +def test_get_all_records_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + """``get_all_records`` returns the stored rows and the pending row together.""" + assert _values(stored_and_pending.get_all_records(PATH)) == [1, 2, 3] + + +def test_get_record_by_id_finds_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + """``get_record_by_id`` finds a stored row and a pending row alike.""" + assert _values(stored_and_pending.get_record_by_id(PATH, b"a")) == [1] + assert _values(stored_and_pending.get_record_by_id(PATH, b"c")) == [3] + + +def test_get_records_by_ids_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + """``get_records_by_ids`` returns the requested stored and pending rows together.""" + assert _values(stored_and_pending.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] + + +def test_get_records_with_column_value_returns_stored_and_pending( + stored_and_pending: DeltaTableDatabase, +) -> None: + """``get_records_with_column_value`` matches stored and pending rows alike.""" + db = stored_and_pending + assert _values(db.get_records_with_column_value(PATH, {"g": 1})) == [2, 3] + assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1] + + +def test_reads_with_no_match_return_none( + stored_and_pending: DeltaTableDatabase, +) -> None: + """A read that matches no stored or pending row returns ``None``.""" + db = stored_and_pending + assert db.get_records_with_column_value(PATH, {"g": 9}) is None + assert db.get_records_by_ids(PATH, [b"zz"]) is None + assert db.get_record_by_id(PATH, b"zz") is None + + +@pytest.mark.parametrize( + ("read", "expected_ids"), + [ + ( + lambda db: db.get_all_records(PATH, record_id_column="rid"), + [b"a", b"b", b"c"], + ), + ( + lambda db: db.get_records_by_ids( + PATH, [b"a", b"c"], record_id_column="rid" + ), + [b"a", b"c"], + ), + ( + lambda db: db.get_records_with_column_value( + PATH, [("g", 1)], record_id_column="rid" + ), + [b"b", b"c"], + ), + ], + ids=["get_all_records", "get_records_by_ids", "get_records_with_column_value"], +) +def test_record_id_column_names_the_merged_ids( + stored_and_pending: DeltaTableDatabase, + read: Callable[[DeltaTableDatabase], pa.Table | None], + expected_ids: list[bytes], +) -> None: + """With ``record_id_column`` the ids of stored and pending rows come back under that name.""" + table = read(stored_and_pending) + assert table is not None + assert "__record_id" not in table.column_names + assert sorted(cast(list[bytes], table.column("rid").to_pylist())) == expected_ids + + +@pytest.mark.parametrize("skip_duplicates", [False, True]) +def test_pending_row_with_stored_id_reads_as_flush_keeps_it( + tmp_path: Path, skip_duplicates: bool +) -> None: + """A pending row whose id is already stored is never returned; the stored row is.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["a"], [1]), flush=True) + db.add_records(PATH, _rows(["a", "d"], [10, 4]), skip_duplicates=skip_duplicates) + + for when in ("before flush", "after flush"): + assert _values(db.get_all_records(PATH)) == [1, 4], when + assert _values(db.get_record_by_id(PATH, b"a")) == [1], when + assert _values(db.get_records_by_ids(PATH, [b"a", b"d"])) == [1, 4], when + assert _values(db.get_records_with_column_value(PATH, {"g": 0})) == [1, 4], when + db.flush() + + +@pytest.fixture +def fail_id_refresh_once(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the next stored-ID refresh fail once, as a transient storage error would. + + ``DeltaTable.to_pyarrow_table`` is the backend call the stored-ID refresh makes; the + read methods scan through ``to_pyarrow_dataset``, so only the refresh fails. + """ + original = deltalake.DeltaTable.to_pyarrow_table + calls = {"n": 0} + + def to_pyarrow_table(self: deltalake.DeltaTable) -> pa.Table: + """Raise on the first call, then read the table as the backend does.""" + calls["n"] += 1 + if calls["n"] == 1: + raise OSError("transient storage error") + return original(self) + + monkeypatch.setattr(deltalake.DeltaTable, "to_pyarrow_table", to_pyarrow_table) + + +@pytest.mark.usefixtures("fail_id_refresh_once") +@pytest.mark.parametrize("method", list(READS)) +def test_failed_id_refresh_is_retried_by_the_next_read( + tmp_path: Path, method: str +) -> None: + """After a failed stored-ID refresh, a pending row with a stored id is still not returned.""" + db = DeltaTableDatabase(tmp_path).at(SCOPE) + db.add_records(PATH, _rows(["b"], [2]), flush=True) + db.add_records(PATH, _rows(["a", "b"], [1, 20])) + + # The failed refresh is a failed stored read (ITL-884): never "no stored ids". + with pytest.raises(StoreReadError) as info: + _ = db.get_all_records(PATH) + assert isinstance(info.value.__cause__, OSError) + + assert _values(READS[method](db, PATH)) == EXPECTED[method] + + +@pytest.mark.usefixtures("fail_id_refresh_once") +def test_get_records_by_ids_raises_on_failed_id_refresh( + stored_and_pending: DeltaTableDatabase, +) -> None: + """A failed stored-ID refresh raises ``StoreReadError``, never "no records" (ITL-884).""" + db = stored_and_pending + + with pytest.raises(StoreReadError) as info: + _ = db.get_records_by_ids(PATH, [b"a", b"c"]) + assert isinstance(info.value.__cause__, OSError) + assert _values(db.get_records_by_ids(PATH, [b"a", b"c"])) == [1, 3] + + +@pytest.mark.parametrize("cache", ["cold", "warm"]) +@pytest.mark.parametrize("method", list(READS)) +def test_unreadable_table_with_sibling_pending_rows_raises_and_drops_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, method: str, cache: str +) -> None: + """A read never answers from pending rows alone when the stored table is unreadable. + + ``b`` is stored and ``a`` is pending through a sibling handle. While the stored + table cannot be read, every read method raises ``StoreReadError`` (ITL-884) + rather than returning only the pending rows (ITL-864); once it is readable again + the same read returns both rows, so nothing was dropped from the pending batch. + """ + DeltaTableDatabase(tmp_path).at(SCOPE).add_records( + PATH, _rows(["b"], [2]), flush=True + ) + root = DeltaTableDatabase(tmp_path) + reader, writer = root.at(SCOPE), root.at(SCOPE) + if cache == "warm": + # The family has loaded the table and its stored ids before the failure. + assert _values(reader.get_all_records(PATH)) == [2] + assert reader.get_record_by_id(PATH, b"a") is None + writer.add_records(PATH, _rows(["a"], [1])) + + def unreadable(*_args: object, **_kwargs: object) -> pa.Table: + """Fail as a storage outage would.""" + raise OSError("storage unavailable") + + with monkeypatch.context() as patch: + patch.setattr(deltalake.DeltaTable, "to_pyarrow_dataset", unreadable) + patch.setattr(deltalake.DeltaTable, "to_pyarrow_table", unreadable) + with pytest.raises(StoreReadError) as info: + _ = READS[method](reader, PATH) + assert isinstance(info.value.__cause__, OSError) + + assert _values(READS[method](reader, PATH)) == EXPECTED[method] + root.flush() + assert _values(READS[method](reader, PATH)) == EXPECTED[method] + + +@pytest.mark.parametrize("method", list(READS)) +def test_fresh_handle_drops_pending_rows_with_stored_ids( + tmp_path: Path, method: str +) -> None: + """A handle that loaded a table written by another root still drops pending rows with stored ids.""" + DeltaTableDatabase(tmp_path).add_records(PATH, _rows(["b"], [2]), flush=True) + db = DeltaTableDatabase(tmp_path) + assert _values(db.get_all_records(PATH)) == [2] # loads the table, no ids yet + + db.add_records(PATH, _rows(["a", "b"], [1, 20])) + + assert _values(READS[method](db, PATH)) == EXPECTED[method] + + +def _table(record_id: str, x: pa.Array[Any], x_nullable: bool = True) -> pa.Table: + """Build one row keyed by ``record_id`` with ``v=0`` and the column ``x``.""" + schema = pa.schema( + [ + pa.field("__record_id", pa.large_binary()), + pa.field("v", pa.int64()), + pa.field("x", x.type, nullable=x_nullable), + ] + ) + return pa.table( + [ + pa.array([record_id.encode()], type=pa.large_binary()), + pa.array([0], type=pa.int64()), + x, + ], + schema=schema, + ) + + +def _without_x(table: pa.Table) -> pa.Table: + """Return ``table`` without its ``x`` column.""" + return table.drop_columns(["x"]) + + +# (stored row, pending row) pairs whose schemas differ in a way a flush accepts. +SCHEMA_CHANGES: dict[str, tuple[pa.Table, pa.Table]] = { + "pending_adds_column": ( + _without_x(_table("a", pa.array([1], pa.int64()))), + _table("c", pa.array([3], pa.int64())), + ), + "pending_lacks_column": ( + _table("a", pa.array([1], pa.int64())), + _without_x(_table("c", pa.array([3], pa.int64()))), + ), + "non_null_pending_into_nullable": ( + _table("a", pa.array([1], pa.int64())), + _table("c", pa.array([3], pa.int64()), x_nullable=False), + ), + "struct_child_added": ( + _table("a", pa.array([{"p": 1}])), + _table("c", pa.array([{"p": 3, "q": 4}])), + ), + "non_large_string": ( + _table("a", pa.array(["s"], pa.string())), + _table("c", pa.array(["t"], pa.string())), + ), + "non_large_list": ( + _table("a", pa.array([[1]], pa.list_(pa.int64()))), + _table("c", pa.array([[3]], pa.list_(pa.int64()))), + ), +} + +ReadAc = Callable[[DeltaTableDatabase], "pa.Table | None"] + +# Each read method that can return rows ``a`` and ``c`` together, with the record id kept. +READS_AC: dict[str, ReadAc] = { + "get_all_records": lambda db: db.get_all_records(PATH, record_id_column="rid"), + "get_records_by_ids": lambda db: db.get_records_by_ids( + PATH, [b"a", b"c"], record_id_column="rid" + ), + "get_records_with_column_value": lambda db: db.get_records_with_column_value( + PATH, {"v": 0}, record_id_column="rid" + ), +} + + +def _read_sorted(db: DeltaTableDatabase, method: str) -> pa.Table: + """Read rows ``a`` and ``c`` with ``method`` and sort them by record id.""" + table = READS_AC[method](db) + assert table is not None + return table.sort_by("rid") + + +@pytest.mark.parametrize("method", list(READS_AC)) +@pytest.mark.parametrize("change", list(SCHEMA_CHANGES)) +def test_read_before_flush_matches_read_after_flush_across_schema_change( + tmp_path: Path, change: str, method: str +) -> None: + """Stored and pending rows whose schemas differ read back as the flushed table does.""" + stored, pending = SCHEMA_CHANGES[change] + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, stored, flush=True) + db.add_records(PATH, pending) + + before = _read_sorted(db, method) + db.flush() + after = _read_sorted(db, method) + + assert before.num_rows == 2 + assert before.schema == after.schema + assert before.to_pylist() == after.to_pylist() + + +@pytest.mark.parametrize( + ("stored_x", "pending_x"), + [ + (pa.array([1], pa.int64()), pa.array([3], pa.int32())), + (pa.array([1], pa.int64()), pa.array(["t"], pa.large_string())), + (pa.array([1], pa.timestamp("us")), pa.array([3], pa.timestamp("ms"))), + ( + pa.array([1], pa.decimal128(10, 2)), + pa.array([3], pa.decimal128(12, 2)), + ), + ], + ids=["int64_vs_int32", "int64_vs_string", "timestamp_unit", "decimal_precision"], +) +@pytest.mark.parametrize("method", list(READS_AC)) +def test_read_of_stored_and_pending_rows_with_different_types_raises( + tmp_path: Path, method: str, stored_x: pa.Array[Any], pending_x: pa.Array[Any] +) -> None: + """A column whose stored and pending types differ is never widened: the read raises.""" + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, _table("a", stored_x), flush=True) + db.add_records(PATH, _table("c", pending_x)) + + with pytest.raises(pa.ArrowTypeError, match="Field x has incompatible types"): + _ = READS_AC[method](db) + + +def test_pending_null_typed_column_reads_back_but_does_not_flush( + tmp_path: Path, +) -> None: + """A new all-null ``null``-typed column reads back as ``null``; Delta cannot store it. + + The flush fails and the batch stays pending, so the read is unchanged after it. + """ + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, _without_x(_table("a", pa.array([1]))), flush=True) + db.add_records(PATH, _table("c", pa.nulls(1))) + + before = _read_sorted(db, "get_all_records") + assert before.schema.field("x").type == pa.null() + assert before.column("x").to_pylist() == [None, None] + + with pytest.raises(RuntimeError, match="Failed to flush"): + db.flush() + assert _read_sorted(db, "get_all_records").equals(before) + + +@pytest.mark.parametrize( + ("stored", "pending", "query", "expected_ids"), + [ + ( + _without_x(_table("a", pa.array([1], pa.int64()))), + _table("c", pa.array([3], pa.int64())), + {"x": 3}, + [b"c"], + ), + ( + _table("a", pa.array([1], pa.int64())), + _without_x(_table("c", pa.array([3], pa.int64()))), + {"x": 1}, + [b"a"], + ), + ], + ids=["column_only_pending", "column_only_stored"], +) +def test_query_on_column_one_side_lacks_matches_read_after_flush( + tmp_path: Path, + stored: pa.Table, + pending: pa.Table, + query: dict[str, int], + expected_ids: list[bytes], +) -> None: + """A query on a column only one side has returns before flush what it returns after.""" + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, stored, flush=True) + db.add_records(PATH, pending) + + before = db.get_records_with_column_value(PATH, query, record_id_column="rid") + db.flush() + after = db.get_records_with_column_value(PATH, query, record_id_column="rid") + + assert before is not None and after is not None + assert before.column("rid").to_pylist() == expected_ids + assert before.schema == after.schema + assert before.to_pylist() == after.to_pylist() + + +@pytest.mark.parametrize("flushed", [False, True]) +def test_query_on_column_neither_side_has_raises(tmp_path: Path, flushed: bool) -> None: + """A query on a column neither the stored nor the pending rows have raises, flushed or not.""" + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, _rows(["a"], [1]), flush=True) + db.add_records(PATH, _rows(["c"], [3])) + if flushed: + db.flush() + + with pytest.raises(pa.ArrowInvalid, match="No match for FieldRef"): + _ = db.get_records_with_column_value(PATH, {"y": 1}) + + +def test_pending_row_with_null_id_reads_back_like_after_flush(tmp_path: Path) -> None: + """A pending row with a null record id is returned when a stored row also has one. + + Delta's merge matches ids with ``=``, and ``null = null`` is not true, so a flush + inserts a pending row whose id is null even when a stored row's id is null too. + """ + null_id = pa.table( + { + "__record_id": pa.array([None], type=pa.large_binary()), + "v": pa.array([5], type=pa.int64()), + "g": pa.array([0], type=pa.int64()), + } + ) + db = DeltaTableDatabase(tmp_path) + db.add_records(PATH, pa.concat_tables([_rows(["a"], [1]), null_id]), flush=True) + db.add_records(PATH, null_id.set_column(1, "v", pa.array([6], type=pa.int64()))) + + before = _values(db.get_all_records(PATH)) + db.flush() + + assert before == _values(db.get_all_records(PATH)) == [1, 5, 6] diff --git a/tests/test_pipeline/test_delta_node_records_after_run.py b/tests/test_pipeline/test_delta_node_records_after_run.py new file mode 100644 index 000000000..15c925f1f --- /dev/null +++ b/tests/test_pipeline/test_delta_node_records_after_run.py @@ -0,0 +1,46 @@ +"""A function node on a Delta store returns every row right after its first run (ITL-864). + +``PipelineJob.run`` flushes through handles other than the node's own, so a node +handle that cached the table during the run used to keep reading an old snapshot +and return only some of the rows it had just written. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +@pytest.mark.parametrize("n_rows", [2, 3]) +def test_node_returns_every_row_right_after_first_run( + tmp_path: Path, + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], + n_rows: int, +) -> None: + source = op.sources.DictSource( + [{"id": i, "x": 10 * i} for i in range(n_rows)], tag_columns=["id"] + ) + + @op.function_pod(output_keys=["result"]) + def double(x: int) -> int: + return x * 2 + + job = PipelineJob(store=DeltaTableDatabase(tmp_path)) + with job: + double.pod(source, label="double") + job.run(orchestrator=orchestrator()) + + records = job.nodes["double"].get_all_records() + assert records is not None + assert sorted(records.column("result").to_pylist()) == [ + 20 * i for i in range(n_rows) + ]