From 596889fbc7d3940645e0fd386bf088bb60d0a3ac Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:12:59 -0800 Subject: [PATCH 1/4] fix(delta): later commits to a table holding a list of a binary-stored type (ITL-984) Co-Authored-By: Claude Sonnet 5.5 --- changelog.d/ITL-984.md | 8 + src/orcapod/databases/delta_lake_databases.py | 49 ++++-- .../2026-11-12-itl-984-delta-second-commit.md | 24 +++ .../test_delta_list_binary_commits.py | 139 ++++++++++++++++++ 4 files changed, 207 insertions(+), 13 deletions(-) create mode 100644 changelog.d/ITL-984.md create mode 100644 superpowers/plans/2026-11-12-itl-984-delta-second-commit.md create mode 100644 tests/test_databases/test_delta_list_binary_commits.py diff --git a/changelog.d/ITL-984.md b/changelog.d/ITL-984.md new file mode 100644 index 000000000..0d6cd7db3 --- /dev/null +++ b/changelog.d/ITL-984.md @@ -0,0 +1,8 @@ +### Fixed + +- A second commit to a Delta table that holds a list of a type stored as binary + (`list[uuid.UUID]`, `list[bytes]`) no longer fails with `type_coercion ... LargeList(LargeBinary) + vs List(Binary)`. It failed whichever runner made it (a run on `job.dag`, which commits per row, + or a later `job.run()` that adds rows). The append check now compares storage types, and a batch + that holds ids already stored is appended without those rows instead of going to the MERGE. + Stored tables, paths and hashes are unchanged. diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index f80302f29..2257b630f 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -1701,18 +1701,31 @@ def _flush_by_key(self, record_key: str) -> None: assert base.ids is not None incoming_ids = self._non_null_ids(combined_table) inserted_ids = incoming_ids - base.ids - if len(inserted_ids) == len(incoming_ids) and self._stores_as_is( - delta_table, combined_table - ): - # Every pending id is new in the version just loaded and the rows - # have the stored schema: append (ITL-919). A MERGE reads the whole - # stored table on each commit; it remains for every other batch. - deltalake.write_deltalake( - table_uri, - combined_table, - mode="append", - storage_options=self._storage_options or None, - ) + if self._stores_as_is(delta_table, combined_table): + # The rows have the stored schema: append (ITL-919). A MERGE reads + # the whole stored table on each commit; it remains for every + # other batch. A row whose id is stored already is left out, as + # the MERGE leaves it (ITL-984: the MERGE cannot take a large + # list of large binary, so a batch holding stored ids is not + # handed to it); a row with no id never matches, so it is kept. + if len(inserted_ids) == len(incoming_ids): + new_rows = combined_table + else: + is_stored = pc.is_in( + combined_table[self.RECORD_ID_COLUMN], + value_set=pa.array( + sorted(base.ids), + type=combined_table[self.RECORD_ID_COLUMN].type, + ), + ) + new_rows = combined_table.filter(pc.invert(is_stored)) + if new_rows.num_rows: + deltalake.write_deltalake( + table_uri, + new_rows, + mode="append", + storage_options=self._storage_options or None, + ) else: # Some pending ids are stored already: insert only the others. delta_table.merge( @@ -1796,8 +1809,18 @@ def _stores_as_is(delta_table: deltalake.DeltaTable, batch: pa.Table) -> bool: Returns: Whether an append stores ``batch`` exactly as the MERGE would. """ + # Delta stores a logical column as its storage type (ITL-984): compare that, + # or a batch with a logical list never equals the stored plain list. + schema = pa.schema( + [ + f.with_type(f.type.storage_type) + if isinstance(f.type, pa.ExtensionType) + else f + for f in batch.schema + ] + ) try: - incoming = _as_delta_writer_stores(batch.schema) + incoming = _as_delta_writer_stores(schema) except Exception: # A type Delta cannot store directly (null): leave it to the MERGE. return False diff --git a/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md b/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md new file mode 100644 index 000000000..f750e7da3 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md @@ -0,0 +1,24 @@ +# ITL-984 plan: second commit to a Delta table holding a list of a binary-stored type + +Diagnosis: `.untracked/2026-11-12-itl-984-delta-second-commit-diagnosis.md` (copied from the +ITL-858 worktree) and the third Linear comment. Scope: `DeltaTableDatabase` only. + +## Fix (src/orcapod/databases/delta_lake_databases.py) +1. `_stores_as_is`: reduce extension types to their storage types before `_as_delta_writer_stores` + (logical list `list[uuid]` -> `large_list` -> Delta `list`), so new ids append. +2. In `_flush_by_key`, a batch that `_stores_as_is` but holds stored ids: drop the rows whose id is in + `base.ids` and append the rest (no write if none is left). Chosen over a cast MERGE source because + it keeps today's semantics (an id already stored is skipped, as `when_not_matched_insert_all` + does; rows with a null id never match, so they are kept), writes the batch exactly as the first + commit does (stored field metadata of logical columns kept), and does not read the stored table. + The MERGE stays for batches that are not stored as-is (null-typed columns, schema evolution). + +## Tests first (tests/test_databases/test_delta_list_binary_commits.py) +- pod `-> list[uuid.UUID]`: SyncPipelineOrchestrator().run(dag), AsyncPipelineOrchestrator().run(dag), + second job.run() adding rows; stored table equals a single commit; read back has logical type. +- plain `list[bytes]` with an id already stored (add_records twice). +- regression: scalar uuid, `list[str]`. + +## Finish +Golden suite unchanged, full suite, changelog.d/ITL-984.md, design_issues entry only if one exists. +PR body: ITL-871 (#300, tests/test_torch/test_tensor_pipeline_stores.py) / ITL-981 (#328) xfails. diff --git a/tests/test_databases/test_delta_list_binary_commits.py b/tests/test_databases/test_delta_list_binary_commits.py new file mode 100644 index 000000000..5c21d8388 --- /dev/null +++ b/tests/test_databases/test_delta_list_binary_commits.py @@ -0,0 +1,139 @@ +"""A later commit to a Delta table holding a list of a binary-stored type (ITL-984). + +Delta stores ``large_list`` as ``list``. A second commit used to +fail in the MERGE (``LargeList(LargeBinary)`` vs ``List(Binary)``), whichever runner +made it. The stored table must be what a single commit stores. +""" + +from __future__ import annotations + +import uuid +from pathlib import Path +from typing import Any + +import deltalake +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +N_ROWS = 4 + + +@op.function_pod(output_keys=["ids"]) +def make_uuids(x: int) -> list[uuid.UUID]: + return [uuid.uuid5(uuid.NAMESPACE_DNS, f"{x}-{i}") for i in range(2)] + + +@op.function_pod(output_keys=["ids"]) +def make_strs(x: int) -> list[str]: + return [f"{x}-{i}" for i in range(2)] + + +@op.function_pod(output_keys=["ids"]) +def make_uuid(x: int) -> uuid.UUID: + return uuid.uuid5(uuid.NAMESPACE_DNS, str(x)) + + +POD_CASES = [make_uuids, make_strs, make_uuid] + + +def _build(store: Path, pod: Any, n: int) -> PipelineJob: + job = PipelineJob(name="j", store=DeltaTableDatabase(base_path=store)) + with job: + src = op.sources.DictSource( + [{"id": i, "x": i} for i in range(n)], tag_columns=["id"], source_id="s" + ) + pod.pod(src, label="node") + return job + + +def _result_table(store: Path) -> pa.Table: + """The stored function-result table, read with Delta directly, sorted by record id.""" + (path,) = [p for p in store.rglob("rdb_v2") if (p / "_delta_log").is_dir()] + table = deltalake.DeltaTable(str(path)).to_pyarrow_table() + return table.sort_by("__record_id") + + +def _stored_form(table: pa.Table) -> list[tuple[str, str, bool]]: + return [(f.name, str(f.type), f.nullable) for f in table.schema] + + +def _values(job: PipelineJob) -> list[Any]: + return sorted( + (row.as_dict()["ids"] for _, row in job.nodes["node"].iter_data()), key=repr + ) + + +@pytest.mark.parametrize("pod", POD_CASES, ids=lambda p: p.__name__) +@pytest.mark.parametrize( + "runner", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +def test_one_commit_per_row_stores_what_a_single_commit_stores( + tmp_path: Path, pod: Any, runner: Any +) -> None: + single = tmp_path / "single" + reference = _build(single, pod, N_ROWS) + reference.run() + + many = tmp_path / "many" + job = _build(many, pod, N_ROWS) + runner().run(job.dag) # the result cache flushes once per row + + expected, actual = _result_table(single), _result_table(many) + assert _stored_form(actual) == _stored_form(expected) + assert actual.num_rows == N_ROWS + assert _values(job) == _values(reference) + + +@pytest.mark.parametrize("pod", POD_CASES, ids=lambda p: p.__name__) +def test_second_run_that_adds_rows(tmp_path: Path, pod: Any) -> None: + _build(tmp_path / "grown", pod, 2).run() + job = _build(tmp_path / "grown", pod, N_ROWS) + job.run() + + single = tmp_path / "single" + reference = _build(single, pod, N_ROWS) + reference.run() + + assert _stored_form(_result_table(tmp_path / "grown")) == _stored_form( + _result_table(single) + ) + assert _values(job) == _values(reference) + + +def test_read_back_has_the_logical_type(tmp_path: Path) -> None: + job = _build(tmp_path, make_uuids, N_ROWS) + AsyncPipelineOrchestrator().run(job.dag) + reopened = _build(tmp_path, make_uuids, N_ROWS) + reopened.run() # every row is stored: it reads them back + rows = [row.as_dict()["ids"] for _, row in reopened.nodes["node"].iter_data()] + assert len(rows) == N_ROWS + assert all(isinstance(u, uuid.UUID) for ids in rows for u in ids) + + +def _bytes_rows(ids: list[bytes], values: list[list[bytes]]) -> pa.Table: + return pa.table( + { + "__record_id": pa.array(ids, pa.large_binary()), + "v": pa.array(values, pa.large_list(pa.large_binary())), + } + ) + + +def test_list_bytes_batch_holding_a_stored_id(tmp_path: Path) -> None: + db = DeltaTableDatabase(base_path=tmp_path) + path = ("t",) + db.add_records(path, _bytes_rows([b"r0", b"r1"], [[b"a"], [b"b"]]), flush=True) + db.add_records(path, _bytes_rows([b"r1", b"r2"], [[b"X"], [b"c"]]), flush=True) + + table = db.get_all_records(path, record_id_column="__record_id") + assert table is not None + got = dict(zip(table["__record_id"].to_pylist(), table["v"].to_pylist())) + # the stored id keeps its stored row (skipped, as when_not_matched_insert_all does) + assert got == {b"r0": [b"a"], b"r1": [b"b"], b"r2": [b"c"]} From ef0ad76af1627ddd7b61fabb91019279e7547566 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:22:15 -0800 Subject: [PATCH 2/4] fix(delta): give the MERGE its list columns in the stored layout (ITL-984) Co-Authored-By: Claude Sonnet 5.5 --- changelog.d/ITL-984.md | 4 +- src/orcapod/databases/delta_lake_databases.py | 89 +++++++++++++------ .../2026-11-12-itl-984-delta-second-commit.md | 11 ++- 3 files changed, 70 insertions(+), 34 deletions(-) diff --git a/changelog.d/ITL-984.md b/changelog.d/ITL-984.md index 0d6cd7db3..ef89efb7d 100644 --- a/changelog.d/ITL-984.md +++ b/changelog.d/ITL-984.md @@ -3,6 +3,6 @@ - A second commit to a Delta table that holds a list of a type stored as binary (`list[uuid.UUID]`, `list[bytes]`) no longer fails with `type_coercion ... LargeList(LargeBinary) vs List(Binary)`. It failed whichever runner made it (a run on `job.dag`, which commits per row, - or a later `job.run()` that adds rows). The append check now compares storage types, and a batch - that holds ids already stored is appended without those rows instead of going to the MERGE. + or a later `job.run()` that adds rows). The append check now compares storage types, and the MERGE + that handles a batch holding ids already stored is given its list columns in the stored layout. Stored tables, paths and hashes are unchanged. diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 2257b630f..fe8e44a81 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -251,6 +251,54 @@ def _as_delta_writer_stores(schema: pa.Schema) -> pa.Schema: ) +def _merge_source_in_stored_layout( + delta_table: deltalake.DeltaTable, batch: pa.Table +) -> pa.Table: + """Return ``batch`` with its list columns in the layout the table stores. + + Delta stores ``large_list`` as ``list``, and the MERGE + cannot plan a ``CASE WHEN`` over the two (ITL-984). A list column, a logical + list included, is therefore handed to the MERGE as its storage type cast to + the stored type. The values are unchanged and the stored field metadata stays + as it is; every other column, and a list column the table does not have yet, + is passed as it is. + + Args: + delta_table: The loaded stored table. + batch: The rows about to be merged. + + Returns: + ``batch``, with the cast list columns. + """ + stored = _stored_arrow_schema(delta_table) + for index, field in enumerate(batch.schema): + column = batch.column(index) + if isinstance(field.type, pa.ExtensionType): + plain = column.combine_chunks().storage # pyright: ignore[reportAttributeAccessIssue] # an extension array + is_list = pa.types.is_large_list(plain.type) or pa.types.is_list(plain.type) + else: + plain = column + is_list = pa.types.is_large_list(field.type) or pa.types.is_list(field.type) + if not is_list or field.name not in stored.names: + continue + target = stored.field(field.name).type + try: + as_stored = _as_delta_writer_stores(pa.schema([pa.field("c", plain.type)])) + except Exception: + continue + if as_stored.field("c").type != target: + # More than the layout differs (schema evolution): the MERGE decides. + continue + batch = batch.set_column( + index, + pa.field(field.name, target, nullable=field.nullable), + pa.chunked_array([plain]).cast(target) + if isinstance(plain, pa.Array) + else plain.cast(target), + ) + return batch + + def _without_field_metadata(delta_type: Any) -> Any: """Return a Delta type with the metadata of every nested field dropped. @@ -1701,35 +1749,24 @@ def _flush_by_key(self, record_key: str) -> None: assert base.ids is not None incoming_ids = self._non_null_ids(combined_table) inserted_ids = incoming_ids - base.ids - if self._stores_as_is(delta_table, combined_table): - # The rows have the stored schema: append (ITL-919). A MERGE reads - # the whole stored table on each commit; it remains for every - # other batch. A row whose id is stored already is left out, as - # the MERGE leaves it (ITL-984: the MERGE cannot take a large - # list of large binary, so a batch holding stored ids is not - # handed to it); a row with no id never matches, so it is kept. - if len(inserted_ids) == len(incoming_ids): - new_rows = combined_table - else: - is_stored = pc.is_in( - combined_table[self.RECORD_ID_COLUMN], - value_set=pa.array( - sorted(base.ids), - type=combined_table[self.RECORD_ID_COLUMN].type, - ), - ) - new_rows = combined_table.filter(pc.invert(is_stored)) - if new_rows.num_rows: - deltalake.write_deltalake( - table_uri, - new_rows, - mode="append", - storage_options=self._storage_options or None, - ) + if len(inserted_ids) == len(incoming_ids) and self._stores_as_is( + delta_table, combined_table + ): + # Every pending id is new in the version just loaded and the rows + # have the stored schema: append (ITL-919). A MERGE reads the whole + # stored table on each commit; it remains for every other batch. + deltalake.write_deltalake( + table_uri, + combined_table, + mode="append", + storage_options=self._storage_options or None, + ) else: # Some pending ids are stored already: insert only the others. delta_table.merge( - source=combined_table, + source=_merge_source_in_stored_layout( + delta_table, combined_table + ), predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}", source_alias="source", target_alias="target", diff --git a/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md b/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md index f750e7da3..590e1e7e1 100644 --- a/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md +++ b/superpowers/plans/2026-11-12-itl-984-delta-second-commit.md @@ -6,12 +6,11 @@ ITL-858 worktree) and the third Linear comment. Scope: `DeltaTableDatabase` only ## Fix (src/orcapod/databases/delta_lake_databases.py) 1. `_stores_as_is`: reduce extension types to their storage types before `_as_delta_writer_stores` (logical list `list[uuid]` -> `large_list` -> Delta `list`), so new ids append. -2. In `_flush_by_key`, a batch that `_stores_as_is` but holds stored ids: drop the rows whose id is in - `base.ids` and append the rest (no write if none is left). Chosen over a cast MERGE source because - it keeps today's semantics (an id already stored is skipped, as `when_not_matched_insert_all` - does; rows with a null id never match, so they are kept), writes the batch exactly as the first - commit does (stored field metadata of logical columns kept), and does not read the stored table. - The MERGE stays for batches that are not stored as-is (null-typed columns, schema evolution). +2. MERGE source in the stored layout (`_merge_source_in_stored_layout`): list columns (logical lists + reduced to storage) cast to the stored type. Chosen over drop-ids-and-append because it keeps the + MERGE (`when_not_matched_insert_all` skips a stored id, a null id never matches) exactly as today + for every case; the first attempt (drop and append) changed the operation pinned by + `test_a_stored_id_goes_through_merge_and_keeps_the_stored_row`, so it was reverted. ## Tests first (tests/test_databases/test_delta_list_binary_commits.py) - pod `-> list[uuid.UUID]`: SyncPipelineOrchestrator().run(dag), AsyncPipelineOrchestrator().run(dag), From 92d58e91a9bd323018ab1c0f7996a81661a74934 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:38:27 -0800 Subject: [PATCH 3/4] fix(delta): cast a list only when the stored type differs by layout (ITL-984) Co-Authored-By: Claude Sonnet 5.5 --- src/orcapod/databases/delta_lake_databases.py | 11 +++++------ .../test_delta_list_binary_commits.py | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index fe8e44a81..37485da48 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -282,12 +282,11 @@ def _merge_source_in_stored_layout( if not is_list or field.name not in stored.names: continue target = stored.field(field.name).type - try: - as_stored = _as_delta_writer_stores(pa.schema([pa.field("c", plain.type)])) - except Exception: - continue - if as_stored.field("c").type != target: - # More than the layout differs (schema evolution): the MERGE decides. + if arrow_utils.normalize_to_large_types( + plain.type + ) != arrow_utils.normalize_to_large_types(target): + # More than the layout differs (a timestamp unit, a width, an evolved + # struct): the MERGE decides, and no cast may change a value. continue batch = batch.set_column( index, diff --git a/tests/test_databases/test_delta_list_binary_commits.py b/tests/test_databases/test_delta_list_binary_commits.py index 5c21d8388..512f5453e 100644 --- a/tests/test_databases/test_delta_list_binary_commits.py +++ b/tests/test_databases/test_delta_list_binary_commits.py @@ -130,10 +130,22 @@ def test_list_bytes_batch_holding_a_stored_id(tmp_path: Path) -> None: db = DeltaTableDatabase(base_path=tmp_path) path = ("t",) db.add_records(path, _bytes_rows([b"r0", b"r1"], [[b"a"], [b"b"]]), flush=True) - db.add_records(path, _bytes_rows([b"r1", b"r2"], [[b"X"], [b"c"]]), flush=True) + db.add_records( + path, + _bytes_rows([b"r1", b"r2", b"r3", b"r4"], [[b"X"], [b"c"], None, [None, b"d"]]), + flush=True, + ) - table = db.get_all_records(path, record_id_column="__record_id") + table = DeltaTableDatabase(base_path=tmp_path).get_all_records( + path, record_id_column="__record_id" + ) # reopened assert table is not None got = dict(zip(table["__record_id"].to_pylist(), table["v"].to_pylist())) # the stored id keeps its stored row (skipped, as when_not_matched_insert_all does) - assert got == {b"r0": [b"a"], b"r1": [b"b"], b"r2": [b"c"]} + assert got == { + b"r0": [b"a"], + b"r1": [b"b"], + b"r2": [b"c"], + b"r3": None, + b"r4": [None, b"d"], + } From 72d04f1bbd59d219110de0b9136fdf641dc415c5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:40:09 -0800 Subject: [PATCH 4/4] test(delta): type the null-bearing list rows (ITL-984) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_databases/test_delta_list_binary_commits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_databases/test_delta_list_binary_commits.py b/tests/test_databases/test_delta_list_binary_commits.py index 512f5453e..51d83dd6c 100644 --- a/tests/test_databases/test_delta_list_binary_commits.py +++ b/tests/test_databases/test_delta_list_binary_commits.py @@ -117,7 +117,7 @@ def test_read_back_has_the_logical_type(tmp_path: Path) -> None: assert all(isinstance(u, uuid.UUID) for ids in rows for u in ids) -def _bytes_rows(ids: list[bytes], values: list[list[bytes]]) -> pa.Table: +def _bytes_rows(ids: list[bytes], values: list[list[bytes | None] | None]) -> pa.Table: return pa.table( { "__record_id": pa.array(ids, pa.large_binary()),