Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions changelog.d/ITL-984.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
### Fixed

- A second commit to a Delta table that holds a list of a type stored as binary
(`list[uuid.UUID]`, `list[bytes]`) no longer fails with `type_coercion ... LargeList(LargeBinary)
vs List(Binary)`. It failed whichever runner made it (a run on `job.dag`, which commits per row,
or a later `job.run()` that adds rows). The append check now compares storage types, and the MERGE
that handles a batch holding ids already stored is given its list columns in the stored layout.
Stored tables, paths and hashes are unchanged.
63 changes: 61 additions & 2 deletions src/orcapod/databases/delta_lake_databases.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,53 @@ def _as_delta_writer_stores(schema: pa.Schema) -> pa.Schema:
)


def _merge_source_in_stored_layout(
delta_table: deltalake.DeltaTable, batch: pa.Table
) -> pa.Table:
"""Return ``batch`` with its list columns in the layout the table stores.

Delta stores ``large_list<large_binary>`` as ``list<binary>``, and the MERGE
cannot plan a ``CASE WHEN`` over the two (ITL-984). A list column, a logical
list included, is therefore handed to the MERGE as its storage type cast to
the stored type. The values are unchanged and the stored field metadata stays
as it is; every other column, and a list column the table does not have yet,
is passed as it is.

Args:
delta_table: The loaded stored table.
batch: The rows about to be merged.

Returns:
``batch``, with the cast list columns.
"""
stored = _stored_arrow_schema(delta_table)
for index, field in enumerate(batch.schema):
column = batch.column(index)
if isinstance(field.type, pa.ExtensionType):
plain = column.combine_chunks().storage # pyright: ignore[reportAttributeAccessIssue] # an extension array
is_list = pa.types.is_large_list(plain.type) or pa.types.is_list(plain.type)
else:
plain = column
is_list = pa.types.is_large_list(field.type) or pa.types.is_list(field.type)
if not is_list or field.name not in stored.names:
continue
target = stored.field(field.name).type
if arrow_utils.normalize_to_large_types(
plain.type
) != arrow_utils.normalize_to_large_types(target):
# More than the layout differs (a timestamp unit, a width, an evolved
# struct): the MERGE decides, and no cast may change a value.
continue
batch = batch.set_column(
index,
pa.field(field.name, target, nullable=field.nullable),
pa.chunked_array([plain]).cast(target)
if isinstance(plain, pa.Array)
else plain.cast(target),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
)
return batch


def _without_field_metadata(delta_type: Any) -> Any:
"""Return a Delta type with the metadata of every nested field dropped.

Expand Down Expand Up @@ -1716,7 +1763,9 @@ def _flush_by_key(self, record_key: str) -> None:
else:
# Some pending ids are stored already: insert only the others.
delta_table.merge(
source=combined_table,
source=_merge_source_in_stored_layout(
delta_table, combined_table
),
predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}",
source_alias="source",
target_alias="target",
Expand Down Expand Up @@ -1796,8 +1845,18 @@ def _stores_as_is(delta_table: deltalake.DeltaTable, batch: pa.Table) -> bool:
Returns:
Whether an append stores ``batch`` exactly as the MERGE would.
"""
# Delta stores a logical column as its storage type (ITL-984): compare that,
# or a batch with a logical list never equals the stored plain list.
schema = pa.schema(
[
f.with_type(f.type.storage_type)
if isinstance(f.type, pa.ExtensionType)
else f
for f in batch.schema
]
)
try:
incoming = _as_delta_writer_stores(batch.schema)
incoming = _as_delta_writer_stores(schema)
except Exception:
# A type Delta cannot store directly (null): leave it to the MERGE.
return False
Expand Down
23 changes: 23 additions & 0 deletions superpowers/plans/2026-11-12-itl-984-delta-second-commit.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# ITL-984 plan: second commit to a Delta table holding a list of a binary-stored type

Diagnosis: `.untracked/2026-11-12-itl-984-delta-second-commit-diagnosis.md` (copied from the
ITL-858 worktree) and the third Linear comment. Scope: `DeltaTableDatabase` only.

## Fix (src/orcapod/databases/delta_lake_databases.py)
1. `_stores_as_is`: reduce extension types to their storage types before `_as_delta_writer_stores`
(logical list `list[uuid]` -> `large_list<large_binary>` -> Delta `list<binary>`), so new ids append.
2. MERGE source in the stored layout (`_merge_source_in_stored_layout`): list columns (logical lists
reduced to storage) cast to the stored type. Chosen over drop-ids-and-append because it keeps the
MERGE (`when_not_matched_insert_all` skips a stored id, a null id never matches) exactly as today
for every case; the first attempt (drop and append) changed the operation pinned by
`test_a_stored_id_goes_through_merge_and_keeps_the_stored_row`, so it was reverted.

## Tests first (tests/test_databases/test_delta_list_binary_commits.py)
- pod `-> list[uuid.UUID]`: SyncPipelineOrchestrator().run(dag), AsyncPipelineOrchestrator().run(dag),
second job.run() adding rows; stored table equals a single commit; read back has logical type.
- plain `list[bytes]` with an id already stored (add_records twice).
- regression: scalar uuid, `list[str]`.

## Finish
Golden suite unchanged, full suite, changelog.d/ITL-984.md, design_issues entry only if one exists.
PR body: ITL-871 (#300, tests/test_torch/test_tensor_pipeline_stores.py) / ITL-981 (#328) xfails.
151 changes: 151 additions & 0 deletions tests/test_databases/test_delta_list_binary_commits.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
"""A later commit to a Delta table holding a list of a binary-stored type (ITL-984).

Delta stores ``large_list<large_binary>`` as ``list<binary>``. A second commit used to
fail in the MERGE (``LargeList(LargeBinary)`` vs ``List(Binary)``), whichever runner
made it. The stored table must be what a single commit stores.
"""

from __future__ import annotations

import uuid
from pathlib import Path
from typing import Any

import deltalake
import pyarrow as pa
import pytest

import orcapod as op
from orcapod.databases import DeltaTableDatabase
from orcapod.pipeline import AsyncPipelineOrchestrator, PipelineJob
from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator

N_ROWS = 4


@op.function_pod(output_keys=["ids"])
def make_uuids(x: int) -> list[uuid.UUID]:
return [uuid.uuid5(uuid.NAMESPACE_DNS, f"{x}-{i}") for i in range(2)]


@op.function_pod(output_keys=["ids"])
def make_strs(x: int) -> list[str]:
return [f"{x}-{i}" for i in range(2)]


@op.function_pod(output_keys=["ids"])
def make_uuid(x: int) -> uuid.UUID:
return uuid.uuid5(uuid.NAMESPACE_DNS, str(x))


POD_CASES = [make_uuids, make_strs, make_uuid]


def _build(store: Path, pod: Any, n: int) -> PipelineJob:
job = PipelineJob(name="j", store=DeltaTableDatabase(base_path=store))
with job:
src = op.sources.DictSource(
[{"id": i, "x": i} for i in range(n)], tag_columns=["id"], source_id="s"
)
pod.pod(src, label="node")
return job


def _result_table(store: Path) -> pa.Table:
"""The stored function-result table, read with Delta directly, sorted by record id."""
(path,) = [p for p in store.rglob("rdb_v2") if (p / "_delta_log").is_dir()]
table = deltalake.DeltaTable(str(path)).to_pyarrow_table()
return table.sort_by("__record_id")


def _stored_form(table: pa.Table) -> list[tuple[str, str, bool]]:
return [(f.name, str(f.type), f.nullable) for f in table.schema]


def _values(job: PipelineJob) -> list[Any]:
return sorted(
(row.as_dict()["ids"] for _, row in job.nodes["node"].iter_data()), key=repr
)


@pytest.mark.parametrize("pod", POD_CASES, ids=lambda p: p.__name__)
@pytest.mark.parametrize(
"runner",
[SyncPipelineOrchestrator, AsyncPipelineOrchestrator],
ids=["sync", "async"],
)
def test_one_commit_per_row_stores_what_a_single_commit_stores(
tmp_path: Path, pod: Any, runner: Any
) -> None:
single = tmp_path / "single"
reference = _build(single, pod, N_ROWS)
reference.run()

many = tmp_path / "many"
job = _build(many, pod, N_ROWS)
runner().run(job.dag) # the result cache flushes once per row

expected, actual = _result_table(single), _result_table(many)
assert _stored_form(actual) == _stored_form(expected)
assert actual.num_rows == N_ROWS
assert _values(job) == _values(reference)


@pytest.mark.parametrize("pod", POD_CASES, ids=lambda p: p.__name__)
def test_second_run_that_adds_rows(tmp_path: Path, pod: Any) -> None:
_build(tmp_path / "grown", pod, 2).run()
job = _build(tmp_path / "grown", pod, N_ROWS)
job.run()

single = tmp_path / "single"
reference = _build(single, pod, N_ROWS)
reference.run()

assert _stored_form(_result_table(tmp_path / "grown")) == _stored_form(
_result_table(single)
)
assert _values(job) == _values(reference)


def test_read_back_has_the_logical_type(tmp_path: Path) -> None:
job = _build(tmp_path, make_uuids, N_ROWS)
AsyncPipelineOrchestrator().run(job.dag)
reopened = _build(tmp_path, make_uuids, N_ROWS)
reopened.run() # every row is stored: it reads them back
rows = [row.as_dict()["ids"] for _, row in reopened.nodes["node"].iter_data()]
assert len(rows) == N_ROWS
assert all(isinstance(u, uuid.UUID) for ids in rows for u in ids)


def _bytes_rows(ids: list[bytes], values: list[list[bytes | None] | None]) -> pa.Table:
return pa.table(
{
"__record_id": pa.array(ids, pa.large_binary()),
"v": pa.array(values, pa.large_list(pa.large_binary())),
}
)


def test_list_bytes_batch_holding_a_stored_id(tmp_path: Path) -> None:
db = DeltaTableDatabase(base_path=tmp_path)
path = ("t",)
db.add_records(path, _bytes_rows([b"r0", b"r1"], [[b"a"], [b"b"]]), flush=True)
db.add_records(
path,
_bytes_rows([b"r1", b"r2", b"r3", b"r4"], [[b"X"], [b"c"], None, [None, b"d"]]),
flush=True,
)

table = DeltaTableDatabase(base_path=tmp_path).get_all_records(
path, record_id_column="__record_id"
) # reopened
assert table is not None
got = dict(zip(table["__record_id"].to_pylist(), table["v"].to_pylist()))
# the stored id keeps its stored row (skipped, as when_not_matched_insert_all does)
assert got == {
b"r0": [b"a"],
b"r1": [b"b"],
b"r2": [b"c"],
b"r3": None,
b"r4": [None, b"d"],
}
Loading