diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 695708a20..aad884ef1 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2382,6 +2382,117 @@ newcomer. Re-checked on `b197544e` (v0.2.0 + #291): --- +## `src/orcapod/databases/delta_lake_databases.py` — reading back a handle family's own writes + +### DW1 — A read through one `at()` handle could miss rows its family had just written +**Status:** resolved +**Severity:** high +**Issue:** ITL-864 + +Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` +returned 1 of 2 or 3 rows (plain `int -> int` pod, sync and async orchestrators), while +`iter_data` and a fresh handle saw every row. Every `at()` handle shares the root's pending +batches, and any handle's `flush()` writes every pending key, so the result-store handle +flushed the pipeline table's rows. But each handle kept its own loaded-table cache, so the +node's pipeline handle kept reading the table version it had loaded during the run. Every read +method missed the rows a sibling flushed. Separately, the read methods merged pending and stored +rows inconsistently: `get_records_by_ids` ignored pending rows, `get_records_with_column_value` +ignored stored rows whenever a pending batch existed (and returned an empty table instead of +`None`), and a pending row whose record id was already stored was returned as well, although a +flush keeps the stored row and drops the pending one. + +**Fix:** the loaded-table cache and the committed snapshots (`_CommittedSnapshot`, which +replaced the existing-ids and cache-dirty dicts in ITL-919) are shared across the `at()` +family exactly as pending state (pending batches and their indexes) is. Every read method returns what a flush would leave: the +stored rows plus the pending rows whose record id is not stored (`_pending_not_stored`), and +`None` when there are none. These dicts are touched only from the thread that drives the +pipeline; the async orchestrator runs only user functions in worker threads. Regression tests: +`tests/test_databases/test_delta_read_your_writes.py` and +`tests/test_pipeline/test_delta_node_records_after_run.py`. Stored ids come from +`_stored_ids`: the snapshot of the loaded table version reads them on first use, through an +id-only scan (a one-column schema); a failed scan raises `StoreReadError` and leaves the +snapshot's ids unset (`None`), so the next read scans again instead of treating every stored +id as unstored and returning duplicates. A handle that loaded the table before it had any +stored ids (a fresh root over an existing table) reads them on its first read with pending +rows (`_pending_not_stored`). After the family's own commit the ids are extended from the +commit itself, with no re-read (ITL-919). + +When stored and pending rows are read together, the result has the schema a flush would give: +both parts are large-normalized (`null` kept), then combined with +`pa.concat_tables(promote_options="default")`. A column missing on one side is null-filled, and a +`null`-typed column takes the other side's type. Columns whose types differ (`int64` vs `int32`, +a string vs an int, a timestamp unit, a decimal precision) raise `pyarrow.ArrowTypeError`; nothing +is widened. Before this, a read of stored rows next to pending rows with any schema difference, +including a pending `string` column next to the stored `large_string`, raised `ArrowInvalid`; +`get_all_records` already did so on `main`. +A `get_records_with_column_value` query on a column that only the stored or only the pending +rows have skips the side without it (after a flush that side holds null there, and +`column == value` never matches null), so it returns before a flush what it returns after; a +column neither side has still raises `ArrowInvalid`. + +Recorded for the nullability audit (ITL-842) and ITL-890: Delta's merge relaxes nullability (a +non-null pending column into a nullable stored one stays nullable) and evolves struct children +(`struct
` stored + `struct
` pending becomes `struct
`, stored rows `q = null`). The
+read path now mirrors both. Whether Orcapod should allow either at all is for those issues. A
+pending all-null `null`-typed column reads back as `null`, but Delta cannot store it: the flush
+fails and the batch stays pending (unchanged, pinned by
+`test_pending_null_typed_column_reads_back_but_does_not_flush`).
+
+Still open: a handle does not see writes made by another process, or by a separate
+`DeltaTableDatabase` root on the same path, until its own next commit to that table reloads it;
+this is unchanged. When it does reload (another writer committed in between), the stored ids
+are read again with the id-only scan, and the stored rows on the next lookup, each once per
+version (D3, now bounded by ITL-919's snapshot).
+
+---
+
+## `src/orcapod/databases/delta_lake_databases.py`, `src/orcapod/core/sources/delta_table_source.py` — exit after a Delta read
+
+### DX1 — A process that read a Delta table can hang or abort at interpreter exit
+**Status:** resolved (local stores); cloud stores remain exposed upstream
+**Severity:** high
+**Issue:** ITL-892
+
+`DeltaTable.to_pyarrow_dataset()` (and `to_pyarrow_table()`) reads through deltalake's
+`PyFileSystem(DeltaStorageHandler)`, a filesystem implemented in Python. After a scan returns,
+Arrow finishes tearing down the scan on a thread-pool worker. If that worker drops the last
+reference to a Python-backed buffer during interpreter finalization, CPython stops the thread,
+and `ThreadPool::~ThreadPool` in `__cxa_finalize` waits for it forever. On Linux with Python
+3.11/3.12 the process may abort with SIGABRT instead. This is upstream:
+apache/arrow#50482 and #54536, with an open fix PR, #54467. The failure follows a scan, not a
+write.
+
+Measured over 100 runs each, locked versions: one write plus one scan fails 57% of runs on
+macOS arm64 and 5–38% on Linux x86_64. A fresh `DeltaTableDatabase.get_all_records` fails 53%
+on macOS and 8–23% on Linux. `DeltaTableSource` fails 4–10%. A `PipelineJob` run fails 0–4%.
+Every pyarrow 20.0.0–26.0.1 × deltalake 1.0.2/1.6.6 combination fails, so no version pin helps.
+Scanning local tables through a native pyarrow filesystem (`SubTreeFileSystem(table_path,
+LocalFileSystem())`) gives 0/100 in every case. Its results are byte-identical. Cloud stores
+keep the Python filesystem and stay exposed until pyarrow ships the fix. Findings, scripts and
+raw results: `superpowers/specs/itl-892-exit-hang/FINDINGS.md`.
+
+**Fix:** `storage_utils.delta_to_pyarrow_dataset` opens local Delta tables on a native pyarrow
+filesystem (`SubTreeFileSystem` over `LocalFileSystem`). Every scan in `DeltaTableDatabase` and
+`DeltaTableSource` goes through it. Cloud URIs keep deltalake's reader. The results are
+schema- and value-identical: `tests/test_databases/test_delta_native_fs_read.py` compares them
+with deltalake's reader for logical types (Path, dataclass, numpy, pandas), partitioned tables
+and escaped partition values, and for table roots whose names need percent-encoding
+(`table_uri` is percent-encoded, so the helper decodes it with `pyarrow.fs.FileSystem.from_uri`;
+a root that is neither a `file://` URI nor an absolute POSIX path, such as a Windows drive path,
+is untested and keeps deltalake's reader). `tests/test_databases/test_delta_exit.py` runs fresh
+interpreters and fails if one does not exit cleanly; it failed before the fix. The remaining
+cloud-store exposure is documented in `docs/troubleshooting.md`.
+
+**Considered alternative, not implemented:** an `atexit` hook that releases the GIL briefly
+(`time.sleep(0.05)`) before finalization, the idea of apache/arrow#54467. It measured 0/100 on
+macOS and Linux and also covers cloud stores. But it is timing-based, so a loaded machine or a
+large scan can outlast it. It would also run in every process that imports Orcapod. If cloud
+stores need covering before pyarrow ships the fix, register it lazily on the first cloud-store
+scan. When pyarrow ships #54467, bump the floor and reconsider whether the helper is still
+needed.
+
+---
+
## `src/orcapod/core/operators/filters.py` — `PolarsFilter` identity and nullability (ITL-828)
### PF1 — An expression predicate could not be hashed, and a filter broke a join inside a job
@@ -2415,6 +2526,58 @@ function pod downstream of a filter, so a 0.2 store recomputes those results onc
---
+## `src/orcapod/databases/` — PipelineJob run time quadratic in rows
+
+### PQ1 — Every row a function node processes scans the rows already stored
+**Status:** resolved
+**Severity:** high
+**Issue:** ITL-919
+
+A function node in a `PipelineJob` does, per row: `ResultCache.lookup` (lookup by input data
+hash in the result table), `ResultCache.store` with a flush of the whole store (both
+tables), and `add_pipeline_record` (lookup by base entry id for the recomputation index,
+then `add_record(skip_duplicates=True)`). Sync and async orchestrators share that path;
+operator nodes write with one `add_records` and are not affected. Measured on `c8efc092`,
+in-memory store, sync: 7.2 / 20.9 / 64 ms per row at 100 / 400 / 1,000 rows (standalone
+pod: 0.09 ms).
+
+- `InMemoryArrowDatabase`: each lookup filtered the whole table, the duplicate check built
+ the set of every committed id, and each flush ran `is_in` over all ids, filtered and
+ concatenated, adding one chunk per row, so every later Arrow operation cost O(rows).
+ **Fix:** lazily built hash indexes over committed rows (binary and string columns, record
+ ids), extended on append and rebuilt only after a flush overwrites an id; flush keeps the
+ committed table at O(log N) record batches by binary-counter compaction. Stored rows, row
+ order, ids and schemas are unchanged (differential check against `main` on a joined
+ multi-node pipeline, both orchestrators). Store work per row is now independent of rows
+ stored (tests count Arrow bytes allocated per row).
+- `DeltaTableDatabase`: one Delta MERGE commit per row per table (cost grows with the
+ table and leaves one file per row), lookups re-opened and read every file, the
+ existing-id cache was re-read after every flush, and the pending batch was scanned per
+ lookup (main: 80 / 198 / 509 ms per row at 100 / 400 / 1,000 rows, sync). **Fix:** function
+ nodes inside `PipelineJob.run` commit every `commit_every_rows` rows (default 100), at node
+ end, best-effort on failure and at job end, results before pipeline records; a node used
+ on its own still commits per row. The store keeps, per loaded table version, the stored
+ ids and rows in memory (indexed), extended after its own commits from the new data files
+ only; the pending batch is compacted and indexed; a commit appends when every pending id is
+ new in the version loaded for that commit, and MERGEs otherwise. Stored rows, ids, hashes,
+ paths and schemas are unchanged (differential against `main`). Store failures that only a
+ commit reveals now surface at the next commit point and name no row (developer decision,
+ November 11, 2026). Found on the way: an async run hung on any node failure with more rows
+ in flight than the channel buffers (ITL-932, fixed: closing a channel never blocks), and an
+ async rerun of an ephemeral-result node skipped its rows (ITL-923, fixed). Remaining
+ (ITL-923 item 3): a pending row written with `skip_duplicates=False` whose id is already
+ stored is still dropped by the commit (Delta keeps the stored row; the in-memory store
+ overwrites it). Memory: a store object keeps the committed rows of each table it looks rows
+ up in for as long as it lives, all of them at once (main read one whole table per lookup
+ and released it), so memory is cumulative over the tables: measured for an `int -> int`
+ pod, about 820 bytes per stored row for the result table and 746 for the pipeline-record
+ table, so about 1.6 GB per function node at 1M rows. Keeping only the lookup columns would
+ need a read per matched row. A follow-up to bound it is for the coordinator to file. The store's own
+ size-triggered flush (`batch_size`) now commits only the full table, and `PipelineJob.run`
+ refuses `commit_every_rows >= batch_size`, so the results-first order holds.
+
+---
+
## `src/orcapod/logical_types/numpy_type.py`, `src/orcapod/logical_types/array_format.py` — array format
### AR1 — `numpy.ndarray` stored as a `.npy` file and hashed by file bytes
diff --git a/changelog.d/ITL-864.md b/changelog.d/ITL-864.md
new file mode 100644
index 000000000..cfc4bc3dc
--- /dev/null
+++ b/changelog.d/ITL-864.md
@@ -0,0 +1,16 @@
+### Fixed
+
+#### Delta reads return every row the database's handles wrote (ITL-864)
+
+Right after a first `PipelineJob` run on a `DeltaTableDatabase`, `node.get_all_records()` could
+return only some of the rows (1 of 2 or 3) on both orchestrators. Handles made with `at()` share
+their pending writes, and any handle's flush writes all of them, but each handle kept its own
+loaded copy of the table, so a sibling kept reading the version it had loaded before the flush.
+The loaded tables and the stored-id sets are now shared by every handle of one root. Every read
+method (`get_all_records`, `get_record_by_id`, `get_records_by_ids`,
+`get_records_with_column_value`) returns what a flush would leave: the stored rows plus the
+pending rows whose record id is not stored, or `None` when there are none. Before,
+`get_records_by_ids` ignored pending rows, `get_records_with_column_value` ignored stored rows
+while a pending batch existed, and a pending row whose id was already stored could be returned
+twice or instead of the stored row. Writes made by another process or another root on the same
+path are still seen only after the table is reloaded. (ITL-864 #293)
diff --git a/changelog.d/ITL-892.md b/changelog.d/ITL-892.md
new file mode 100644
index 000000000..f6510cb02
--- /dev/null
+++ b/changelog.d/ITL-892.md
@@ -0,0 +1,16 @@
+### Fixed
+
+#### A script that read a local Delta store no longer hangs at exit (ITL-892)
+
+A process that had read a Delta table through `DeltaTableDatabase` or `DeltaTableSource` could
+finish its work and then never exit: it hung in Arrow's thread-pool destructor, or, on Linux with
+Python 3.11 or 3.12, aborted with `terminate called without an active exception`. It happened in
+up to three runs in four on macOS and about one in ten on Linux, on every supported pyarrow and
+deltalake version. The cause is upstream (apache/arrow#50482, apache/arrow#54536). deltalake's
+default reader goes through a filesystem written in Python, and Arrow can release one of its
+buffers on a worker thread while the interpreter is shutting down.
+
+Orcapod now reads local Delta tables through a native pyarrow filesystem, which returns the same
+schema and values. Tables on cloud storage (`s3://`, `gs://`, `az://`) still use deltalake's
+reader and stay exposed until pyarrow ships the fix; see
+[Troubleshooting](docs/troubleshooting.md).
diff --git a/changelog.d/ITL-919.md b/changelog.d/ITL-919.md
new file mode 100644
index 000000000..f7312d516
--- /dev/null
+++ b/changelog.d/ITL-919.md
@@ -0,0 +1,52 @@
+### Added
+
+- **`PipelineJob.run(commit_every_rows=100)`** — how many rows a function node processes
+ between commits of its results and pipeline records (an `int >= 1`). Nodes also commit when
+ they finish, before a failed run raises, and at the end of the job; `1` commits after every
+ row. A hard crash on a durable store loses at most the rows the running node processed
+ since its last commit, and the rerun computes only those. (ITL-919)
+
+### Fixed
+
+#### A `PipelineJob` on an in-memory store runs in time linear in its rows (ITL-919)
+
+A `PipelineJob` on an `InMemoryArrowDatabase` runs in time linear in its rows. Every row a
+function node processed used to filter, copy or rebuild the whole stored table (the
+result lookup, the recomputation-index lookup, the duplicate check and the flush, each
+over a table that gained one chunk per row), so 1,000 rows took about a minute. Lookups
+now use hash indexes over the stored rows and a flush appends without copying them;
+stored rows, ids and hashes are unchanged.
+
+#### A `PipelineJob` on a Delta store runs in time linear in its rows (ITL-919)
+
+A `PipelineJob` on a `DeltaTableDatabase` committed every row of every function node as
+its own Delta version (a MERGE that reads the whole table) and re-read the stored table for
+each per-row lookup, so 1,000 rows took about eight minutes. Function nodes inside
+`job.run()` now commit every `commit_every_rows` rows (default 100, see *Added*); a batch
+whose record ids are all new is appended instead of merged; and lookups are answered from
+an indexed in-memory copy of the committed rows, read once and extended with each commit.
+Stored rows, record ids, hashes, paths and schemas are unchanged. Behaviour changes: a store
+failure that only a commit reveals (an unreadable table, a failed commit) surfaces at the
+node's next commit point and names the node but no row, and the rows computed since the
+last commit are computed again on the next run; `commit_every_rows=1` restores per-row
+surfacing. A `DeltaTableDatabase` object keeps answering from the table version it loaded
+until its own next commit (as before; it no longer re-reads the table per lookup).
+
+#### An asynchronous run whose node fails raises instead of hanging (ITL-932)
+
+On the asynchronous orchestrator, a node failure while more rows were in flight than the
+channel buffers hold (about 16 to 64 rows) hung the run forever: the failure cancelled the
+readers, and closing a full channel waited for room that never came. Closing a channel no
+longer blocks; readers still receive every item sent before the close.
+
+#### An asynchronous rerun of an ephemeral-result node recomputes its rows (ITL-923)
+
+Rerunning a node with `NodeConfig(is_result_ephemeral=True)` on the asynchronous
+orchestrator, after its ephemeral results were gone, neither recomputed nor recorded the
+rows, and reading the node afterwards raised `EmptyDataAccessError`. It now recomputes them,
+as the synchronous orchestrator does.
+
+#### A failed side-effect completion lookup names the row
+
+A `StoreReadError` from a side-effect pod's check of earlier deliveries now names the pod
+and the row, as a failed write of the delivery record does.
diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md
index 4a6a5aa77..a79cb207f 100644
--- a/docs/concepts/function-pods.md
+++ b/docs/concepts/function-pods.md
@@ -248,6 +248,52 @@ print("function calls:", len(calls))
flush tries them again. A flush that fails for any other reason raises `RuntimeError`,
with the first failure as its cause.
+### When results are committed
+
+Inside `job.run()`, each function node commits its results and pipeline records to the
+store in batches: every `commit_every_rows` rows (default 100), when the node finishes,
+before a failed run raises, and when the job ends. Committing in batches keeps a run's time
+linear in its rows; on a `DeltaTableDatabase` each commit is one Delta version per table, so
+a 1,000-row node writes 10 versions per table instead of 1,000.
+
+```python
+job.run(commit_every_rows=500) # fewer, larger commits
+```
+
+`commit_every_rows=1` commits after every row. A node or a cached pod used on its own,
+outside `job.run()`, also commits after every row. On a `DeltaTableDatabase`,
+`commit_every_rows` must be smaller than the store's `batch_size` (default 1000):
+the store commits every pending table by itself once one table holds that many pending
+rows, which would not keep the order described below, so `job.run()` refuses a larger value
+with `ValueError`. Create the store with a larger `batch_size` to commit less often. For the same reason, `job.run()` first commits
+whatever such a store already holds pending (writes made without a flush, or the batches of a
+run that failed); if that fails, the run stops before computing anything.
+
+**Memory.** A `DeltaTableDatabase` keeps the committed rows of every table it looks rows up
+in (each function node's result and pipeline-record tables) in memory for as long as the
+store object lives, so that a lookup never re-reads the table. These copies are held
+together, so plan memory for all of them at once: for a function returning one `int`, about
+1.6 KB per stored row per function node (result and pipeline-record tables together), more
+for larger outputs. Before, each lookup read one whole table and released it. To release
+the copies, use a new store object, for example one per job.
+
+**What a hard crash loses.** If the process dies without raising (power loss, `kill -9`,
+the machine restarts), a durable store such as `DeltaTableDatabase` keeps every batch
+committed before the crash; at most the rows the running node processed since its last
+commit are lost, and nothing stored is damaged. Run the job again: it computes only the rows
+whose results were not committed. A node commits its results before their pipeline records,
+so a crash between those two commits leaves results stored without pipeline records. Reading
+the node (`as_table()`, `get_all_records()`) does not show those rows until the next run,
+which finds their results in the store, computes nothing for them and writes their pipeline
+records.
+
+**Reads see the version this process loaded.** A `DeltaTableDatabase` keeps the committed
+rows of each table it has read in memory and answers lookups from them, extending them with
+its own commits. Rows another process commits to the same table are not seen by a store
+object that has already loaded the table until its own next commit to that table reloads it;
+a new `DeltaTableDatabase` object reads them at once. This was already the case before
+batching; only the re-reading per lookup is gone.
+
A node from a job that has run also provides:
- **`get_all_records()`** -- every record this step of the job has stored, as a PyArrow
@@ -263,13 +309,20 @@ A node from a job that has run also provides:
A failed write to the store always fails the run. If a node's results or records cannot be
stored -- the disk is full, the connection drops, or the database cannot hold one of the
output types -- `job.run()` raises `StoreWriteError` (from `orcapod.errors`) at the first
-failed write, on both the synchronous and the asynchronous orchestrator. The error names the
-node, the row being stored and the table, and the database's own exception is chained as its
-cause. When committing pending writes fails, the error has `flush=True` and names the store
-instead of a row and table: every view of a store shares its pending writes, so the write
-that failed may belong to an earlier row or another table. A run that returns normally has
-stored every result that could be turned into a record; a result rejected before that is a
-row failure (see below).
+failed write, on both the synchronous and the asynchronous orchestrator. A row's write that
+fails names the node, the row being stored and the table, and the database's own exception
+is chained as its cause. When committing pending writes fails, the error has `flush=True`
+and names the node and the store instead of a row and table: a commit holds a batch of rows
+(see *When results are committed*), and every view of a store shares its pending writes, so
+the write that failed may belong to any row of the batch or another table. A run that
+returns normally has stored every result that could be turned into a record; a result
+rejected before that is a row failure (see below).
+
+With batched commits, a store failure that only committing reveals (an unreadable table, a
+full disk) surfaces at the node's next commit point, not at the row that caused it: the rows
+the node computed since its last commit are not stored and are computed again on the next
+run. `job.run(commit_every_rows=1)` commits after every row, so such a failure surfaces at
+its row, at the cost of one commit per row.
This holds whatever `error_policy` the orchestrator was given. `error_policy="continue"` (the
default) skips rows whose *function* raised and keeps going; it never skips a row because it
@@ -285,8 +338,10 @@ first in the pipeline, the one the synchronous orchestrator would have stopped a
failures are logged and noted on it. A node whose failure the asynchronous orchestrator
cancelled before it happened is not reported.
-After the failure, rows stored before it are kept: `job.run()` commits the writes that
-succeeded before it re-raises, and nothing already stored is rewritten. Once the cause is
+After the failure, rows stored before it are kept: every batch committed before the failure
+stays, and `job.run()` tries once more to commit the writes still pending before it
+re-raises (a failed commit's batch stays pending, so this is its one retry; if it fails
+again, that is noted on the error), and nothing already stored is rewritten. Once the cause is
fixed, run the job again: stored function results are not recomputed, only the missing ones
are. An operator with `cache_mode=LOG` recomputes its output on every run, as it always does.
A side effect is different: if its delivery completed but the write to its invocation log
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
new file mode 100644
index 000000000..b0e5f3d94
--- /dev/null
+++ b/docs/troubleshooting.md
@@ -0,0 +1,39 @@
+# Troubleshooting
+
+## A script finishes its work but the process never exits
+
+**Symptom.** A Python script that used a Delta store prints all of its output and then hangs,
+and has to be killed. On Linux with Python 3.11 or 3.12 the process may instead end with
+`terminate called without an active exception` and exit code 134 (SIGABRT). Notebooks are not
+affected in the same way, because the kernel keeps running.
+
+**Cause.** This is a bug in pyarrow, not in your pipeline
+([apache/arrow#50482](https://github.com/apache/arrow/issues/50482),
+[apache/arrow#54536](https://github.com/apache/arrow/issues/54536)). deltalake's default reader
+goes through a filesystem written in Python. After a read returns, Arrow finishes cleaning it up
+on a worker thread. If that cleanup runs while the interpreter is shutting down, the worker
+cannot take the GIL and never finishes, and Arrow's thread pool waits for it forever. Your data
+is not affected: the hang happens after every write has completed.
+
+**What Orcapod does.** `DeltaTableDatabase` and `DeltaTableSource` read **local** Delta tables
+through a native pyarrow filesystem, so they are not affected. Tables on cloud storage
+(`s3://`, `gs://`, `az://` and similar) still use deltalake's reader and can still hang at exit.
+Reading a Delta table yourself with `deltalake.DeltaTable(...).to_pyarrow_table()` or
+`to_pyarrow_dataset()` is affected too, whatever the storage.
+
+**Workarounds.**
+
+- Upgrade pyarrow once a release includes the fix
+ ([apache/arrow#54467](https://github.com/apache/arrow/pull/54467)).
+- In a script that reads a cloud Delta store, end it by flushing your outputs and calling
+ `os._exit(0)`. That skips interpreter shutdown, where the hang happens. It also skips
+ `atexit` handlers and other cleanup, so close files and flush stores first.
+- When reading local Delta tables directly with deltalake, pass a native filesystem. Resolve
+ `dt.table_uri` first: it is percent-encoded, and newer deltalake versions return a
+ `file://` URI.
+
+ ```python
+ uri = dt.table_uri if "://" in dt.table_uri else "file://" + dt.table_uri
+ fs, root = pyarrow.fs.FileSystem.from_uri(uri)
+ dataset = dt.to_pyarrow_dataset(filesystem=pyarrow.fs.SubTreeFileSystem(root, fs))
+ ```
diff --git a/mkdocs.yml b/mkdocs.yml
index 5b4709c30..a33a00059 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -53,3 +53,4 @@ nav:
- Types: api/types.md
- Reference:
- Hashing: reference/hashing.md
+ - Troubleshooting: troubleshooting.md
diff --git a/src/orcapod/channels.py b/src/orcapod/channels.py
index 68cfd028d..8111912f4 100644
--- a/src/orcapod/channels.py
+++ b/src/orcapod/channels.py
@@ -99,6 +99,10 @@ def __init__(self, channel: Channel[T]) -> None:
async def receive(self) -> T:
if self._channel._drained:
raise ChannelClosed()
+ if self._channel._close_pending and self._channel._queue.empty():
+ # Closed while the queue was full: every item is received; the end now.
+ self._channel._drained = True
+ raise ChannelClosed()
item = await self._channel._queue.get()
if isinstance(item, _Sentinel):
self._channel._drained = True
@@ -143,9 +147,20 @@ async def send(self, item: T) -> None:
await self._channel._queue.put(item)
async def close(self) -> None:
+ """Signal that no more items will be sent, without waiting.
+
+ Never blocks (ITL-932): on a failed run every reader is cancelled while
+ writers close their outputs in ``finally`` blocks, and a close that waited
+ for room in a full queue would hang the run. When the queue is full, the
+ close is recorded instead of queued; readers receive every queued item and
+ then ``ChannelClosed``, exactly as with a queued close.
+ """
if not self._channel._closed.is_set():
self._channel._closed.set()
- await self._channel._queue.put(_CLOSED)
+ try:
+ self._channel._queue.put_nowait(_CLOSED)
+ except asyncio.QueueFull:
+ self._channel._close_pending = True
# ---------------------------------------------------------------------------
@@ -166,6 +181,8 @@ class Channel(Generic[T]):
_queue: asyncio.Queue[T | _Sentinel] = field(init=False)
_closed: asyncio.Event = field(init=False, default_factory=asyncio.Event)
_drained: bool = field(init=False, default=False)
+ # Closed while the queue was full: the close is not queued (ITL-932).
+ _close_pending: bool = field(init=False, default=False)
def __post_init__(self) -> None:
self._queue = asyncio.Queue(maxsize=self.buffer_size)
@@ -193,16 +210,21 @@ class _BroadcastReader(Generic[T]):
multiple downstream consumers can read at their own pace.
"""
- __slots__ = ("_queue",)
+ __slots__ = ("_closed", "_queue")
def __init__(self, buffer_size: int) -> None:
self._queue: asyncio.Queue[T | _Sentinel] = asyncio.Queue(maxsize=buffer_size)
+ # Set once this reader's stream has ended, or by a close that found its
+ # queue full (ITL-932): then the end comes once the queue is drained.
+ self._closed = False
async def receive(self) -> T:
+ if self._closed and self._queue.empty():
+ raise ChannelClosed()
item = await self._queue.get()
if isinstance(item, _Sentinel):
- # Re-enqueue so repeated receive() calls also raise
- await self._queue.put(item)
+ # Remember the end so repeated receive() calls also raise.
+ self._closed = True
raise ChannelClosed()
return item # type: ignore[return-value]
@@ -264,7 +286,15 @@ async def send(self, item: T) -> None:
await reader._queue.put(item)
async def close(self) -> None:
+ """Signal every reader that no more items will be sent, without waiting.
+
+ Never blocks on a slow or cancelled reader (ITL-932); see
+ ``_ChannelWriter.close``.
+ """
if not self._broadcast._closed:
self._broadcast._closed = True
for reader in self._broadcast._readers:
- await reader._queue.put(_CLOSED)
+ try:
+ reader._queue.put_nowait(_CLOSED)
+ except asyncio.QueueFull:
+ reader._closed = True
diff --git a/src/orcapod/core/cached_function_pod.py b/src/orcapod/core/cached_function_pod.py
index 757b43692..b8cbaacfe 100644
--- a/src/orcapod/core/cached_function_pod.py
+++ b/src/orcapod/core/cached_function_pod.py
@@ -76,6 +76,15 @@ def set_ignore_schema(self, ignore_schema: tuple[str, ...] | None) -> None:
"""
self._cache.set_ignore_schema(ignore_schema)
+ def set_auto_flush(self, on: bool = True) -> None:
+ """Set whether each stored result is committed (flushed) right away.
+
+ Args:
+ on: ``True`` (the default) commits after every stored result;
+ ``False`` leaves the commit to the caller.
+ """
+ self._cache.set_auto_flush(on)
+
def lookup_cached_data(self, data: DataProtocol) -> DataProtocol | None:
"""Look up a cached result for ``data`` without triggering computation.
diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py
index 3454c8856..398ec851c 100644
--- a/src/orcapod/core/nodes/function_node.py
+++ b/src/orcapod/core/nodes/function_node.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import asyncio
+import contextlib
import logging
import uuid
from abc import abstractmethod
@@ -39,7 +40,7 @@
StoreReadError,
StoreWriteError,
)
-from orcapod.utils.store_errors import keep_store_error
+from orcapod.utils.store_errors import commit_best_effort, keep_store_error
from orcapod.protocols.core_protocols import (
FunctionPodProtocol,
DataFunctionExecutorProtocol,
@@ -49,6 +50,7 @@
TagProtocol,
TrackerManagerProtocol,
)
+from orcapod.databases.utils import as_table_committing
from orcapod.protocols.database_protocols import ArrowDatabaseProtocol
from orcapod.protocols.observability_protocols import (
DataExecutionLoggerProtocol,
@@ -822,6 +824,11 @@ def __init__(
# .node_config setter or PipelineJob.apply_node_config().
self._node_config: NodeConfig = NodeConfig()
+ # Batched commits, set only while PipelineJob.run runs the node (ITL-919).
+ self._commit_every_rows: int | None = None
+ self._rows_since_commit = 0
+ self._commit_failed = False
+
self.attach_databases(
pipeline_database=pipeline_database,
result_database=result_database,
@@ -1105,6 +1112,9 @@ def from_descriptor(
# FunctionJobNode — orchestrator config
node._node_config = NodeConfig()
+ node._commit_every_rows = None
+ node._rows_since_commit = 0
+ node._commit_failed = False
# FunctionJobNode — DB persistence state (wired by attach_databases)
node._pipeline_database = None
@@ -1193,6 +1203,155 @@ def _require_pipeline_database(self) -> ArrowDatabaseProtocol:
)
return self._pipeline_database
+ # ------------------------------------------------------------------
+ # Batched commits (ITL-919)
+ # ------------------------------------------------------------------
+
+ @contextlib.contextmanager
+ def _batched_commits(self, rows_per_commit: int) -> Iterator[None]:
+ """Commit this node's writes every ``rows_per_commit`` rows inside the block.
+
+ ``PipelineJob.run`` runs each function node inside this block. Its result
+ caches stop committing after every stored result, and the node commits its
+ pending writes (ephemeral results, then its result table, then the pipeline
+ records) every ``rows_per_commit`` rows and at its end. Outside the block
+ (a node run on its own) every result is committed as it is stored.
+
+ Args:
+ rows_per_commit: How many processed rows' writes one commit holds.
+
+ Yields:
+ Nothing; the node commits in batches while the block runs.
+ """
+ pods = [
+ pod
+ for pod in (self._cached_function_pod, self._ephemeral_cached_pod)
+ if isinstance(pod, CachedFunctionPod)
+ ]
+ self._commit_every_rows = rows_per_commit
+ self._rows_since_commit = 0
+ self._commit_failed = False
+ for pod in pods:
+ pod.set_auto_flush(False)
+ try:
+ yield
+ finally:
+ self._commit_every_rows = None
+ for pod in pods:
+ pod.set_auto_flush(True)
+
+ def _ephemeral_store_in_use(self) -> ArrowDatabaseProtocol | None:
+ """Return the ephemeral store this node writes results to, if it does."""
+ if self._node_config.is_result_ephemeral and self._ephemeral_cached_pod:
+ return self.ephemeral_result_store
+ return None
+
+ def _commit_result_table(self) -> None:
+ """Commit this node's pending result rows, before its pipeline records.
+
+ Every view of a store shares one set of pending writes, and a flush commits
+ them table by table in no guaranteed order. Committing the result table on
+ its own first means a crash between the two commits leaves results without
+ pipeline records (found in the result cache on the rerun, so nothing is
+ recomputed), never pipeline records pointing at results that were lost.
+ Only a store that commits table by table
+ (``TableCommittingDatabaseProtocol``, the Delta store) needs and offers this;
+ for another store the pipeline flush that follows commits everything.
+ """
+ pod = self._cached_function_pod
+ if not isinstance(pod, CachedFunctionPod):
+ return
+ result_database = as_table_committing(pod.result_database)
+ if result_database is not None:
+ result_database.flush_batch(pod.record_path)
+
+ def _commit_pending_writes(self) -> None:
+ """Commit this node's pending writes, results first (see ``_batched_commits``).
+
+ Raises:
+ StoreReadError: If a stored table could not be read; it names this node.
+ StoreWriteError: If a commit failed, naming this node and the store
+ (``flush=True``, no row: the batch holds many rows). The failed
+ writes stay pending, and earlier commits are kept.
+ """
+ pipeline_database = self._require_pipeline_database()
+ ephemeral = self._ephemeral_store_in_use()
+ try:
+ if ephemeral is not None:
+ try:
+ ephemeral.flush()
+ except StoreReadError:
+ raise
+ except Exception as exc:
+ raise StoreWriteError(
+ ephemeral.base_path, node_label=self.label, flush=True
+ ) from exc
+ try:
+ self._commit_result_table()
+ pipeline_database.flush()
+ except StoreReadError:
+ raise
+ except Exception as exc:
+ raise StoreWriteError(
+ pipeline_database.base_path, node_label=self.label, flush=True
+ ) from exc
+ except StoreError as exc:
+ self._commit_failed = True
+ exc.add_context(node_label=self.label)
+ raise
+ self._rows_since_commit = 0
+
+ def _row_processed(self) -> None:
+ """Count a processed row; commit once a batch is full (only inside a job run)."""
+ if self._commit_every_rows is None:
+ return
+ self._rows_since_commit += 1
+ if self._rows_since_commit >= self._commit_every_rows:
+ self._commit_pending_writes()
+
+ def _commit_at_node_end(self) -> None:
+ """Commit what is still pending when the node finishes (only inside a job run)."""
+ if self._commit_every_rows is not None:
+ self._commit_pending_writes()
+
+ def _commit_after_failure(self, run_error: BaseException) -> None:
+ """Commit what succeeded before a failure, never replacing the run's error.
+
+ The job flushes its store after a failed run, but not a node's ephemeral
+ store, a separate database, and not table by table: so the node commits its
+ ephemeral results and then its result table here, best-effort (a failure
+ is noted on ``run_error``). Skipped when the failure was this node's own
+ commit: the job's flush is that commit's one retry.
+
+ Args:
+ run_error: The error the node is failing with.
+ """
+ if self._commit_every_rows is None or self._commit_failed:
+ return
+ ephemeral = self._ephemeral_store_in_use()
+ if ephemeral is not None:
+ commit_best_effort(run_error, ephemeral.flush)
+ commit_best_effort(run_error, self._commit_result_table)
+
+ @staticmethod
+ def _commit_or_end_node(
+ commit: Any, ctx_obs: Any, node_label: str, node_hash: str
+ ) -> None:
+ """Run a commit step; if it fails, end the node for the observer and re-raise.
+
+ Args:
+ commit: ``_row_processed`` or ``_commit_at_node_end``.
+ ctx_obs: The node's contextualized observer.
+ node_label: The node's label, for the observer.
+ node_hash: The node's content hash, for the observer.
+ """
+ try:
+ commit()
+ except StoreError as exc:
+ with keep_store_error(exc, "Observer on_node_end"):
+ ctx_obs.on_node_end(node_label, node_hash)
+ raise
+
# ------------------------------------------------------------------
# as_node — return the lightweight FunctionNode equivalent
# ------------------------------------------------------------------
@@ -1291,21 +1450,27 @@ def execute(
self.get_cached_results(base_entry_ids=base_entry_ids)
output: list[tuple[TagProtocol, DataProtocol]] = []
- for tag, data, base_entry_id in upstream_entries:
- ctx_obs.on_data_start(node_label, tag, data)
-
- if base_entry_id in self._cached_output_datas and not isinstance(
- self._cached_output_datas[base_entry_id][1], EmptyData
- ):
- tag_out, result = self._cached_output_datas[base_entry_id]
- ctx_obs.on_data_end(node_label, tag, data, result, cached=True)
- if result is not None:
- output.append((tag_out, result))
- else:
+ try:
+ for tag, data, base_entry_id in upstream_entries:
+ ctx_obs.on_data_start(node_label, tag, data)
+
+ if base_entry_id in self._cached_output_datas and not isinstance(
+ self._cached_output_datas[base_entry_id][1], EmptyData
+ ):
+ tag_out, result = self._cached_output_datas[base_entry_id]
+ ctx_obs.on_data_end(node_label, tag, data, result, cached=True)
+ if result is not None:
+ output.append((tag_out, result))
+ continue
+
pkt_logger = ctx_obs.create_data_logger(tag, data)
try:
tag_out, result = self._process_data_internal(
- tag, data, logger=pkt_logger, run_id=run_id
+ tag,
+ data,
+ logger=pkt_logger,
+ run_id=run_id,
+ base_entry_id=base_entry_id,
)
except StoreError as exc:
# A failed store read or write fails the run whatever the error
@@ -1328,10 +1493,21 @@ def execute(
if error_policy == "fail_fast":
ctx_obs.on_node_end(node_label, node_hash)
raise
- else:
- ctx_obs.on_data_end(node_label, tag, data, result, cached=False)
- if result is not None:
- output.append((tag_out, result))
+ continue
+
+ ctx_obs.on_data_end(node_label, tag, data, result, cached=False)
+ if result is not None:
+ output.append((tag_out, result))
+ self._commit_or_end_node(
+ self._row_processed, ctx_obs, node_label, node_hash
+ )
+
+ self._commit_or_end_node(
+ self._commit_at_node_end, ctx_obs, node_label, node_hash
+ )
+ except Exception as run_error:
+ self._commit_after_failure(run_error)
+ raise
ctx_obs.on_node_end(node_label, node_hash)
# Mark this node as freshly computed so subsequent iter_data() calls
@@ -1346,6 +1522,7 @@ def _process_data_internal(
*,
logger: DataExecutionLoggerProtocol | None = None,
run_id: str | None = None,
+ base_entry_id: bytes | None = None,
) -> tuple[TagProtocol, DataProtocol | None]:
"""Core compute + persist + cache.
@@ -1359,6 +1536,8 @@ def _process_data_internal(
logger: Optional data execution logger.
run_id: Optional pipeline run identifier threaded through to
``InvocationContext.pipeline_run_id``.
+ base_entry_id: The row's ``compute_base_entry_id``, when the caller
+ already has it; computed here otherwise.
When ``node_config.is_result_ephemeral=True``:
- Uses ``_ephemeral_cached_pod`` for both compute and storage.
@@ -1383,11 +1562,12 @@ def _process_data_internal(
# add_pipeline_record is intentionally skipped for EmptyData inputs because
# EmptyData.as_table() raises EmptyDataAccessError (no payload to read source
# columns from). Pipeline DB write for this path is deferred.
+ if base_entry_id is None:
+ base_entry_id = self.compute_base_entry_id(tag, data)
if isinstance(data, EmptyData):
if self._cached_function_pod is not None:
cached = self._cached_function_pod.lookup_cached_data(data)
if cached is not None:
- base_entry_id = self.compute_base_entry_id(tag, data)
self._cached_output_datas[base_entry_id] = (tag, cached)
self._cached_output_table = None
self._cached_content_hash_column = None
@@ -1422,9 +1602,10 @@ def _process_data_internal(
)
)
if self._pipeline_database is not None:
- self.add_pipeline_record(
+ self._add_pipeline_record(
tag,
data,
+ base_entry_id,
data_record_id=output_data.datagram_uuid,
computed=result_computed,
is_ephemeral=True,
@@ -1440,9 +1621,10 @@ def _process_data_internal(
self._cached_function_pod.RESULT_COMPUTED_FLAG, False
)
)
- self.add_pipeline_record(
+ self._add_pipeline_record(
tag,
data,
+ base_entry_id,
data_record_id=output_data.datagram_uuid,
computed=result_computed,
output_data_hash=output_data.content_hash(),
@@ -1453,7 +1635,6 @@ def _process_data_internal(
)
# Store by base_entry_id (stable across recomputation cycles) and invalidate caches
- base_entry_id = self.compute_base_entry_id(tag, data)
self._cached_output_datas[base_entry_id] = (tag_out, output_data)
self._cached_output_table = None
self._cached_content_hash_column = None
@@ -1509,22 +1690,24 @@ async def _async_process_data_internal(
data: DataProtocol,
*,
logger: DataExecutionLoggerProtocol | None = None,
+ base_entry_id: bytes | None = None,
) -> tuple[TagProtocol, DataProtocol | None]:
"""Async counterpart of ``_process_data_internal``.
Mirrors the sync ephemeral/persistent branch logic using async variants
of ``process_data``. See ``_process_data_internal`` for full behaviour
- documentation.
+ documentation, including ``base_entry_id``.
Returns:
A ``(tag, output_data)`` 2-tuple.
"""
# Guard: same logic as sync _process_data_internal — see inline comment there.
+ if base_entry_id is None:
+ base_entry_id = self.compute_base_entry_id(tag, data)
if isinstance(data, EmptyData):
if self._cached_function_pod is not None:
cached = self._cached_function_pod.lookup_cached_data(data)
if cached is not None:
- base_entry_id = self.compute_base_entry_id(tag, data)
self._cached_output_datas[base_entry_id] = (tag, cached)
self._cached_output_table = None
self._cached_content_hash_column = None
@@ -1559,9 +1742,10 @@ async def _async_process_data_internal(
)
)
if self._pipeline_database is not None:
- self.add_pipeline_record(
+ self._add_pipeline_record(
tag,
data,
+ base_entry_id,
data_record_id=output_data.datagram_uuid,
computed=result_computed,
is_ephemeral=True,
@@ -1577,9 +1761,10 @@ async def _async_process_data_internal(
self._cached_function_pod.RESULT_COMPUTED_FLAG, False
)
)
- self.add_pipeline_record(
+ self._add_pipeline_record(
tag,
data,
+ base_entry_id,
data_record_id=output_data.datagram_uuid,
computed=result_computed,
output_data_hash=output_data.content_hash(),
@@ -1590,7 +1775,6 @@ async def _async_process_data_internal(
)
# Store by base_entry_id (stable across recomputation cycles) and invalidate caches
- base_entry_id = self.compute_base_entry_id(tag, data)
self._cached_output_datas[base_entry_id] = (tag_out, output_data)
self._cached_output_table = None
self._cached_content_hash_column = None
@@ -1724,9 +1908,39 @@ def add_pipeline_record(
so that downstream ``EmptyData`` flow-through can locate its cached
result keyed by the same hash (= the downstream's ``INPUT_DATA_HASH_COL``).
"""
+ self._add_pipeline_record(
+ tag,
+ input_data,
+ self.compute_base_entry_id(tag, input_data),
+ data_record_id=data_record_id,
+ computed=computed,
+ is_ephemeral=is_ephemeral,
+ output_data_hash=output_data_hash,
+ )
+
+ def _add_pipeline_record(
+ self,
+ tag: TagProtocol,
+ input_data: DataProtocol,
+ base_entry_id: bytes,
+ data_record_id: uuid.UUID,
+ computed: bool,
+ is_ephemeral: bool = False,
+ output_data_hash: ContentHash | None = None,
+ ) -> None:
+ """``add_pipeline_record`` with the row's base entry id already computed.
+
+ Args:
+ tag: The tag associated with the input data.
+ input_data: The input data that was processed.
+ base_entry_id: ``compute_base_entry_id(tag, input_data)``.
+ data_record_id: UUID of the result record in the result database.
+ computed: Whether the result was freshly computed.
+ is_ephemeral: Whether the result is stored in the ephemeral store.
+ output_data_hash: Content hash of the output, if any.
+ """
pipeline_database = self._require_pipeline_database()
self._ensure_pdb_schema()
- base_entry_id = self.compute_base_entry_id(tag, input_data)
# Determine the next recomputation index by querying all existing rows
# for this base_entry_id. No await is used here, so within a single-threaded
@@ -2417,14 +2631,22 @@ async def async_execute(
)
# Local dict: correlation_key → (original_tag, original_input_data)
- input_store: dict[bytes, tuple[TagProtocol, DataProtocol]] = {}
+ input_store: dict[bytes, tuple[TagProtocol, DataProtocol, bytes]] = {}
async def route_inputs() -> None:
"""Stage 1: send cache hits to output; stamp misses for computation."""
try:
async for tag, data in input_channel:
base_entry_id = self.compute_base_entry_id(tag, data)
- if base_entry_id in cached_by_base_entry_id:
+ # A cached EmptyData is a result that is gone (an
+ # ephemeral result from an earlier run): compute the row
+ # again, as the sync path does (ITL-923).
+ if base_entry_id in cached_by_base_entry_id and not (
+ isinstance(
+ cached_by_base_entry_id[base_entry_id][1],
+ EmptyData,
+ )
+ ):
cached_tag, cached_data = cached_by_base_entry_id[
base_entry_id
]
@@ -2435,7 +2657,11 @@ async def route_inputs() -> None:
await output.send((cached_tag, cached_data))
else:
correlation_key = uuid.uuid4().bytes
- input_store[correlation_key] = (tag, data)
+ input_store[correlation_key] = (
+ tag,
+ data,
+ base_entry_id,
+ )
stamped_tag = tag.with_meta_columns(
**{_TAG_NODE_INPUT_REF: correlation_key}
)
@@ -2447,16 +2673,19 @@ async def record_and_forward() -> None:
"""Stage 3: record pipeline entry, strip key, emit to output."""
async for output_tag, output_data in result_channel.reader:
correlation_key = output_tag.get_meta_value(_TAG_NODE_INPUT_REF)
- original_tag, input_data = input_store.pop(correlation_key)
+ original_tag, input_data, base_entry_id = input_store.pop(
+ cast(bytes, correlation_key)
+ )
result_computed = bool(
output_data.get_meta_value(
execution_pod.RESULT_COMPUTED_FLAG, False
)
)
try:
- self.add_pipeline_record(
+ self._add_pipeline_record(
original_tag,
input_data,
+ base_entry_id,
data_record_id=output_data.datagram_uuid,
computed=result_computed,
is_ephemeral=is_ephemeral,
@@ -2470,9 +2699,6 @@ async def record_and_forward() -> None:
)
raise
# Update in-memory cache so iter_data() sees the result.
- base_entry_id = self.compute_base_entry_id(
- original_tag, input_data
- )
clean_tag = output_tag.drop_meta_columns(
_TAG_NODE_INPUT_REF, ignore_missing=True
)
@@ -2482,6 +2708,9 @@ async def record_and_forward() -> None:
)
self._cached_output_table = None
self._cached_content_hash_column = None
+ # Commits every commit_every_rows rows inside a job (ITL-919);
+ # no await in between, so no other row interleaves.
+ self._row_processed()
await output.send((clean_tag, output_data))
# Capture outer self so _NodeLabelObserver methods can
@@ -2541,10 +2770,7 @@ def on_data_end(
if correlation_key is not None:
original = input_store.pop(correlation_key, None)
if original is not None:
- orig_tag, orig_data = original
- base_entry_id = fn_node.compute_base_entry_id(
- orig_tag, orig_data
- )
+ _, _, base_entry_id = original
fn_node._cached_output_datas[base_entry_id] = (
clean,
None,
@@ -2588,6 +2814,8 @@ def create_data_logger(self, tag: Any, data: Any) -> Any:
)
)
tg.create_task(record_and_forward())
+ # Commit the last rows inside a job (ITL-919); a no-op otherwise.
+ self._commit_at_node_end()
else:
# No-DB path: delegate dispatch entirely to the pod.
# The pod's async_execute() closes `output` in its own finally.
@@ -2596,5 +2824,8 @@ def create_data_logger(self, tag: Any, data: Any) -> Any:
)
ctx_obs.on_node_end(node_label, node_hash)
+ except Exception as run_error:
+ self._commit_after_failure(run_error)
+ raise
finally:
await output.close()
diff --git a/src/orcapod/core/sources/delta_table_source.py b/src/orcapod/core/sources/delta_table_source.py
index bfc2b0520..86a187932 100644
--- a/src/orcapod/core/sources/delta_table_source.py
+++ b/src/orcapod/core/sources/delta_table_source.py
@@ -6,6 +6,7 @@
from orcapod.core.sources.base import RootSource
from orcapod.core.sources.stream_builder import SourceStreamBuilder
+from orcapod.databases.storage_utils import delta_to_pyarrow_dataset
from orcapod.types import PathLike
from orcapod.utils.lazy_module import LazyModule
from orcapod.utils.schema_utils import _normalize_column_list
@@ -49,8 +50,8 @@ def __init__(
except deltalake.exceptions.TableNotFoundError:
raise ValueError(f"Delta table not found at {self._delta_table_path}")
- table: pa.Table = self._delta_table.to_pyarrow_dataset(
- as_large_types=True
+ table: pa.Table = delta_to_pyarrow_dataset(
+ self._delta_table, as_large_types=True
).to_table()
# The Delta log stores field-level nullable flags; to_pyarrow_dataset()
# preserves them exactly — no inference needed.
diff --git a/src/orcapod/databases/_row_index.py b/src/orcapod/databases/_row_index.py
new file mode 100644
index 000000000..b9da1f6e8
--- /dev/null
+++ b/src/orcapod/databases/_row_index.py
@@ -0,0 +1,170 @@
+"""Hash indexes over a table's rows, shared by the Arrow databases (ITL-919).
+
+A database that answers per-row lookups by filtering its whole stored table makes a run
+quadratic in its rows. These helpers keep, per stored table, lazily built hash indexes from
+a column value to the positions of the rows holding it, extended in place when rows are
+appended, and keep a growing table at a logarithmic number of record batches.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from orcapod.utils.lazy_module import LazyModule
+
+if TYPE_CHECKING:
+ import pyarrow as pa
+ import pyarrow.compute as pc
+else:
+ pa = LazyModule("pyarrow")
+ pc = LazyModule("pyarrow.compute")
+
+
+@dataclass
+class KeyIndex:
+ """Hash indexes over the rows of one stored table.
+
+ Each column index maps a value to the positions, in ascending order, of the rows of
+ ``table`` holding it. The indexes describe ``table`` only: they are valid while
+ ``table`` is the table they were built for, and are rebuilt for any other table.
+
+ Attributes:
+ table: The indexed table.
+ columns: Column name to its index, for the columns indexed so far.
+ """
+
+ table: "pa.Table"
+ columns: dict[str, dict[Any, list[int]]] = field(default_factory=dict)
+
+ def column(self, name: str) -> dict[Any, list[int]]:
+ """Return the index of column ``name``, building it on first use.
+
+ Args:
+ name: A column of ``table``.
+
+ Returns:
+ The value-to-positions index of the column.
+ """
+ positions = self.columns.get(name)
+ if positions is None:
+ positions = {}
+ for position, value in enumerate(self.table.column(name).to_pylist()):
+ positions.setdefault(value, []).append(position)
+ self.columns[name] = positions
+ return positions
+
+ def append(self, rows: "pa.Table", appended: "pa.Table") -> None:
+ """Extend the indexes for ``rows`` appended after the indexed table's rows.
+
+ Args:
+ rows: The rows that were appended, with every indexed column.
+ appended: The new table: the indexed table followed by ``rows``. It
+ becomes the indexed table.
+ """
+ offset = self.table.num_rows
+ for name, positions in self.columns.items():
+ for position, value in enumerate(rows.column(name).to_pylist(), offset):
+ positions.setdefault(value, []).append(position)
+ self.table = appended
+
+
+def is_indexed_lookup(arrow_type: "pa.DataType", value: Any) -> bool:
+ """Whether an equality lookup of ``value`` in a column can use a hash index.
+
+ Only binary and string columns queried with a value of the matching Python type
+ qualify: for those, Python equality of the values is exactly Arrow's equality. Every
+ other lookup filters the table, as Arrow defines that comparison.
+
+ Args:
+ arrow_type: The column's Arrow type.
+ value: The value looked up.
+
+ Returns:
+ Whether the lookup can be answered from a hash index.
+ """
+ if pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type):
+ return type(value) is bytes
+ if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type):
+ return type(value) is str
+ return False
+
+
+def compact_tail(table: "pa.Table") -> "pa.Table":
+ """Merge the table's trailing record batches while an earlier one is not larger.
+
+ Called after each append, this keeps batch sizes strictly decreasing, like a binary
+ counter: a table of N rows has at most log2(N) + 1 batches, and each row is copied
+ O(log N) times over the table's life. Every Arrow operation on a table costs time per
+ batch, so without it a table grown one append at a time makes every later operation
+ cost time proportional to the appends so far.
+
+ Args:
+ table: The table just appended to.
+
+ Returns:
+ The same rows, in the same order, with the trailing batches merged.
+ """
+ batches = [batch for batch in table.to_batches() if batch.num_rows > 0]
+ while len(batches) >= 2 and batches[-2].num_rows <= batches[-1].num_rows:
+ merged = pa.Table.from_batches(batches[-2:]).combine_chunks()
+ batches[-2:] = merged.to_batches()
+ return pa.Table.from_batches(batches, schema=table.schema)
+
+
+def rows_at(table: "pa.Table", positions: Sequence[int]) -> "pa.Table":
+ """Return the rows of ``table`` at ``positions``, in the given order.
+
+ Uses zero-copy one-row slices: ``take`` on a multi-chunk table concatenates every
+ chunk first, copying all rows of the table.
+
+ Args:
+ table: The table to select from.
+ positions: Row positions.
+
+ Returns:
+ The selected rows, with ``table``'s schema.
+ """
+ rows = [table.slice(position, 1) for position in positions]
+ return pa.concat_tables(rows) if rows else table.slice(0, 0)
+
+
+def matching_rows(
+ key_index: KeyIndex,
+ pairs: Sequence[tuple[str, Any]],
+ expression: "pc.Expression",
+) -> "pa.Table":
+ """Return the rows of ``key_index.table`` where every column equals its value.
+
+ The pairs that can use a hash index select the rows at the intersection of their
+ indexes' positions. When every pair can, those rows are the result; when only some
+ can, just those rows are filtered by ``expression``; when none can, the whole table
+ is. Rows come back in table order.
+
+ Args:
+ key_index: The indexes of the table to search.
+ pairs: ``(column, value)`` equality conditions, all to hold.
+ expression: The same conditions as an Arrow expression.
+
+ Returns:
+ The matching rows (possibly none).
+ """
+ table = key_index.table
+ indexed = [
+ (column, value)
+ for column, value in pairs
+ if column in table.column_names
+ and is_indexed_lookup(table.schema.field(column).type, value)
+ ]
+ if not indexed:
+ return table.filter(expression)
+
+ matches: set[int] | None = None
+ for column, value in indexed:
+ found = key_index.column(column).get(value, [])
+ matches = set(found) if matches is None else matches.intersection(found)
+ candidates = rows_at(table, sorted(matches or ()))
+ if len(indexed) == len(pairs):
+ return candidates
+ return candidates.filter(expression)
diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py
index 8c34c1f04..dba0fc3e8 100644
--- a/src/orcapod/databases/delta_lake_databases.py
+++ b/src/orcapod/databases/delta_lake_databases.py
@@ -3,12 +3,24 @@
import logging
from collections import defaultdict
from collections.abc import Collection, Mapping
+from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, cast
+from orcapod.databases._row_index import (
+ KeyIndex,
+ compact_tail,
+ is_indexed_lookup,
+ matching_rows,
+ rows_at,
+)
from orcapod.databases.utils import coerce_record_id
from orcapod.errors import InputValidationError, StoreReadError
-from orcapod.databases.storage_utils import is_cloud_uri, parse_base_path
+from orcapod.databases.storage_utils import (
+ delta_to_pyarrow_dataset,
+ is_cloud_uri,
+ parse_base_path,
+)
from orcapod.utils import arrow_utils
from orcapod.utils.lazy_module import LazyModule
@@ -17,6 +29,7 @@
import polars as pl
import pyarrow as pa
import pyarrow.compute as pc
+ import pyarrow.dataset as pa_ds
from upath import UPath
else:
deltalake = LazyModule("deltalake")
@@ -272,14 +285,29 @@ def _store_error_to_raise(errors: list[Exception]) -> Exception | None:
return None
-class _DeltaTableReadError(Exception):
- """A Delta table's data could not be read; the backend error is the ``__cause__``.
+@dataclass
+class _CommittedSnapshot:
+ """What one version of a stored Delta table holds, kept in memory (ITL-919).
- Raised by ``DeltaTableDatabase._read_delta_table`` so that its callers, which
- know the table's path, can tell a failed read from a bad filter and raise
- ``StoreReadError``.
+ Per-row lookups are answered from this snapshot instead of re-reading the table.
+ Each part is read on first use and extended in place after this store's own
+ commits, so its cost follows the rows written, not the rows stored.
+
+ Attributes:
+ version: The table version described.
+ ids: The stored record ids (never null), or ``None`` until first needed.
+ rows: Indexes over every stored row, read exactly as ``get_all_records``
+ reads them, or ``None`` until first needed.
+ files: The data files ``rows`` was read from (set together with ``rows``).
+ dataset_schema: The schema of the dataset ``rows`` was read from.
"""
+ version: int
+ ids: set[bytes] | None = None
+ rows: KeyIndex | None = None
+ files: frozenset[str] = frozenset()
+ dataset_schema: "pa.Schema | None" = None
+
class DeltaTableDatabase:
"""
@@ -311,6 +339,9 @@ def __init__(
_scoped_path: tuple[str, ...] = (),
_shared_pending_batches: "dict[str, pa.Table] | None" = None,
_shared_pending_record_ids: "defaultdict[str, set[bytes]] | None" = None,
+ _shared_delta_table_cache: "dict[str, deltalake.DeltaTable] | None" = None,
+ _shared_snapshots: "dict[str, _CommittedSnapshot] | None" = None,
+ _shared_pending_indexes: "dict[str, KeyIndex] | None" = None,
):
self._root_uri, self._storage_options = parse_base_path(
base_path, storage_options
@@ -335,9 +366,16 @@ def __init__(
)
# For cloud paths: create_base_path is silently ignored (no directory needed).
- self._delta_table_cache: dict[str, deltalake.DeltaTable] = {}
- # Shared pending state — when constructed via at(), these dicts are shared
- # with the root instance so that root.flush() can flush all scoped writes.
+ # Shared state — when constructed via at(), these dicts are shared with the
+ # root instance. Any handle's flush() writes every pending key of the family
+ # and refreshes the loaded table and existing-ids entries for it, so these
+ # caches must be family-wide too: a per-handle cache would keep serving the
+ # snapshot a sibling loaded before the flush (ITL-864). All of them are
+ # touched only from the thread that drives the pipeline (the async
+ # orchestrator runs only user functions in worker threads).
+ self._delta_table_cache: dict[str, deltalake.DeltaTable] = (
+ _shared_delta_table_cache if _shared_delta_table_cache is not None else {}
+ )
if _shared_pending_batches is not None:
self._pending_batches = _shared_pending_batches
else:
@@ -346,8 +384,15 @@ def __init__(
self._pending_record_ids = _shared_pending_record_ids
else:
self._pending_record_ids: dict[str, set[bytes]] = defaultdict(set)
- self._existing_ids_cache: dict[str, set[str]] = defaultdict(set)
- self._cache_dirty: dict[str, bool] = defaultdict(lambda: True)
+ # The committed rows of each loaded table version (ITL-919), shared like the
+ # loaded tables: a snapshot describes the cached table's version only.
+ self._snapshots: dict[str, _CommittedSnapshot] = (
+ _shared_snapshots if _shared_snapshots is not None else {}
+ )
+ # Indexes over each pending batch, valid while it is the key's batch.
+ self._pending_indexes: dict[str, KeyIndex] = (
+ _shared_pending_indexes if _shared_pending_indexes is not None else {}
+ )
def _clear_pending(self):
"""Clear all pending state."""
@@ -544,79 +589,113 @@ def _handle_record_id_column(
f"Record ID column '{self.RECORD_ID_COLUMN}' not found in the table and cannot be renamed."
)
- def _create_record_id_filter(self, record_id: str | bytes) -> list:
- """
- Create a proper filter expression for Delta Lake.
+ def _snapshot(
+ self, record_key: str, delta_table: deltalake.DeltaTable
+ ) -> _CommittedSnapshot:
+ """Return the snapshot of ``delta_table``'s version, starting an empty one if needed.
Args:
- record_id: The entry ID to filter by
+ record_key: The table's absolute record key.
+ delta_table: The loaded table the snapshot must describe.
Returns:
- List containing the filter expression for Delta Lake
+ The snapshot for the table's version; its parts are read on first use.
"""
- return [(self.RECORD_ID_COLUMN, "=", coerce_record_id(record_id))]
+ version = delta_table.version()
+ snapshot = self._snapshots.get(record_key)
+ if snapshot is None or snapshot.version != version:
+ snapshot = _CommittedSnapshot(version)
+ self._snapshots[record_key] = snapshot
+ return snapshot
- def _create_record_ids_filter(self, record_ids: list[bytes]) -> list:
- """
- Create a proper filter expression for multiple entry IDs.
+ @classmethod
+ def _scan_ids(cls, delta_table: deltalake.DeltaTable) -> set[bytes]:
+ """Read the record ids stored in ``delta_table`` (nulls left out).
Args:
- record_ids: List of entry IDs to filter by
+ delta_table: The loaded table.
Returns:
- List containing the filter expression for Delta Lake
+ The stored record ids.
+
+ Raises:
+ Exception: The backend's error when the table's data cannot be read.
"""
- return [(self.RECORD_ID_COLUMN, "in", record_ids)]
+ stored = _stored_arrow_schema(delta_table)
+ if cls.RECORD_ID_COLUMN not in stored.names:
+ logger.warning(f"Delta table missing {cls.RECORD_ID_COLUMN} column")
+ return set()
+ # Read through a one-column schema: the other columns' types are never
+ # converted, so a logical type stored in a list cannot break the id read.
+ id_field = stored.field(cls.RECORD_ID_COLUMN)
+ id_schema = pa.schema(
+ [id_field.with_type(arrow_utils.normalize_to_large_types(id_field.type))]
+ )
+ dataset = delta_to_pyarrow_dataset(delta_table, schema=id_schema)
+ ids = dataset.to_table()[cls.RECORD_ID_COLUMN]
+ return {record_id for record_id in ids.to_pylist() if record_id is not None}
- def _refresh_existing_ids_cache(self, record_path: tuple[str, ...]) -> None:
- """Refresh the cache of existing IDs in the Delta table."""
- record_key = self._get_record_key(record_path)
+ @staticmethod
+ def _scan_rows(
+ delta_table: deltalake.DeltaTable,
+ ) -> tuple[pa.Table, frozenset[str], pa.Schema]:
+ """Read every row of ``delta_table`` as ``get_all_records`` reads it.
- delta_table = self._get_delta_table(record_path)
+ Args:
+ delta_table: The loaded table.
- if delta_table is None:
- self._existing_ids_cache[record_key] = set()
- self._cache_dirty[record_key] = False
- return
+ Returns:
+ The rows (view types normalized: PyArrow has no comparison kernels for
+ ``string_view``, ENG-601), the data
+ files they were read from and the dataset's schema.
- try:
- # Get all existing IDs from Delta table using standard RECORD_ID_COLUMN
- # TODO: replace this with more targetted loading of only the target column and in batches
- arrow_table = delta_table.to_pyarrow_table()
- except Exception as e:
- # Leave the cache dirty so the next call reads the table again.
- raise self._read_failed(record_path) from e
+ Raises:
+ Exception: The backend's error when the table's data cannot be read.
+ """
+ # deltalake builds a FileSystemDataset; its data files identify a version.
+ dataset = cast(
+ "pa_ds.FileSystemDataset",
+ delta_to_pyarrow_dataset(delta_table, as_large_types=True),
+ )
+ table = arrow_utils.normalize_table_view_types(dataset.to_table())
+ return table.combine_chunks(), frozenset(dataset.files), dataset.schema
- if arrow_table.num_rows == 0:
- self._existing_ids_cache[record_key] = set()
- elif self.RECORD_ID_COLUMN not in arrow_table.column_names:
- # TODO: replace this with proper checking of the table schema first!
- logger.warning(f"Delta table missing {self.RECORD_ID_COLUMN} column")
- self._existing_ids_cache[record_key] = set()
- else:
- existing_ids = cast(
- set[str], set(arrow_table[self.RECORD_ID_COLUMN].to_pylist())
- )
- self._existing_ids_cache[record_key] = existing_ids
+ def _stored_ids(self, record_path: tuple[str, ...]) -> set[bytes]:
+ """Return the record ids stored at ``record_path`` (in the loaded version).
- self._cache_dirty[record_key] = False
- logger.debug(
- f"Refreshed existing IDs cache: {len(self._existing_ids_cache)} IDs"
- )
+ Raises:
+ StoreReadError: If the table exists but its ids could not be read.
+ """
+ delta_table = self._get_delta_table(record_path)
+ if delta_table is None:
+ return set()
+ snapshot = self._snapshot(self._get_record_key(record_path), delta_table)
+ if snapshot.ids is None:
+ try:
+ snapshot.ids = self._scan_ids(delta_table)
+ except Exception as e:
+ raise self._read_failed(record_path) from e
+ return snapshot.ids
- def _get_existing_ids(self, record_path: tuple[str, ...]) -> set[str]:
- """Get the set of existing IDs in the Delta table, using cache when possible."""
- record_key = self._get_record_key(record_path)
- if (
- self._cache_dirty.get(record_key)
- or record_key not in self._delta_table_cache
- ):
- self._refresh_existing_ids_cache(record_path)
- return self._existing_ids_cache.get(record_key) or set()
+ def _stored_rows(self, record_path: tuple[str, ...]) -> KeyIndex | None:
+ """Return the indexed rows stored at ``record_path``, or ``None`` if no table.
- def _invalidate_cache(self, record_path: tuple[str, ...]) -> None:
- """Mark the existing IDs cache as dirty."""
- self._cache_dirty[self._get_record_key(record_path)] = True
+ Raises:
+ StoreReadError: If the table exists but could not be read.
+ """
+ delta_table = self._get_delta_table(record_path)
+ if delta_table is None:
+ return None
+ snapshot = self._snapshot(self._get_record_key(record_path), delta_table)
+ if snapshot.rows is None:
+ try:
+ table, files, schema = self._scan_rows(delta_table)
+ except Exception as e:
+ raise self._read_failed(record_path) from e
+ snapshot.rows = KeyIndex(table)
+ snapshot.files = files
+ snapshot.dataset_schema = schema
+ return snapshot.rows
def add_record(
self,
@@ -706,9 +785,15 @@ def add_records(
# Step 5: Add to pending batch (no overwrite logic needed)
self._add_to_pending_batch(record_path, schema_compatible_table)
- # Step 6: Auto-flush if needed
- if flush or self._should_auto_flush(record_path):
+ # Step 6: Flush if asked to; otherwise commit this table alone once its
+ # pending batch reaches batch_size. Only the full table: committing every
+ # pending table here would commit another table's rows, such as pipeline
+ # records whose results are still pending, out of their writer's order
+ # (ITL-919).
+ if flush:
self.flush()
+ elif self._should_auto_flush(record_path):
+ self.flush_batch(record_path)
def _deduplicate_within_table(self, table: pa.Table) -> pa.Table:
# TODO: consider erroring out if duplicates are found
@@ -747,7 +832,7 @@ def _filter_existing_records(
existing_in_pending = input_ids.intersection(
self._pending_record_ids[record_key]
)
- existing_in_delta = input_ids.intersection(self._get_existing_ids(record_path))
+ existing_in_delta = input_ids.intersection(self._stored_ids(record_path))
all_existing = existing_in_pending.union(existing_in_delta)
if not all_existing:
@@ -777,7 +862,7 @@ def _check_all_conflicts(
)
# Check conflicts with Delta table
- existing_ids = self._get_existing_ids(record_path)
+ existing_ids = self._stored_ids(record_path)
delta_conflicts = input_ids.intersection(existing_ids)
if delta_conflicts:
raise ValueError(
@@ -871,9 +956,15 @@ def _add_to_pending_batch(self, record_path: tuple[str, ...], table: pa.Table):
if pending_batch is None:
self._pending_batches[record_key] = table
else:
- self._pending_batches[record_key] = pa.concat_tables([pending_batch, table])
-
- pending_ids = cast(list[str], table[self.RECORD_ID_COLUMN].to_pylist())
+ # Kept at O(log n) record batches, and its indexes extended in place, so a
+ # lookup's cost does not grow with the rows pending (ITL-919).
+ grown = compact_tail(pa.concat_tables([pending_batch, table]))
+ self._pending_batches[record_key] = grown
+ index = self._pending_indexes.get(record_key)
+ if index is not None and index.table is pending_batch:
+ index.append(table, grown)
+
+ pending_ids = cast(list[bytes], table[self.RECORD_ID_COLUMN].to_pylist())
self._pending_record_ids[record_key].update(pending_ids)
def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool:
@@ -884,6 +975,130 @@ def _should_auto_flush(self, record_path: tuple[str, ...]) -> bool:
and self._pending_batches[record_key].num_rows >= self.batch_size
)
+ def _pending_rows(self, record_path: tuple[str, ...]) -> KeyIndex | None:
+ """Return the indexed pending rows at ``record_path``, or ``None`` if none.
+
+ The indexes are built on first use and describe the current pending batch
+ only (``_add_to_pending_batch`` extends them; a flush or schema change
+ replaces the batch, and with it the indexes).
+ """
+ record_key = self._get_record_key(record_path)
+ pending = self._pending_batches.get(record_key)
+ if pending is None or pending.num_rows == 0:
+ return None
+ index = self._pending_indexes.get(record_key)
+ if index is None or index.table is not pending:
+ index = KeyIndex(pending)
+ self._pending_indexes[record_key] = index
+ return index
+
+ def _without_stored_ids(
+ self, record_path: tuple[str, ...], pending: pa.Table
+ ) -> pa.Table | None:
+ """Return the pending rows a flush would insert: those whose id is not stored.
+
+ A flush inserts only rows whose record id is not stored yet (the merge is
+ ``when_not_matched_insert_all``), so a pending row with a stored id is
+ never returned by a read: the stored row is. The stored ids are read even
+ when ``pending`` is empty, so a read with rows pending raises
+ ``StoreReadError`` whenever the stored ids cannot be read.
+
+ Args:
+ record_path: Path components relative to this database view.
+ pending: Pending rows of the table at ``record_path``.
+
+ Returns:
+ The rows whose record id is not stored, or ``None`` if there are none.
+ """
+ stored_ids = self._stored_ids(record_path)
+ if stored_ids and pending.num_rows > 0:
+ # The mask is built from the pending ids against the cached set, so its
+ # cost follows the pending rows, not the stored table. A null id is
+ # never matched by the merge (``null = null`` is not true), so its row
+ # is kept.
+ keep = [
+ record_id is None or record_id not in stored_ids
+ for record_id in pending[self.RECORD_ID_COLUMN].to_pylist()
+ ]
+ if not all(keep):
+ pending = pending.filter(pa.array(keep, type=pa.bool_()))
+ return pending if pending.num_rows > 0 else None
+
+ def _pending_not_stored(self, record_path: tuple[str, ...]) -> pa.Table | None:
+ """Return every pending row a flush would insert at ``record_path``, or ``None``.
+
+ Args:
+ record_path: Path components relative to this database view.
+
+ Returns:
+ The pending rows whose record id is not stored, or ``None``.
+ """
+ pending = self._pending_rows(record_path)
+ if pending is None:
+ return None
+ return self._without_stored_ids(record_path, pending.table)
+
+ def _combine_stored_and_pending(
+ self,
+ stored: pa.Table | None,
+ pending: pa.Table | None,
+ record_id_column: str | None,
+ ) -> pa.Table | None:
+ """Combine stored rows with pending rows into one read result.
+
+ Combined rows have the schema a flush would give them: both parts take
+ their large layouts (a stored table reads back large), a column missing
+ on one side is null-filled, a ``null``-typed column takes the other
+ side's type, nullability is merged (nullable if either side is), and
+ struct children are merged, as Delta's merge evolves the stored schema.
+ A column whose types still differ raises ``pyarrow.ArrowTypeError``;
+ nothing is widened. A single part is returned as it is.
+
+ Args:
+ stored: Rows read from the Delta table, or ``None``.
+ pending: Pending rows that are not stored (see ``_pending_not_stored``),
+ already filtered by the read's condition, or ``None``.
+ record_id_column: Name to give the record id column, or ``None`` to
+ drop it.
+
+ Returns:
+ The combined rows, or ``None`` if there are none.
+ """
+ parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0]
+ if not parts:
+ return None
+ if len(parts) == 1:
+ table = parts[0]
+ else:
+ table = pa.concat_tables(
+ [self._large_layout(t) for t in parts], promote_options="default"
+ )
+ return self._handle_record_id_column(table, record_id_column)
+
+ @staticmethod
+ def _large_layout(table: pa.Table) -> pa.Table:
+ """Return ``table`` with every column in its large layout.
+
+ Only layouts change (``string`` to ``large_string``, ``list`` to
+ ``large_list``, recursively); ``null`` stays ``null``, and names,
+ nullability and metadata are kept.
+
+ Args:
+ table: The table to normalize.
+
+ Returns:
+ The table with large-normalized column types.
+ """
+ schema = table.schema
+ for index, field in enumerate(table.schema):
+ schema = schema.set(
+ index,
+ field.with_type(
+ arrow_utils.normalize_to_large_types(field.type, keep_null=True)
+ ),
+ )
+ return table if schema.equals(table.schema) else table.cast(schema)
+
def get_all_records(
self,
record_path: tuple[str, ...],
@@ -893,45 +1108,33 @@ def get_all_records(
"""
Get all records from both pending batch and Delta table.
+ Every read method returns the rows a flush would leave: the stored rows
+ plus the pending rows whose record id is not stored yet, as seen through
+ any handle of this ``at()`` family.
+
Returns:
Combined Arrow table with all records, or None if no records exist
Raises:
StoreReadError: If the Delta table exists but could not be read.
"""
- record_key = self._get_record_key(record_path)
-
tables_to_combine = []
# Add Delta table data
if (delta_table := self._get_delta_table(record_path)) is not None:
try:
- delta_table_data = delta_table.to_pyarrow_dataset(
- as_large_types=True
+ delta_table_data = delta_to_pyarrow_dataset(
+ delta_table, as_large_types=True
).to_table()
except Exception as e:
raise self._read_failed(record_path) from e
if delta_table_data.num_rows > 0:
tables_to_combine.append(delta_table_data)
- # Add pending batch data
- if (
- retrieve_pending
- and (pending_batch := self._pending_batches.get(record_key)) is not None
- ):
- if pending_batch.num_rows > 0:
- tables_to_combine.append(pending_batch)
-
- if not tables_to_combine:
- return None
-
- if len(tables_to_combine) == 1:
- table_to_return = tables_to_combine[0]
- else:
- table_to_return = pa.concat_tables(tables_to_combine)
-
- # Handle record_id_column if specified
- return self._handle_record_id_column(table_to_return, record_id_column)
+ # Add the pending rows a flush would insert
+ pending = self._pending_not_stored(record_path) if retrieve_pending else None
+ stored = pa.concat_tables(tables_to_combine) if tables_to_combine else None
+ return self._combine_stored_and_pending(stored, pending, record_id_column)
def get_records_with_column_value(
self,
@@ -942,14 +1145,12 @@ def get_records_with_column_value(
) -> "pa.Table | None":
if flush:
self.flush_batch(record_path)
- # check if record_id is found in pending batches
- record_key = self._get_record_key(record_path)
- pending_batch = self._pending_batches.get(record_key)
+ pair_list: list[tuple[str, Any]]
if isinstance(column_values, Mapping):
# Convert Mapping to list of tuples
- pair_list = list(column_values.items())
- elif isinstance(column_values, Collection):
+ pair_list = list(cast(Mapping[str, Any], column_values).items())
+ else:
# Ensure it's a list of tuples
pair_list = cast(list[tuple[str, Any]], list(column_values))
@@ -958,26 +1159,40 @@ def get_records_with_column_value(
for next_expression in expressions[1:]:
combined_expression = combined_expression & next_expression
- if pending_batch is not None:
- filtered_table = pending_batch.filter(combined_expression)
- return self._handle_record_id_column(filtered_table, record_id_column)
-
- # Now check the Delta table
- delta_table = self._get_delta_table(record_path)
- if delta_table is None:
- return None
-
- # Use schema-preserving read
- try:
- result = self._read_delta_table(delta_table, expression=combined_expression)
- except _DeltaTableReadError as e:
- raise self._read_failed(record_path) from e.__cause__
-
- if len(result) == 0:
- return None
-
- # Handle (remove/rename) the record id column before returning
- return self._handle_record_id_column(result, record_id_column)
+ pending = self._pending_rows(record_path)
+ if pending is not None:
+ # Rows are pending: the stored ids decide which of them a read returns.
+ self._stored_ids(record_path)
+ stored = self._stored_rows(record_path)
+
+ # A side without a queried column holds null there once flushed, and
+ # ``column == value`` never matches null, so it contributes no rows. A
+ # column neither side has is unknown: the filter raises as before.
+ queried = {c for c, _ in pair_list}
+ stored_columns = set() if stored is None else set(stored.table.column_names)
+ pending_columns = set() if pending is None else set(pending.table.column_names)
+ if queried <= stored_columns | pending_columns:
+ if not queried <= pending_columns:
+ pending = None
+ if not queried <= stored_columns:
+ stored = None
+
+ # Both sides are answered from indexes (ITL-919): no re-read, no scan.
+ pending_matches = (
+ None
+ if pending is None
+ else self._without_stored_ids(
+ record_path, matching_rows(pending, pair_list, combined_expression)
+ )
+ )
+ result = (
+ None
+ if stored is None
+ else matching_rows(stored, pair_list, combined_expression)
+ )
+ return self._combine_stored_and_pending(
+ result, pending_matches, record_id_column
+ )
def get_record_by_id(
self,
@@ -1001,14 +1216,19 @@ def get_record_by_id(
self.flush_batch(record_path)
record_id = coerce_record_id(record_id)
- # check if record_id is found in pending batches
+ # Serve a pending record only if its id is not stored: a flush keeps the
+ # stored row and drops the pending one.
record_key = self._get_record_key(record_path)
- if record_id in self._pending_record_ids[record_key]:
+ if record_id in self._pending_record_ids[
+ record_key
+ ] and record_id not in self._stored_ids(record_path):
# Return the pending record after removing the entry id column
- pending_batch = self._pending_batches[record_key]
- assert pending_batch is not None, "Pending batch should not be None"
- filtered_table = pending_batch.filter(
- pc.field(self.RECORD_ID_COLUMN) == record_id
+ pending = self._pending_rows(record_path)
+ assert pending is not None, "Pending batch should not be None"
+ filtered_table = self._rows_with_ids(
+ pending,
+ [record_id],
+ pending.table.schema.field(self.RECORD_ID_COLUMN).type,
)
if filtered_table.num_rows != 1:
raise ValueError(
@@ -1016,19 +1236,12 @@ def get_record_by_id(
)
return self._handle_record_id_column(filtered_table, record_id_column)
- # Now check the Delta table
- delta_table = self._get_delta_table(record_path)
- if delta_table is None:
+ # Now check the stored rows (the indexed snapshot, ITL-919)
+ stored = self._stored_rows(record_path)
+ if stored is None:
return None
-
- # Use schema-preserving read
- filter_expr = self._create_record_id_filter(record_id)
- try:
- result = self._read_delta_table(delta_table, filters=filter_expr)
- except _DeltaTableReadError as e:
- raise self._read_failed(record_path) from e.__cause__
-
- if len(result) == 0:
+ result = self._rows_with_ids(stored, [record_id])
+ if result.num_rows == 0:
return None
# Handle (remove/rename) the record id column before returning
@@ -1070,84 +1283,23 @@ def get_records_by_ids(
if len(record_ids) == 0:
return None
- # check inside the batch
- delta_table = self._get_delta_table(record_path)
- if delta_table is None:
- return None
- # Use schema-preserving read with filters
- filter_expr = self._create_record_ids_filter(record_ids_list)
- try:
- result = self._read_delta_table(delta_table, filters=filter_expr)
- except _DeltaTableReadError as e:
- raise self._read_failed(record_path) from e.__cause__
-
- if len(result) == 0:
- return None
-
- # Handle record_id column based on parameter
- return self._handle_record_id_column(result, record_id_column)
-
- def _read_delta_table(
- self,
- delta_table: deltalake.DeltaTable,
- filters: list | None = None,
- expression: pc.Expression | None = None,
- ) -> pa.Table:
- """
- Read table using to_pyarrow_dataset with original schema preservation.
-
- Args:
- delta_table: The Delta table to read from
- filters: Optional filters to apply
- expression: Optional filter expression, used instead of ``filters``
-
- Returns:
- Arrow table with preserved schema
-
- Raises:
- _DeltaTableReadError: If the table's data could not be read, with the
- backend's error as its ``__cause__``. An error in the filter itself
- (an unknown column, say) is raised as is.
- """
- filter_expr = None
- # Use to_pyarrow_dataset with as_large_types for Polars compatible arrow table loading
- try:
- dataset = delta_table.to_pyarrow_dataset(as_large_types=True)
- except Exception as e:
- raise _DeltaTableReadError from e
- if filters and expression is None:
- for filt in filters:
- if len(filt) == 3:
- col, op, val = filt
- if op == "=":
- expr = pc.equal(pc.field(col), pa.scalar(val)) # type: ignore
- elif op == "in":
- expr = pc.is_in(pc.field(col), pa.array(val)) # type: ignore
- else:
- logger.warning(
- f"Unsupported filter operation: {op}. Falling back to table-level filter application which may be less efficient."
- )
- # Fallback to table-level filtering
- return dataset.to_table()(filters=filters)
-
- if filter_expr is None:
- filter_expr = expr
- else:
- filter_expr = pc.and_(filter_expr, expr) # type: ignore
- elif expression is not None:
- filter_expr = expression
-
- # Convert view types before filtering: PyArrow has no comparison/sort
- # kernels for string_view, so filtering the scan directly crashes
- # (ENG-601). Trades predicate pushdown for correctness.
- try:
- scanned = dataset.to_table()
- except Exception as e:
- raise _DeltaTableReadError from e
- table = arrow_utils.normalize_table_view_types(scanned)
- if filter_expr is not None:
- return table.filter(filter_expr)
- return table
+ stored = self._stored_rows(record_path)
+ pending = self._pending_rows(record_path)
+ result = None
+ if stored is not None:
+ result = self._rows_with_ids(stored, record_ids_list)
+
+ pending_matches = None
+ if pending is not None:
+ id_type = pending.table.schema.field(self.RECORD_ID_COLUMN).type
+ # Reading the stored ids drops the pending rows a flush would not
+ # insert; it raises StoreReadError when the stored table cannot be read.
+ pending_matches = self._without_stored_ids(
+ record_path, self._rows_with_ids(pending, record_ids_list, id_type)
+ )
+ return self._combine_stored_and_pending(
+ result, pending_matches, record_id_column
+ )
def to_config(self, db_registry: Any = None) -> dict[str, Any]:
"""Serialize database configuration to a JSON-compatible dict."""
@@ -1272,7 +1424,7 @@ def _flush_by_key(self, record_key: str) -> None:
try:
# Combine all tables in the batch. Convert view types so they never
- # reach storage (see _read_delta_table / ENG-601).
+ # reach storage (ENG-601).
combined_table = arrow_utils.normalize_table_view_types(
pending_batch.combine_chunks()
)
@@ -1306,6 +1458,8 @@ def _flush_by_key(self, record_key: str) -> None:
schema_mode="merge" if self.allow_schema_evolution else "overwrite",
storage_options=self._storage_options or None,
)
+ base = _CommittedSnapshot(version=-1, ids=set())
+ inserted_ids = self._non_null_ids(combined_table)
logger.debug(
f"Created new Delta table for {record_key} with {len(combined_table)} records"
)
@@ -1320,28 +1474,38 @@ def _flush_by_key(self, record_key: str) -> None:
f"{combined_table.schema.field(column).type}). "
"The stored rows are unchanged and nothing was written."
)
- delta_table.merge(
- source=combined_table,
- predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}",
- source_alias="source",
- target_alias="target",
- merge_schema=self.allow_schema_evolution,
- ).when_not_matched_insert_all().execute()
+ base = self._snapshot_for_commit(
+ record_key, delta_table, table_uri, abs_path_components
+ )
+ assert base.ids is not None
+ incoming_ids = self._non_null_ids(combined_table)
+ inserted_ids = incoming_ids - base.ids
+ if len(inserted_ids) == len(incoming_ids) and self._stores_as_is(
+ delta_table, combined_table
+ ):
+ # Every pending id is new in the version just loaded and the rows
+ # have the stored schema: append (ITL-919). A MERGE reads the whole
+ # stored table on each commit; it remains for every other batch.
+ deltalake.write_deltalake(
+ table_uri,
+ combined_table,
+ mode="append",
+ storage_options=self._storage_options or None,
+ )
+ else:
+ # Some pending ids are stored already: insert only the others.
+ delta_table.merge(
+ source=combined_table,
+ predicate=f"target.{self.RECORD_ID_COLUMN} = source.{self.RECORD_ID_COLUMN}",
+ source_alias="source",
+ target_alias="target",
+ merge_schema=self.allow_schema_evolution,
+ ).when_not_matched_insert_all().execute()
logger.debug(
f"Appended batch of {len(combined_table)} records to {record_key}"
)
- # Update cache
- self._delta_table_cache[record_key] = deltalake.DeltaTable(
- table_uri, storage_options=self._storage_options or None
- )
-
- # Invalidate the existing-IDs cache directly by record_key to avoid
- # double-prefixing that would occur if _invalidate_cache() were used
- # (it prepends _path_prefix again via _get_record_key).
- self._cache_dirty[record_key] = True
-
except Exception as e:
logger.error(f"Error flushing batch for {record_key}: {e}")
# Put the tables back in the pending queue
@@ -1349,6 +1513,185 @@ def _flush_by_key(self, record_key: str) -> None:
self._pending_record_ids[record_key] = pending_ids
raise
+ # The rows are committed: nothing below may put them back in the queue.
+ try:
+ committed = deltalake.DeltaTable(
+ table_uri, storage_options=self._storage_options or None
+ )
+ self._delta_table_cache[record_key] = committed
+ self._snapshots[record_key] = self._extend_snapshot(
+ base, committed, inserted_ids
+ )
+ except Exception as e:
+ # The next read loads the table again and reports a failure itself.
+ logger.warning(f"Could not reload {record_key} after committing: {e}")
+ self._delta_table_cache.pop(record_key, None)
+ self._snapshots.pop(record_key, None)
+
+ def _rows_with_ids(
+ self,
+ rows: KeyIndex,
+ record_ids: list[bytes],
+ value_type: pa.DataType | None = None,
+ ) -> pa.Table:
+ """Return the rows whose record id is one of ``record_ids``, in table order.
+
+ Served from the record-id index (ITL-919); a table whose id column is not
+ binary is filtered instead, with Arrow's comparison.
+
+ Args:
+ rows: The indexed rows (stored or pending).
+ record_ids: The ids looked up.
+ value_type: The Arrow type to build the looked-up ids with for the
+ filter, or ``None`` to infer it from the ids.
+
+ Returns:
+ The matching rows (possibly none).
+ """
+ id_type = rows.table.schema.field(self.RECORD_ID_COLUMN).type
+ if not all(is_indexed_lookup(id_type, r) for r in record_ids):
+ return rows.table.filter(
+ pc.is_in(
+ pc.field(self.RECORD_ID_COLUMN),
+ pa.array(record_ids, type=value_type),
+ )
+ )
+ id_index = rows.column(self.RECORD_ID_COLUMN)
+ positions = {p for r in set(record_ids) for p in id_index.get(r, ())}
+ return rows_at(rows.table, sorted(positions))
+
+ @staticmethod
+ def _stores_as_is(delta_table: deltalake.DeltaTable, batch: pa.Table) -> bool:
+ """Whether ``batch`` has exactly the stored schema once the Delta writer converts it.
+
+ Only such a batch is appended: an append stores it unchanged, as the MERGE
+ would. A batch that needs a cast (a ``null``-typed column, a nullability
+ difference) or evolves the schema goes through the MERGE, which handles it.
+
+ Args:
+ delta_table: The loaded stored table.
+ batch: The rows about to be committed.
+
+ Returns:
+ Whether an append stores ``batch`` exactly as the MERGE would.
+ """
+ try:
+ incoming = _as_delta_writer_stores(batch.schema)
+ except Exception:
+ # A type Delta cannot store directly (null): leave it to the MERGE.
+ return False
+ return incoming.equals(_stored_arrow_schema(delta_table), check_metadata=False)
+
+ @classmethod
+ def _non_null_ids(cls, table: pa.Table) -> set[bytes]:
+ """Return the record ids of ``table``'s rows, nulls left out."""
+ return {
+ record_id
+ for record_id in table[cls.RECORD_ID_COLUMN].to_pylist()
+ if record_id is not None
+ }
+
+ def _snapshot_for_commit(
+ self,
+ record_key: str,
+ delta_table: deltalake.DeltaTable,
+ table_uri: str,
+ abs_path_components: tuple[str, ...],
+ ) -> _CommittedSnapshot:
+ """Return the stored ids of the version a commit is about to build on.
+
+ ``delta_table`` was loaded for this commit, so whether the pending ids are
+ new is decided against the table as it is now, never against an older
+ cached version. The cached snapshot is reused when it describes this
+ version; otherwise the ids are read from it.
+
+ Args:
+ record_key: The table's absolute record key.
+ delta_table: The table freshly loaded for the commit.
+ table_uri: The table's location, for the error.
+ abs_path_components: The table's absolute path, for the error.
+
+ Returns:
+ A snapshot of the loaded version with its ``ids`` read.
+
+ Raises:
+ StoreReadError: If the stored ids could not be read.
+ """
+ cached = self._snapshots.get(record_key)
+ if (
+ cached is not None
+ and cached.ids is not None
+ and cached.version == delta_table.version()
+ ):
+ return cached
+ try:
+ ids = self._scan_ids(delta_table)
+ except Exception as e:
+ raise StoreReadError(table_uri, abs_path_components) from e
+ return _CommittedSnapshot(delta_table.version(), ids=ids)
+
+ def _extend_snapshot(
+ self,
+ base: _CommittedSnapshot,
+ committed: deltalake.DeltaTable,
+ inserted_ids: set[bytes],
+ ) -> _CommittedSnapshot:
+ """Return the snapshot of ``committed``, built from ``base`` plus this commit.
+
+ When ``committed`` is the version right after ``base`` (no other writer
+ committed in between), its ids are ``base``'s plus the inserted ones, and
+ its rows, if ``base`` has them, are ``base``'s plus the rows of the data
+ files the commit added, read as ``get_all_records`` reads them. Anything
+ else (a skipped version, a removed file, a changed schema) leaves that
+ part to be read again on first use.
+
+ Args:
+ base: The snapshot of the version the commit built on.
+ committed: The table reloaded after the commit.
+ inserted_ids: The non-null record ids the commit inserted.
+
+ Returns:
+ The new snapshot.
+ """
+ version = committed.version()
+ if version != base.version + 1:
+ return _CommittedSnapshot(version)
+ assert base.ids is not None
+ snapshot = _CommittedSnapshot(version, ids=base.ids | inserted_ids)
+ if base.rows is None:
+ return snapshot
+ dataset = cast(
+ "pa_ds.FileSystemDataset",
+ delta_to_pyarrow_dataset(committed, as_large_types=True),
+ )
+ files = frozenset(dataset.files)
+ if (
+ base.dataset_schema is None
+ or not base.files <= files
+ or not dataset.schema.equals(base.dataset_schema)
+ ):
+ return snapshot
+ added = files - base.files
+ new_rows = arrow_utils.normalize_table_view_types(
+ pa.concat_tables(
+ [
+ fragment.to_table(schema=dataset.schema)
+ for fragment in dataset.get_fragments()
+ if fragment.path in added
+ ]
+ )
+ if added
+ else dataset.schema.empty_table()
+ )
+ if not new_rows.schema.equals(base.rows.table.schema):
+ return snapshot
+ rows = base.rows
+ rows.append(new_rows, compact_tail(pa.concat_tables([rows.table, new_rows])))
+ snapshot.rows = rows
+ snapshot.files = files
+ snapshot.dataset_schema = dataset.schema
+ return snapshot
+
@staticmethod
def _incompatible_append_column(
delta_table: deltalake.DeltaTable, batch: pa.Table
@@ -1453,6 +1796,9 @@ def at(self, *path_components: str) -> "DeltaTableDatabase":
_scoped_path=new_scoped_path,
_shared_pending_batches=root_instance._pending_batches,
_shared_pending_record_ids=root_instance._pending_record_ids,
+ _shared_delta_table_cache=root_instance._delta_table_cache,
+ _shared_snapshots=root_instance._snapshots,
+ _shared_pending_indexes=root_instance._pending_indexes,
)
def list_sources(self) -> list[tuple[str, ...]]:
diff --git a/src/orcapod/databases/extension_aware_database.py b/src/orcapod/databases/extension_aware_database.py
index fd96817d5..608d00e08 100644
--- a/src/orcapod/databases/extension_aware_database.py
+++ b/src/orcapod/databases/extension_aware_database.py
@@ -22,7 +22,7 @@
from __future__ import annotations
-from collections.abc import Collection, Mapping
+from collections.abc import Callable, Collection, Mapping
from typing import TYPE_CHECKING, Any
from orcapod.logical_types.database_hooks import register_discovered_logical_types
@@ -167,6 +167,26 @@ def add_records(
def flush(self) -> None:
self._db.flush()
+ @property
+ def batch_size(self) -> int:
+ """The wrapped database's ``batch_size``.
+
+ Raises:
+ AttributeError: If the wrapped database has none, so this wrapper
+ satisfies ``TableCommittingDatabaseProtocol`` only when the
+ wrapped database does.
+ """
+ return getattr(self._db, "batch_size") # noqa: B009 # AttributeError when absent, by design
+
+ @property
+ def flush_batch(self) -> Callable[[tuple[str, ...]], None]:
+ """The wrapped database's ``flush_batch`` (commits one table's writes).
+
+ Raises:
+ AttributeError: If the wrapped database has none (see ``batch_size``).
+ """
+ return getattr(self._db, "flush_batch") # noqa: B009 # AttributeError when absent, by design
+
# ── Structural delegation ─────────────────────────────────────────────────
@property
diff --git a/src/orcapod/databases/in_memory_databases.py b/src/orcapod/databases/in_memory_databases.py
index d25fd1b5a..ed2a7a68a 100644
--- a/src/orcapod/databases/in_memory_databases.py
+++ b/src/orcapod/databases/in_memory_databases.py
@@ -5,6 +5,7 @@
from collections.abc import Collection, Mapping
from typing import TYPE_CHECKING, Any, cast
+from orcapod.databases._row_index import KeyIndex, compact_tail, matching_rows
from orcapod.databases.utils import coerce_record_id
from orcapod.utils.lazy_module import LazyModule
@@ -29,6 +30,10 @@ class InMemoryArrowDatabase:
Supports the same pending-batch semantics as DeltaTableDatabase:
records are buffered in a pending batch and become part of the committed
store only after flush() is called (or flush=True is passed to a write method).
+
+ Lookups by record ID or by a binary or string column value use hash indexes over
+ the committed rows, and each flush appends without copying the stored rows, so the
+ cost of a per-row write, flush or lookup does not grow with the rows already stored.
"""
RECORD_ID_COLUMN = "__record_id"
@@ -40,6 +45,7 @@ def __init__(
_shared_tables: "dict[str, pa.Table] | None" = None,
_shared_pending_batches: "dict[str, pa.Table] | None" = None,
_shared_pending_record_ids: "dict[str, set[bytes]] | None" = None,
+ _shared_indexes: "dict[str, KeyIndex] | None" = None,
_root: InMemoryArrowDatabase | None = None,
_scoped_path: tuple[str, ...] = (),
):
@@ -58,6 +64,9 @@ def __init__(
if _shared_pending_record_ids is not None
else defaultdict(set)
)
+ self._indexes: dict[str, KeyIndex] = (
+ _shared_indexes if _shared_indexes is not None else {}
+ )
# ------------------------------------------------------------------
# Path helpers
@@ -146,22 +155,37 @@ def _deduplicate_within_table(self, table: "pa.Table") -> "pa.Table":
# Internal helpers for duplicate detection
# ------------------------------------------------------------------
- def _committed_ids(self, record_key: str) -> "set[bytes]":
+ def _key_index(self, record_key: str, committed: "pa.Table") -> KeyIndex:
+ """Return the indexes of ``committed``, the key's committed table.
+
+ Starts them on first use; ``flush`` extends them as rows are appended.
+ """
+ key_index = self._indexes.get(record_key)
+ if key_index is None or key_index.table is not committed:
+ key_index = KeyIndex(committed)
+ self._indexes[record_key] = key_index
+ return key_index
+
+ def _committed_id_index(self, record_key: str) -> dict[Any, list[int]]:
+ """Return the record-ID index of the key's committed rows (empty if none)."""
committed = self._tables.get(record_key)
if committed is None or committed.num_rows == 0:
- return set()
- existing_ids = committed[self.RECORD_ID_COLUMN].to_pylist()
- # TODO: evaluate the efficiency of this implementation
- return {id for id in existing_ids if id is not None}
+ return {}
+ return self._key_index(record_key, committed).column(self.RECORD_ID_COLUMN)
def _filter_existing_records(
self, record_key: str, table: "pa.Table"
) -> "pa.Table":
"""Filter out records whose IDs are already in pending or committed store."""
input_ids = set(table[self.RECORD_ID_COLUMN].to_pylist())
- all_existing = input_ids & (
- self._pending_record_ids[record_key] | self._committed_ids(record_key)
- )
+ pending_ids = self._pending_record_ids[record_key]
+ committed_ids = self._committed_id_index(record_key)
+ all_existing = {
+ record_id
+ for record_id in input_ids
+ if record_id in pending_ids
+ or (record_id is not None and record_id in committed_ids)
+ }
if not all_existing:
return table
mask = pc.invert(
@@ -272,16 +296,30 @@ def flush(self) -> None:
committed = self._tables.get(record_key)
if committed is None:
- self._tables[record_key] = pending
- else:
- # Insert-if-not-exists: keep committed rows not overwritten by new batch,
- # then append the new batch on top.
- new_ids = set(pending[self.RECORD_ID_COLUMN].to_pylist())
+ self._tables[record_key] = pending.combine_chunks()
+ continue
+
+ # Insert-if-not-exists: keep committed rows not overwritten by new batch,
+ # then append the new batch on top.
+ new_ids = pending[self.RECORD_ID_COLUMN].to_pylist()
+ committed_ids = self._committed_id_index(record_key)
+ if any(record_id in committed_ids for record_id in new_ids):
mask = pc.invert(
pc.is_in(committed[self.RECORD_ID_COLUMN], pa.array(list(new_ids)))
)
kept = committed.filter(mask)
- self._tables[record_key] = pa.concat_tables([kept, pending])
+ self._tables[record_key] = pa.concat_tables(
+ [kept, pending]
+ ).combine_chunks()
+ # Rows moved: the indexes are rebuilt on the next lookup.
+ self._indexes.pop(record_key, None)
+ continue
+
+ appended = compact_tail(pa.concat_tables([committed, pending]))
+ self._tables[record_key] = appended
+ key_index = self._indexes.get(record_key)
+ if key_index is not None and key_index.table is committed:
+ key_index.append(pending, appended)
# ------------------------------------------------------------------
# Path scoping
@@ -322,6 +360,7 @@ def at(self, *path_components: str) -> "InMemoryArrowDatabase":
_shared_tables=self._tables,
_shared_pending_batches=self._pending_batches,
_shared_pending_record_ids=self._pending_record_ids,
+ _shared_indexes=self._indexes,
_root=new_root,
_scoped_path=new_scoped_path,
)
@@ -492,12 +531,18 @@ def get_records_with_column_value(
self.flush()
record_key = self._get_record_key(record_path)
- table = self._combined_table(record_key)
- if table is None:
+ committed = self._tables.get(record_key)
+ if committed is not None and committed.num_rows == 0:
+ committed = None
+ pending = self._pending_batches.get(record_key)
+ if pending is not None and pending.num_rows == 0:
+ pending = None
+ if committed is None and pending is None:
return None
+ pair_list: list[tuple[str, Any]]
if isinstance(column_values, Mapping):
- pair_list = list(column_values.items())
+ pair_list = list(cast(Mapping[str, Any], column_values).items())
else:
pair_list = cast(list[tuple[str, Any]], list(column_values))
@@ -506,7 +551,26 @@ def get_records_with_column_value(
for expr in expressions[1:]:
combined_expr = combined_expr & expr
- filtered = table.filter(combined_expr)
+ parts = []
+ if committed is not None:
+ parts.append(
+ self._filter_committed(record_key, committed, pair_list, combined_expr)
+ )
+ if pending is not None:
+ parts.append(pending.filter(combined_expr))
+ filtered = parts[0] if len(parts) == 1 else pa.concat_tables(parts)
if filtered.num_rows == 0:
return None
return self._handle_record_id_column(filtered, record_id_column)
+
+ def _filter_committed(
+ self,
+ record_key: str,
+ committed: "pa.Table",
+ pair_list: list[tuple[str, Any]],
+ expression: "pc.Expression",
+ ) -> "pa.Table":
+ """Return the committed rows matching ``expression``, in table order."""
+ return matching_rows(
+ self._key_index(record_key, committed), pair_list, expression
+ )
diff --git a/src/orcapod/databases/storage_utils.py b/src/orcapod/databases/storage_utils.py
index 544d20e69..69410c432 100644
--- a/src/orcapod/databases/storage_utils.py
+++ b/src/orcapod/databases/storage_utils.py
@@ -1,20 +1,30 @@
"""
Storage utility helpers for DeltaTableDatabase.
-Provides two public functions:
+Provides three public functions:
- parse_base_path: Converts any path type (str, Path, UPath) into a URI string
and a merged dict of deltalake-compatible storage_options.
- is_cloud_uri: Returns True for cloud storage schemes (s3://, gs://, etc.).
+- delta_to_pyarrow_dataset: Opens a Delta table as a pyarrow dataset, reading local
+ tables through a native filesystem.
"""
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
+
+from orcapod.utils.lazy_module import LazyModule
if TYPE_CHECKING:
from pathlib import Path
+
+ import deltalake
+ import pyarrow.dataset as pa_ds
+ import pyarrow.fs as pa_fs
from upath import UPath
+else:
+ pa_fs = LazyModule("pyarrow.fs")
logger = logging.getLogger(__name__)
@@ -38,6 +48,42 @@ def is_cloud_uri(uri: str) -> bool:
return scheme in _CLOUD_SCHEMES
+def delta_to_pyarrow_dataset(
+ delta_table: deltalake.DeltaTable, **kwargs: Any
+) -> pa_ds.Dataset:
+ """Open a Delta table as a pyarrow dataset, natively for local tables.
+
+ By default ``DeltaTable.to_pyarrow_dataset`` reads through
+ ``PyFileSystem(DeltaStorageHandler)``, a filesystem implemented in Python. Arrow
+ finishes tearing down a scan on a thread-pool worker after the scan returns. If that
+ worker releases a Python-backed buffer while the interpreter is finalizing, the process
+ hangs at exit in Arrow's thread-pool destructor, or aborts on Linux with Python
+ 3.11/3.12 (ITL-892; apache/arrow#50482, apache/arrow#54536).
+
+ A local table is therefore read through a native pyarrow filesystem rooted at the
+ table, which returns the same schema and values without Python file objects. Any other
+ URI, a cloud store or a ``filesystem`` passed by the caller, keeps deltalake's reader.
+
+ ``table_uri`` is percent-encoded: deltalake 1.0.2 and 1.2.1 return a bare path such as
+ ``/data/a%20b/``, and 1.6.6 returns ``file:///data/a%20b/``. pyarrow's own URI parser
+ decodes it to the native path. A root that is neither a ``file://`` URI nor an absolute
+ POSIX path (a Windows drive path, untested) keeps deltalake's reader.
+
+ Args:
+ delta_table: The Delta table to open.
+ **kwargs: Passed to ``DeltaTable.to_pyarrow_dataset`` unchanged.
+
+ Returns:
+ The pyarrow dataset over the table's current version.
+ """
+ uri = delta_table.table_uri
+ file_uri = f"file://{uri}" if uri.startswith("/") else uri
+ if kwargs.get("filesystem") is None and file_uri.startswith("file://"):
+ filesystem, root = pa_fs.FileSystem.from_uri(file_uri)
+ kwargs["filesystem"] = pa_fs.SubTreeFileSystem(root, filesystem)
+ return delta_table.to_pyarrow_dataset(**kwargs)
+
+
def _extract_upath_options(upath: UPath) -> dict[str, str]:
"""
Extract storage options from a UPath and translate to deltalake's AWS_* format.
diff --git a/src/orcapod/databases/utils.py b/src/orcapod/databases/utils.py
index 22da24227..62c9b34a5 100644
--- a/src/orcapod/databases/utils.py
+++ b/src/orcapod/databases/utils.py
@@ -2,6 +2,11 @@
from __future__ import annotations
+from typing import TYPE_CHECKING, cast
+
+if TYPE_CHECKING:
+ from orcapod.protocols.database_protocols import TableCommittingDatabaseProtocol
+
def coerce_record_id(record_id: str | bytes) -> bytes:
"""Encode ``record_id`` to bytes if it is a ``str``.
@@ -20,3 +25,31 @@ def coerce_record_id(record_id: str | bytes) -> bytes:
if isinstance(record_id, str):
return record_id.encode()
return record_id
+
+
+def as_table_committing(database: object) -> TableCommittingDatabaseProtocol | None:
+ """Return ``database`` if it commits table by table, else ``None`` (ITL-919).
+
+ The one check for ``TableCommittingDatabaseProtocol``, used by ``PipelineJob.run``
+ (the ``batch_size`` rule) and by function nodes (results committed before their
+ pipeline records). It looks the attributes up on the object, so a wrapper that
+ forwards them only when the database it wraps has them (``ExtensionAwareDatabase``)
+ qualifies exactly when that database does; ``isinstance`` on the protocol would
+ not, because it inspects the class.
+
+ Args:
+ database: Any database object.
+
+ Returns:
+ ``database``, typed as the protocol, or ``None``.
+ """
+ try:
+ batch_size = getattr(database, "batch_size") # noqa: B009 # AttributeError means "no"
+ flush_batch = getattr(database, "flush_batch") # noqa: B009
+ except AttributeError:
+ return None
+ if isinstance(batch_size, bool) or not isinstance(batch_size, int):
+ return None
+ if not callable(flush_batch):
+ return None
+ return cast("TableCommittingDatabaseProtocol", database)
diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py
index b2bb855ba..448990fb6 100644
--- a/src/orcapod/pipeline/job.py
+++ b/src/orcapod/pipeline/job.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import contextlib
import logging
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
@@ -12,11 +13,13 @@
from orcapod.core.nodes.source_node import SourceJobNode
from orcapod.side_effects import SideEffectJobNode
from orcapod.core.tracker import AutoRegisteringContextBasedTracker
-from orcapod.errors import StoreReadError, StoreWriteError
+from orcapod.errors import StoreError, StoreReadError, StoreWriteError
from orcapod.pipeline.base import AbstractPipelineBase
from orcapod.pipeline.dag import OrcaDAG
from orcapod.protocols import core_protocols as cp
+from orcapod.databases.utils import as_table_committing
from orcapod.types import CacheMode, NodeConfig
+from orcapod.utils.store_errors import flush_best_effort
if TYPE_CHECKING:
from pathlib import Path
@@ -30,34 +33,11 @@
logger = logging.getLogger(__name__)
-
-def _flush_best_effort(
- run_error: BaseException, *databases: ArrowDatabaseProtocol
-) -> None:
- """Flush each database, noting instead of raising if a flush fails.
-
- Used when a run has already failed: the run's own error is the one to report,
- and a second failure while committing the writes that did succeed must not
- replace it. A flush failure is logged and added as a note to ``run_error``,
- so the caller learns that pending results may not have been stored.
-
- Args:
- run_error: The error the run failed with, which the caller re-raises.
- *databases: The databases to flush.
- """
- for database in databases:
- try:
- database.flush()
- except Exception as flush_error:
- logger.exception(
- "Could not flush pending writes after the run failed; "
- "the rows they hold were not stored"
- )
- run_error.add_note(
- "Flushing pending writes after this failure also failed "
- + f"({type(flush_error).__name__}: {flush_error}); results that were "
- + "still pending may not have been stored."
- )
+_PENDING_BEFORE_RUN_NOTE = (
+ "These writes were left pending in the store before this run started (written "
+ "without a flush, or by a run that failed). The run stopped before computing "
+ "anything."
+)
class PipelineJob(AbstractPipelineBase[JobNode]):
@@ -734,6 +714,7 @@ def run(
orchestrator: "SyncPipelineOrchestrator | AsyncPipelineOrchestrator | None" = None,
observer: "ExecutionObserverProtocol | None" = None,
materialize_results: bool = True,
+ commit_every_rows: int = 100,
) -> "PipelineJob":
"""Execute the resolvable subgraph of this job in place.
@@ -758,19 +739,37 @@ def run(
result access — DB records are written regardless, and
``job.nodes["label"].get_all_records()`` works the same in
either case.
+ commit_every_rows: How many rows a function node processes between
+ commits of its results and pipeline records to the store
+ (default 100). Each node also commits when it finishes, before a
+ failed run raises, and the job commits at its end. On a durable
+ store (``DeltaTableDatabase``) a hard crash (power loss, killed
+ process) loses at most the rows of the running node since its
+ last commit; re-running the job recomputes only those. A larger
+ value commits less often (faster, fewer files); ``1`` commits
+ after every row. A node run on its own, outside ``run()``,
+ commits after every row. It must be smaller than the store's
+ ``batch_size`` when the store has one (``DeltaTableDatabase``,
+ default 1000): the store commits every pending table by itself
+ once one table's pending rows reach that size, which would
+ break the node's results-before-records commit order.
Returns:
``self`` — the same ``PipelineJob`` instance, with
``_has_run`` and ``_run_id`` set.
Raises:
- ValueError: If no store is set.
+ ValueError: If no store is set, or ``commit_every_rows`` is not a
+ positive integer smaller than the store's ``batch_size``.
RuntimeError: If no pipeline has been compiled.
StoreWriteError: If a node's results or records could not be written
to the store, whatever the orchestrator's ``error_policy``. The run
stops at the first failed write. Writes that succeeded before it
are flushed first, so rows stored before the failure are kept, and
re-running the job does not recompute stored function results.
+ A failed commit of a node's batch names the node but no row
+ (``flush=True``): the batch's writes stay pending and are retried
+ once by the flush that follows the failure.
StoreReadError: If a stored table exists but could not be read,
whatever the orchestrator's ``error_policy``, also when the read
was needed to write a node's results. Its rows are not recomputed
@@ -782,11 +781,32 @@ def run(
from orcapod.pipeline.observer import NoOpObserver
from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator
+ if (
+ isinstance(commit_every_rows, bool)
+ or not isinstance(commit_every_rows, int)
+ or commit_every_rows < 1
+ ):
+ raise ValueError(
+ "commit_every_rows must be a positive integer (rows per commit), "
+ f"got {commit_every_rows!r}"
+ )
if not self._compiled:
raise RuntimeError(
"PipelineJob has no compiled pipeline. "
"Record invocations inside a 'with job:' block first."
)
+ table_committing = as_table_committing(self._store)
+ if (
+ table_committing is not None
+ and commit_every_rows >= table_committing.batch_size
+ ):
+ raise ValueError(
+ f"commit_every_rows={commit_every_rows} must be smaller than the "
+ f"store's batch_size={table_committing.batch_size}: the store commits a "
+ "table by itself once it holds batch_size pending rows, which could "
+ "commit pipeline records before their results. Use a smaller "
+ "commit_every_rows, or create the store with a larger batch_size."
+ )
if self._store is None:
raise ValueError(
"PipelineJob.run() requires a store. "
@@ -869,19 +889,42 @@ def run(
pipeline_db = self._store.at(*self._name)
result_db = pipeline_db.at("_result")
+ # A store that commits a table by itself at batch_size: commit what it already
+ # holds pending (writes made without a flush, or a failed run's batches) before
+ # this run writes. Those rows would otherwise count towards a table's
+ # batch_size, which could then be reached mid-run and commit this run's pipeline
+ # records ahead of their results (ITL-919).
+ if table_committing is not None:
+ try:
+ pipeline_db.flush()
+ except StoreError as exc:
+ exc.add_note(_PENDING_BEFORE_RUN_NOTE)
+ raise
+ except Exception as exc:
+ flush_error = StoreWriteError(pipeline_db.base_path, flush=True)
+ flush_error.add_note(_PENDING_BEFORE_RUN_NOTE)
+ raise flush_error from exc
+
effective_orchestrator = orchestrator or SyncPipelineOrchestrator()
try:
- effective_orchestrator.run(
- exec_dag,
- observer=effective_observer,
- run_id=run_id,
- pipeline_uri=pipeline_uri,
- materialize_results=materialize_results,
- )
+ with contextlib.ExitStack() as batched:
+ # Function nodes commit every commit_every_rows rows (ITL-919).
+ for node in exec_dag.nodes():
+ if isinstance(node, FunctionJobNode):
+ batched.enter_context(
+ node._batched_commits(commit_every_rows) # pyright: ignore[reportPrivateUsage] # the job owns its nodes' commit cadence
+ )
+ effective_orchestrator.run(
+ exec_dag,
+ observer=effective_observer,
+ run_id=run_id,
+ pipeline_uri=pipeline_uri,
+ materialize_results=materialize_results,
+ )
except Exception as run_error:
# Commit the writes that did succeed before the failure, so their rows
# do not look dropped, then report the failure itself.
- _flush_best_effort(run_error, pipeline_db, result_db)
+ flush_best_effort(run_error, pipeline_db, result_db)
raise
# Flush databases. Every view of the store shares one set of pending
diff --git a/src/orcapod/protocols/database_protocols.py b/src/orcapod/protocols/database_protocols.py
index 3215a342c..98ad15e41 100644
--- a/src/orcapod/protocols/database_protocols.py
+++ b/src/orcapod/protocols/database_protocols.py
@@ -159,6 +159,27 @@ class ArrowDatabaseWithMetadataProtocol(
pass
+class TableCommittingDatabaseProtocol(Protocol):
+ """A database that commits table by table (ITL-919).
+
+ Such a database (``DeltaTableDatabase``) can commit one table's pending writes
+ on its own, and commits a table by itself once that table holds ``batch_size``
+ pending rows. A function node uses ``flush_batch`` to commit its results before
+ its pipeline records; ``PipelineJob.run`` keeps ``commit_every_rows`` below
+ ``batch_size``. A wrapper satisfies this protocol only when the database it
+ wraps does. Check with ``orcapod.databases.utils.as_table_committing``.
+ """
+
+ @property
+ def batch_size(self) -> int:
+ """Pending rows of one table at which the database commits that table."""
+ ...
+
+ def flush_batch(self, record_path: tuple[str, ...]) -> None:
+ """Commit the pending writes of the table at ``record_path`` only."""
+ ...
+
+
@runtime_checkable
class DatabaseRegistryProtocol(Protocol):
"""Protocol for a database config registry used during pipeline save/load.
diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py
index 854eab5ba..0cc8218b4 100644
--- a/src/orcapod/side_effects.py
+++ b/src/orcapod/side_effects.py
@@ -238,7 +238,12 @@ def _execute_side_effect_row(
and pipeline_database is not None
and table_path is not None
):
- prior = pipeline_database.get_record_by_id(table_path, record_id)
+ try:
+ prior = pipeline_database.get_record_by_id(table_path, record_id)
+ except StoreError as exc:
+ # A failed lookup fails the run like a failed write: name pod and row.
+ exc.add_context(node_label=pod_name, tag=tag.as_dict())
+ raise
if prior is not None:
return (tag, data) # already completed — re-emit without re-delivery
diff --git a/src/orcapod/utils/store_errors.py b/src/orcapod/utils/store_errors.py
index a28c1a640..20d447d18 100644
--- a/src/orcapod/utils/store_errors.py
+++ b/src/orcapod/utils/store_errors.py
@@ -8,11 +8,15 @@
from __future__ import annotations
import logging
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
from contextlib import contextmanager
+from typing import TYPE_CHECKING
from orcapod.errors import StoreError
+if TYPE_CHECKING:
+ from orcapod.protocols.database_protocols import ArrowDatabaseProtocol
+
logger = logging.getLogger(__name__)
@@ -63,3 +67,44 @@ def keep_store_error(store_error: StoreError | None, during: str) -> Iterator[No
yield
except Exception as failure:
note_callback_failure(store_error, during, failure)
+
+
+def commit_best_effort(run_error: BaseException, commit: Callable[[], None]) -> None:
+ """Commit pending writes after a run failed, noting instead of raising a failure.
+
+ The run's own error is the one to report, and a second failure while committing
+ the writes that did succeed must not replace it. A failed commit is logged and
+ added as a note to ``run_error``, so the caller learns that pending results may
+ not have been stored.
+
+ Args:
+ run_error: The error the run failed with, which the caller re-raises.
+ commit: Commits the pending writes, such as a database's ``flush``.
+ """
+ try:
+ commit()
+ except Exception as flush_error:
+ logger.exception(
+ "Could not flush pending writes after the run failed; "
+ "the rows they hold were not stored"
+ )
+ run_error.add_note(
+ "Flushing pending writes after this failure also failed "
+ + f"({type(flush_error).__name__}: {flush_error}); results that were "
+ + "still pending may not have been stored."
+ )
+
+
+def flush_best_effort(
+ run_error: BaseException, *databases: ArrowDatabaseProtocol
+) -> None:
+ """Flush each database after a run failed, noting instead of raising a failure.
+
+ See ``commit_best_effort``.
+
+ Args:
+ run_error: The error the run failed with, which the caller re-raises.
+ *databases: The databases to flush.
+ """
+ for database in databases:
+ commit_best_effort(run_error, database.flush)
diff --git a/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md
new file mode 100644
index 000000000..546c5e252
--- /dev/null
+++ b/superpowers/plans/2026-11-10-itl-864-delta-read-your-writes.md
@@ -0,0 +1,174 @@
+# ITL-864 DeltaTableDatabase read-your-writes Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Every read method of `DeltaTableDatabase` returns exactly the rows a flush would leave, through every handle of an `at()` family, so `node.get_all_records()` returns every row right after the first run on a Delta store.
+
+**Architecture:** Two defects, one contract. (1) `at()` handles share the root's pending batches, and any handle's `flush()` writes every pending key, but each handle keeps its own `_delta_table_cache`, so a handle that loaded a table earlier keeps serving a stale `DeltaTable` snapshot after a sibling flushed. The fix shares `_delta_table_cache`, `_existing_ids_cache` and `_cache_dirty` across the family, exactly as pending state is already shared. (2) The read methods merge pending and stored rows inconsistently (`get_records_by_ids` ignores pending; `get_records_with_column_value` ignores stored whenever a pending batch exists; overlapping ids show twice). The fix: every read returns stored rows plus pending rows whose record id is not stored (Delta's flush is `when_not_matched_insert_all`, so the stored row wins), `None` when empty.
+
+**Tech Stack:** Python, pyarrow, deltalake, pytest, `uv run`.
+
+Approved by the coordinator (option A, Nov 10, 2026). No public API, hash, stored schema or dependency change. Error handling is untouched (PR #290 / ITL-884 rewrites it in the same file; merge main after it lands).
+
+---
+
+## Reproduction (Step 1 of the brief)
+
+`PipelineJob`, `DictSource` of 2 or 3 ints, `int -> int` pod, `DeltaTableDatabase`:
+
+| store | orchestrator | n | `get_all_records` | `iter_data` |
+|---|---|---|---|---|
+| delta | sync | 2 / 3 | 1 / 1 | 2 / 3 |
+| delta | async | 2 / 3 | 1 / 2 | 2 / 3 |
+| memory | both | 2 / 3 | 2 / 3 | 2 / 3 |
+
+Trace: the result-store handle (`at("pipeline", "_result")`) flushes the pipeline table's rows; the node's pipeline handle (`at("pipeline")`) cached the table at version 0 and still reads version 0 while the table is at version 2. Its pending batch is empty; a fresh handle reads all rows.
+
+## Contract and per-method findings (database level)
+
+Contract: a read through a handle reflects all writes of its `at()` family, pending and flushed together, and equals what a flush would leave: no duplicates, no missing rows.
+
+| case | `get_all_records` | `get_record_by_id` | `get_records_by_ids` | `get_records_with_column_value` |
+|---|---|---|---|---|
+| A sibling handle flushed | stale | stale | stale | stale |
+| B stored + pending, one handle | ok | ok | misses pending | misses stored when pending exists; empty table instead of `None` |
+| C pending id already stored, `skip_duplicates=False` | duplicate | returns pending row, flush keeps stored | (dup) | (dup) |
+| D same, `skip_duplicates=True` | ok | ok | ok | ok |
+| `record_id_column=` | returns (no hang) | | | |
+
+Other backends: `InMemoryArrowDatabase` shares `_tables` across `at()` and `ConnectorArrowDatabase` re-reads committed rows on every call, so neither goes stale; both merge pending + committed in every method. Both flush with the NEW row winning on id overlap (Delta keeps the stored row) and show both rows before flush under `skip_duplicates=False`. Report only.
+
+Thread safety: the async orchestrator runs only the user function in `run_in_executor` (`core/executors/local.py`); every database read and write happens on the event-loop thread, as for the already-shared pending dicts, so sharing the caches adds no cross-thread access.
+
+## File structure
+
+- Modify: `src/orcapod/databases/delta_lake_databases.py` — `__init__` / `at()` share caches; new helpers `_pending_not_stored` and `_combine_stored_and_pending`; the four read methods use them.
+- Create: `tests/test_databases/test_delta_read_your_writes.py` — database-level regression tests.
+- Create: `tests/test_pipeline/test_delta_node_records_after_run.py` — `PipelineJob` regression, both orchestrators.
+- Modify: `DESIGN_ISSUES.md` — one entry appended at the end.
+
+---
+
+### Task 1: Failing database-level tests
+
+**Files:** Create `tests/test_databases/test_delta_read_your_writes.py`
+
+- [ ] **Step 1: Write the tests**
+
+Helpers: `_rows(db, ids, vals, groups)` builds a table with `__record_id` (`large_binary`), `v` (`int64`), `g` (`int64`); `_values(table)` returns sorted `v` or `None`. A `READS` parametrization maps each read method to a call and its expected values:
+
+```python
+READS = {
+ "get_all_records": (lambda db: db.get_all_records(PATH), [1, 2]),
+ "get_record_by_id": (lambda db: db.get_record_by_id(PATH, b"b"), [2]),
+ "get_records_by_ids": (lambda db: db.get_records_by_ids(PATH, [b"a", b"b"]), [1, 2]),
+ "get_records_with_column_value": (
+ lambda db: db.get_records_with_column_value(PATH, {"g": 0}), [1, 2]),
+}
+```
+
+Tests:
+1. `test_sibling_handle_sees_rows_flushed_by_another_handle[method][flusher]`: `root = DeltaTableDatabase(tmp_path)`, `h1, h2 = root.at("s"), root.at("s")`; `h1` writes `a=1` with `flush=True` and reads once (loads its snapshot); `h2` writes `b=2`; read through `h1` and through `root` (path `("s",) + PATH`) before flush; flush through `flusher` in (`h2`, `root`, `h1`); read through `h1`, `root` and a new `root.at("s")` after flush. All equal the expected values.
+2. `test_pending_and_stored_rows_are_both_returned[method]`: one handle, stored `a=1 (g=0), b=2 (g=1)`, pending `c=3 (g=1)`: `get_records_by_ids([a, c]) == [1, 3]`, `get_records_with_column_value({"g": 1}) == [2, 3]`, `{"g": 0} == [1]`, `get_all_records == [1, 2, 3]`, `get_record_by_id(c) == [3]`.
+3. `test_no_match_returns_none`: stored + pending, `get_records_with_column_value({"g": 9})` and `get_records_by_ids([b"zz"])` are `None`.
+4. `test_pending_row_with_stored_id_reads_as_flush_keeps_it[skip_duplicates]`: stored `a=1`; pending add of `a=10, d=4`; before flush each method returns `a=1` and `d=4` once (`get_all_records == [1, 4]`, `get_record_by_id(a) == [1]`, `get_records_by_ids([a, d]) == [1, 4]`, `get_records_with_column_value({"g": 0}) == [1, 4]`); after `flush()` the same values.
+5. `test_record_id_column_is_returned_for_merged_rows`: stored + pending, `get_all_records(PATH, record_id_column="rid")` has column `rid` with all three ids and no `__record_id`; same for `get_records_by_ids` and `get_records_with_column_value`.
+
+- [ ] **Step 2: Run, verify the expected failures**
+
+Run: `uv run pytest tests/test_databases/test_delta_read_your_writes.py -v`
+Expected: test 1 fails for every method after flush; test 2 fails for `get_records_by_ids` / `get_records_with_column_value`; test 3 fails (empty table, not `None`); test 4 fails for `skip_duplicates=False`.
+
+### Task 2: Failing PipelineJob test
+
+**Files:** Create `tests/test_pipeline/test_delta_node_records_after_run.py`
+
+- [ ] **Step 1: Write the test**
+
+```python
+@pytest.mark.parametrize("orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator])
+def test_node_returns_every_row_right_after_first_run(tmp_path, orchestrator):
+ source = op.sources.DictSource([{"id": i, "x": 10 * i} for i in range(3)], tag_columns=["id"])
+
+ @op.function_pod(output_keys=["result"])
+ def double(x: int) -> int:
+ return x * 2
+
+ job = PipelineJob(store=DeltaTableDatabase(tmp_path))
+ with job:
+ double.pod(source, label="double")
+ job.run(orchestrator=orchestrator())
+
+ records = job.nodes["double"].get_all_records()
+ assert records is not None
+ assert sorted(records.column("result").to_pylist()) == [0, 20, 40]
+```
+
+- [ ] **Step 2: Run, verify it fails** (`1 != 3` rows) for both orchestrators.
+
+- [ ] **Step 3: Commit the failing tests**
+
+```bash
+git add tests/test_databases/test_delta_read_your_writes.py tests/test_pipeline/test_delta_node_records_after_run.py
+git commit -m "test(databases): regression tests for Delta reads missing own writes (ITL-864)"
+```
+
+### Task 3: Share the snapshot caches across the `at()` family
+
+**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (`__init__`, `at()`)
+
+- [ ] **Step 1:** Add keyword-only private constructor parameters `_shared_delta_table_cache`, `_shared_existing_ids_cache`, `_shared_cache_dirty`, used when given (same pattern as `_shared_pending_batches`); `at()` passes the root instance's dicts. Comment why: any family member's `flush()` writes every pending key and refreshes the cache entry for it, so the snapshot cache must be family-wide or a sibling keeps reading the old version.
+- [ ] **Step 2:** Run Task 1 test 1 and the Task 2 test: expected PASS for test 1 and Task 2.
+
+### Task 4: Merge stored and pending rows the way flush does
+
+**Files:** Modify `src/orcapod/databases/delta_lake_databases.py` (read methods)
+
+- [ ] **Step 1:** Add helpers:
+
+```python
+def _pending_not_stored(self, record_path):
+ """Pending rows whose record id is not stored; flush inserts only these."""
+ pending = self._pending_batches.get(self._get_record_key(record_path))
+ if pending is None or pending.num_rows == 0:
+ return None
+ stored_ids = self._get_existing_ids(record_path)
+ if stored_ids:
+ id_type = pending.schema.field(self.RECORD_ID_COLUMN).type
+ pending = pending.filter(pc.invert(pc.is_in(
+ pending[self.RECORD_ID_COLUMN], value_set=pa.array(list(stored_ids), type=id_type))))
+ return pending if pending.num_rows > 0 else None
+
+def _combine_stored_and_pending(self, stored, pending, record_id_column):
+ parts = [t for t in (stored, pending) if t is not None and t.num_rows > 0]
+ if not parts:
+ return None
+ table = parts[0] if len(parts) == 1 else pa.concat_tables(parts)
+ return self._handle_record_id_column(table, record_id_column)
+```
+
+- [ ] **Step 2:** `get_all_records`: pending part uses `_pending_not_stored`. `get_record_by_id`: serve from pending only when the id is pending and not stored. `get_records_by_ids`: stored read (existing try/except unchanged) plus pending filtered by the ids. `get_records_with_column_value`: stored read (existing try/except unchanged) plus pending filtered by the expression; `None` when empty.
+- [ ] **Step 3:** Run `uv run pytest tests/test_databases tests/test_pipeline -q`: all pass.
+- [ ] **Step 4:** Commit `fix(databases): Delta reads return every row the handle family wrote (ITL-864)`.
+
+### Task 5: Design issue entry, full verification, PR
+
+- [ ] Append one `DESIGN_ISSUES.md` entry (resolved, with **Fix:**; note that writes from other processes are still not seen until a handle reloads the table — follow-up for the coordinator to file).
+- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/` (golden hashes unchanged).
+- [ ] Merge `origin/main`, rerun if it moved, push, PR "Fixes ITL-864", `orca linear attach`, `orca worktree set --pr`.
+
+---
+
+## Review rounds and continuation notes (successor-readable)
+
+Done and pushed (verify, do not redo):
+- Round 1 (e9c78b5c): a failed ID refresh keeps the cache dirty; `get_records_by_ids` refresh shares the stored-read handler (8f9adf8b).
+- Round 2 (8f9adf8b): refresh when the record key has no ID-cache entry; stored + pending combined with `normalize_table_to_large_types` + `pa.concat_tables(promote_options="default")` (b8bcc817). Tests: `test_fresh_handle_drops_pending_rows_with_stored_ids`, evolved-schema tests.
+
+Round 3 (CodeRabbit review of b8bcc817, outside-diff, `get_records_with_column_value`), approved fix:
+- A side (stored schema, read without loading data; pending column names) that lacks a queried column contributes no rows; after a flush its rows hold null there and `col == value` never matches null. A column on neither side keeps today's `ArrowInvalid` ("No match for FieldRef").
+- Tests: `test_query_on_column_one_side_lacks_matches_read_after_flush[column_only_pending|column_only_stored]`, `test_query_on_column_neither_side_has_raises[flushed]`.
+- Also the round-1 outside-diff trivial: `_add_to_pending_batch` cast `list[str]` -> `list[bytes]`.
+- Reply: signed top-level PR comment (no thread for outside-diff findings), and one for the round-1 trivial.
+
+Then: merge origin/main (b197544e); wait for PR #290 (ITL-884, StoreReadError) to merge; merge main again, take #290's error handling around `_pending_not_stored` / `_combine_stored_and_pending`; a failed ID refresh must raise `StoreReadError` on every read method; full suite; worker_done only after the post-#290 merge is green.
diff --git a/superpowers/plans/2026-11-10-itl-892-exit-hang.md b/superpowers/plans/2026-11-10-itl-892-exit-hang.md
new file mode 100644
index 000000000..2cdd8d916
--- /dev/null
+++ b/superpowers/plans/2026-11-10-itl-892-exit-hang.md
@@ -0,0 +1,295 @@
+# ITL-892 Exit Hang Investigation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Reproduce and characterise the interpreter-exit hang after a Delta read (ITL-892), evaluate
+mitigations with hang rates over at least 40 runs each, and report to the coordinator before any
+`src/` or `uv.lock` change.
+
+**Architecture:** One timeout harness (`harness.py`) runs a target script N times as child processes,
+counts exits that miss the deadline, and samples the native stack of the first hung child (`sample`
+on macOS, `py-spy`/`gdb` not required). Target scripts are small and parameterised by argv so one
+harness drives every matrix cell. Version cells run in throwaway environments
+(`uv run --no-project --with pyarrow==X --with deltalake==Y`) so `uv.lock` never changes.
+
+**Tech Stack:** Python 3.11/3.12/3.13 (uv-managed), pyarrow, deltalake, polars, Orcapod
+`PipelineJob` on `DeltaTableDatabase`, GitHub Actions (temporary, non-required job) for Linux.
+
+Constraints (from the dispatch brief): no change to `uv.lock` or `src/` before the coordinator
+approves a mitigation. All experiments live in `superpowers/specs/itl-892-exit-hang/`. Draft PR body
+says `Refs ITL-892`.
+
+---
+
+### Task 1: Harness and minimal pure reproduction
+
+**Files:**
+- Create: `superpowers/specs/itl-892-exit-hang/harness.py`
+- Create: `superpowers/specs/itl-892-exit-hang/repro_pure.py`
+
+- [ ] Write `repro_pure.py`: builds a small Delta table in a temp dir and performs one operation
+ chosen by `--op` (`write`, `merge`, `to_table`, `to_dataset`, `dataset_scan`, `write_and_read`
+ — the last is the ITL-864 worker's script: 1 write, 4 merges, a `to_pyarrow_dataset().to_table()`
+ after each). Flags: `--import-polars`, `--cpu-threads N` (`pa.set_cpu_count`), `--io-threads N`
+ (`pa.set_io_thread_count`), `--mitigation {none,del,gc,os_exit,pool_shutdown}`, `--mp {none,fork,spawn}`.
+ Prints `done` with flush then returns.
+- [ ] Write `harness.py`: `harness.py N TIMEOUT -- , LocalFileSystem())` when the store is
+ local (`not self._is_cloud`). Do the same in `DeltaTableSource`, which already requires a
+ local path.
+2. Keep deltalake's default reader for cloud URIs. A native `S3FileSystem` has its own exit
+ deadlock ([apache/arrow#50188](https://github.com/apache/arrow/issues/50188)), and the
+ storage-option mapping is not one-to-one. Document the remaining exposure.
+3. Revisit when pyarrow ships #54467. Bump the floor then, and decide whether the helper can
+ go.
+
+Why (c) over (b3). (b3) also measured 0% everywhere, and it is the only option that also
+covers cloud stores. But it does not remove the cause. It only gives the pool time to finish
+before finalization starts. The 50 ms is a guess that held on these machines; a loaded machine
+or a large scan could outlast it, and the failure would come back silently. It would also run
+in every process that imported Orcapod.
+
+(c) removes the Python objects from the scan, so there is nothing for a worker to release
+late. Its results are byte-identical by check. A possible follow-up, not proposed for v0.3
+unless the developer wants cloud covered now: register the (b3) hook lazily, only the first
+time a cloud-store scan runs, so local users never pay for it.
+
+## 7. Regression check
+
+`regression_check_proto.py` (prototype, not in `tests/` yet): for each of two modes
+(`DeltaTableDatabase` write, flush and fresh-handle read; `DeltaTableSource` read), it starts 40
+fresh interpreters, 4 at a time. It fails if any run does not exit within 60 s or exits
+non-zero, which covers both the hang and the Linux abort. `ITL892_PATCH` lets the same check
+validate a prototype.
+
+| | macOS | Linux 3.11 / 3.12 / 3.13 |
+|---|---|---|
+| unpatched (today's `main`) | fails both cases (wall time 497 s, bounded by the 60 s deadlines) | fails both cases on each version (162 / 198 / 511 s) |
+| native-fs patch | passes, 10 s | passes on each version, 39 / 50 / 50 s |
+
+Detection. Measured per-run failure rates without a fix: macOS 53–77% (plain-read) and 8%
+(source-read); Linux 8–23% and 4–10%. With 40 runs per case, both cases miss a regression with
+probability below 1e-12 on macOS and below 1% per Linux Python job, which is about 1e-6 across
+CI's three Python jobs.
+
+False failures. A clean run takes 1–2 s, so the 60 s deadline leaves a wide margin. Every
+patched run passed (one macOS and three Linux runs of the check, plus thousands of patched
+harness runs).
+
+Recommendation: add it to `tests/` as an ordinary test that runs in CI, for example
+`tests/test_databases/test_delta_exit.py`, with the child script inline and only the public API.
+When it passes it costs about 40–50 s per Python job on GitHub runners (10 s on this Mac) and
+needs no infrastructure. If that time is unwelcome, halve `RUNS` to 20: a miss is then about 8%
+likely per job and 0.05% across the three jobs. It is reliable enough for CI. An opt-in marker
+would be justified only by its run time, not by flakiness.
+
+## 8. Not verified
+
+- Linux arm64 and Windows.
+- Cloud stores (S3/MinIO): not run. The analysis says they stay exposed with (c), since they
+ keep the Python filesystem.
+- Jupyter kernels: not affected in the same way, since the kernel does not exit.
+- The mechanism on 3.11/3.12 macOS is inferred: the worker had already vanished in every
+ sample, so its last frame was not captured. The 3.13 stacks show it directly.
+
+## 8a. Out-of-scope findings
+
+- `delta_lake_databases.py:868`: `return dataset.to_table()(filters=filters)` calls the
+ returned `Table`, which raises `TypeError` whenever `_read_delta_table` gets a filter
+ operator other than `=` or `in`.
+
+## 9. Files
+
+| file | purpose |
+|---|---|
+| `harness.py` | runs a command N times with a deadline; counts hangs and exit codes; samples the first hung child's native stack |
+| `repro_pure.py` | pyarrow + deltalake only; `--op`, `--read-via`, thread, polars, mp and mitigation switches |
+| `repro_orcapod.py` | the same through Orcapod's public API; `--patch` prototypes options (b) and (c) |
+| `run_matrix.sh` | the macOS matrix groups |
+| `check_native_fs_equivalence.py` | option (c) returns identical schemas and values |
+| `native_fs_pytest_plugin.py` | runs the suite with option (c) applied |
+| `regression_check_proto.py` | proposed regression check |
+| `summarize.py` | renders the tables below |
+| `results.jsonl` | macOS raw results (one line per cell) |
+| `samples/` | representative macOS stacks: py3.12 (`op=dataset_scan_py3.12_locked.txt`), py3.13 with the worker in `PyGILState_Ensure` (`dataset_scan_py3.13_pa23.0.1_dl1.0.2.txt`), Orcapod with no worker left (`orcapod_plain_read_hang_manual.txt`). The other stacks named in `results.jsonl` are not committed (`*.txt` is gitignored; about 4 MB) |
+| `linux/round*/` | Linux raw results from the temporary CI probe, with two py3.13 `gdb` stacks. On 3.11/3.12, `gdb` could not attach because the child aborted first |
+
+## 10. All results
+
+Each cell is one `harness.py` run (`results.jsonl`, `linux/*/results.jsonl`). "hung" means the run missed its deadline. In almost every cell it had already printed `done`; the exception is the `spawn` cell, where the parent waits on a hung child. Early exploratory cells (`baseline`, `load`, `amplify`, `itl864 ...`) used the ITL-864 script, which fails less often than the one-scan cell (`dataset_scan`) used afterwards.
+
+#### macOS arm64 (this machine)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| baseline itl864 py3.12 locked | 40 | 2 | 0 | 0 | **5%** |
+| load j=1 itl864 py3.12 locked | 60 | 2 | 0 | 0 | **3%** |
+| load j=4 itl864 py3.12 locked | 60 | 3 | 0 | 0 | **5%** |
+| amplify rounds=12 itl864 py3.12 locked | 40 | 0 | 0 | 0 | **0%** |
+| amplify rounds=24 itl864 py3.12 locked | 40 | 0 | 0 | 0 | **0%** |
+| load j=10 itl864 py3.12 locked | 60 | 6 | 0 | 0 | **10%** |
+| op=pyarrow_only py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| op=write py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| op=merge py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| op=to_dataset py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| op=to_table py3.12 locked | 100 | 57 | 0 | 0 | **57%** |
+| op=dataset_scan py3.12 locked | 100 | 57 | 0 | 0 | **57%** |
+| op=itl864 py3.12 locked | 100 | 12 | 0 | 0 | **12%** |
+| itl864 read-via=native_fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| itl864 read-via=no_threads py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| itl864 read-via=querybuilder py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| itl864 read-via=files py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| itl864 read-via=polars py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| itl864 cpu_threads=1 | 100 | 6 | 0 | 0 | **6%** |
+| dataset_scan read-via=native_fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan read-via=no_threads py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan read-via=querybuilder py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan read-via=files py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan read-via=polars py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan cpu_threads=1 | 100 | 33 | 0 | 0 | **33%** |
+| dataset_scan io_threads=1 | 100 | 53 | 0 | 0 | **53%** |
+| dataset_scan OMP_NUM_THREADS=1 | 100 | 45 | 0 | 0 | **45%** |
+| dataset_scan RAYON/TOKIO threads=1 | 100 | 49 | 0 | 0 | **49%** |
+| dataset_scan import polars after pyarrow | 100 | 48 | 0 | 0 | **48%** |
+| dataset_scan import polars first | 100 | 27 | 0 | 0 | **27%** |
+| dataset_scan in fork child | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan in spawn child | 100 | 90 | 0 | 0 | **90%** |
+| dataset_scan mitigation=del | 100 | 2 | 0 | 0 | **2%** |
+| dataset_scan mitigation=atexit_del | 100 | 1 | 0 | 0 | **1%** |
+| dataset_scan mitigation=os_exit | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan mitigation=atexit_sleep | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan mitigation=atexit_sleep_long | 100 | 0 | 0 | 0 | **0%** |
+| orcapod plain-read py3.12 locked | 100 | 53 | 0 | 0 | **53%** |
+| orcapod source-read py3.12 locked | 100 | 8 | 0 | 0 | **8%** |
+| orcapod pipeline-sync py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod rerun py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan py3.11 pa23.0.1 dl1.0.2 | 100 | 57 | 0 | 0 | **57%** |
+| dataset_scan native_fs py3.11 pa23.0.1 dl1.0.2 | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan py3.12 pa23.0.1 dl1.0.2 | 100 | 50 | 0 | 0 | **50%** |
+| dataset_scan native_fs py3.12 pa23.0.1 dl1.0.2 | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan py3.13 pa23.0.1 dl1.0.2 | 100 | 65 | 0 | 0 | **65%** |
+| dataset_scan native_fs py3.13 pa23.0.1 dl1.0.2 | 100 | 0 | 0 | 0 | **0%** |
+| dataset_scan py3.12 pa20.0.0 dl1.6.6 | 100 | 59 | 0 | 0 | **59%** |
+| dataset_scan py3.12 pa22.0.0 dl1.0.2 | 100 | 55 | 0 | 0 | **55%** |
+| dataset_scan py3.12 pa22.0.0 dl1.6.6 | 100 | 55 | 0 | 0 | **55%** |
+| dataset_scan py3.12 pa23.0.1 dl1.6.6 | 100 | 61 | 0 | 0 | **61%** |
+| dataset_scan py3.12 pa24.0.0 dl1.0.2 | 100 | 53 | 0 | 0 | **53%** |
+| dataset_scan py3.12 pa24.0.0 dl1.6.6 | 100 | 58 | 0 | 0 | **58%** |
+| dataset_scan py3.12 pa25.0.1 dl1.0.2 | 100 | 36 | 0 | 0 | **36%** |
+| dataset_scan py3.12 pa25.0.1 dl1.6.6 | 100 | 39 | 0 | 0 | **39%** |
+| dataset_scan py3.12 pa26.0.1 dl1.0.2 | 100 | 56 | 0 | 0 | **56%** |
+| dataset_scan py3.12 pa26.0.1 dl1.6.6 | 100 | 68 | 0 | 0 | **68%** |
+| dataset_scan py3.12 pa20.0.0 dl1.0.2 | 100 | 67 | 0 | 0 | **67%** |
+| orcapod plain-read patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod source-read patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-sync patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod plain-read patch=no-threads py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod source-read patch=no-threads py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-sync patch=no-threads py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async patch=no-threads py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod plain-read patch=atexit-gc py3.12 locked | 100 | 27 | 0 | 0 | **27%** |
+| orcapod source-read patch=atexit-gc py3.12 locked | 100 | 5 | 0 | 0 | **5%** |
+| orcapod pipeline-sync patch=atexit-gc py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async patch=atexit-gc py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod plain-read patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod source-read patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-sync patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod plain-read py3.11 locked | 100 | 77 | 0 | 0 | **77%** |
+| orcapod pipeline-sync py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod plain-read py3.13 locked | 100 | 76 | 0 | 0 | **76%** |
+| orcapod pipeline-sync py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| orcapod pipeline-async py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+
+#### Linux x86_64, GitHub ubuntu-latest (round1-py3.11)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| linux op=dataset_scan py3.11 locked | 100 | 8 | 13 | 0 | **21%** |
+| linux op=to_table py3.11 locked | 100 | 4 | 19 | 0 | **23%** |
+| linux op=itl864 py3.11 locked | 100 | 8 | 0 | 0 | **8%** |
+| linux dataset_scan native_fs py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux dataset_scan no_threads py3.11 locked | 100 | 11 | 0 | 0 | **11%** |
+| linux dataset_scan io_threads=1 py3.11 locked | 100 | 18 | 0 | 0 | **18%** |
+| linux dataset_scan mitigation=del py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read py3.11 locked | 100 | 12 | 0 | 0 | **12%** |
+| linux orcapod source-read py3.11 locked | 100 | 4 | 0 | 0 | **4%** |
+| linux orcapod pipeline-sync py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-async py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux op=dataset_scan py3.11 pa22.0.0 dl1.0.2 | 100 | 6 | 0 | 0 | **6%** |
+| linux op=dataset_scan py3.11 pa26.0.1 dl1.6.6 | 100 | 2 | 2 | 0 | **4%** |
+
+#### Linux x86_64, GitHub ubuntu-latest (round1-py3.12)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| linux op=dataset_scan py3.12 locked | 100 | 10 | 8 | 0 | **18%** |
+| linux op=to_table py3.12 locked | 100 | 3 | 17 | 0 | **20%** |
+| linux op=itl864 py3.12 locked | 100 | 8 | 0 | 0 | **8%** |
+| linux dataset_scan native_fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux dataset_scan no_threads py3.12 locked | 100 | 20 | 1 | 0 | **21%** |
+| linux dataset_scan io_threads=1 py3.12 locked | 100 | 18 | 0 | 0 | **18%** |
+| linux dataset_scan mitigation=del py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read py3.12 locked | 100 | 8 | 0 | 0 | **8%** |
+| linux orcapod source-read py3.12 locked | 100 | 5 | 0 | 0 | **5%** |
+| linux orcapod pipeline-sync py3.12 locked | 100 | 4 | 0 | 0 | **4%** |
+| linux orcapod pipeline-async py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux op=dataset_scan py3.12 pa22.0.0 dl1.0.2 | 100 | 13 | 0 | 0 | **13%** |
+| linux op=dataset_scan py3.12 pa26.0.1 dl1.6.6 | 100 | 2 | 1 | 0 | **3%** |
+
+#### Linux x86_64, GitHub ubuntu-latest (round1-py3.13)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| linux op=dataset_scan py3.13 locked | 100 | 5 | 0 | 0 | **5%** |
+| linux op=to_table py3.13 locked | 100 | 38 | 0 | 0 | **38%** |
+| linux op=itl864 py3.13 locked | 100 | 10 | 0 | 0 | **10%** |
+| linux dataset_scan native_fs py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux dataset_scan no_threads py3.13 locked | 100 | 8 | 0 | 0 | **8%** |
+| linux dataset_scan io_threads=1 py3.13 locked | 100 | 32 | 0 | 0 | **32%** |
+| linux dataset_scan mitigation=del py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read py3.13 locked | 100 | 15 | 0 | 0 | **15%** |
+| linux orcapod source-read py3.13 locked | 100 | 10 | 0 | 0 | **10%** |
+| linux orcapod pipeline-sync py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-async py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux op=dataset_scan py3.13 pa22.0.0 dl1.0.2 | 100 | 14 | 0 | 0 | **14%** |
+| linux op=dataset_scan py3.13 pa26.0.1 dl1.6.6 | 100 | 5 | 0 | 0 | **5%** |
+
+#### Linux x86_64, GitHub ubuntu-latest (round2-py3.11)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| linux dataset_scan mitigation=atexit_sleep py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux dataset_scan mitigation=atexit_sleep_long py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=none py3.11 locked | 100 | 9 | 0 | 0 | **9%** |
+| linux orcapod source-read patch=none py3.11 locked | 100 | 8 | 0 | 0 | **8%** |
+| linux orcapod pipeline-sync patch=none py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=native-fs py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod source-read patch=native-fs py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-sync patch=native-fs py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=atexit-sleep py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod source-read patch=atexit-sleep py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-sync patch=atexit-sleep py3.11 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=atexit-gc py3.11 locked | 100 | 8 | 13 | 0 | **21%** |
+| linux orcapod source-read patch=atexit-gc py3.11 locked | 100 | 4 | 1 | 0 | **5%** |
+| linux orcapod pipeline-sync patch=atexit-gc py3.11 locked | 100 | 1 | 0 | 0 | **1%** |
+
+#### Linux x86_64, GitHub ubuntu-latest (round2-py3.12)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| linux dataset_scan mitigation=atexit_sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux dataset_scan mitigation=atexit_sleep_long py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=none py3.12 locked | 100 | 8 | 11 | 0 | **19%** |
+| linux orcapod source-read patch=none py3.12 locked | 100 | 8 | 2 | 0 | **10%** |
+| linux orcapod pipeline-sync patch=none py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod source-read patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-sync patch=native-fs py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod source-read patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-sync patch=atexit-sleep py3.12 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=atexit-gc py3.12 locked | 100 | 11 | 1 | 0 | **12%** |
+| linux orcapod source-read patch=atexit-gc py3.12 locked | 100 | 6 | 0 | 0 | **6%** |
+| linux orcapod pipeline-sync patch=atexit-gc py3.12 locked | 100 | 1 | 0 | 0 | **1%** |
+
+#### Linux x86_64, GitHub ubuntu-latest (round2-py3.13)
+
+| cell | runs | hung | aborted (rc<0) | other rc!=0 | failed |
+|---|---:|---:|---:|---:|---:|
+| linux dataset_scan mitigation=atexit_sleep py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux dataset_scan mitigation=atexit_sleep_long py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=none py3.13 locked | 100 | 23 | 0 | 0 | **23%** |
+| linux orcapod source-read patch=none py3.13 locked | 100 | 8 | 0 | 0 | **8%** |
+| linux orcapod pipeline-sync patch=none py3.13 locked | 100 | 1 | 0 | 0 | **1%** |
+| linux orcapod plain-read patch=native-fs py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod source-read patch=native-fs py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-sync patch=native-fs py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=atexit-sleep py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod source-read patch=atexit-sleep py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod pipeline-sync patch=atexit-sleep py3.13 locked | 100 | 0 | 0 | 0 | **0%** |
+| linux orcapod plain-read patch=atexit-gc py3.13 locked | 100 | 17 | 0 | 0 | **17%** |
+| linux orcapod source-read patch=atexit-gc py3.13 locked | 100 | 6 | 0 | 0 | **6%** |
+| linux orcapod pipeline-sync patch=atexit-gc py3.13 locked | 100 | 1 | 0 | 0 | **1%** |
+
+
+## 11. Proposed delta-rs issue (not filed; for the developer)
+
+**Title:** `to_pyarrow_dataset()`'s default `PyFileSystem(DeltaStorageHandler)` makes processes hang or abort at interpreter exit
+
+**Body:**
+
+> A process that scans a Delta table with `DeltaTable.to_pyarrow_dataset().to_table()` or
+> `to_pyarrow_table()` and then exits normally often fails during interpreter shutdown. It
+> hangs forever (macOS, any Python; Linux, Python 3.13), or on Linux with Python 3.11/3.12 it
+> aborts with `terminate called without an active exception`. All of its work and output are
+> done by then.
+>
+> ```python
+> import tempfile, deltalake, pyarrow as pa
+> p = tempfile.mkdtemp()
+> deltalake.write_deltalake(p, pa.table({"k": [1, 2, 3]}))
+> deltalake.DeltaTable(p).to_pyarrow_dataset().to_table()
+> print("done") # prints, then the process may never exit
+> ```
+>
+> Rate over 100 fresh processes: about 57% on macOS arm64 and 5–38% on Linux x86_64. It
+> happens on every combination we tried: pyarrow 20.0.0–26.0.1 × deltalake 1.0.2 and 1.6.6,
+> Python 3.11–3.13. Writes and merges alone never trigger it.
+>
+> Native stacks (Python 3.13): the main thread is in `exit → __cxa_finalize →
+> arrow::internal::ThreadPool::~ThreadPool → Shutdown`. A pool worker is in
+> `parquet::arrow::FileReaderImpl::~FileReaderImpl → ReadRangeCache::~ReadRangeCache →
+> arrow::py::PyBuffer::~PyBuffer → PyGILState_Ensure → PyThread_hang_thread`. The cached
+> buffers come from `DeltaStorageHandler` (Python `bytes`; `pre_buffer=True`), and Arrow
+> releases them on a worker after `to_table()` returns. If that happens during finalization,
+> the worker cannot take the GIL. The root bug is in Arrow: apache/arrow#50482, #54536, fix PR
+> #54467.
+>
+> Passing a native filesystem avoids it completely (0/100 on macOS and Linux, identical
+> results, partitioned tables included). `table_uri` is percent-encoded (a bare path on 1.0.2
+> and 1.2.1, a `file://` URI on 1.6.6), so it has to be decoded first:
+>
+> ```python
+> uri = dt.table_uri if "://" in dt.table_uri else "file://" + dt.table_uri
+> fs, root = pyarrow.fs.FileSystem.from_uri(uri)
+> dt.to_pyarrow_dataset(filesystem=pyarrow.fs.SubTreeFileSystem(root, fs))
+> ```
+>
+> Suggestion: default to a native pyarrow filesystem for local tables (and possibly the native
+> object-store filesystems), or document the issue and the workaround in `to_pyarrow_dataset`
+> until Arrow's fix is released.
diff --git a/superpowers/specs/itl-892-exit-hang/check_native_fs_equivalence.py b/superpowers/specs/itl-892-exit-hang/check_native_fs_equivalence.py
new file mode 100644
index 000000000..068a3befd
--- /dev/null
+++ b/superpowers/specs/itl-892-exit-hang/check_native_fs_equivalence.py
@@ -0,0 +1,121 @@
+"""Option (c) check: a native filesystem returns exactly what deltalake's default returns.
+
+Writes a Delta table with a spread of Arrow types (narrow ints, decimals, timestamps,
+nested lists and structs, binary, nulls, non-nullable fields), appends and merges into
+it, then reads it with ``to_pyarrow_dataset(as_large_types=True).to_table()`` twice:
+once through the default ``PyFileSystem(DeltaStorageHandler)`` and once through
+``SubTreeFileSystem(table_path, LocalFileSystem())``. Schemas (with field metadata and
+nullability) and values must be identical, with and without a filter expression.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import decimal
+import tempfile
+
+import deltalake
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.fs as pa_fs
+
+
+def make(i: int) -> pa.Table:
+ """Build 50 rows covering narrow, decimal, timestamp, binary, nested and null columns.
+
+ Args:
+ i: Batch number, which keeps the ids of each batch distinct.
+
+ Returns:
+ The table.
+ """
+ schema = pa.schema(
+ [
+ pa.field("id", pa.large_string(), nullable=False),
+ pa.field("i8", pa.int8()),
+ pa.field("f32", pa.float32()),
+ pa.field("dec", pa.decimal128(12, 3)),
+ pa.field("ts", pa.timestamp("us", tz="UTC")),
+ pa.field("bin", pa.large_binary()),
+ pa.field("lst", pa.large_list(pa.int16())),
+ pa.field(
+ "st",
+ pa.struct(
+ [pa.field("a", pa.int32()), pa.field("b", pa.large_string())]
+ ),
+ ),
+ pa.field("n", pa.large_string()),
+ ]
+ )
+ rows = 50
+ return pa.table(
+ {
+ "id": [f"r{i}-{k}" for k in range(rows)],
+ "i8": [k % 100 for k in range(rows)],
+ "f32": [k / 3 for k in range(rows)],
+ "dec": [decimal.Decimal(k * 1001) / 1000 for k in range(rows)],
+ "ts": [dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc)] * rows,
+ "bin": [bytes([k % 256]) * 3 for k in range(rows)],
+ "lst": [[k, k + 1] if k % 5 else None for k in range(rows)],
+ "st": [{"a": k, "b": str(k)} for k in range(rows)],
+ "n": [None] * rows,
+ },
+ schema=schema,
+ )
+
+
+def read(
+ table: deltalake.DeltaTable, native: bool, flt: pc.Expression | None
+) -> pa.Table:
+ """Read ``table`` through the default or the native filesystem, sorted by ``id``.
+
+ Args:
+ table: The Delta table.
+ native: Read through a native ``SubTreeFileSystem`` instead of deltalake's.
+ flt: Optional filter applied by the scan.
+
+ Returns:
+ The scanned rows.
+ """
+ kwargs = {}
+ if native:
+ kwargs["filesystem"] = pa_fs.SubTreeFileSystem(
+ table.table_uri, pa_fs.LocalFileSystem()
+ )
+ ds = table.to_pyarrow_dataset(as_large_types=True, **kwargs)
+ return ds.to_table(filter=flt).sort_by("id")
+
+
+def main() -> None:
+ """Compare default and native reads of one table across versions and filters."""
+ path = tempfile.mkdtemp()
+ deltalake.write_deltalake(path, make(0))
+ deltalake.write_deltalake(path, make(1), mode="append")
+ deltalake.DeltaTable(path).merge(
+ source=make(2),
+ predicate="target.id = source.id",
+ source_alias="source",
+ target_alias="target",
+ ).when_not_matched_insert_all().execute()
+ table = deltalake.DeltaTable(path)
+ checks = 0
+ rows = 0
+ for flt in (
+ None,
+ pc.field("i8") > 10,
+ pc.is_in(pc.field("id"), pa.array(["r1-3", "r2-4"])),
+ ):
+ a, b = read(table, False, flt), read(table, True, flt)
+ assert a.schema.equals(b.schema, check_metadata=True), (a.schema, b.schema)
+ assert a.equals(b), "values differ"
+ checks += 1
+ rows = a.num_rows
+ print(
+ f"files={len(table.file_uris())} last_filter_rows={rows} "
+ f"checks={checks} identical schema+values"
+ )
+ print("done", flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/superpowers/specs/itl-892-exit-hang/harness.py b/superpowers/specs/itl-892-exit-hang/harness.py
new file mode 100644
index 000000000..a1c215523
--- /dev/null
+++ b/superpowers/specs/itl-892-exit-hang/harness.py
@@ -0,0 +1,150 @@
+"""Run a command many times and count the runs that do not exit before a deadline.
+
+Usage:
+ harness.py --label LABEL -n 40 -t 20 -j 4 --