diff --git a/CHANGELOG.md b/CHANGELOG.md index b9688f8a4..d03a7c9e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,16 @@ that raised. See *When the store cannot be written* in the function-pods guide. - `PolarsFilter(predicates=[...])` and `stream.polars_filter(...)` with the same predicates now have the same hashes (`predicates` is stored as a tuple). +#### A `PipelineJob` on an in-memory store runs in time linear in its rows (ITL-919) + +A `PipelineJob` on an `InMemoryArrowDatabase` runs in time linear in its rows. Every row a +function node processed used to filter, copy or rebuild the whole stored table (the +result lookup, the recomputation-index lookup, the duplicate check and the flush, each +over a table that gained one chunk per row), so 1,000 rows took about a minute. Lookups +now use hash indexes over the stored rows and a flush appends without copying them; +stored rows, ids and hashes are unchanged. A `DeltaTableDatabase` store is still slow +(one Delta commit per row) and is tracked separately (ITL-919). + ## [0.2.0] — 2026-11-10 **Distribution:** Orcapod is MIT licensed starting with `0.2.0` (it was proprietary on diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 616fc31fe..b81d6da38 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2380,6 +2380,8 @@ newcomer. Re-checked on `b197544e` (v0.2.0 + #291): column ... because it's a binary field` lines on every Delta write still appear. **Open:** awaiting the developer's choice of how to silence them. +--- + ## `src/orcapod/core/operators/filters.py` — `PolarsFilter` identity and nullability (ITL-828) ### PF1 — An expression predicate could not be hashed, and a filter broke a join inside a job @@ -2410,3 +2412,37 @@ source row. 2026). A filtered row's data hash now equals its source row's. The pinned `pipeline:polars_filter` row data hashes moved (approved); they are the cache keys of any function pod downstream of a filter, so a 0.2 store recomputes those results once. + +--- + +## `src/orcapod/databases/` — PipelineJob run time quadratic in rows + +### PQ1 — Every row a function node processes scans the rows already stored +**Status:** in progress +**Severity:** high +**Issue:** ITL-919 + +A function node in a `PipelineJob` does, per row: `ResultCache.lookup` (lookup by input data +hash in the result table), `ResultCache.store` with a flush of the whole store (both +tables), and `add_pipeline_record` (lookup by base entry id for the recomputation index, +then `add_record(skip_duplicates=True)`). Sync and async orchestrators share that path; +operator nodes write with one `add_records` and are not affected. Measured on `c8efc092`, +in-memory store, sync: 7.2 / 20.9 / 64 ms per row at 100 / 400 / 1,000 rows (standalone +pod: 0.09 ms). + +- `InMemoryArrowDatabase`: each lookup filtered the whole table, the duplicate check built + the set of every committed id, and each flush ran `is_in` over all ids, filtered and + concatenated, adding one chunk per row, so every later Arrow operation cost O(rows). + **Fix:** lazily built hash indexes over committed rows (binary and string columns, record + ids), extended on append and rebuilt only after a flush overwrites an id; flush keeps the + committed table at O(log N) record batches by binary-counter compaction. Stored rows, row + order, ids and schemas are unchanged (differential check against `main` on a joined + multi-node pipeline, both orchestrators). Store work per row is now independent of rows + stored (tests count Arrow bytes allocated per row). +- `DeltaTableDatabase`: **open.** One Delta MERGE commit per row per table (cost grows with + the table and leaves one file per row), lookups re-open and read every file, the + existing-id cache is refreshed after every flush, and the pending batch is scanned per + lookup. Making it fast changes when rows are committed; awaiting the developer's decision. + Related: `get_records_with_column_value` searches only the pending batch when one exists + (ignores committed rows) and `get_records_by_ids` ignores pending rows; reported to the + coordinator and filed as ITL-923, not fixed here. diff --git a/src/orcapod/databases/in_memory_databases.py b/src/orcapod/databases/in_memory_databases.py index d25fd1b5a..dc0e77140 100644 --- a/src/orcapod/databases/in_memory_databases.py +++ b/src/orcapod/databases/in_memory_databases.py @@ -3,6 +3,7 @@ import logging from collections import defaultdict from collections.abc import Collection, Mapping +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast from orcapod.databases.utils import coerce_record_id @@ -19,6 +20,49 @@ logger = logging.getLogger(__name__) +@dataclass +class _KeyIndex: + """Hash indexes over the committed rows of one record key. + + Each column index maps a value to the positions, in ascending order, of the rows of + ``table`` holding it. The indexes describe ``table`` only: they are valid while + ``table`` is the key's committed table, and are rebuilt for any other table. + """ + + table: "pa.Table" + columns: dict[str, dict[Any, list[int]]] = field(default_factory=dict) + + +def _is_indexed_lookup(arrow_type: "pa.DataType", value: Any) -> bool: + """Whether an equality lookup of ``value`` in a column can use a hash index. + + Only binary and string columns queried with a value of the matching Python type + qualify: for those, Python equality of the values is exactly Arrow's equality. Every + other lookup filters the table, as Arrow defines that comparison. + """ + if pa.types.is_binary(arrow_type) or pa.types.is_large_binary(arrow_type): + return type(value) is bytes + if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type): + return type(value) is str + return False + + +def _compact_tail(table: "pa.Table") -> "pa.Table": + """Merge the table's trailing record batches while an earlier one is not larger. + + Called after each append, this keeps batch sizes strictly decreasing, like a binary + counter: a table of N rows has at most log2(N) + 1 batches, and each row is copied + O(log N) times over the table's life. Every Arrow operation on a table costs time per + batch, so without it a table grown one row per flush makes every later operation + cost time proportional to the rows already stored. + """ + batches = [batch for batch in table.to_batches() if batch.num_rows > 0] + while len(batches) >= 2 and batches[-2].num_rows <= batches[-1].num_rows: + merged = pa.Table.from_batches(batches[-2:]).combine_chunks() + batches[-2:] = merged.to_batches() + return pa.Table.from_batches(batches, schema=table.schema) + + class InMemoryArrowDatabase: """ A pure in-memory implementation of the ArrowDatabaseProtocol protocol. @@ -29,6 +73,10 @@ class InMemoryArrowDatabase: Supports the same pending-batch semantics as DeltaTableDatabase: records are buffered in a pending batch and become part of the committed store only after flush() is called (or flush=True is passed to a write method). + + Lookups by record ID or by a binary or string column value use hash indexes over + the committed rows, and each flush appends without copying the stored rows, so the + cost of a per-row write, flush or lookup does not grow with the rows already stored. """ RECORD_ID_COLUMN = "__record_id" @@ -40,6 +88,7 @@ def __init__( _shared_tables: "dict[str, pa.Table] | None" = None, _shared_pending_batches: "dict[str, pa.Table] | None" = None, _shared_pending_record_ids: "dict[str, set[bytes]] | None" = None, + _shared_indexes: "dict[str, _KeyIndex] | None" = None, _root: InMemoryArrowDatabase | None = None, _scoped_path: tuple[str, ...] = (), ): @@ -58,6 +107,9 @@ def __init__( if _shared_pending_record_ids is not None else defaultdict(set) ) + self._indexes: dict[str, _KeyIndex] = ( + _shared_indexes if _shared_indexes is not None else {} + ) # ------------------------------------------------------------------ # Path helpers @@ -146,22 +198,45 @@ def _deduplicate_within_table(self, table: "pa.Table") -> "pa.Table": # Internal helpers for duplicate detection # ------------------------------------------------------------------ - def _committed_ids(self, record_key: str) -> "set[bytes]": + def _column_index( + self, record_key: str, committed: "pa.Table", column: str + ) -> dict[Any, list[int]]: + """Return the index of ``column`` over ``committed``, the key's committed table. + + Builds it on first use; ``flush`` extends it as rows are appended. + """ + key_index = self._indexes.get(record_key) + if key_index is None or key_index.table is not committed: + key_index = _KeyIndex(committed) + self._indexes[record_key] = key_index + positions = key_index.columns.get(column) + if positions is None: + positions = {} + for position, value in enumerate(committed.column(column).to_pylist()): + positions.setdefault(value, []).append(position) + key_index.columns[column] = positions + return positions + + def _committed_id_index(self, record_key: str) -> dict[Any, list[int]]: + """Return the record-ID index of the key's committed rows (empty if none).""" committed = self._tables.get(record_key) if committed is None or committed.num_rows == 0: - return set() - existing_ids = committed[self.RECORD_ID_COLUMN].to_pylist() - # TODO: evaluate the efficiency of this implementation - return {id for id in existing_ids if id is not None} + return {} + return self._column_index(record_key, committed, self.RECORD_ID_COLUMN) def _filter_existing_records( self, record_key: str, table: "pa.Table" ) -> "pa.Table": """Filter out records whose IDs are already in pending or committed store.""" input_ids = set(table[self.RECORD_ID_COLUMN].to_pylist()) - all_existing = input_ids & ( - self._pending_record_ids[record_key] | self._committed_ids(record_key) - ) + pending_ids = self._pending_record_ids[record_key] + committed_ids = self._committed_id_index(record_key) + all_existing = { + record_id + for record_id in input_ids + if record_id in pending_ids + or (record_id is not None and record_id in committed_ids) + } if not all_existing: return table mask = pc.invert( @@ -272,16 +347,35 @@ def flush(self) -> None: committed = self._tables.get(record_key) if committed is None: - self._tables[record_key] = pending - else: - # Insert-if-not-exists: keep committed rows not overwritten by new batch, - # then append the new batch on top. - new_ids = set(pending[self.RECORD_ID_COLUMN].to_pylist()) + self._tables[record_key] = pending.combine_chunks() + continue + + # Insert-if-not-exists: keep committed rows not overwritten by new batch, + # then append the new batch on top. + new_ids = pending[self.RECORD_ID_COLUMN].to_pylist() + committed_ids = self._committed_id_index(record_key) + if any(record_id in committed_ids for record_id in new_ids): mask = pc.invert( pc.is_in(committed[self.RECORD_ID_COLUMN], pa.array(list(new_ids))) ) kept = committed.filter(mask) - self._tables[record_key] = pa.concat_tables([kept, pending]) + self._tables[record_key] = pa.concat_tables( + [kept, pending] + ).combine_chunks() + # Rows moved: the indexes are rebuilt on the next lookup. + self._indexes.pop(record_key, None) + continue + + appended = _compact_tail(pa.concat_tables([committed, pending])) + self._tables[record_key] = appended + key_index = self._indexes.get(record_key) + if key_index is not None and key_index.table is committed: + offset = committed.num_rows + for column, positions in key_index.columns.items(): + values = pending.column(column).to_pylist() + for position, value in enumerate(values, start=offset): + positions.setdefault(value, []).append(position) + key_index.table = appended # ------------------------------------------------------------------ # Path scoping @@ -322,6 +416,7 @@ def at(self, *path_components: str) -> "InMemoryArrowDatabase": _shared_tables=self._tables, _shared_pending_batches=self._pending_batches, _shared_pending_record_ids=self._pending_record_ids, + _shared_indexes=self._indexes, _root=new_root, _scoped_path=new_scoped_path, ) @@ -492,12 +587,18 @@ def get_records_with_column_value( self.flush() record_key = self._get_record_key(record_path) - table = self._combined_table(record_key) - if table is None: + committed = self._tables.get(record_key) + if committed is not None and committed.num_rows == 0: + committed = None + pending = self._pending_batches.get(record_key) + if pending is not None and pending.num_rows == 0: + pending = None + if committed is None and pending is None: return None + pair_list: list[tuple[str, Any]] if isinstance(column_values, Mapping): - pair_list = list(column_values.items()) + pair_list = list(cast(Mapping[str, Any], column_values).items()) else: pair_list = cast(list[tuple[str, Any]], list(column_values)) @@ -506,7 +607,49 @@ def get_records_with_column_value( for expr in expressions[1:]: combined_expr = combined_expr & expr - filtered = table.filter(combined_expr) + parts = [] + if committed is not None: + parts.append( + self._filter_committed(record_key, committed, pair_list, combined_expr) + ) + if pending is not None: + parts.append(pending.filter(combined_expr)) + filtered = parts[0] if len(parts) == 1 else pa.concat_tables(parts) if filtered.num_rows == 0: return None return self._handle_record_id_column(filtered, record_id_column) + + def _filter_committed( + self, + record_key: str, + committed: "pa.Table", + pair_list: list[tuple[str, Any]], + expression: "pc.Expression", + ) -> "pa.Table": + """Return the committed rows matching ``expression``, in table order. + + The pairs that can use a hash index select the rows at the intersection of + their indexes' positions. When every pair can, those rows are the result; + when only some can, just those rows are filtered by ``expression``; when none + can, the whole table is. + """ + indexed = [ + (column, value) + for column, value in pair_list + if column in committed.column_names + and _is_indexed_lookup(committed.schema.field(column).type, value) + ] + if not indexed: + return committed.filter(expression) + + matches: set[int] | None = None + for column, value in indexed: + found = self._column_index(record_key, committed, column).get(value, []) + matches = set(found) if matches is None else matches.intersection(found) + # Zero-copy row slices in table order: ``take`` on a multi-chunk table + # concatenates every chunk first, copying all stored rows. + rows = [committed.slice(p, 1) for p in sorted(matches or ())] + candidates = pa.concat_tables(rows) if rows else committed.slice(0, 0) + if len(indexed) == len(pair_list): + return candidates + return candidates.filter(expression) diff --git a/superpowers/plans/2026-11-11-itl-919-in-memory-store-linear.md b/superpowers/plans/2026-11-11-itl-919-in-memory-store-linear.md new file mode 100644 index 000000000..7c26bf1af --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-919-in-memory-store-linear.md @@ -0,0 +1,123 @@ +# ITL-919 (part 1): linear PipelineJob runs on InMemoryArrowDatabase — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make every per-row `InMemoryArrowDatabase` operation a `PipelineJob` performs cost +O(rows touched), not O(rows already stored), so job run time is linear in rows. Stored rows, +row order, record ids, hashes, paths, schemas, public API and write timing stay identical. + +**Architecture:** Two internal changes to `src/orcapod/databases/in_memory_databases.py` only: +(1) the committed table is kept at O(log N) chunks by binary-counter compaction on `flush`; +(2) per-record-key, per-column hash indexes over committed rows (value → sorted row +positions), built lazily, extended on append, dropped when a flush overwrites an existing id. +Lookups (`get_records_with_column_value`, the skip-duplicates check, flush's overwrite check) +use the index and filter only candidate rows. + +**Tech Stack:** PyArrow 23, pytest. + +**Scope:** In-memory store only (coordinator decision, November 11, 2026). Delta is deferred +pending the developer's decision on write timing (options D1/D2/D3 in the ask); do NOT touch +`delta_lake_databases.py`, `result_cache.py` flush timing, `function_node.py` or `job.py`. + +--- + +## Cause (STEP 1 findings, main c8efc092) + +Only function nodes are affected (operator nodes write with one `add_records`). Sync and async +orchestrators both run, per row: `ResultCache.lookup` → `store` (+ auto-flush) → +`add_pipeline_record`. Per row, against the result table and the pipeline table: + +1. `ResultCache.lookup` → `get_records_with_column_value(input hash)`: filters the whole table. +2. `add_pipeline_record` → `get_records_with_column_value(base_entry_id)`: filters the whole + table; then `add_record(skip_duplicates=True)` → `_committed_ids` → `to_pylist` of every id. +3. `ResultCache.store` auto-flushes every row; in-memory `flush` does `is_in` over all committed + ids + `filter` + `concat_tables`. Every flush adds a chunk, so after N rows the table has N + chunks and every Arrow op (`filter`, `take`) costs O(N). + +Delta (deferred): one MERGE commit per row per table (grows with table), lookups re-open and +read all files (one per row), existing-id cache refreshed after every flush; pending batch also +scanned per lookup. `get_records_with_column_value` on Delta returns only pending matches when a +pending batch exists (ignores committed rows) — repro requested by coordinator. + +Timings on main (sync, seconds per row): in-memory 100/400/1000 = 7.2/20.9/64 ms; +Delta 100/200/400 = 78/104/150 ms; standalone pod 0.09 ms flat. + +## Files + +- Modify: `src/orcapod/databases/in_memory_databases.py` +- Create: `tests/test_databases/test_in_memory_database_scaling.py` (DB-level + job-level + scaling tests, deterministic: Arrow bytes allocated per row through a counting wrapper) +- Create: `tests/test_databases/test_in_memory_database_lookup_equivalence.py` (index results + equal a plain filter: multi-column queries, overwrite path, pending + committed, types not + indexable, str vs bytes values) +- Modify: `CHANGELOG.md` (`[0.3.0] — unreleased`), `DESIGN_ISSUES.md` (one appended entry) + +## Task 1: failing scaling tests + +- [ ] Counting wrapper: a delegating `ArrowDatabaseProtocol` wrapper whose `at()` returns a + wrapped view and whose methods add `pa.default_memory_pool().total_bytes_allocated()` deltas + to a shared counter. Deterministic (bytes requested, not time). +- [ ] DB-level test: add N single-row records one by one, each followed by + `get_records_with_column_value` on an indexed binary column, `add_record(skip_duplicates=True)` + and `flush()` (the job's per-row pattern). Arrow bytes per row for the last 100 rows at + N=1600 must be < 2x those at N=200 (main: ~8x, grows linearly). +- [ ] Job-level test: `PipelineJob` with `InMemoryArrowDatabase` wrapped by the counter, one + function pod; run 200 rows and 800 rows; store bytes per row ratio < 2 (main ~4). + Run with both sync and async orchestrators. +- [ ] Run, confirm both fail on main. Commit (test only) as WIP. + +## Task 2: compaction on flush + +```python +def _compact_tail(table: pa.Table) -> pa.Table: + """Merge trailing record batches while the earlier one is not larger (binary counter).""" + batches = [b for b in table.to_batches() if b.num_rows > 0] + while len(batches) >= 2 and batches[-2].num_rows <= batches[-1].num_rows: + merged = pa.Table.from_batches(batches[-2:]).combine_chunks().to_batches() + batches[-2:] = merged + return pa.Table.from_batches(batches, schema=table.schema) +``` + +In `flush`: append path `pa.concat_tables([committed, pending])` (keeps today's schema errors) +then `_compact_tail`; overwrite path (rare) filters then `combine_chunks()`. + +## Task 3: column indexes + +- `_indexes: dict[str, _KeyIndex]` shared across `at()` views (`_shared_indexes`). +- `_KeyIndex(table, columns: dict[str, dict[Any, list[int]]])`; valid only while + `index.table is self._tables[key]` (rebuilt otherwise). +- Indexable: Arrow binary/large_binary/string/large_string/integer/bool columns queried with a + value of the matching Python type (`bytes`, `str`, `int` not `bool`, `bool`). Anything else + falls back to today's filter. +- `get_records_with_column_value`: pick the first indexable pair, `take` sorted candidate + positions from committed, `filter` the full expression on them; pending part filtered as + today; concat parts exactly as `_combined_table` did (same inclusion rule, same errors). +- `_filter_existing_records` and flush overwrite detection use membership in the + `RECORD_ID_COLUMN` index instead of building a full id set. +- Flush append path extends existing column indexes with the pending rows' values at + `old_len + i`. + +## Task 4: verification + +- [ ] New tests pass; `uv run pytest tests/test_databases tests/test_pipeline tests/test_core -q`. +- [ ] Differential check vs main (exported copy of main's `src` on `PYTHONPATH`): multi-node + pipeline with a join (two sources → join → function pod → function pod), in-memory store, + sync and async: compare every stored table (`list_sources` + `get_all_records`) — same + schema, same rows (excluding run-dependent columns: result record UUIDs, timestamps, which + are random/now() on main too), same pipeline record ids, content hashes, entry ids. +- [ ] After timings 100/400/1000/5000 for mem sync/async (+ Delta unchanged numbers). +- [ ] Golden hashes, full suite, ruff, basedpyright. +- [ ] CHANGELOG + DESIGN_ISSUES entry; PR "Fixes"? — No: this PR is part 1 of ITL-919, the + Delta part follows; use "Part of ITL-919" unless the coordinator says otherwise (ask). + +## Notes / progress log + +- 2026-11-11: cause established, coordinator approved in-memory fix; Delta pending developer. +- Main 5000-row timings running in background (scratchpad before_mem.txt / before_delta.txt). +- 2026-11-11: in-memory fix implemented (commit "perf(databases): ..."). Scaling tests pass; + equivalence tests pass on main and on the fix; differential check vs main identical + (scratchpad diff/ scripts: dump_store.py + compare.py). Lookup uses zero-copy row slices + (``take`` on a multi-chunk table copies every chunk) and skips the Arrow filter when every + pair is indexed (acero filter has ~0.35 ms fixed overhead). +- Delta repro of pending-shadow bug reported to coordinator (msg_01ac95ae10d6). +- PR wording: part 1 of ITL-919 (Delta follows) — confirm Fixes vs Part of with coordinator. diff --git a/tests/test_databases/arrow_work_counter.py b/tests/test_databases/arrow_work_counter.py new file mode 100644 index 000000000..4ebcb03ca --- /dev/null +++ b/tests/test_databases/arrow_work_counter.py @@ -0,0 +1,64 @@ +"""A database wrapper that counts the Arrow memory its calls allocate (ITL-919). + +The scaling tests use the bytes a store's calls allocate from the Arrow memory pool as a +deterministic measure of the work those calls do: a call that filters, copies or +concatenates the rows already stored allocates in proportion to them, while a call that +touches only the rows it reads or writes does not. Unlike wall-clock time, the count does +not depend on machine load. +""" + +from __future__ import annotations + +from typing import Any + +import pyarrow as pa + + +class ArrowWorkCounter: + """Shared counter of Arrow bytes allocated inside wrapped database calls.""" + + def __init__(self) -> None: + self.bytes_allocated = 0 + self._depth = 0 + + def measure(self, method: Any) -> Any: + """Wrap ``method`` so the Arrow bytes it allocates are added to the count.""" + + def wrapped(*args: Any, **kwargs: Any) -> Any: + pool = pa.default_memory_pool() + outermost = self._depth == 0 + before = pool.total_bytes_allocated() + self._depth += 1 + try: + return method(*args, **kwargs) + finally: + self._depth -= 1 + if outermost: + self.bytes_allocated += pool.total_bytes_allocated() - before + + return wrapped + + +class CountingDatabase: + """Delegates to an Arrow database and counts the Arrow bytes its calls allocate. + + ``at()`` returns a counting view sharing the same counter, so every scoped view a + pipeline creates is counted. + + Args: + inner: The database to delegate to. + counter: The counter to add to; a new one when omitted. + """ + + def __init__(self, inner: Any, counter: ArrowWorkCounter | None = None) -> None: + self._inner = inner + self.counter = counter if counter is not None else ArrowWorkCounter() + + def at(self, *path_components: str) -> CountingDatabase: + return CountingDatabase(self._inner.at(*path_components), self.counter) + + def __getattr__(self, name: str) -> Any: + attr = getattr(self._inner, name) + if callable(attr): + return self.counter.measure(attr) + return attr diff --git a/tests/test_databases/test_in_memory_database_lookup_equivalence.py b/tests/test_databases/test_in_memory_database_lookup_equivalence.py new file mode 100644 index 000000000..1f7d6c3b0 --- /dev/null +++ b/tests/test_databases/test_in_memory_database_lookup_equivalence.py @@ -0,0 +1,181 @@ +"""InMemoryArrowDatabase's indexed lookups return exactly what a filter would (ITL-919). + +Lookups by a binary or string column value use hash indexes over the committed rows. These +tests check, through the public interface, that every lookup returns the same rows, in the +same order and with the same schema, as filtering all stored rows, across many flushes, +pending rows, overwritten ids, scoped views and lookups the indexes do not serve. +""" + +from __future__ import annotations + +from typing import Any + +import pyarrow as pa +import pyarrow.compute as pc +import pytest + +from orcapod.databases import InMemoryArrowDatabase + +PATH = ("results", "v1") + + +def _row(i: int, group: str, score: int) -> pa.Table: + """Return row ``i`` with a binary key, a group, a score and a nullable string.""" + return pa.table( + { + "key": pa.array([f"key-{i % 7}".encode()], type=pa.large_binary()), + "group": pa.array([group], type=pa.large_string()), + "score": pa.array([score], type=pa.int64()), + "maybe": pa.array( + [None if i % 3 == 0 else f"m{i}"], type=pa.large_string() + ), + } + ) + + +def _fill(db: InMemoryArrowDatabase, n: int, flush_every: int = 1) -> None: + """Write ``n`` rows one at a time, flushing after every ``flush_every`` rows.""" + for i in range(n): + db.add_record(PATH, f"id-{i}".encode(), _row(i, f"g{i % 4}", i % 5)) + if (i + 1) % flush_every == 0: + db.flush() + + +def _expected(db: InMemoryArrowDatabase, pairs: dict[str, Any]) -> pa.Table | None: + """Return what a filter of every stored row by ``pairs`` gives, or ``None``.""" + everything = db.get_all_records(PATH, record_id_column="rid") + assert everything is not None + expression = None + for column, value in pairs.items(): + term = pc.field(column) == value + expression = term if expression is None else expression & term + assert expression is not None + matched = everything.filter(expression) + return None if matched.num_rows == 0 else matched + + +def _assert_same(actual: pa.Table | None, expected: pa.Table | None) -> None: + """Assert two lookup results have the same schema and rows in the same order.""" + if expected is None: + assert actual is None + return + assert actual is not None + assert actual.schema == expected.schema + assert actual.to_pylist() == expected.to_pylist() + + +LOOKUPS: list[dict[str, Any]] = [ + {"key": b"key-3"}, # indexed binary + {"group": "g1"}, # indexed string + {"key": b"key-3", "group": "g1"}, # two indexed columns + {"key": b"key-2", "score": 2}, # indexed + filtered + {"score": 4}, # integer: filtered, not indexed + {"maybe": "m4"}, # string column holding nulls + {"key": b"absent"}, # no match + {"key": b"key-1", "group": "g9"}, # indexed columns that never co-occur +] + + +@pytest.mark.parametrize("pairs", LOOKUPS, ids=lambda p: "-".join(p)) +@pytest.mark.parametrize("flush_every", [1, 3, 1000]) +def test_lookup_matches_a_filter_of_all_rows(pairs: dict[str, Any], flush_every: int): + """Lookups over many flushes return what a filter of all rows returns.""" + db = InMemoryArrowDatabase() + _fill(db, 90, flush_every=flush_every) + actual = db.get_records_with_column_value(PATH, pairs, record_id_column="rid") + _assert_same(actual, _expected(db, pairs)) + + +@pytest.mark.parametrize("pairs", LOOKUPS, ids=lambda p: "-".join(p)) +def test_lookup_includes_pending_rows_after_committed_rows(pairs: dict[str, Any]): + """Pending matches follow committed matches, as in one filter of both.""" + db = InMemoryArrowDatabase() + _fill(db, 60) + for i in range(60, 75): + db.add_record(PATH, f"id-{i}".encode(), _row(i, f"g{i % 4}", i % 5)) + actual = db.get_records_with_column_value(PATH, pairs, record_id_column="rid") + _assert_same(actual, _expected(db, pairs)) + + +def test_lookup_accepts_a_list_of_pairs(): + """A list of (column, value) pairs works like a mapping.""" + db = InMemoryArrowDatabase() + _fill(db, 30) + pairs = [("key", b"key-4"), ("group", "g0")] + actual = db.get_records_with_column_value(PATH, pairs, record_id_column="rid") + _assert_same(actual, _expected(db, dict(pairs))) + + +@pytest.mark.parametrize("pairs", LOOKUPS, ids=lambda p: "-".join(p)) +def test_lookup_after_a_flush_overwrites_a_committed_id(pairs: dict[str, Any]): + """A rewritten committed id moves its row to the end and lookups follow it.""" + db = InMemoryArrowDatabase() + _fill(db, 40) + _ = db.get_records_with_column_value(PATH, pairs) # build the indexes first + # A committed id written again replaces its row, which moves to the end. + db.add_record(PATH, b"id-5", _row(5, "g1", 4)) + db.add_record(PATH, b"id-40", _row(40, "g1", 4)) + db.flush() + actual = db.get_records_with_column_value(PATH, pairs, record_id_column="rid") + _assert_same(actual, _expected(db, pairs)) + everything = db.get_all_records(PATH, record_id_column="rid") + assert everything is not None + assert everything.num_rows == 41 + assert everything["rid"].to_pylist()[-2:] == [b"id-5", b"id-40"] + + +def test_indexes_follow_appends_after_they_are_built(): + """Indexes built early stay correct as later flushes append rows.""" + db = InMemoryArrowDatabase() + _fill(db, 20) + _ = db.get_records_with_column_value(PATH, {"key": b"key-1"}) # build indexes + for i in range(20, 50): + db.add_record(PATH, f"id-{i}".encode(), _row(i, f"g{i % 4}", i % 5)) + db.flush() + for pairs in LOOKUPS: + actual = db.get_records_with_column_value( + PATH, pairs, record_id_column="rid" + ) + _assert_same(actual, _expected(db, pairs)) + + +def test_scoped_views_share_records_and_indexes(): + """Writes through a scoped view are found through the root and the view.""" + root = InMemoryArrowDatabase() + view = root.at("pipeline") + for i in range(30): + view.add_record(PATH, f"id-{i}".encode(), _row(i, f"g{i % 4}", i % 5)) + root.flush() + found = root.get_records_with_column_value( + ("pipeline",) + PATH, {"key": f"key-{i % 7}".encode()} + ) + assert found is not None + assert found.num_rows == i // 7 + 1 + from_view = view.get_records_with_column_value(PATH, {"group": "g2"}) + from_root = root.get_records_with_column_value( + ("pipeline",) + PATH, {"group": "g2"} + ) + assert from_view is not None and from_root is not None + assert from_view.to_pylist() == from_root.to_pylist() + + +def test_skip_duplicates_skips_committed_and_pending_ids(): + """``skip_duplicates`` drops ids already committed or pending.""" + db = InMemoryArrowDatabase() + _fill(db, 20) + db.add_record(PATH, b"id-new", _row(99, "g0", 0)) + db.add_record(PATH, b"id-3", _row(3, "changed", 0), skip_duplicates=True) + db.add_record(PATH, b"id-new", _row(99, "changed", 0), skip_duplicates=True) + db.flush() + everything = db.get_all_records(PATH, record_id_column="rid") + assert everything is not None + assert everything.num_rows == 21 + assert "changed" not in everything["group"].to_pylist() + + +def test_lookup_of_a_missing_column_still_raises(): + """Looking up a column the table lacks raises, as a filter would.""" + db = InMemoryArrowDatabase() + _fill(db, 5) + with pytest.raises(pa.ArrowInvalid): + db.get_records_with_column_value(PATH, {"no_such_column": b"x"}) diff --git a/tests/test_databases/test_in_memory_database_scaling.py b/tests/test_databases/test_in_memory_database_scaling.py new file mode 100644 index 000000000..2c97654f3 --- /dev/null +++ b/tests/test_databases/test_in_memory_database_scaling.py @@ -0,0 +1,59 @@ +"""The work of InMemoryArrowDatabase's per-row calls must not grow with stored rows (ITL-919). + +A ``PipelineJob`` function node does, for every row: a lookup by column value on the +result table, a write with ``flush()``, a lookup by column value on the pipeline table and +a write with ``skip_duplicates=True``. Each of those used to filter, copy or rebuild the +whole stored table, so a run was quadratic in its rows. These tests count the Arrow bytes +the calls allocate (deterministic, unlike timing) and require the cost of the last rows +written to stay flat as the table grows. +""" + +from __future__ import annotations + +import hashlib + +import pyarrow as pa + +from orcapod.databases import InMemoryArrowDatabase + +from tests.test_databases.arrow_work_counter import CountingDatabase + +PATH = ("node", "v1") +WINDOW = 100 + + +def _key(i: int) -> bytes: + """Return the binary lookup key of row ``i``, like an input data hash.""" + return hashlib.sha256(str(i).encode()).digest() + + +def _row(i: int) -> pa.Table: + """Return row ``i`` as a one-row table with a binary key, a string and an integer.""" + return pa.table( + { + "key": pa.array([_key(i)], type=pa.large_binary()), + "label": pa.array([f"row-{i}"], type=pa.large_string()), + "value": pa.array([i], type=pa.int64()), + } + ) + + +def _bytes_per_row_at(stored_rows: int) -> float: + """Arrow bytes per row the job's per-row calls allocate after ``stored_rows`` rows.""" + db = CountingDatabase(InMemoryArrowDatabase()) + for i in range(stored_rows + WINDOW): + if i == stored_rows: + db.counter.bytes_allocated = 0 + db.get_records_with_column_value(PATH, {"key": _key(i)}) + db.add_record(PATH, _key(i), _row(i), skip_duplicates=True) + db.flush() + return db.counter.bytes_allocated / WINDOW + + +def test_per_row_work_does_not_grow_with_stored_rows() -> None: + """Per-row lookup, write and flush cost the same after 1,600 stored rows as after 200.""" + small = _bytes_per_row_at(200) + large = _bytes_per_row_at(1600) + # Eight times the rows stored: a scan of the stored rows costs about eight times + # as much per row; work proportional to the rows touched stays flat. + assert large < 2 * small, (small, large) diff --git a/tests/test_pipeline/test_job_store_scaling.py b/tests/test_pipeline/test_job_store_scaling.py new file mode 100644 index 000000000..040f07437 --- /dev/null +++ b/tests/test_pipeline/test_job_store_scaling.py @@ -0,0 +1,52 @@ +"""A PipelineJob run must do work per row independent of the rows already stored (ITL-919). + +On an ``InMemoryArrowDatabase`` a function node's per-row lookups, writes and flushes used +to filter, copy or rebuild the whole stored table, so a run was quadratic in its rows +(400 rows took 7 s; the same pod outside a job took 0.04 s). The tests count the Arrow +bytes the store's calls allocate during a run, which is deterministic, and require the +cost per row to stay flat when the rows quadruple. +""" + +from __future__ import annotations + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.sources import DictSource +from tests.test_databases.arrow_work_counter import CountingDatabase + + +@function_pod(output_keys="y") +def double(x: int) -> int: + """Return twice ``x``.""" + return 2 * x + + +def _store_bytes_per_row(n_rows: int, orchestrator: str) -> float: + """Run a one-pod job over ``n_rows`` rows and return the store's Arrow bytes per row.""" + store = CountingDatabase(InMemoryArrowDatabase()) + job = PipelineJob(name="scaling", store=store) # pyright: ignore[reportArgumentType] # counting wrapper of a real store + with job: + src = DictSource( + data=[{"k": i, "x": i} for i in range(n_rows)], + tag_columns=["k"], + source_id="src", + ) + double.pod(src, label="dbl") + job.run( + orchestrator=AsyncPipelineOrchestrator() if orchestrator == "async" else None + ) + assert job.nodes["dbl"].as_table().num_rows == n_rows + return store.counter.bytes_allocated / n_rows + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +def test_store_work_per_row_does_not_grow_with_rows(orchestrator: str) -> None: + """A job's store work per row stays flat when its rows quadruple.""" + small = _store_bytes_per_row(150, orchestrator) + large = _store_bytes_per_row(600, orchestrator) + # Four times the rows: a per-row scan of the stored rows costs about four times as + # much per row; work proportional to the rows touched stays flat. + assert large < 2 * small, (small, large)