From ae88c116cff2235f8d73625df47843e9adfe4efb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 13:59:22 -0800 Subject: [PATCH 01/17] test(spiraldb): scan log in the pyspiral fake and streaming source tests (ITL-186) The fake records every scan and hands out batches on demand. Two of the new tests are strict xfails until the source stops reading the table when built. Adds the design note, the job-memory finding and the plan. Co-Authored-By: Claude Fable 5.2 --- ...6-11-12-itl-186-streaming-spiral-source.md | 143 +++++++++ ...-itl-186-streaming-spiral-source-design.md | 282 ++++++++++++++++++ .../2026-11-12-itl-186-what-a-job-keeps.md | 41 +++ .../test_spiraldb_table_source_streaming.py | 86 ++++++ tests/test_databases/fake_pyspiral.py | 64 +++- 5 files changed, 603 insertions(+), 13 deletions(-) create mode 100644 superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md create mode 100644 superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md create mode 100644 superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md create mode 100644 tests/test_core/sources/test_spiraldb_table_source_streaming.py diff --git a/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md b/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md new file mode 100644 index 000000000..f5d93602e --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md @@ -0,0 +1,143 @@ +# ITL-186 Streaming Spiral Table Source Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `SpiralDBTableSource` builds from metadata only and streams its rows in bounded batches. + +**Architecture:** Design note `superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md`. +The source stops inheriting the eager load of `DBTableSource`; it keeps a zero-row template +stream for schema questions and opens one Spiral scan per read. Tests run on the disk-backed +pyspiral fake, which gains a scan log. + +**Tech Stack:** Python 3.11+, pyarrow, pyspiral 0.16.33 (`spiraldb` extra), pytest, `uv run`. + +--- + +## State (keep current; a successor continues from here) + +* Branch `eywalker/itl-186-add-dynamic-mode-to-spiraldbsource`, from `origin/main` 25650c42. +* Nov 12: design sent to the coordinator. Approved by the coordinator: D4 (batches of 8,192), + D7 (no orchestrator / Join change), D8 (full re-scan), D9 (PR scope), D10 (tests). + **Waiting on the developer: D1, D2, D3, D5, D6.** No library code until the reply. +* Allowed meanwhile: Phase A below (fake + tests that do not depend on D1/D3/D5), local only. +* Live probe scripts: `.untracked/itl186_probe_{sizes,scan,lazy,ids}.py` (untracked). +* Finding passed to ITL-858: `superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md`. + +## File structure + +| File | Role | +|---|---| +| `tests/test_databases/fake_pyspiral.py` | fake: scan log, lazy batches, `limit` (Phase A); `where`, `asof`, `table_id`, `last_modified_at`, row count (Phase B) | +| `tests/test_core/sources/test_spiraldb_table_source_streaming.py` | new: streaming behaviour through the public interface | +| `src/orcapod/databases/spiraldb_connector.py` | Phase B: metadata-only reads and a restricted scan | +| `src/orcapod/core/sources/spiraldb_table_source.py` | Phase B: the source | +| `docs/concepts/sources.md`, `changelog.d/ITL-186.md`, `design_issues/ITL-186-SP4.md` | Phase B | + +## Phase A (approved now) + +### Task A1: scan log and lazy batches in the fake + +**Files:** Modify `tests/test_databases/fake_pyspiral.py` (`FakeScan`, `FakeSpiral.scan`, module docstring). + +- [ ] **Step 1: add the log and the batch-size switch** after `_MAX_ENTRIES`: + +```python +@dataclass +class ScanRecord: + """One ``Spiral.scan(...)`` call: the table, its keyword arguments, batches handed out.""" + + identifier: str + kwargs: dict[str, Any] + batches_read: int = 0 + + +# Every scan opened in this process, oldest first. Tests clear it and read it. +scan_log: list[ScanRecord] = [] +# Rows per batch a scan hands out (SpiralDB's batches follow its stored fragments). +scan_batch_rows = 2 +``` + +- [ ] **Step 2: replace `FakeScan`** + +```python +class FakeScan: + """What ``Spiral.scan(table.select(), limit=...)`` returns.""" + + def __init__(self, table: FakeTable, record: ScanRecord, limit: int | None) -> None: + self._table = table + self._record = record + self._limit = limit + + def _rows(self) -> pa.Table: + rows = self._table._read_all() + return rows if self._limit is None else rows.slice(0, self._limit) + + def to_table(self) -> pa.Table: + """Return the scanned rows, in key order.""" + return self._rows() + + def to_record_batches(self, **kwargs: Any) -> pa.RecordBatchReader: + """Return the rows in key order, ``scan_batch_rows`` per batch, read on demand.""" + rows = self._rows() + + def batches() -> Iterator[pa.RecordBatch]: + for batch in rows.to_batches(max_chunksize=scan_batch_rows): + self._record.batches_read += 1 + yield batch + + return pa.RecordBatchReader.from_batches(rows.schema, batches()) +``` + +(`Iterator` from `collections.abc`.) + +- [ ] **Step 3: `FakeSpiral.scan` records the call and supports `limit`** + +```python + def scan(self, *projections: Any, **kwargs: Any) -> FakeScan: + """Return a scan of one ``table.select()``, restricted by ``limit`` if given.""" + if len(projections) != 1 or not isinstance(projections[0], FakeExpr): + raise NotImplementedError( + "fake Spiral.scan supports only scan(table.select())" + ) + unsupported = {k for k, v in kwargs.items() if v is not None} - {"limit"} + if unsupported: + raise NotImplementedError(f"fake Spiral.scan: {sorted(unsupported)}") + table = projections[0].table + record = ScanRecord(table._identifier, dict(kwargs)) + scan_log.append(record) + return FakeScan(table, record, kwargs.get("limit")) +``` + +- [ ] **Step 4:** run `uv run pytest tests/test_databases tests/test_core/sources -q -k "spiral or fake"`; expected: all pass (the fake's behaviour for existing callers is unchanged). +- [ ] **Step 5:** commit `test(spiraldb): the pyspiral fake logs scans and reads batches on demand (ITL-186)`. + +### Task A2: failing streaming tests + +**Files:** Create `tests/test_core/sources/test_spiraldb_table_source_streaming.py`. + +- [ ] **Step 1: write the tests** (fixture: a 5-row `default.event` table keyed on + `(subject_id, timestamp)`, all columns nullable; `scan_batch_rows` set to 2 and + `scan_log` cleared by the fixture): + + * `test_construction_reads_no_rows`: build the source, call `output_schema()`, `keys()`, + `content_hash()`, `pipeline_hash()`, `to_config()`; assert `fake_pyspiral.scan_log == []`. + * `test_iter_data_reads_batches_only_as_rows_are_consumed`: `rows = iter(source.iter_data())`; + `next(rows)`; assert exactly one scan is logged and its `batches_read == 1`; exhaust the + iterator; assert `batches_read == 3` and 5 rows were seen. + * `test_streamed_rows_equal_as_table_rows`: for every row, the tag and data values with + `all_info=True` equal the same columns of `source.as_table(all_info=True)`. (Passes today + too: it pins the equality the streaming path must keep.) + +- [ ] **Step 2:** run `uv run pytest tests/test_core/sources/test_spiraldb_table_source_streaming.py -q`; + expected: the first two FAIL (a scan is logged at construction, with all batches read), the + third passes. +- [ ] **Step 3:** commit `test(spiraldb): streaming source tests, failing until ITL-186 lands`. + The two failing tests are marked `xfail(strict=True, reason="ITL-186")` in the commit so + the suite stays green; the marks are removed in Phase B. + +## Phase B (gated on D1, D2, D3, D5, D6; detailed after the reply) + +Written out task by task once the decisions are known, because the constructor signature, +the source id, the token format and the cap all depend on them. Order: connector metadata +and restricted scan; per-batch enrichment; the source; save/load; cap; async; docs, +changelog fragment, design-issue entry; live opt-in tests; full suite, lint, types; PR. diff --git a/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md b/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md new file mode 100644 index 000000000..3d3c69b9b --- /dev/null +++ b/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md @@ -0,0 +1,282 @@ +# ITL-186: streaming Spiral table source, design for decision (November 12, 2026) + +Status: proposal. No library code is written. It replaces +`2026-11-11-itl-186-spiral-streaming-source-design.md` where the two differ (the differences +are listed at the end). Decisions are numbered D1 to D10; each has options and a +recommendation. Rulings the developer already gave on November 12 are marked FIXED. + +## Summary of the proposal + +`SpiralDBTableSource(project, table)` reads no rows when it is built. It reads the schema, the +key columns and Spiral's table id (three metadata calls, about 1 s). Rows are read only when +asked for, in batches of at most 8,192 rows, and each row's record id comes from its primary +key. A read can be restricted with `where=` and `limit=`, which Spiral applies before any row +reaches Orcapod. A cap (`max_rows`, default 100,000) stops a read that would be too large for +a job, with a message that says how to restrict it. + +One finding limits what "streaming" can deliver by November 16: the source can stream a table +of any size in bounded memory, but **a pipeline job cannot**, because every node keeps every +row it produced (about 5 KB per row). That is in the nodes, not in the source, and is not +part of the first PR. + +## What was measured + +Live, read-only, pyspiral 0.16.33, project `edi-tutorial-v1-841226` (largest tables: +`lfp_timeseries` 86,331 rows, `spike_event` 63,349 rows). Scripts: `.untracked/itl186_probe_*.py`. + +| Question | Result on `spike_event` (63,349 rows, 8 columns) | +|---|---| +| Metadata without a scan | row count, key columns, table id, last-modified time: 0.4 to 1.5 s per table | +| Does a scan yield batches? | Yes: 2 batches of 14,771 and 48,578 rows. Batch size follows Spiral's stored fragments and cannot be set | +| Time to first batch | 0.02 to 0.24 s; whole scan 0.3 to 0.9 s, 13.2 MB of Arrow | +| `limit=1000` | 1,000 rows, 0.21 MB: applied by Spiral | +| Column selection | one column: 0.25 MB instead of 13.2 MB: applied by Spiral | +| `where=` equality | works on `uint32`, string and timestamp key columns, and as a conjunction | +| `key_ranges=` | **not usable**: a key holding a timestamp raises `TypeError` (datetime and Arrow scalar both); every large EDI table has `session_start_datetime` in its key | +| `asof=` (a table version) | works; `Table.last_modified_at()` gives the current version | +| `Table.to_polars_lazy_frame()` | raises `ImportError` in our lock (`betterproto2` 0.9.1). With 0.10: column selection and `head` are pushed down, but `filter(unit_id == 106)` fails ("Substrait is only capable of representing unsafe casts") | +| `Table.to_arrow_dataset()` | fails on `count_rows` ("Reading key columns without ... column group") | +| `Scan.lazy()` | experimental; refuses a scan with a limit | +| Table id on rename | unchanged (`table_6p9S6T9WzymR` before and after `rename_table`) | +| Table id on drop and recreate, same name | new id (`table_8tEglDEJMS5P`) | + +The last two rows used one throwaway table, `scratch.itl186_idprobe`, created and dropped in +`edi-tutorial-v1-841226`. Nothing else was written. + +Offline, on spike_event-shaped synthetic data (script in the session scratchpad): + +| Step | Speed | Memory | +|---|---|---| +| Build key tokens, record ids, source info and system tags for a batch, in Arrow | 565,000 rows/s | bounded by the batch | +| `iter_data()`: one `(Tag, Data)` pair per row | about 9,000 rows/s (1,000,000 rows in 113 s) | about 10 KB per row of the batch being iterated: +671 MB at 65,536 rows per batch, +90 MB at 8,192 | +| Keeping the pairs (what a job node does) | | about 5.3 KB per row: 200,000 rows = +1,065 MB | + +So the proposed read path costs about 90 MB whatever the table size (today: about 4.6 GB at +construction for the 8,087,906-row `spike_event`). Iterating that table row by row would +still take about 15 minutes before any pod runs. + +**What a job keeps (read from the code).** `OperatorNode.async_execute` appends every output +row to `collected` (ITL-896). `FunctionJobNode` keeps every output in `_cached_output_datas` +and starts each run by loading all stored records (`_load_cached_entries`). The sync +orchestrator does `list(iter_data())` on the source. The async `Join` keeps every row of both +inputs. With `materialize_results=True` (the default) the orchestrator keeps a further copy. +So on either orchestrator a job over N rows holds about N × 5 KB per node: 100,000 rows is +about 0.5 GB per node, 8,087,906 rows about 43 GB. + +## D1. Locator and source id + +FIXED: identity comes from how the table is located, never from its rows; Spiral's opaque +table id (with the project id) guarantees uniqueness; a readable form is what a person sees. + +**Locator** = `(project id, table id)`. `.` is how the user names the table; +it is resolved to the table id when the source is built (`project.table(name).table_id`, no +scan). Not in the locator: credentials, tokens, `overrides` (server URL and other client +options), `batch_rows`, `max_rows`. + +**Today.** Spiral's default source id is already `//
` (ITL-926), so +two projects do not collide. A table dropped and recreated under the same name does: same +source id, same content hash, same record ids. `PostgreSQLTableSource` and `SQLiteTableSource` +still default to the bare table name (`db_table_source.py:79`), so two databases with a +same-named table of the same schema share identity. Not changed here; listed for a follow-up. + +Options for where the table id lives: + +* **A (recommended): inside the source id.** Default + `edi-tutorial-v1-841226/default.spike_event@table_W7C01nkLTmEJ`. Readable and unique + everywhere the source id goes (source-info tokens, record ids, `SourceRegistry`). + `identity_structure` keeps its form `(class, schema, source_id)`. Cost: a renamed table gets + a new source id, so its pipeline records are entered again (function results are still + reused, because they are found by row content). +* **B: source id is `/
` only**, the readable name shown in `label`, `repr` + and the saved file. A renamed table keeps its whole identity. Cost: every provenance token + in stored data reads `edi-tutorial-v1-841226/table_W7C01nkLTmEJ::...`, which no longer says + `spike_event`. +* **C: readable source id, table id only in `identity_structure`.** Rejected: the content hash + is unique, but record ids and tokens of a recreated table collide with the old table's. + +Under A and B a table dropped and recreated under the same name is a different source and +nothing cached is reused. Under B only, a renamed table keeps its identity. + +A `source_id=` given by the user is used as it is, as for every source. + +**What moves.** The default source id of Spiral sources changes, so their `content_hash`, +record ids and source-info tokens change. `pipeline_hash` does not. No golden value moves +(no Spiral source is in the golden set). Stores written through a Spiral source so far are +the local ITL-926 verification runs. + +## D2. Restrictions given at construction (`where`, `limit`, `asof`) + +They select different rows of the same table. Options: + +* **(recommended) In the content hash, not in the source id.** + `identity_structure = (class, schema, source_id, restriction)`, where the restriction is + the canonical form of `where`, `limit` and an explicit `asof`; with none given it is + absent, so an unrestricted source hashes as `(class, schema, source_id)`. A row has the + same record id whatever restriction read it, so results are shared; two jobs with different + restrictions have different node hashes, so each reads back only its own rows. +* In neither: a job built with `limit=1000` would read back the rows of another job's + `limit=5000` run from a shared store. +* In the source id: the same row would get a different record id under each restriction. + +When no `asof` is given, the source pins the table's current version when it is built, so all +reads from one source object see one version. That pin is not hashed and is not saved: a +loaded job reads the latest version. + +## D3. Record identity per row + +FIXED: from the primary key, never from row content. + +* The token is built from **Spiral's key columns** (always present: Spiral requires a key + schema, and keys are unique and non-null), not from `tag_columns`, which the user may set + to fewer, non-unique columns. `record_id_column` is no longer needed for this source and is + removed from it. +* Token format, options: + * **(recommended) readable**: `subject_id=Goliath,unit_id=106,...` in key order, values as + Arrow renders them, with `%`, `,`, `=` and `:` percent-escaped. Escaping is required: + the real `spike_extraction_hash` values contain `::`, the separator inside source-info + values. About 290 characters for `spike_event`. + * short: a hex digest of the key values. Fixed length, but a person can no longer find the + row from a provenance token. +* Record id stays `uuid5(source_id + "::" + token)`, as for every source. +* A table with no key schema cannot exist in Spiral. If pyspiral ever returns an empty key + schema the source raises; it never falls back to row position, because position changes + with every restriction. +* This replaces positional `row_` tokens for Spiral sources, so their record ids change + (together with D1). + +## D4. How rows arrive + +One scan per read: `scan(table.select(), where=, limit=, asof=).to_record_batches()`. Each +Spiral batch is cut into slices of at most `batch_rows` rows (default **8,192**; Spiral's own +batches were up to 48,578 rows), normalized to large types, checked against the nullability +record (#308), enriched in Arrow, and yielded row by row. Nothing is accumulated. No +`to_pylist()` of a table, no table hash. Memory: about 90 MB at the default. + +Column selection is not offered in the first PR (it changes the schema and so the pipeline +hash; `SelectDataColumns` exists downstream). + +## D5. What each public method does + +| Method | Behaviour | +|---|---| +| construction, `output_schema`, `keys`, `content_hash`, `pipeline_hash`, `source_id`, `to_config`, `repr` | metadata only, never a scan. Schema from Spiral's schema plus the nullability record | +| `row_count_estimate` (new, read-only property) | Spiral's manifest count for the whole table; no scan | +| `iter_data`, `async_iter_data` | stream batches as in D4 | +| `as_table`, `as_df`, `as_polars_df`, `as_pandas_df`, `as_lazy_frame` | read the (restricted) rows into one Arrow table; `as_lazy_frame` stays `as_polars_df().lazy()` | +| `len` / count | none exists today; none added | + +First N rows: `SpiralDBTableSource(project, "spike_event", limit=1000)`. A subset: +`where={"subject_id": "Goliath", "unit_id": 106}` (equality only, combined with "and"; it is +saved in the job file). Key ranges are not offered (see the measurements). + +The size guard, options: + +* **(recommended) One cap on every read**: `max_rows=100_000`. A read with no `where` whose + manifest count (or `limit`) is above the cap fails before scanning; a read with `where` + fails when row `max_rows + 1` arrives. The message gives the count and the three ways out: + `limit=`, `where=`, or a larger `max_rows` (`None` removes the cap). 100,000 rows is about + 0.5 GB per job node. +* Cap only on the `as_table` family: `job.run()` on the full `spike_event` would then exhaust + a laptop's memory. +* No cap. + +## D6. One class, a sibling, or `PollingSource`? + +* **(recommended) `SpiralDBTableSource` itself, with no mode.** It always builds without + reading and always streams. "Static, whole table" is `as_table()` on a source within the + cap. It no longer inherits from `DBTableSource` (whose constructor is the eager load); + PostgreSQL and SQLite sources are untouched. + + ```python + SpiralDBTableSource("edi-tutorial-v1-841226", "spike_event", + where={"unit_id": 106}, limit=None, max_rows=100_000) + ``` + Saved form (`source_type` stays `spiraldb_table`): + `{project_id, dataset, table_name, table_id, where, limit, asof, max_rows, batch_rows, + tag_columns, system_tag_columns, source_id, overrides, + identity fields}`. On load the name + is resolved again; if it now resolves to another table id, loading raises an error naming + both ids. Loading reads no rows (today it reads the whole table). +* A mode switch (`streaming=True/False/None`): `None` would pick by table size, which makes + behaviour depend on data. Not recommended. +* A sibling class `SpiralDBStreamSource(project, table, ...)` with its own `source_type`: + two classes to teach, and the same table would have two content hashes (class name is + hashed). +* `PollingSource(SpiralReader(project, table), tag_columns=...)`: `PollingSource` keeps every + batch, has no end of data, reads on a timer, numbers rows per batch (record ids collide + across batches), and is not in the load registry. Making it fit changes it for its + existing users. + +Pyspiral's lazy APIs (developer's question): Orcapod should not build on +`to_polars_lazy_frame`, `to_arrow_dataset`, `to_duckdb_relation` or `Scan.lazy()` now. They +fail in the locked environment, filters are not pushed down reliably, and the record id +(`uuid5`) cannot be a lazy Polars expression. `scan(where=, limit=, asof=)` is the stable base. +A lazy frame with pushdown is later work. + +## D7. Sync, async and joins + +* **Async**: `async_iter_data` reads each batch in a worker thread and the bounded channel + (64 rows) applies backpressure, so the source holds one batch. +* **Sync**: `SourceJobNode.execute` does `list(iter_data())`; it works and holds every row. +* **Join**: unchanged. It keeps both inputs, and the operator node keeps the output. +* Recommended for the first PR: no orchestrator, node or Join change. The cap of D5 is what + keeps both orchestrators safe. Releasing a Join side early (the earlier D6) saves little + while the operator node keeps every output row. + +## D8. Caching and re-runs + +Second run with a store: the source scans again (the restricted rows, latest version). For +each row the function node finds its record and computes nothing; a row appended to the Spiral +table since the first run has a new key, so it is computed and recorded; existing rows keep +their record ids even when the new key sorts first. This is the issue's "full re-scan" +strategy, and it is enough for the workshop, whose tables are read-only. Watermark reads and +polling for new rows inside one run are not needed for the workshop and wait. + +## D9. First PR (before November 16) and what waits + +First PR, one class plus the connector's read side: + +1. Construction from metadata only; locator, source id and restriction identity (D1, D2). +2. Batch streaming for `iter_data` / `async_iter_data` / `as_table`, key-based record ids (D3, D4). +3. `where=`, `limit=`, `asof=`, `max_rows`, `batch_rows`, `row_count_estimate` (D5). +4. `to_config` / `from_config` with the table id check (D6). +5. Tests on the fake, opt-in live tests, `docs/concepts/sources.md`, changelog fragment, + design-issue entry. + +Waits (each its own issue): bounded-memory job runs (nodes keeping rows), Join buffering, +watermark and polling, column selection, a lazy frame with pushdown, key ranges, the same +lazy treatment for PostgreSQL and SQLite sources and their default source ids, `betterproto2` +in the lock. + +Hashes: no golden value moves. Spiral sources' default source id, content hash, record ids +and tokens change (D1, D3); pipeline hashes and every other source are unchanged. + +## D10. Tests + +Without a connection, on `tests/test_databases/fake_pyspiral.py` extended with `table_id`, +`last_modified_at`, row count, `scan(where=, limit=, asof=)`, several batches, and a scan +counter: + +* construction, schema, hashes, `to_config` and load make zero scans; +* streamed rows equal `as_table()` rows (types, nullability, tokens, record ids); +* no batch handed to enrichment exceeds `batch_rows`; +* a row's record id is the same under `limit`, under `where`, and after a row with a smaller + key is added; +* the cap: early failure, failure while streaming, `max_rows=None`; +* drop and recreate gives a new identity; a saved job whose name resolves to another table id + fails to load; +* a job on both orchestrators: function pod, Join with a small source, second run computes + nothing, appended row is the only one computed; save and load. + +Opt-in live tests (skipped unless `ORCAPOD_SPIRAL_LIVE_PROJECT` is set), read-only on +`edi-tutorial-v1-841226`: construction of `spike_event` makes no scan and takes under 5 s; +`limit=1000`; one `where`; an async job over `limit=2000`. One write test confined to a +`scratch.` table of that project: append rows between two runs. + +## Differences from the November 11 note + +* No size-dependent mode (`streaming=None` choosing by row count); one behaviour. +* Record ids from the key (ruling), not global row offsets. +* The table id enters identity (ruling); restrictions enter the content hash. +* `head(n)` is replaced by `limit=`; the dataframe methods work under the cap instead of + always raising. +* No Join or orchestrator change: measured to be ineffective while nodes keep their rows. diff --git a/superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md b/superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md new file mode 100644 index 000000000..74f13557a --- /dev/null +++ b/superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md @@ -0,0 +1,41 @@ +# What a pipeline job keeps in memory (finding from ITL-186, input to ITL-858) + +November 12, 2026. Read from the code on `main` at 25650c42 and measured on synthetic +spike_event-shaped rows (7 tag columns, 1 data column). Not a proposal. + +## Measured + +* One `(Tag, Data)` pair kept in a Python list costs about **5.3 KB**: 200,000 pairs add + 1,065 MB of peak RSS. The same rows as Arrow are 62 bytes each. +* `ArrowTableStream.iter_data()` produces about **9,000 rows per second**, and holds about + 10 KB per row of the table it is iterating (65,536-row table: +671 MB; 8,192-row table: + +90 MB). + +## Where rows are kept + +| Place | What is kept | Until | +|---|---|---| +| `SyncPipelineOrchestrator` (`sync_orchestrator.py:91`) | `node.execute()` → `list(iter_data())`: every source row, then every node's output in `buffers` | end of run; released per node only with `materialize_results=False` | +| `AsyncPipelineOrchestrator._run_async` | `_CollectingWriter` copies every row of every node into `collectors` | end of run, unless `materialize_results=False` (default is `True`) | +| `OperatorNode.async_execute` (`operator_node.py:1457`) | `collected`: every output row, then `_cached_output_stream` (ITL-896, so `iter_data()` works after an async run) | lifetime of the node | +| `FunctionJobNode.async_execute` (`function_node.py:3071`, `3161`) | `_load_cached_entries()` loads every stored record of the node before the first input; every new output is added to `_cached_output_datas` | lifetime of the node | +| `Join._binary_streaming_join` (`join.py:564`) | `buffers`: every row of both inputs, also after one input has closed | end of the join | + +Channels themselves are bounded (64 rows), so backpressure works; the memory is in the lists +above. + +## Consequence + +On either orchestrator a job over N rows holds roughly N × 5 KB for each node that keeps its +output. 100,000 rows: about 0.5 GB per node. The 8,087,906-row `spike_event`: about 43 GB per +node, and 15 minutes of row iteration before any pod runs. A source that streams in bounded +memory (ITL-186) does not change this. + +## What a bounded-memory run would need (not designed here) + +* Nodes that do not keep their output when the store already has it (function nodes), or + when nothing asks for it (operator nodes with `cache_mode=OFF`). +* A function node that looks up stored records per input instead of loading all of them. +* A Join that keeps only the side it needs, or only the smaller side. +* Possibly channels that carry Arrow batches instead of one `(Tag, Data)` pair per row: the + per-row objects are both the memory and the 9,000 rows/s limit. diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py new file mode 100644 index 000000000..59ab46369 --- /dev/null +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -0,0 +1,86 @@ +"""SpiralDBTableSource reads no rows until asked and then reads them in batches (ITL-186). + +Runs against the disk-backed pyspiral fake, whose ``scan_log`` records every scan a source +opens and how many batches were taken from it. +""" + +from __future__ import annotations + +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod.sources import SpiralDBTableSource +from tests.test_databases import fake_pyspiral + +_PROJECT = "edi-sample" +_KEYS = pa.schema( + [pa.field("subject_id", pa.string()), pa.field("timestamp", pa.uint64())] +) +_ROWS = 5 + + +@pytest.fixture() +def event_table(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Provide a fake root holding a 5-row ``event`` table read 2 rows per batch.""" + monkeypatch.setattr(fake_pyspiral, "scan_batch_rows", 2) + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project(_PROJECT) + table = project.create_table("default.event", key_schema=_KEYS) + rows = pa.table( + { + "subject_id": ["m1"] * _ROWS, + "timestamp": pa.array(range(_ROWS), pa.uint64()), + "amplitude": pa.array([0.5, 1.5, None, 3.5, 4.5], pa.float32()), + } + ) + table.write( + rows.cast(pa.schema([pa.field(f.name, f.type, True) for f in rows.schema])) + ) + fake_pyspiral.scan_log.clear() + yield + fake_pyspiral.scan_log.clear() + + +@pytest.mark.xfail(strict=True, reason="ITL-186: the source reads the table when built") +def test_construction_reads_no_rows(event_table): + source = SpiralDBTableSource(_PROJECT, "event") + source.output_schema(all_info=True) + source.keys(all_info=True) + source.content_hash() + source.pipeline_hash() + source.to_config() + + assert fake_pyspiral.scan_log == [] + + +@pytest.mark.xfail(strict=True, reason="ITL-186: the source reads the table when built") +def test_iter_data_reads_batches_only_as_rows_are_consumed(event_table): + source = SpiralDBTableSource(_PROJECT, "event") + rows = iter(source.iter_data()) + + next(rows) + assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [1] + + assert sum(1 for _ in rows) == _ROWS - 1 + assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [3] + + +def test_streamed_rows_equal_as_table_rows(event_table): + source = SpiralDBTableSource(_PROJECT, "event") + + streamed = [ + {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} + for tag, data in source.iter_data() + ] + table_rows = source.as_table(all_info=True).to_pylist() + + assert len(streamed) == _ROWS + assert len(table_rows) == _ROWS + for streamed_row, table_row in zip(streamed, table_rows, strict=True): + shared = streamed_row.keys() & table_row.keys() + assert {"subject_id", "timestamp", "amplitude"} <= shared + assert any(name.startswith("_tag_record_id::") for name in shared) + assert any(name.startswith("_source_") for name in shared) + assert {k: streamed_row[k] for k in shared} == {k: table_row[k] for k in shared} diff --git a/tests/test_databases/fake_pyspiral.py b/tests/test_databases/fake_pyspiral.py index 49ee4480f..a0ce8a465 100644 --- a/tests/test_databases/fake_pyspiral.py +++ b/tests/test_databases/fake_pyspiral.py @@ -8,7 +8,8 @@ Only the calls the connector makes are implemented: -* ``Spiral(overrides=...)``, ``Spiral.project(id)``, ``Spiral.scan(expr)`` +* ``Spiral(overrides=...)``, ``Spiral.project(id)``, ``Spiral.scan(expr, limit=...)`` + (each call is recorded in ``scan_log``; batches are read on demand) * ``Project.list_tables()`` (``TableResource`` with ``.dataset`` / ``.table``), ``Project.table(id)``, ``Project.create_table(id, key_schema=..., exist_ok=...)``, ``Project.drop_table(id)`` @@ -49,6 +50,7 @@ import json import os import types +from collections.abc import Iterator from contextlib import contextmanager from dataclasses import asdict, dataclass from pathlib import Path @@ -66,6 +68,21 @@ _MAX_ENTRIES = 1024 +@dataclass +class ScanRecord: + """One ``Spiral.scan(...)`` call: the table, its keyword arguments, batches handed out.""" + + identifier: str + kwargs: dict[str, Any] + batches_read: int = 0 + + +# Every scan opened in this process, oldest first. Tests clear it and read it. +scan_log: list[ScanRecord] = [] +# Rows per batch a scan hands out (SpiralDB's batches follow its stored fragments). +scan_batch_rows = 2 + + @dataclass(frozen=True) class StorageBehaviour: """How the fake transforms data, mirroring SpiralDB's storage behaviour.""" @@ -332,21 +349,32 @@ def __init__(self, table: FakeTable) -> None: class FakeScan: - """What ``Spiral.scan(table.select())`` returns: reads the whole stored table.""" + """What ``Spiral.scan(table.select(), limit=...)`` returns.""" - def __init__(self, table: FakeTable) -> None: + def __init__(self, table: FakeTable, record: ScanRecord, limit: int | None) -> None: self._table = table + self._record = record + self._limit = limit + + def _rows(self) -> pa.Table: + rows = self._table._read_all() + return rows if self._limit is None else rows.slice(0, self._limit) def to_table(self) -> pa.Table: - """Return every stored row, in key order.""" - return self._table._read_all() + """Return the scanned rows, in key order.""" + return self._rows() + + def to_record_batches(self, **kwargs: Any) -> pa.RecordBatchReader: + """Return the rows in key order, ``scan_batch_rows`` per batch, read on demand.""" + rows = self._rows() + + def batches() -> Iterator[pa.RecordBatch]: + # SpiralDB yields more than one batch for larger tables; split to exercise that. + for batch in rows.to_batches(max_chunksize=scan_batch_rows): + self._record.batches_read += 1 + yield batch - def to_record_batches(self) -> pa.RecordBatchReader: - """Return the stored rows in key order, as batches of at most two rows.""" - t = self._table._read_all() - # SpiralDB yields more than one batch for larger tables; split to exercise that. - batches = t.to_batches(max_chunksize=2) if t.num_rows else [] - return pa.RecordBatchReader.from_batches(t.schema, batches) + return pa.RecordBatchReader.from_batches(rows.schema, batches()) class FakeTable: @@ -597,12 +625,22 @@ def project(self, project_id: str) -> FakeProject: return FakeProject(self._store, project_id) def scan(self, *projections: Any, **kwargs: Any) -> FakeScan: - """Return a scan of one ``table.select()``; other projections are not modelled.""" + """Return a scan of one ``table.select()``, restricted by ``limit`` if given. + + Every call is appended to ``scan_log``. Other projections and the other + restrictions of ``Spiral.scan`` are not modelled. + """ if len(projections) != 1 or not isinstance(projections[0], FakeExpr): raise NotImplementedError( "fake Spiral.scan supports only scan(table.select())" ) - return FakeScan(projections[0].table) + unsupported = {k for k, v in kwargs.items() if v is not None} - {"limit"} + if unsupported: + raise NotImplementedError(f"fake Spiral.scan: {sorted(unsupported)}") + table = projections[0].table + record = ScanRecord(table._identifier, dict(kwargs)) + scan_log.append(record) + return FakeScan(table, record, kwargs.get("limit")) fake_spiral_module = types.SimpleNamespace(Spiral=FakeSpiral) From 042d5c5c7db8942e4defe39b0cc464fb67b77066 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:08:15 -0800 Subject: [PATCH 02/17] docs(specs): ITL-186 D3 key digest definition and D2 measurements Co-Authored-By: Claude Fable 5.2 --- ...-itl-186-streaming-spiral-source-design.md | 84 +++++++++++ .../d2_restriction_identity.py | 137 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 superpowers/specs/itl-186-experiments/d2_restriction_identity.py diff --git a/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md b/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md index 3d3c69b9b..638f96fac 100644 --- a/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md +++ b/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md @@ -280,3 +280,87 @@ Opt-in live tests (skipped unless `ORCAPOD_SPIRAL_LIVE_PROJECT` is set), read-on * `head(n)` is replaced by `limit=`; the dataframe methods work under the cap instead of always raising. * No Join or orchestrator change: measured to be ineffective while nodes keep their rows. + +--- + +# Addendum, November 12, 2026: decisions, the D3 definition, and D2 measured + +Developer decisions: **D1 option A**, **D5 cap approved for now** (default 100,000, `None` +removes it), **D6 one class, no mode**. D4, D7, D8, D9, D10 approved by the coordinator. +D3 changed (below). D2 is open; its facts are below. + +## D3: the row token is a digest of the primary key + +**Does Spiral have its own per-row id?** No. Read from pyspiral 0.16.33: a row is addressed +only by its key (`Table.key(*parts)`, `KeyRange`, `key_table`). The one row-index name in the +package, `__row_idx__`, is a key column the Hugging Face import helper adds when a dataset +has no key. There is no row uuid, row version or hidden row id in scan output. + +**Definition.** + +* Input: the row's key columns, in key-schema order, as a one-row Arrow table. Each field has + the key column's name and its Arrow type normalized to the large variant. Each field is + declared **non-nullable**, whatever Spiral or a nullability record reports (a Spiral key is + never null). This matters: the Arrow hasher gives a different digest for a nullable and a + non-nullable field (measured), so without this rule a nullability record written later + would move every token. +* Hasher: the source's data context's `arrow_hasher.hash_table(...)`, so the algorithm is + covered by the data context version. `string` and `large_string` input give the same + digest (measured), and a row gives the same digest in any batch or slice (measured). +* Written form: `ContentHash.to_string()`, the labelled full digest, for example + `arrow_v1:000002069471f502581c53b8f4efb4400ae7d8a1edf280582bdc8923c6b95acbcd9bfd` + (79 characters, no `::`). No truncation. +* Source-info value: `::::`. Record id: + `uuid5( + "::" + )`, unchanged recipe. +* A null in a key column raises, naming the table and the column. + +**Cost, and one deviation from "in Arrow per batch".** The data context's hasher has no +per-row entry point: its unit is a table. The token therefore needs one `hash_table` call +per row on a one-row slice of the batch. The values never become Python objects, but the +loop is in Python. Measured: 183 µs per row (8,192 rows in 1.5 s), on top of about 110 µs +per row for building the row. 100,000 rows: about 18 s of hashing. A vectorized per-row +digest would need a new hasher entry point (a data context matter); not proposed now. + +**One point to confirm.** The labelled token goes into the `uuid5` recipe, so a future +change of the `arrow_v1` label would move record ids. That is the same position the labelled +default source id (`dict:arrow_v1:`) already has in the recipe. + +## D2 measured: two jobs, one store, the restriction in or out of the content hash + +Script and full output: `superpowers/specs/itl-186-experiments/d2_restriction_identity.py`, +`d2_output.txt`. A stand-in source over one 10-row table with key-based record ids; pipeline +`source -> double(v)`; job A runs first, then job B, same job name, same Delta store. + +**Limit case: A reads k=0..4, B reads k=0..1.** + +| | OUT, sync | OUT, async | IN, sync and async | +|---|---|---|---| +| B's function calls | 0 | 0 | 0 | +| B `node.as_table()` / `iter_data()` after `run()` | 0,1 | **0,1,2,3,4** | 0,1 | +| B `node.get_all_records()` | **0,1,2,3,4** | **0,1,2,3,4** | 0,1 | +| B saved and loaded, `get_all_records()` | **0,1,2,3,4** | **0,1,2,3,4** | 0,1 | +| node content hash, A vs B | same | same | different | +| table paths (pipeline hash) | same | same | same | +| stored result rows / pipeline-record rows | 5 / 5 | 5 / 5 | 5 / 7 | + +**Where case: A reads even k, B reads k<3.** B computes only k=1 in every variant. OUT: B's +`get_all_records()` returns 0,1,2,4,6,8 (the union), and so does `as_table()` after an async +run; after a sync run `as_table()` returns 0,1,2. IN: 0,1,2 everywhere; 6 result rows, +8 pipeline-record rows. + +What this says: + +* **OUT.** Nothing is wrong in a value: every row returned is a real row of the table with + its correct result, and nothing is computed twice. The read-back is larger than what job B + asked for: it is every row any job with that pipeline has read from that table. Rows sent + downstream during the run are only B's rows, on both orchestrators. +* **OUT is what happens today.** Two `DictSource`s with the same schema and source id and + different rows give exactly the OUT columns above, including the difference between + `as_table()` after a sync run (B's rows) and after an async run (all rows). That + sync/async difference exists on `main` today and is not caused by this design. +* **IN.** Each job reads back exactly its rows. Results are not recomputed (the result table + is shared, found by row content). B's pipeline records are written again under B's node + hash (2 more rows in the limit case), in the same table. Node content hashes differ, so + the two jobs are different node instances, which is the developer's objection. +* In the code the choice is one function, `SpiralDBTableSource._restriction_identity()`, + returning nothing (OUT) or the canonical restriction (IN). diff --git a/superpowers/specs/itl-186-experiments/d2_restriction_identity.py b/superpowers/specs/itl-186-experiments/d2_restriction_identity.py new file mode 100644 index 000000000..5347df4e3 --- /dev/null +++ b/superpowers/specs/itl-186-experiments/d2_restriction_identity.py @@ -0,0 +1,137 @@ +"""D2 experiment (ITL-186): a source's row restriction in or out of its content hash. + +Run from the repository root: + + uv run python superpowers/specs/itl-186-experiments/d2_restriction_identity.py + +No SpiralDB needed. ``Restricted`` stands for a Spiral source over one 10-row table: the +same source id and key-based record ids whatever rows it reads, with the restriction either +left out of ``identity_structure`` (OUT) or appended to it (IN). Two jobs with the same +pipeline run one after the other on one Delta store: job A reads a larger part of the +table, job B a smaller one. +""" + +from __future__ import annotations + +import sys +import tempfile +from pathlib import Path +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource, DictSource + +FULL = pa.table( + {"k": pa.array(range(10), pa.int64()), "v": pa.array(range(10), pa.int64())} +).cast(pa.schema([pa.field("k", pa.int64(), False), pa.field("v", pa.int64(), False)])) +SOURCE_ID = "proj/default.t@table_X" + + +class Restricted(ArrowTableSource): + """One table, read through a restriction that may or may not enter identity.""" + + def __init__(self, keys: list[int], restriction: str, in_identity: bool) -> None: + self._restriction = restriction + self._in_identity = in_identity + rows = FULL.filter(pa.compute.is_in(FULL["k"], pa.array(keys, pa.int64()))) + super().__init__( + rows, tag_columns=["k"], source_id=SOURCE_ID, record_id_column="k" + ) + + def identity_structure(self) -> Any: + base = super().identity_structure() + return (*base, self._restriction) if self._in_identity else base + + +def build(store: Path, source: Any, calls: list[int]) -> PipelineJob: + @function_pod(output_keys="w") + def double(v: int) -> int: + calls.append(v) + return 2 * v + + job = PipelineJob(name="job", store=DeltaTableDatabase(store)) + with job: + double.pod(source, label="double") + return job + + +def keys_of(table: pa.Table | None) -> list[int] | None: + return None if table is None else sorted(table.column("k").to_pylist()) + + +def report(name: str, job: PipelineJob, calls: list[int], store: Path) -> None: + node = job.nodes["double"] + print(f" {name}: function calls this run = {sorted(calls)}") + print(f" node.as_table() keys = {keys_of(node.as_table())}") + print(f" node.iter_data() rows = {sum(1 for _ in node.iter_data())}") + print(f" node.get_all_records() keys = {keys_of(node.get_all_records())}") + saved = store.parent / f"{store.name}-{name}.json" + job.save(saved) + loaded = PipelineJob.load(saved, store=DeltaTableDatabase(store)).nodes["double"] + print(f" loaded job get_all_records = {keys_of(loaded.get_all_records())}") + print(f" node content hash = {node.content_hash().to_hex()[:12]}") + print(f" node pipeline hash = {node.pipeline_hash().to_hex()[:12]}") + + +def tables(store: Path) -> None: + import deltalake + + for log in sorted(store.rglob("_delta_log")): + table = deltalake.DeltaTable(str(log.parent)).to_pyarrow_table() + extra = "" + if "__node_content_hash" in table.column_names: + distinct = len(set(table.column("__node_content_hash").to_pylist())) + extra = f", {distinct} distinct node content hash(es)" + print( + f" table .../{log.parent.name} ({'results' if '_result' in log.parts else 'pipeline records'}): {table.num_rows} rows{extra}" + ) + + +def scenario(title: str, make_a: Any, make_b: Any, orchestrator: Any) -> None: + print(f"\n== {title} ({'async' if orchestrator else 'sync'})") + store = Path(tempfile.mkdtemp()) / "store" + for name, make in (("A", make_a), ("B", make_b)): + calls: list[int] = [] + job = build(store, make(), calls) + job.run(orchestrator=orchestrator() if orchestrator else None) + report(name, job, calls, store) + print(" stored tables:") + tables(store) + + +LIMIT_A, LIMIT_B = [0, 1, 2, 3, 4], [0, 1] +WHERE_A, WHERE_B = [0, 2, 4, 6, 8], [0, 1, 2] +for orch in (None, AsyncPipelineOrchestrator): + for in_identity in (False, True): + tag = "IN the content hash" if in_identity else "OUT of the content hash" + scenario( + f"limit: A reads k=0..4, B reads k=0..1; restriction {tag}", + lambda: Restricted(LIMIT_A, "limit=5", in_identity), + lambda: Restricted(LIMIT_B, "limit=2", in_identity), + orch, + ) + scenario( + f"where: A reads even k, B reads k<3; restriction {tag}", + lambda: Restricted(WHERE_A, "where=even", in_identity), + lambda: Restricted(WHERE_B, "where=k<3", in_identity), + orch, + ) + scenario( + "today: two DictSources, same schema and source id, different rows (positional record ids)", + lambda: DictSource( + [{"k": k, "v": k} for k in LIMIT_A], + tag_columns=["k"], + source_id=SOURCE_ID, + ), + lambda: DictSource( + [{"k": k, "v": k} for k in LIMIT_B], + tag_columns=["k"], + source_id=SOURCE_ID, + ), + orch, + ) +sys.stdout.flush() From cbcf80657054ceede02793d4f00de7fbf88d2de8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:14:34 -0800 Subject: [PATCH 03/17] wip(spiraldb): streaming SpiralDBTableSource built from metadata (ITL-186) Co-Authored-By: Claude Fable 5.2 --- .../core/sources/spiraldb_table_source.py | 484 ++++++++++++++---- src/orcapod/core/sources/stream_builder.py | 71 ++- src/orcapod/databases/spiraldb_connector.py | 146 +++++- src/orcapod/errors.py | 8 + .../test_spiraldb_table_source_streaming.py | 452 ++++++++++++++-- tests/test_databases/fake_pyspiral.py | 124 ++++- 6 files changed, 1093 insertions(+), 192 deletions(-) diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 167277016..1fb7afa29 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -1,7 +1,9 @@ -"""SpiralDBTableSource — a read-only RootSource backed by a SpiralDB table. +"""SpiralDBTableSource — a read-only RootSource that streams a SpiralDB table. -Wraps a SpiralDB table as an OrcaPod Source. The table's primary-key -(key-schema) columns are used as tag columns by default. +The source is built from the table's metadata alone (schema, key columns, SpiralDB's +table id): no row is read until one is asked for. Rows then arrive in batches, so a +read holds one batch at a time whatever the table's size. The table's primary-key +(key-schema) columns are the tag columns by default. Requires the ``spiraldb`` optional extra: ``pip install orcapod[spiraldb]``. Authentication is handled externally — run ``spiral login`` once; pyspiral keeps @@ -11,94 +13,137 @@ Example:: # Default dataset, PK columns become tag columns automatically - source = SpiralDBTableSource("my-project-123456", "spike_data") + source = SpiralDBTableSource("my-project-123456", "spike_event", limit=1000) - # Explicit dataset + # Only the rows of one unit source = SpiralDBTableSource( - "my-project-123456", "spike_data", dataset="prod" + "my-project-123456", "spike_event", where={"unit_id": 106} ) - # Override tag columns + # Explicit dataset, tag columns overridden source = SpiralDBTableSource( - "my-project-123456", "spike_data", tag_columns=["session_id"] + "my-project-123456", "spike_event", dataset="prod", tag_columns=["unit_id"] ) - -Note: - SpiralDB enforces non-null values in key-schema columns at storage time, - so NULL values in PK (tag) columns are not expected in practice. If a - table is written with NULL key columns via an external tool, those NULLs - will be propagated into the tag columns as-is. - - Tables with no key schema and no explicit ``tag_columns`` will raise - ``ValueError`` at construction time. Either define a key schema on the - SpiralDB table or supply explicit ``tag_columns``. """ from __future__ import annotations -from collections.abc import Collection +import asyncio +import datetime +from collections.abc import AsyncIterator, Collection, Iterator, Mapping +from contextlib import contextmanager from typing import TYPE_CHECKING, Any -from orcapod.core.sources.db_table_source import DBTableSource +from orcapod.core.sources.base import RootSource +from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.databases.spiraldb_connector import SpiralDBConnector -from orcapod.errors import ( - NullabilityRecordError, - NullabilityRecordMismatchError, - NullInNonNullableColumnError, - UnsupportedValueError, -) +from orcapod.errors import SourceTooLargeError +from orcapod.types import ColumnConfig +from orcapod.utils import arrow_utils +from orcapod.utils.lazy_module import LazyModule from orcapod.utils.schema_utils import _normalize_column_list if TYPE_CHECKING: + import pyarrow as pa + from orcapod import contexts from orcapod.config import OrcapodConfig + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + from orcapod.protocols.core_protocols import DataProtocol, TagProtocol +else: + pa = LazyModule("pyarrow") + +DEFAULT_MAX_ROWS = 100_000 +DEFAULT_BATCH_ROWS = 8_192 + +# How a datetime or date value of ``where`` is written in a saved config. +_DATETIME_KEY = "datetime" +_DATE_KEY = "date" + + +def _where_to_config(where: Mapping[str, object]) -> dict[str, Any]: + """Return ``where`` with its date and datetime values in a JSON form.""" + out: dict[str, Any] = {} + for column, value in where.items(): + if isinstance(value, datetime.datetime): + out[column] = {_DATETIME_KEY: value.isoformat()} + elif isinstance(value, datetime.date): + out[column] = {_DATE_KEY: value.isoformat()} + else: + out[column] = value + return out + +def _where_from_config(saved: Mapping[str, Any]) -> dict[str, object]: + """Return the ``where`` mapping that ``_where_to_config`` wrote.""" + out: dict[str, object] = {} + for column, value in saved.items(): + if isinstance(value, dict) and set(value) == {_DATETIME_KEY}: + out[column] = datetime.datetime.fromisoformat(value[_DATETIME_KEY]) + elif isinstance(value, dict) and set(value) == {_DATE_KEY}: + out[column] = datetime.date.fromisoformat(value[_DATE_KEY]) + else: + out[column] = value + return out -class SpiralDBTableSource(DBTableSource): - """A read-only Source backed by a table in a SpiralDB dataset. - At construction time the source: +class SpiralDBTableSource(RootSource): + """A read-only Source over a table in a SpiralDB dataset, read in batches. - 1. Opens a ``SpiralDBConnector`` for *project_id* and *dataset*. - 2. Validates the table exists. - 3. Resolves tag columns: + Building the source reads the table's metadata only: that the table exists, its + key columns, its schema (with the table's nullability record applied) and + SpiralDB's own id of the table. No row is read. ``output_schema``, ``keys``, + ``content_hash``, ``pipeline_hash`` and ``to_config`` never read a row either. - - If *tag_columns* is provided, uses them as-is. - - Otherwise uses the table's primary-key (key-schema) columns. - - Raises ``ValueError`` if the table has no key schema and no - explicit *tag_columns* are given. + Rows are read when asked for (``iter_data``, ``async_iter_data``, ``as_table`` and + the dataframe methods). Each read is one scan of the table in key order, taken in + batches of at most ``batch_rows`` rows, so iterating holds one batch at a time. + ``as_table`` and the dataframe methods hold every row they return. Nothing is + cached: every read scans again. - 4. Delegates to ``DBTableSource.__init__`` for fetching and stream - building (source-info provenance, schema hash, system tags). - 5. Closes the connector — all data is eagerly loaded into memory, so - the connection is released immediately. + Identity. The source is identified by where the table is, never by its rows. The + default ``source_id`` is ``/.@
``, where + the table id is SpiralDB's own: a table dropped and created again under the same + name is a different source, and a renamed table gets a new source id too. + ``where``, ``limit`` and ``asof`` choose which rows are read and are not part of + the source's identity. + + Record identity. A row is identified by its primary key: the row's token is the + data context's Arrow hash of its key columns (in key order, as non-nullable + fields), written with its algorithm label. A row therefore has the same record ID + whichever read returned it, and keeps it when other rows are added. Args: project_id: SpiralDB project identifier (e.g. ``"my-project-123456"``). table_name: Name of the SpiralDB table to expose as a source. dataset: Dataset within the project. Defaults to ``"default"``. - tag_columns: Columns to use as tag columns. A bare string is - accepted as a single column name. If ``None`` (default), - the table's primary-key (key-schema) columns are used. Raises - ``ValueError`` if the table has no key schema. + tag_columns: Columns to use as tag columns. A bare string is accepted as a + single column name. If ``None`` (default), the table's primary-key + (key-schema) columns are used. system_tag_columns: Additional system-level tag columns. - record_id_column: Column for stable per-row record IDs in provenance. - source_id: Canonical source name for the registry and provenance - tokens. Defaults to ``"//"``. - The project id is Spiral's project identifier (for example - ``"edi-sample-v1-855632"``: the API's resource id, unique across - Spiral, not a display name), so tables with the same name in - different projects or datasets never share record ids. + where: Column name to value. Only rows in which every named column equals its + value are read; SpiralDB applies this before any row is transferred. + limit: Read at most this many rows: the first in key order (after ``where``). + asof: Read the table as of this version (microseconds, as SpiralDB's + transaction timestamps). If ``None`` (default), the version the table has + when the source is built is used for every read of this source object. + max_rows: The most rows one read may return (default 100,000); ``None`` + removes the cap. A read that would return more raises + ``SourceTooLargeError``. A pipeline job keeps every row it processes in + memory, about 5 KB per row and node. + batch_rows: The most rows held at once while iterating (default 8,192). + source_id: Canonical source name for the registry and provenance tokens. + Defaults to ``"/.@
"``. label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: OrcaPod configuration. - overrides: Optional pyspiral client config overrides passed through - to ``SpiralDBConnector``, e.g. - ``{"server.url": "http://api.spiraldb.dev"}``. + overrides: Optional pyspiral client config overrides passed through to + ``SpiralDBConnector``, e.g. ``{"server.url": "http://api.spiraldb.dev"}``. Raises: - ValueError: If the table is not found, has no primary-key columns - and no explicit *tag_columns* are given, or the table is empty. + ValueError: If the table is not found, has no key schema, a tag column or a + ``where`` column is not in the table, or ``limit``, ``max_rows`` or + ``batch_rows`` is not a positive integer. """ def __init__( @@ -108,74 +153,279 @@ def __init__( dataset: str = "default", tag_columns: str | Collection[str] | None = None, system_tag_columns: Collection[str] = (), - record_id_column: str | None = None, + where: Mapping[str, object] | None = None, + limit: int | None = None, + asof: int | None = None, + max_rows: int | None = DEFAULT_MAX_ROWS, + batch_rows: int = DEFAULT_BATCH_ROWS, source_id: str | None = None, label: str | None = None, data_context: "str | contexts.DataContext | None" = None, config: "OrcapodConfig | None" = None, overrides: dict[str, str] | None = None, ) -> None: + for name, value in (("limit", limit), ("max_rows", max_rows)): + if value is not None and (not isinstance(value, int) or value < 1): + raise ValueError( + f"{name} must be a positive integer or None, got {value!r}" + ) + if not isinstance(batch_rows, int) or batch_rows < 1: + raise ValueError( + f"batch_rows must be a positive integer, got {batch_rows!r}" + ) + + super().__init__( + source_id=source_id, label=label, data_context=data_context, config=config + ) self._project_id = project_id self._dataset = dataset + self._table_name = table_name self._overrides = overrides - if source_id is None: - source_id = f"{project_id}/{dataset}/{table_name}" + self._where: dict[str, object] = dict(where) if where else {} + self._limit = limit + self._max_rows = max_rows + self._batch_rows = batch_rows + self._given_asof = asof + where_ = f"SpiralDB project {project_id!r}, dataset {dataset!r}" - connector = SpiralDBConnector( - project_id=project_id, - dataset=dataset, - overrides=overrides, + with self._connector() as connector: + if table_name not in connector.get_table_names(): + raise ValueError( + f"{where_}: Table {table_name!r} not found in database." + ) + self._key_columns: tuple[str, ...] = tuple( + connector.get_pk_columns(table_name) + ) + if not self._key_columns: + raise ValueError( + f"{where_}: Table {table_name!r} has no primary key columns, so " + "its rows cannot be identified." + ) + self._table_id = connector.get_spiral_table_id(table_name) + schema = connector.get_arrow_schema(table_name) + # One version for every read of this source object. A table that was + # never written has no version to pin. + self._asof = ( + asof if asof is not None else connector.get_table_version(table_name) + ) or None + + resolved_tags = ( + list(self._key_columns) + if tag_columns is None + else _normalize_column_list(tag_columns) ) + unknown = [c for c in self._where if c not in schema.names] + if unknown: + raise ValueError( + f"{where_}: where names columns that table {table_name!r} does not " + f"have: {unknown}. Columns: {schema.names}" + ) + if self._source_id is None: + self._source_id = f"{project_id}/{dataset}.{table_name}@{self._table_id}" + # A zero-row stream with the table's columns answers every schema question. + self._builder = SourceStreamBuilder(self.data_context, self.orcapod_config) try: - resolved_tags: list[str] | None = ( - _normalize_column_list(tag_columns) if tag_columns is not None else None + template = self._builder.build( + schema.empty_table(), + tag_columns=resolved_tags, + source_id=self._source_id, + system_tag_columns=system_tag_columns, ) + except ValueError as exc: + raise ValueError(f"{where_}, table {table_name!r}: {exc}") from exc + self._stream = template.stream + self._schema_hash = template.schema_hash + self._tag_columns = template.tag_columns + self._system_tag_columns = template.system_tag_columns + self._table_schema: pa.Schema = arrow_utils.drop_system_columns( + schema.empty_table() + ).schema + + # ------------------------------------------------------------------------- + # Identity + # ------------------------------------------------------------------------- + + @property + def table_id(self) -> str: + """SpiralDB's own id of the table this source reads.""" + return self._table_id + + def _restriction_identity(self) -> None: + """Return what ``where``, ``limit`` and ``asof`` add to the source's identity. + + Nothing: a source is identified by where its table is, and a restriction + chooses rows of that same table. This is the one place that decides it. + """ + return None + + def identity_structure(self) -> Any: + """Identity from the class, the schema and the source id (the table's locator).""" + base = super().identity_structure() + restriction = self._restriction_identity() + return base if restriction is None else (*base, restriction) + + # ------------------------------------------------------------------------- + # Reading + # ------------------------------------------------------------------------- - # DBTableSource handles PK resolution and raises ValueError when - # the table has no key schema and no explicit tag_columns are given. - try: - super().__init__( - connector, - table_name, - tag_columns=resolved_tags, - system_tag_columns=system_tag_columns, - record_id_column=record_id_column, - source_id=source_id, - label=label, - data_context=data_context, - config=config, + @contextmanager + def _connector(self) -> Iterator[SpiralDBConnector]: + """Open a connector to the source's dataset and close it afterwards.""" + connector = SpiralDBConnector( + project_id=self._project_id, + dataset=self._dataset, + overrides=self._overrides, + ) + try: + yield connector + finally: + connector.close() + + @property + def _qualified(self) -> str: + """The table as error messages name it.""" + return f"{self._project_id}/{self._dataset}.{self._table_name}" + + @property + def row_count_estimate(self) -> int: + """The whole table's row count as SpiralDB's manifests record it. + + No row is read. ``where`` and ``limit`` are not taken into account, and the + count can be above what a scan returns when rows were deleted or overwritten + and not yet compacted. + """ + with self._connector() as connector: + return connector.get_row_count_estimate(self._table_name) + + def _too_large(self, found: str) -> SourceTooLargeError: + """Return the error for a read that goes over ``max_rows``.""" + return SourceTooLargeError( + f"SpiralDB table {self._qualified!r}: {found}, more than " + f"max_rows={self._max_rows}. Read fewer rows with limit= or " + "where={column: value}, or raise max_rows (max_rows=None removes the " + "cap; a pipeline job keeps every row in memory)." + ) + + def _key_tokens(self, table: "pa.Table") -> list[str]: + """Return each row's token: the labelled Arrow hash of its primary key. + + Raises: + ValueError: If a key column holds a null. + """ + keys = table.select(list(self._key_columns)) + for name in self._key_columns: + if keys.column(name).null_count: + raise ValueError( + f"SpiralDB table {self._qualified!r}: key column {name!r} holds " + "a null, so the row cannot be identified." ) - except ( - NullabilityRecordError, - NullabilityRecordMismatchError, - NullInNonNullableColumnError, - UnsupportedValueError, + # A key is never null: hash its fields as non-nullable whatever is reported, + # so a token does not depend on the table's nullability record. + keys = keys.cast(pa.schema([f.with_nullable(False) for f in keys.schema])) + hasher = self.data_context.arrow_hasher + return [ + hasher.hash_table(keys.slice(i, 1)).to_string() + for i in range(keys.num_rows) + ] + + def _batch_streams(self) -> Iterator["ArrowTableStream"]: + """Scan the table once and yield its rows as streams of at most ``batch_rows`` rows. + + Raises: + SourceTooLargeError: If the read would return, or returns, more than + ``max_rows`` rows. + ValueError: If a batch's columns differ from the source's schema, or a + key column holds a null. + """ + with self._connector() as connector: + if self._max_rows is not None and not self._where: + # Nothing but limit bounds the read, so its size is known up front. + expected = ( + self._limit + if self._limit is not None and self._limit <= self._max_rows + else min( + connector.get_row_count_estimate(self._table_name), + self._limit if self._limit is not None else 2**63, + ) + ) + if expected > self._max_rows: + raise self._too_large(f"the read would return {expected:,} rows") + seen = 0 + for batch in connector.scan_batches( + self._table_name, + where=self._where or None, + limit=self._limit, + asof=self._asof, ): - # The connector's own errors already name the table: keep their types. - raise - except ValueError as exc: - # DBTableSource knows only the table name; say where it looked. - raise ValueError( - f"SpiralDB project {project_id!r}, dataset {dataset!r}: {exc}" - ) from exc + for start in range(0, batch.num_rows, self._batch_rows): + rows = arrow_utils.drop_system_columns( + pa.Table.from_batches([batch.slice(start, self._batch_rows)]) + ) + if not rows.schema.equals(self._table_schema): + raise ValueError( + f"SpiralDB table {self._qualified!r} returned columns " + f"{rows.schema} but the source was built with " + f"{self._table_schema}." + ) + seen += rows.num_rows + if self._max_rows is not None and seen > self._max_rows: + raise self._too_large("the read returned more rows") + yield self._builder.attach_provenance( + rows, + tag_columns=self._tag_columns, + system_tag_columns=self._system_tag_columns, + schema_hash=self._schema_hash, + source_id=self.source_id, + provenance_tokens=self._key_tokens(rows), + ) + + def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: + """Scan the table and yield its ``(tag, data)`` pairs, one batch held at a time.""" + for stream in self._batch_streams(): + yield from stream.iter_data() + + async def async_iter_data( + self, + ) -> AsyncIterator["tuple[TagProtocol, DataProtocol]"]: + """Yield the table's ``(tag, data)`` pairs, reading each batch off the event loop.""" + streams = self._batch_streams() + try: + while True: + stream = await asyncio.to_thread(next, streams, None) + if stream is None: + break + for pair in stream.iter_data(): + yield pair finally: - try: - connector.close() - except Exception: - # Suppress connector close errors to avoid masking __init__ - # failures. - pass + streams.close() + + def as_table( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "pa.Table": + """Scan the table and return its rows as one PyArrow table (all held in memory).""" + tables = [ + stream.as_table(columns=columns, all_info=all_info) + for stream in self._batch_streams() + ] + if not tables: + return self._stream.as_table(columns=columns, all_info=all_info) + return pa.concat_tables(tables) + + # ------------------------------------------------------------------------- + # Serialization + # ------------------------------------------------------------------------- def to_config(self, db_registry=None) -> dict[str, Any]: """Serialize source configuration to a JSON-compatible dict. - Note: - Does **not** call ``super().to_config()`` (i.e. ``DBTableSource.to_config()``) - because that would invoke ``self._connector.to_config()``, which raises - ``RuntimeError`` once the connector has been closed. The connector is always - closed during ``__init__`` after the eager data load, so the full dict is - built from the already-captured instance attributes instead. + The table is saved by its name and by SpiralDB's table id; ``from_config`` + checks that the name still leads to that table. A version pinned by the + source itself is not saved: a loaded source reads the table's then-current + version. An ``asof`` the user gave is saved. """ return { "source_type": "spiraldb_table", @@ -183,9 +433,14 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "dataset": self._dataset, "overrides": self._overrides, "table_name": self._table_name, + "table_id": self._table_id, "tag_columns": list(self._tag_columns), "system_tag_columns": list(self._system_tag_columns), - "record_id_column": self._record_id_column, + "where": _where_to_config(self._where), + "limit": self._limit, + "asof": self._given_asof, + "max_rows": self._max_rows, + "batch_rows": self._batch_rows, "source_id": self.source_id, **self._identity_config(), } @@ -194,23 +449,40 @@ def to_config(self, db_registry=None) -> dict[str, Any]: def from_config( cls, config: dict[str, Any], db_registry=None ) -> "SpiralDBTableSource": - """Reconstruct a SpiralDBTableSource from a config dict. + """Reconstruct a SpiralDBTableSource from a config dict, reading no rows. Args: config: Dict as produced by ``to_config()``. Returns: A new ``SpiralDBTableSource`` instance. + + Raises: + ValueError: If the saved table name now leads to a table with another + SpiralDB table id than the one saved (the table was dropped and + created again, or another table took the name). """ - return cls( + source = cls( project_id=config["project_id"], table_name=config["table_name"], dataset=config.get("dataset", "default"), tag_columns=config.get("tag_columns"), system_tag_columns=config.get("system_tag_columns", ()), - record_id_column=config.get("record_id_column"), + where=_where_from_config(config.get("where") or {}), + limit=config.get("limit"), + asof=config.get("asof"), + max_rows=config.get("max_rows", DEFAULT_MAX_ROWS), + batch_rows=config.get("batch_rows", DEFAULT_BATCH_ROWS), source_id=config.get("source_id"), label=config.get("label"), data_context=config.get("data_context"), overrides=config.get("overrides"), ) + saved_id = config.get("table_id") + if saved_id is not None and saved_id != source.table_id: + raise ValueError( + f"SpiralDB table {source._qualified!r} was saved with table id " + f"{saved_id!r} but that name now leads to table id " + f"{source.table_id!r}: it is a different table." + ) + return source diff --git a/src/orcapod/core/sources/stream_builder.py b/src/orcapod/core/sources/stream_builder.py index 0d85f4cb8..5e36d0cd5 100644 --- a/src/orcapod/core/sources/stream_builder.py +++ b/src/orcapod/core/sources/stream_builder.py @@ -276,43 +276,72 @@ def build( _make_provenance_token(record_id_column, i, value) for i, value in enumerate(record_values) ] + stream = self.attach_provenance( + table, + tag_columns=tag_columns_tuple, + system_tag_columns=system_tag_columns_tuple, + schema_hash=schema_hash, + source_id=source_id, + provenance_tokens=provenance_tokens, + ) + + return SourceStreamResult( + stream=stream, + schema_hash=schema_hash, + table_hash=table_hash, + source_id=source_id, + tag_columns=tag_columns_tuple, + system_tag_columns=system_tag_columns_tuple, + ) + + def attach_provenance( + self, + table: pa.Table, + *, + tag_columns: Sequence[str], + system_tag_columns: Sequence[str], + schema_hash: str, + source_id: str, + provenance_tokens: Sequence[str], + ) -> ArrowTableStream: + """Add source-info and system-tag columns to ``table`` and wrap it as a stream. + + Each row's source-info value is ``::`` and its record ID is + the UUID v5 of the same two parts, so a row keeps its record ID wherever and + however often it is read, as long as its token is the same. + + Args: + table: The rows, with nullable flags already set correctly. + tag_columns: Column names forming the tag for each row. + system_tag_columns: Additional system-level tag columns. + schema_hash: The source's schema hash, which names its system-tag columns. + source_id: The canonical source name. + provenance_tokens: One token per row, identifying the row within the source. + + Returns: + The stream over the enriched table. + """ source_info = [ f"{source_id}{constants.BLOCK_SEPARATOR}{token}" for token in provenance_tokens ] # Record IDs stored in the system tag column are stable UUID v5 bytes — # deterministically derived from source_id and per-row provenance token. - # Same source_id + same row position/content → same record_id bytes. record_id_values = [ _make_record_id(source_id, token).bytes for token in provenance_tokens ] - - # 8. Add source-info provenance columns. table = arrow_utils.add_source_info( - table, source_info, exclude_columns=tag_columns_tuple + table, source_info, exclude_columns=tag_columns ) - - # 9. Add system tag columns. table = arrow_utils.add_system_tag_columns( table, schema_hash, source_id, record_id_values, ) - - # 10. Wrap in ArrowTableStream. Nullable flags are already correct — - # the caller set them before calling build(). - stream = ArrowTableStream( + # Nullable flags are already correct — the caller set them. + return ArrowTableStream( table=table, - tag_columns=tag_columns_tuple, - system_tag_columns=system_tag_columns_tuple, - ) - - return SourceStreamResult( - stream=stream, - schema_hash=schema_hash, - table_hash=table_hash, - source_id=source_id, - tag_columns=tag_columns_tuple, - system_tag_columns=system_tag_columns_tuple, + tag_columns=tag_columns, + system_tag_columns=system_tag_columns, ) diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index bb2305a16..cb83e853c 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -22,7 +22,7 @@ import json import logging import re -from collections.abc import Iterator +from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from typing import TYPE_CHECKING, Any @@ -541,8 +541,6 @@ def iter_batches( NullInNonNullableColumnError: If a batch holds a null in a recorded non-nullable column (naming the table and the column). """ - import pyarrow as _pa # noqa: PLC0415 - self._require_open() if params is not None: logger.warning( @@ -557,15 +555,36 @@ def iter_batches( "Ignoring batch_size=%r", batch_size, ) - table_name = _parse_table_name(query) - tbl = self._project.table(self._table_id(table_name)) + yield from self.scan_batches(_parse_table_name(query)) + + def _batch_preparer( + self, tbl: Any, table_name: str + ) -> Callable[[pa.RecordBatch], pa.RecordBatch]: + """Return the function that gives a scanned batch the types Orcapod reports. + + The function restores field and schema metadata from the table's key-value + store, normalizes every type to its large variants and applies the table's + nullability record, exactly as ``iter_batches`` documents. + + Args: + tbl: The pyspiral table. + table_name: Plain table name (no dataset prefix). + + Returns: + A function from a scanned batch to the prepared batch. + + Raises: + NullabilityRecordError: If the table's nullability record cannot be read. + """ + import pyarrow as _pa # noqa: PLC0415 + kv = tbl.get_metadata() schema_meta, field_trees = _load_arrow_metadata(kv) qualified = self._qualified(table_name) with _about_table(qualified): paths = read_nullability_record(kv) - reader = self._spiral.scan(tbl.select()).to_record_batches() - for batch in reader: + + def prepare(batch: pa.RecordBatch) -> pa.RecordBatch: schema = batch.schema new_fields = [] needs_rebuild = False @@ -596,7 +615,118 @@ def iter_batches( if paths is not None: with _about_table(qualified): batch = enforce_nullability_record(batch, paths) - yield batch + return batch + + return prepare + + def scan_batches( + self, + table_name: str, + *, + where: Mapping[str, object] | None = None, + limit: int | None = None, + asof: int | None = None, + ) -> Iterator[pa.RecordBatch]: + """Scan a table, optionally restricted, and yield prepared Arrow batches. + + The restrictions are applied by SpiralDB, before any row reaches this + process. Batches arrive in key order; their sizes follow SpiralDB's stored + fragments and cannot be chosen. Each batch has the types, metadata and + nullability ``iter_batches`` documents. + + Args: + table_name: Plain table name (no dataset prefix). + where: Column name to value. A row is read only if every named column + equals its value. + limit: Read at most this many rows, the first in key order. + asof: Read the table as of this version (a ``get_table_version`` value). + + Yields: + Arrow RecordBatch objects. Nothing for a table with no matching row. + + Raises: + NullabilityRecordError: If the table's nullability record cannot be read or + names a column the table does not have. + NullInNonNullableColumnError: If a batch holds a null in a recorded + non-nullable column (naming the table and the column). + """ + self._require_open() + tbl = self._project.table(self._table_id(table_name)) + prepare = self._batch_preparer(tbl, table_name) + predicate = None + for column, value in (where or {}).items(): + equals = tbl[column] == value + predicate = equals if predicate is None else predicate & equals + scan = self._spiral.scan(tbl.select(), where=predicate, limit=limit, asof=asof) + for batch in scan.to_record_batches(): + yield prepare(batch) + + def get_arrow_schema(self, table_name: str) -> pa.Schema: + """Return the schema of the batches ``scan_batches`` yields, without scanning. + + Args: + table_name: Plain table name (no dataset prefix). + + Returns: + The prepared Arrow schema: large-normalized types, restored metadata and + the nullability the table's nullability record gives. + + Raises: + NullabilityRecordError: If the table's nullability record cannot be read or + names a column the table does not have. + """ + import pyarrow as _pa # noqa: PLC0415 + + self._require_open() + tbl = self._project.table(self._table_id(table_name)) + declared = tbl.schema().to_arrow() + empty = _pa.RecordBatch.from_pylist([], schema=declared) + return self._batch_preparer(tbl, table_name)(empty).schema + + def get_spiral_table_id(self, table_name: str) -> str: + """Return SpiralDB's own id of the table (for example ``table_W7C01nkLTmEJ``). + + The id stays the same when the table is renamed. A table dropped and created + again under the same name gets a new one. + + Args: + table_name: Plain table name (no dataset prefix). + + Returns: + The table id. + """ + self._require_open() + return str(self._project.table(self._table_id(table_name)).table_id) + + def get_table_version(self, table_name: str) -> int: + """Return the table's current version, usable as ``asof`` in ``scan_batches``. + + Args: + table_name: Plain table name (no dataset prefix). + + Returns: + The time of the table's last change, in microseconds; ``0`` for a table + that was never written. + """ + self._require_open() + return int(self._project.table(self._table_id(table_name)).last_modified_at()) + + def get_row_count_estimate(self, table_name: str) -> int: + """Return the table's row count as SpiralDB's manifests record it, without scanning. + + The count is what the manifests hold: it can be above the number of rows a + scan returns when rows were deleted or overwritten and not yet compacted. + + Args: + table_name: Plain table name (no dataset prefix). + + Returns: + The recorded row count. + """ + self._require_open() + tbl = self._project.table(self._table_id(table_name)) + stats = self._spiral.core.manifest_stats(tbl.core.get_snapshot(asof=None)) + return int(stats.key_space_stats.total_rows) def delete_table(self, table_name: str) -> None: """Drop a table from SpiralDB, if it exists. diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 03119f3a8..d6150dd15 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -44,6 +44,14 @@ class RecursiveTypeError(TypeError): """ +class SourceTooLargeError(ValueError): + """Raised when a read of a source would return more rows than its ``max_rows``. + + The message gives the number of rows found or expected, the cap, and the ways to + read fewer rows or lift the cap. + """ + + class DuplicateTagError(ValueError): """Raised when duplicate tag values are found and skip_duplicates=False""" diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 59ab46369..1e763c7f4 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -6,11 +6,18 @@ from __future__ import annotations +import asyncio +import datetime as dt from pathlib import Path import pyarrow as pa import pytest +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import SourceTooLargeError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.serialization import resolve_source_from_config from orcapod.sources import SpiralDBTableSource from tests.test_databases import fake_pyspiral @@ -21,66 +28,421 @@ _ROWS = 5 +def _rows(timestamps: list[int], subject: str = "m1") -> pa.Table: + """Return all-nullable ``event`` rows for ``timestamps``.""" + table = pa.table( + { + "subject_id": [subject] * len(timestamps), + "timestamp": pa.array(timestamps, pa.uint64()), + "amplitude": pa.array([t + 0.5 for t in timestamps], pa.float32()), + } + ) + return table.cast(pa.schema([pa.field(f.name, f.type, True) for f in table.schema])) + + @pytest.fixture() -def event_table(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - """Provide a fake root holding a 5-row ``event`` table read 2 rows per batch.""" +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Provide a fake project holding a 5-row ``event`` table read 2 rows per batch.""" monkeypatch.setattr(fake_pyspiral, "scan_batch_rows", 2) with fake_pyspiral.install(tmp_path / "spiral"): project = fake_pyspiral.FakeSpiral().project(_PROJECT) - table = project.create_table("default.event", key_schema=_KEYS) - rows = pa.table( - { - "subject_id": ["m1"] * _ROWS, - "timestamp": pa.array(range(_ROWS), pa.uint64()), - "amplitude": pa.array([0.5, 1.5, None, 3.5, 4.5], pa.float32()), - } - ) - table.write( - rows.cast(pa.schema([pa.field(f.name, f.type, True) for f in rows.schema])) + project.create_table("default.event", key_schema=_KEYS).write( + _rows([1, 2, 3, 4, 5]) ) fake_pyspiral.scan_log.clear() - yield + yield project fake_pyspiral.scan_log.clear() -@pytest.mark.xfail(strict=True, reason="ITL-186: the source reads the table when built") -def test_construction_reads_no_rows(event_table): - source = SpiralDBTableSource(_PROJECT, "event") - source.output_schema(all_info=True) - source.keys(all_info=True) - source.content_hash() - source.pipeline_hash() - source.to_config() +def _timestamps(source: SpiralDBTableSource) -> list[int]: + return source.as_table().column("timestamp").to_pylist() + + +def _record_ids(source: SpiralDBTableSource) -> dict[int, bytes]: + """Return each row's record ID by its ``timestamp`` key.""" + table = source.as_table(all_info=True) + (column,) = [n for n in table.column_names if n.startswith("_tag_record_id::")] + return dict( + zip(table.column("timestamp").to_pylist(), table.column(column).to_pylist()) + ) + + +class TestNoRowIsReadUntilAsked: + def test_construction_and_identity_read_no_rows(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + source.output_schema(all_info=True) + source.keys(all_info=True) + source.content_hash() + source.pipeline_hash() + source.to_config() + assert source.row_count_estimate == _ROWS + + assert fake_pyspiral.scan_log == [] + + def test_iter_data_reads_batches_only_as_rows_are_consumed(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + rows = iter(source.iter_data()) + + next(rows) + assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [1] + + assert sum(1 for _ in rows) == _ROWS - 1 + assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [3] + + def test_batches_handed_on_hold_at_most_batch_rows(self, project, monkeypatch): + monkeypatch.setattr(fake_pyspiral, "scan_batch_rows", 4) + source = SpiralDBTableSource(_PROJECT, "event", batch_rows=3) + + # About the non-public batching: no public method shows the batch sizes. + sizes = [s.as_table().num_rows for s in source._batch_streams()] # pyright: ignore[reportPrivateUsage] + + assert sizes == [3, 1, 1] + + def test_a_read_that_matches_nothing_has_the_schema_of_one_that_does(self, project): + nothing = SpiralDBTableSource(_PROJECT, "event", where={"subject_id": "nobody"}) + rows = SpiralDBTableSource(_PROJECT, "event") + + assert list(nothing.iter_data()) == [] + empty = nothing.as_table(all_info=True) + assert empty.num_rows == 0 + assert empty.schema == rows.as_table(all_info=True).schema + + def test_a_table_with_key_columns_only_is_refused(self, project): + project.create_table("default.never_written", key_schema=_KEYS) + + with pytest.raises(ValueError, match="never_written.*data column"): + SpiralDBTableSource(_PROJECT, "never_written") + + def test_schema_is_the_tables(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + + assert source.keys() == (("subject_id", "timestamp"), ("amplitude",)) + tag_schema, data_schema = source.output_schema() + assert dict(tag_schema) == {"subject_id": str | None, "timestamp": int | None} + assert dict(data_schema) == {"amplitude": float | None} + assert source.as_table().schema.names == [ + "subject_id", + "timestamp", + "amplitude", + ] + + +class TestRows: + def test_streamed_rows_equal_as_table_rows(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + + streamed = [ + {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} + for tag, data in source.iter_data() + ] + table_rows = source.as_table(all_info=True).to_pylist() + + assert len(streamed) == _ROWS + assert len(table_rows) == _ROWS + for streamed_row, table_row in zip(streamed, table_rows, strict=True): + shared = streamed_row.keys() & table_row.keys() + assert {"subject_id", "timestamp", "amplitude"} <= shared + assert any(name.startswith("_tag_record_id::") for name in shared) + assert any(name.startswith("_source_") for name in shared) + assert {k: streamed_row[k] for k in shared} == { + k: table_row[k] for k in shared + } + + def test_async_iteration_yields_the_same_rows(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + + async def collect() -> list[dict]: + return [ + {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} + async for tag, data in source.async_iter_data() + ] + + expected = [ + {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} + for tag, data in source.iter_data() + ] + assert asyncio.run(collect()) == expected + + def test_dataframe_methods_return_the_rows(self, project): + source = SpiralDBTableSource(_PROJECT, "event", limit=2) + + assert source.as_df().get_column("timestamp").to_list() == [1, 2] + assert source.as_lazy_frame().collect().height == 2 + assert list(source.as_pandas_df()["timestamp"]) == [1, 2] + + +class TestRestrictions: + def test_limit_reads_the_first_rows_in_key_order(self, project): + source = SpiralDBTableSource(_PROJECT, "event", limit=2) + + assert _timestamps(source) == [1, 2] + assert fake_pyspiral.scan_log[-1].kwargs["limit"] == 2 + + def test_where_reads_only_matching_rows(self, project): + project.table("default.event").write(_rows([7, 8], subject="m2")) + source = SpiralDBTableSource(_PROJECT, "event", where={"subject_id": "m2"}) + + assert _timestamps(source) == [7, 8] + + def test_where_and_limit_together(self, project): + project.table("default.event").write(_rows([7, 8], subject="m2")) + source = SpiralDBTableSource( + _PROJECT, "event", where={"subject_id": "m2"}, limit=1 + ) + + assert _timestamps(source) == [7] + + def test_where_on_an_unknown_column_is_refused(self, project): + with pytest.raises(ValueError, match="nope"): + SpiralDBTableSource(_PROJECT, "event", where={"nope": 1}) + + @pytest.mark.parametrize("name", ["limit", "max_rows", "batch_rows"]) + def test_sizes_must_be_positive(self, project, name): + with pytest.raises(ValueError, match=name): + SpiralDBTableSource(_PROJECT, "event", **{name: 0}) + + def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project): + table = project.table("default.event") + version = table.last_modified_at() + source = SpiralDBTableSource(_PROJECT, "event") + table.write(_rows([9])) + + source.as_table() + list(source.iter_data()) + + assert [scan.kwargs["asof"] for scan in fake_pyspiral.scan_log] == [version] * 2 + + def test_a_given_version_is_used(self, project): + source = SpiralDBTableSource(_PROJECT, "event", asof=1) + + source.as_table() + + assert fake_pyspiral.scan_log[-1].kwargs["asof"] == 1 + assert source.to_config()["asof"] == 1 + + +class TestRowCap: + def test_a_read_over_the_cap_is_refused_before_scanning(self, project): + source = SpiralDBTableSource(_PROJECT, "event", max_rows=3) + + with pytest.raises(SourceTooLargeError) as raised: + source.as_table() + + assert fake_pyspiral.scan_log == [] + message = str(raised.value) + assert "5 rows" in message and "max_rows=3" in message + assert "limit=" in message and "where=" in message + with pytest.raises(SourceTooLargeError): + list(source.iter_data()) + + def test_a_limit_within_the_cap_is_read(self, project): + source = SpiralDBTableSource(_PROJECT, "event", max_rows=3, limit=3) + + assert _timestamps(source) == [1, 2, 3] + + def test_a_limit_over_the_cap_is_refused(self, project): + source = SpiralDBTableSource(_PROJECT, "event", max_rows=3, limit=4) + + with pytest.raises(SourceTooLargeError, match="4 rows"): + source.as_table() + + def test_a_filtered_read_over_the_cap_stops_while_reading(self, project): + source = SpiralDBTableSource( + _PROJECT, "event", max_rows=3, where={"subject_id": "m1"} + ) + + with pytest.raises(SourceTooLargeError, match="max_rows=3"): + list(source.iter_data()) + + def test_a_filtered_read_within_the_cap_is_read(self, project): + project.table("default.event").write(_rows([7, 8], subject="m2")) + source = SpiralDBTableSource( + _PROJECT, "event", max_rows=3, where={"subject_id": "m2"} + ) + + assert _timestamps(source) == [7, 8] + + def test_no_cap(self, project): + source = SpiralDBTableSource(_PROJECT, "event", max_rows=None) + + assert len(_timestamps(source)) == _ROWS + + +class TestIdentity: + def test_default_source_id_names_the_table_and_carries_its_spiral_id(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + table_id = project.table("default.event").table_id + + assert source.table_id == table_id + assert source.source_id == f"{_PROJECT}/default.event@{table_id}" + + def test_a_given_source_id_is_used_as_it_is(self, project): + assert SpiralDBTableSource(_PROJECT, "event", source_id="mine").source_id == ( + "mine" + ) + + def test_a_recreated_table_is_a_different_source(self, project): + before = SpiralDBTableSource(_PROJECT, "event") + project.drop_table("default.event") + project.create_table("default.event", key_schema=_KEYS).write( + _rows([1, 2, 3, 4, 5]) + ) + after = SpiralDBTableSource(_PROJECT, "event") + + assert after.source_id != before.source_id + assert after.content_hash() != before.content_hash() + assert after.pipeline_hash() == before.pipeline_hash() + assert _record_ids(after)[1] != _record_ids(before)[1] + + def test_restrictions_do_not_change_identity(self, project): + whole = SpiralDBTableSource(_PROJECT, "event") + part = SpiralDBTableSource( + _PROJECT, "event", where={"subject_id": "m1"}, limit=2, asof=1, max_rows=9 + ) + + assert part.source_id == whole.source_id + assert part.content_hash() == whole.content_hash() + assert part.pipeline_hash() == whole.pipeline_hash() + + +class TestRecordIdentity: + def test_the_row_token_is_the_labelled_hash_of_the_primary_key(self, project): + source = SpiralDBTableSource(_PROJECT, "event", limit=1) + key = pa.table( + { + "subject_id": pa.array(["m1"], pa.large_string()), + "timestamp": pa.array([1], pa.uint64()), + }, + schema=pa.schema( + [ + pa.field("subject_id", pa.large_string(), nullable=False), + pa.field("timestamp", pa.uint64(), nullable=False), + ] + ), + ) + token = source.data_context.arrow_hasher.hash_table(key).to_string() + + (info,) = source.as_table(columns={"source": True}).column("_source_amplitude") + + assert info.as_py() == f"{source.source_id}::{token}::amplitude" + assert "::" not in token + + def test_a_row_has_the_same_record_id_in_every_read(self, project): + whole = _record_ids(SpiralDBTableSource(_PROJECT, "event")) + limited = _record_ids(SpiralDBTableSource(_PROJECT, "event", limit=2)) + filtered = _record_ids( + SpiralDBTableSource(_PROJECT, "event", where={"timestamp": 4}) + ) + + assert len(set(whole.values())) == _ROWS + assert limited == {t: whole[t] for t in (1, 2)} + assert filtered == {4: whole[4]} + + def test_a_row_keeps_its_record_id_when_an_earlier_key_is_added(self, project): + before = _record_ids(SpiralDBTableSource(_PROJECT, "event")) + project.table("default.event").write(_rows([0])) + + after = _record_ids(SpiralDBTableSource(_PROJECT, "event", max_rows=None)) + + assert list(after) == [0, 1, 2, 3, 4, 5] + assert {t: after[t] for t in before} == before + + def test_record_ids_do_not_depend_on_tag_columns(self, project): + by_key = _record_ids(SpiralDBTableSource(_PROJECT, "event")) + by_subject = _record_ids( + SpiralDBTableSource(_PROJECT, "event", tag_columns="subject_id") + ) + + assert by_subject == by_key + + +class TestSaveAndLoad: + def test_config_round_trip_reads_no_rows(self, project): + source = SpiralDBTableSource( + _PROJECT, + "event", + where={"subject_id": "m1", "timestamp": 2}, + limit=3, + max_rows=50, + batch_rows=7, + ) + config = source.to_config() + + loaded = resolve_source_from_config(config) + + assert fake_pyspiral.scan_log == [] + assert isinstance(loaded, SpiralDBTableSource) + assert loaded.to_config() == config + assert loaded.content_hash() == source.content_hash() + assert config["table_id"] == source.table_id + assert _timestamps(loaded) == [2] + + def test_a_date_and_time_in_where_survive_the_round_trip(self, project): + keys = pa.schema([pa.field("start", pa.timestamp("us"))]) + start = dt.datetime(2026, 3, 24, 17, 54, 17) + project.create_table("default.session", key_schema=keys).write( + pa.table({"start": pa.array([start], pa.timestamp("us")), "n": [1]}) + ) + source = SpiralDBTableSource(_PROJECT, "session", where={"start": start}) + + loaded = SpiralDBTableSource.from_config(source.to_config()) + + assert loaded.as_table().num_rows == 1 + + def test_loading_refuses_a_name_that_now_leads_to_another_table(self, project): + config = SpiralDBTableSource(_PROJECT, "event").to_config() + project.drop_table("default.event") + project.create_table("default.event", key_schema=_KEYS).write(_rows([1])) + new_id = project.table("default.event").table_id + + with pytest.raises(ValueError) as raised: + SpiralDBTableSource.from_config(config) + + assert config["table_id"] in str(raised.value) + assert new_id in str(raised.value) + - assert fake_pyspiral.scan_log == [] +class TestInAJob: + def _job(self, store: Path, calls: list[float], **source_kwargs) -> PipelineJob: + @function_pod(output_keys="doubled") + def double(amplitude: float | None) -> float | None: + calls.append(amplitude) + return None if amplitude is None else 2 * amplitude + job = PipelineJob(name="events", store=DeltaTableDatabase(store)) + with job: + double.pod( + SpiralDBTableSource(_PROJECT, "event", **source_kwargs), label="double" + ) + return job -@pytest.mark.xfail(strict=True, reason="ITL-186: the source reads the table when built") -def test_iter_data_reads_batches_only_as_rows_are_consumed(event_table): - source = SpiralDBTableSource(_PROJECT, "event") - rows = iter(source.iter_data()) + @pytest.mark.parametrize("orchestrator", [None, AsyncPipelineOrchestrator]) + def test_run_rerun_and_an_appended_row(self, project, tmp_path, orchestrator): + def run(calls: list[float]) -> PipelineJob: + job = self._job(tmp_path / "store", calls) + job.run(orchestrator=orchestrator() if orchestrator else None) + return job - next(rows) - assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [1] + calls: list[float] = [] + first = run(calls) + assert sorted(calls) == [1.5, 2.5, 3.5, 4.5, 5.5] + assert first.nodes["double"].as_table().num_rows == _ROWS - assert sum(1 for _ in rows) == _ROWS - 1 - assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [3] + calls.clear() + run(calls) + assert calls == [] + # A new row, with a key before every other, is the only one computed. + project.table("default.event").write(_rows([0])) + third = run(calls) + assert calls == [0.5] + assert third.nodes["double"].get_all_records().num_rows == _ROWS + 1 -def test_streamed_rows_equal_as_table_rows(event_table): - source = SpiralDBTableSource(_PROJECT, "event") + def test_a_job_over_the_cap_fails_before_computing(self, project, tmp_path): + calls: list[float] = [] + job = self._job(tmp_path / "store", calls, max_rows=3) - streamed = [ - {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} - for tag, data in source.iter_data() - ] - table_rows = source.as_table(all_info=True).to_pylist() + with pytest.raises(SourceTooLargeError): + job.run() - assert len(streamed) == _ROWS - assert len(table_rows) == _ROWS - for streamed_row, table_row in zip(streamed, table_rows, strict=True): - shared = streamed_row.keys() & table_row.keys() - assert {"subject_id", "timestamp", "amplitude"} <= shared - assert any(name.startswith("_tag_record_id::") for name in shared) - assert any(name.startswith("_source_") for name in shared) - assert {k: streamed_row[k] for k in shared} == {k: table_row[k] for k in shared} + assert calls == [] diff --git a/tests/test_databases/fake_pyspiral.py b/tests/test_databases/fake_pyspiral.py index a0ce8a465..531d6b271 100644 --- a/tests/test_databases/fake_pyspiral.py +++ b/tests/test_databases/fake_pyspiral.py @@ -8,12 +8,14 @@ Only the calls the connector makes are implemented: -* ``Spiral(overrides=...)``, ``Spiral.project(id)``, ``Spiral.scan(expr, limit=...)`` +* ``Spiral(overrides=...)``, ``Spiral.project(id)``, + ``Spiral.scan(expr, where=..., limit=..., asof=...)``, ``Spiral.core.manifest_stats`` (each call is recorded in ``scan_log``; batches are read on demand) * ``Project.list_tables()`` (``TableResource`` with ``.dataset`` / ``.table``), ``Project.table(id)``, ``Project.create_table(id, key_schema=..., exist_ok=...)``, ``Project.drop_table(id)`` -* ``Table.key_schema``, ``Table.schema().to_arrow()``, ``Table.select()``, ``Table.write(t)``, +* ``Table.key_schema``, ``Table.table_id``, ``Table.last_modified_at()``, ``Table[name] == v``, + ``Table.core.get_snapshot()``, ``Table.schema().to_arrow()``, ``Table.select()``, ``Table.write(t)``, ``Table.get_metadata()``, ``Table.set_metadata(kv)``, ``Table.drop_metadata(keys)`` * ``Scan.to_record_batches()``, ``Scan.to_table()`` @@ -50,6 +52,7 @@ import json import os import types +import uuid from collections.abc import Iterator from contextlib import contextmanager from dataclasses import asdict, dataclass @@ -270,10 +273,22 @@ def create(self, project_id: str, identifier: str, key_schema: pa.Schema) -> Non json.dumps(key_schema.names) ) self._path(project_id, identifier, ".kv.json").write_text("{}") + # SpiralDB gives every table its own id; a recreated table gets a new one. + self._path(project_id, identifier, ".id.txt").write_text( + f"table_{uuid.uuid4().hex[:12]}" + ) + self._path(project_id, identifier, ".version.txt").write_text("0") def drop(self, project_id: str, identifier: str) -> None: """Remove every file of table ``identifier``.""" - for suffix in (".keys.json", ".keys.arrow", ".kv.json", ".data.arrow"): + for suffix in ( + ".keys.json", + ".keys.arrow", + ".kv.json", + ".data.arrow", + ".id.txt", + ".version.txt", + ): p = self._path(project_id, identifier, suffix) if p.exists(): p.unlink() @@ -292,10 +307,21 @@ def read(self, project_id: str, identifier: str) -> pa.Table | None: return r.read_all() def write(self, project_id: str, identifier: str, table: pa.Table) -> None: - """Replace table ``identifier``'s rows with ``table``.""" + """Replace table ``identifier``'s rows with ``table`` and move its version on.""" p = self._path(project_id, identifier, ".data.arrow") with pa.ipc.new_file(str(p), table.schema) as w: w.write_table(table) + self._path(project_id, identifier, ".version.txt").write_text( + str(self.version(project_id, identifier) + 1) + ) + + def table_id(self, project_id: str, identifier: str) -> str: + """Return table ``identifier``'s own id.""" + return self._path(project_id, identifier, ".id.txt").read_text() + + def version(self, project_id: str, identifier: str) -> int: + """Return table ``identifier``'s version: ``0`` before the first write.""" + return int(self._path(project_id, identifier, ".version.txt").read_text()) def kv(self, project_id: str, identifier: str) -> dict[str, bytes]: """Return table ``identifier``'s key-value metadata.""" @@ -348,16 +374,54 @@ def __init__(self, table: FakeTable) -> None: self.table = table +class FakePredicate: + """A ``where`` expression: every named column equals its value.""" + + def __init__(self, equalities: list[tuple[str, Any]]) -> None: + self.equalities = equalities + + def __and__(self, other: FakePredicate) -> FakePredicate: + return FakePredicate(self.equalities + other.equalities) + + def __repr__(self) -> str: + return " & ".join(f"{name} == {value!r}" for name, value in self.equalities) + + +class FakeColumn: + """What ``table[name]`` returns: compare it with ``==`` to get a predicate.""" + + def __init__(self, name: str) -> None: + self._name = name + + def __eq__(self, value: object) -> FakePredicate: # pyright: ignore[reportIncompatibleMethodOverride] # as pyspiral's Expr does + return FakePredicate([(self._name, value)]) + + __hash__ = None # pyright: ignore[reportAssignmentType] # an expression, not a value + + class FakeScan: - """What ``Spiral.scan(table.select(), limit=...)`` returns.""" + """What ``Spiral.scan(table.select(), where=..., limit=...)`` returns.""" - def __init__(self, table: FakeTable, record: ScanRecord, limit: int | None) -> None: + def __init__( + self, + table: FakeTable, + record: ScanRecord, + where: FakePredicate | None, + limit: int | None, + ) -> None: self._table = table self._record = record + self._where = where self._limit = limit def _rows(self) -> pa.Table: rows = self._table._read_all() + for name, value in self._where.equalities if self._where else []: + rows = rows.filter( + pc.equal( + rows.column(name), pa.scalar(value, rows.schema.field(name).type) + ) + ) return rows if self._limit is None else rows.slice(0, self._limit) def to_table(self) -> pa.Table: @@ -401,6 +465,24 @@ def key_schema(self) -> pa.Schema: """The key columns, in key order.""" return self._store.key_schema(self._project_id, self._identifier) + @property + def table_id(self) -> str: + """The table's own id: kept on rename, new when the table is recreated.""" + return self._store.table_id(self._project_id, self._identifier) + + @property + def core(self) -> Any: + """The table's low-level handle; only ``get_snapshot`` is modelled.""" + return types.SimpleNamespace(get_snapshot=lambda asof=None: self) + + def last_modified_at(self) -> int: + """Return the table's version: ``0`` before the first write.""" + return self._store.version(self._project_id, self._identifier) + + def __getitem__(self, name: str) -> FakeColumn: + """Return the column expression ``table[name]``.""" + return FakeColumn(name) + def schema(self) -> FakeSchema: """Return the stored schema (only the key columns before the first write).""" stored = self._store.read(self._project_id, self._identifier) @@ -557,7 +639,8 @@ def list_tables(self) -> list[TableResource]: out = [] for ident in self._store.identifiers(self._project_id): dataset, _, table = ident.partition(".") - out.append(TableResource(ident, self._project_id, dataset, table)) + table_id = self._store.table_id(self._project_id, ident) + out.append(TableResource(table_id, self._project_id, dataset, table)) return out def table(self, identifier: str) -> FakeTable: @@ -625,22 +708,39 @@ def project(self, project_id: str) -> FakeProject: return FakeProject(self._store, project_id) def scan(self, *projections: Any, **kwargs: Any) -> FakeScan: - """Return a scan of one ``table.select()``, restricted by ``limit`` if given. + """Return a scan of one ``table.select()``, restricted by ``where`` and ``limit``. - Every call is appended to ``scan_log``. Other projections and the other - restrictions of ``Spiral.scan`` are not modelled. + Every call is appended to ``scan_log``. ``asof`` is accepted and recorded; the + fake keeps one version of a table. Other projections and ``key_ranges`` are + not modelled. """ if len(projections) != 1 or not isinstance(projections[0], FakeExpr): raise NotImplementedError( "fake Spiral.scan supports only scan(table.select())" ) - unsupported = {k for k, v in kwargs.items() if v is not None} - {"limit"} + unsupported = {k for k, v in kwargs.items() if v is not None} - { + "where", + "limit", + "asof", + } if unsupported: raise NotImplementedError(f"fake Spiral.scan: {sorted(unsupported)}") table = projections[0].table record = ScanRecord(table._identifier, dict(kwargs)) scan_log.append(record) - return FakeScan(table, record, kwargs.get("limit")) + return FakeScan(table, record, kwargs.get("where"), kwargs.get("limit")) + + @property + def core(self) -> Any: + """The client's low-level handle; only ``manifest_stats`` is modelled.""" + + def manifest_stats(snapshot: FakeTable) -> Any: + rows = snapshot._read_all().num_rows + return types.SimpleNamespace( + key_space_stats=types.SimpleNamespace(total_rows=rows) + ) + + return types.SimpleNamespace(manifest_stats=manifest_stats) fake_spiral_module = types.SimpleNamespace(Spiral=FakeSpiral) From 2376fed68fa3f0194f9f4930c7a18f91b1d7e0b8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:16:46 -0800 Subject: [PATCH 04/17] wip(spiraldb): tests for the streaming source on the fake (ITL-186) Co-Authored-By: Claude Fable 5.2 --- .../sources/test_spiraldb_table_source.py | 808 +++++------------- .../test_spiraldb_table_source_fake.py | 23 +- 2 files changed, 225 insertions(+), 606 deletions(-) diff --git a/tests/test_core/sources/test_spiraldb_table_source.py b/tests/test_core/sources/test_spiraldb_table_source.py index f03417d5e..580950175 100644 --- a/tests/test_core/sources/test_spiraldb_table_source.py +++ b/tests/test_core/sources/test_spiraldb_table_source.py @@ -1,16 +1,18 @@ -"""Tests for SpiralDBTableSource. +"""Tests for SpiralDBTableSource's interface as a source. -SpiralDB requires network access and credentials, so all tests use -``unittest.mock`` to patch ``SpiralDBConnector`` at the import site inside -``spiraldb_table_source``. No live SpiralDB instance is required. +SpiralDB needs network access and credentials, so these tests run against the +disk-backed pyspiral fake (``tests/test_databases/fake_pyspiral.py``): the source goes +through the real ``SpiralDBConnector``. Reading in batches, restrictions, the row cap and +record identity are in ``test_spiraldb_table_source_streaming.py``; the workshop path is in +``test_spiraldb_table_source_fake.py``. Test sections: 1. Import / export sanity 2. Protocol conformance 3. PK as default tag columns (single and composite) 4. Explicit tag column override - 5. No key schema → ValueError (no ROWID fallback) - 6. Error cases (missing table, empty table) + 5. Error cases (missing table, no key schema) + 6. The connector (arguments forwarded, closed after use) 7. Stream behaviour 8. Deterministic hashing 9. Config round-trip (to_config / from_config) @@ -19,109 +21,60 @@ from __future__ import annotations -from unittest.mock import MagicMock, patch +from pathlib import Path +from unittest.mock import patch import pyarrow as pa import pytest - -# --------------------------------------------------------------------------- -# Helpers — build mock connector behaviour -# --------------------------------------------------------------------------- +from orcapod.core.sources import SpiralDBTableSource +from orcapod.databases import SpiralDBConnector +from tests.test_databases import fake_pyspiral _TABLE_NAME = "spike_data" _PROJECT_ID = "test-project-123456" -_DATASET = "default" - - -def _make_mock_connector( - table_names: list[str] | None = None, - pk_columns: list[str] | None = None, - batches: list[pa.RecordBatch] | None = None, -) -> MagicMock: - """Return a MagicMock that faithfully simulates SpiralDBConnector lifecycle. - - Specifically, ``close()`` flips a closed flag and ``to_config()`` raises - ``RuntimeError`` once closed — matching the real ``SpiralDBConnector`` - behaviour. This ensures tests catch bugs where ``to_config()`` (or any - other post-close call) is accidentally delegated to the connector. - - Args: - table_names: Tables reported by ``get_table_names()``. - pk_columns: PK columns returned by ``get_pk_columns()``. - batches: Arrow batches returned by ``iter_batches()``. - """ - if table_names is None: - table_names = [_TABLE_NAME] - if pk_columns is None: - pk_columns = ["session_id"] - if batches is None: - batches = [ - pa.record_batch( - { - "session_id": pa.array(["s1", "s2", "s3"], type=pa.large_string()), - "firing_rate": pa.array([0.1, 0.2, 0.3], type=pa.float64()), - } - ) - ] - - connector = MagicMock(name="MockSpiralDBConnector") - connector.get_table_names.return_value = table_names - connector.get_pk_columns.return_value = pk_columns - connector.get_column_info.return_value = [] - connector.iter_batches.return_value = iter(batches) - - # Simulate the closed-connector lifecycle: close() flips the flag and - # to_config() raises RuntimeError afterwards, exactly as the real - # SpiralDBConnector does. - connector._closed = False - - def _close(): - connector._closed = True - - def _to_config(): - if connector._closed: - raise RuntimeError("SpiralDBConnector is closed") - return { - "connector_type": "spiraldb", - "project_id": _PROJECT_ID, - "dataset": _DATASET, - "overrides": None, - } - - connector.close.side_effect = _close - connector.to_config.side_effect = _to_config - return connector +_CONNECTOR = "orcapod.core.sources.spiraldb_table_source.SpiralDBConnector" + + +def _make_table( + name: str = _TABLE_NAME, + pk_columns: tuple[str, ...] = ("session_id",), + rows: pa.Table | None = None, + project_id: str = _PROJECT_ID, + dataset: str = "default", +) -> None: + """Create a table in the fake project and write ``rows`` to it.""" + if rows is None: + rows = pa.table( + { + "session_id": pa.array(["s1", "s2", "s3"], type=pa.large_string()), + "firing_rate": pa.array([0.1, 0.2, 0.3], type=pa.float64()), + } + ) + keys = pa.schema([rows.schema.field(name) for name in pk_columns]) + project = fake_pyspiral.FakeSpiral().project(project_id) + project.create_table(f"{dataset}.{name}", key_schema=keys).write(rows) -def _make_composite_pk_connector() -> MagicMock: - """Mock connector with composite PK (session_id, probe_id) and data.""" - batches = [ - pa.record_batch( +def _make_composite_pk_table() -> None: + """Create a table with composite PK (session_id, probe_id) and data.""" + _make_table( + pk_columns=("session_id", "probe_id"), + rows=pa.table( { "session_id": pa.array(["s1", "s1"], type=pa.large_string()), "probe_id": pa.array([1, 2], type=pa.int64()), "neuron_count": pa.array([100, 200], type=pa.int64()), } - ) - ] - return _make_mock_connector( - pk_columns=["session_id", "probe_id"], - batches=batches, + ), ) -# --------------------------------------------------------------------------- -# Shared fixture — patches SpiralDBConnector at the module level -# --------------------------------------------------------------------------- - - -def _patch_connector(mock_connector: MagicMock): - """Context manager: patch SpiralDBConnector to return *mock_connector*.""" - return patch( - "orcapod.core.sources.spiraldb_table_source.SpiralDBConnector", - return_value=mock_connector, - ) +@pytest.fixture(autouse=True) +def fake_spiral(tmp_path: Path): + """Route every source in this module to a fresh fake SpiralDB.""" + with fake_pyspiral.install(tmp_path / "spiral"): + yield # =========================================================================== @@ -154,31 +107,24 @@ def test_in_core_sources_all(): class TestProtocolConformance: def test_is_source_protocol(self): - from orcapod.core.sources import SpiralDBTableSource from orcapod.protocols.core_protocols import SourceProtocol - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert isinstance(src, SourceProtocol) + _make_table() + assert isinstance(SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME), SourceProtocol) def test_is_stream_protocol(self): - from orcapod.core.sources import SpiralDBTableSource from orcapod.protocols.core_protocols import StreamProtocol - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert isinstance(src, StreamProtocol) + _make_table() + assert isinstance(SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME), StreamProtocol) def test_is_pipeline_element_protocol(self): - from orcapod.core.sources import SpiralDBTableSource from orcapod.protocols.hashing_protocols import PipelineElementProtocol - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert isinstance(src, PipelineElementProtocol) + _make_table() + assert isinstance( + SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME), PipelineElementProtocol + ) # =========================================================================== @@ -188,66 +134,20 @@ def test_is_pipeline_element_protocol(self): class TestPKAsDefaultTags: def test_single_pk_is_tag_column(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=["session_id"]) - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - tag_schema, _ = src.output_schema() - assert "session_id" in tag_schema - - def test_pk_not_in_data_schema(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=["session_id"]) - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - _, data_schema = src.output_schema() - assert "session_id" not in data_schema - - def test_non_pk_columns_in_data_schema(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=["session_id"]) - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - _, data_schema = src.output_schema() - assert "firing_rate" in data_schema + _make_table() + tag_schema, data_schema = SpiralDBTableSource( + _PROJECT_ID, _TABLE_NAME + ).output_schema() + assert list(tag_schema) == ["session_id"] + assert list(data_schema) == ["firing_rate"] def test_composite_pk_all_columns_are_tags(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_composite_pk_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - tag_schema, _ = src.output_schema() - assert "session_id" in tag_schema - assert "probe_id" in tag_schema - - def test_composite_pk_data_column_in_data(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_composite_pk_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - _, data_schema = src.output_schema() - assert "neuron_count" in data_schema - - def test_default_source_id_is_fully_qualified(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert src.source_id == f"{_PROJECT_ID}/{_DATASET}/{_TABLE_NAME}" - - def test_explicit_source_id_overrides_default(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, source_id="my_src") - assert src.source_id == "my_src" + _make_composite_pk_table() + tag_schema, data_schema = SpiralDBTableSource( + _PROJECT_ID, _TABLE_NAME + ).output_schema() + assert list(tag_schema) == ["session_id", "probe_id"] + assert list(data_schema) == ["neuron_count"] # =========================================================================== @@ -257,141 +157,80 @@ def test_explicit_source_id_overrides_default(self): class TestExplicitTagOverride: def test_explicit_tag_columns_override_pk(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=["session_id"]) - # Provide explicit tag_columns — PK should be ignored - # Use "firing_rate" as tag by overriding, and "session_id" as data - batches = [ - pa.record_batch( - { - "session_id": pa.array(["s1", "s2"], type=pa.large_string()), - "firing_rate": pa.array([0.1, 0.2], type=pa.float64()), - "amplitude": pa.array([1.0, 2.0], type=pa.float64()), - } - ) - ] - connector.iter_batches.return_value = iter(batches) - with _patch_connector(connector): - src = SpiralDBTableSource( - _PROJECT_ID, _TABLE_NAME, tag_columns=["firing_rate"] - ) - tag_schema, _ = src.output_schema() - assert "firing_rate" in tag_schema - assert "session_id" not in tag_schema - - def test_multiple_explicit_tag_columns(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_composite_pk_connector() - with _patch_connector(connector): - src = SpiralDBTableSource( - _PROJECT_ID, - _TABLE_NAME, - tag_columns=["session_id", "probe_id"], - ) - tag_schema, _ = src.output_schema() - assert "session_id" in tag_schema - assert "probe_id" in tag_schema - - -# =========================================================================== -# 5. No key schema → ValueError -# =========================================================================== + _make_composite_pk_table() + src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=["probe_id"]) + tag_schema, data_schema = src.output_schema() + assert list(tag_schema) == ["probe_id"] + assert set(data_schema) == {"session_id", "neuron_count"} + def test_a_bare_string_names_one_tag_column(self): + _make_composite_pk_table() + src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns="session_id") + assert list(src.output_schema()[0]) == ["session_id"] -class TestNoPKRaisesError: - def test_no_pk_columns_and_no_explicit_tags_raises(self): - """SpiralDB has no ROWID fallback — raise ValueError when no PK.""" - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=[]) - with _patch_connector(connector): - with pytest.raises(ValueError, match="no primary key"): - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - - def test_no_pk_columns_but_explicit_tags_succeeds(self): - """Explicit tag_columns bypass the PK requirement.""" - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=[]) - with _patch_connector(connector): - src = SpiralDBTableSource( - _PROJECT_ID, _TABLE_NAME, tag_columns=["session_id"] - ) - tag_schema, _ = src.output_schema() - assert "session_id" in tag_schema + def test_unknown_tag_column_raises(self): + _make_table() + with pytest.raises(ValueError, match="tag_columns not found"): + SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=["nope"]) # =========================================================================== -# 6. Error cases +# 5. Error cases # =========================================================================== class TestErrorCases: def test_missing_table_raises_value_error(self): - from orcapod.core.sources import SpiralDBTableSource + _make_table() + with pytest.raises(ValueError, match="not found in database") as raised: + SpiralDBTableSource(_PROJECT_ID, "no_such_table", dataset="default") + assert _PROJECT_ID in str(raised.value) + assert "default" in str(raised.value) + + @pytest.mark.parametrize("tag_columns", [None, ["session_id"]]) + def test_a_table_without_a_key_schema_raises(self, tag_columns): + """Rows are identified by their primary key, so a key schema is required.""" + _make_table() + with patch.object(SpiralDBConnector, "get_pk_columns", return_value=[]): + with pytest.raises(ValueError, match="no primary key"): + SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=tag_columns) - connector = _make_mock_connector(table_names=[]) - with _patch_connector(connector): - with pytest.raises(ValueError, match="not found in database"): - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - def test_empty_table_raises_value_error(self): - from orcapod.core.sources import SpiralDBTableSource +# =========================================================================== +# 6. The connector +# =========================================================================== - connector = _make_mock_connector(batches=[]) - with _patch_connector(connector): - with pytest.raises(ValueError, match="is empty"): - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - def test_connector_closed_after_successful_init(self): - """Connector must be closed even on success (eager load).""" - from orcapod.core.sources import SpiralDBTableSource +class TestConnector: + def test_dataset_and_overrides_forwarded_to_connector(self): + overrides = {"server.url": "http://api.spiraldb.dev"} + _make_table(dataset="prod") + with patch(_CONNECTOR, wraps=SpiralDBConnector) as connector_class: + SpiralDBTableSource( + _PROJECT_ID, _TABLE_NAME, dataset="prod", overrides=overrides + ) + connector_class.assert_called_once_with( + project_id=_PROJECT_ID, dataset="prod", overrides=overrides + ) - connector = _make_mock_connector() - with _patch_connector(connector): - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - connector.close.assert_called() + def test_every_connector_is_closed(self): + _make_table() + opened: list[SpiralDBConnector] = [] - def test_connector_closed_after_failed_init(self): - """Connector must be closed even when init raises.""" - from orcapod.core.sources import SpiralDBTableSource + def connect(**kwargs): + opened.append(SpiralDBConnector(**kwargs)) + return opened[-1] - connector = _make_mock_connector(table_names=[]) - with _patch_connector(connector): + with patch(_CONNECTOR, side_effect=connect): + src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + src.as_table() + next(iter(src.iter_data())) # a read that is not finished with pytest.raises(ValueError): - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - connector.close.assert_called() - - def test_dataset_forwarded_to_connector(self): - """dataset argument must be passed through to SpiralDBConnector.""" - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with patch( - "orcapod.core.sources.spiraldb_table_source.SpiralDBConnector", - return_value=connector, - ) as mock_cls: - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, dataset="prod") - mock_cls.assert_called_once_with( - project_id=_PROJECT_ID, dataset="prod", overrides=None - ) - - def test_overrides_forwarded_to_connector(self): - """overrides argument must be passed through to SpiralDBConnector.""" - from orcapod.core.sources import SpiralDBTableSource - - overrides = {"server.url": "http://api.spiraldb.dev"} - connector = _make_mock_connector() - with patch( - "orcapod.core.sources.spiraldb_table_source.SpiralDBConnector", - return_value=connector, - ) as mock_cls: - SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, overrides=overrides) - mock_cls.assert_called_once_with( - project_id=_PROJECT_ID, dataset="default", overrides=overrides - ) + SpiralDBTableSource(_PROJECT_ID, "no_such_table") + del src + assert len(opened) == 4 + # About the connector's own closed flag: nothing public reports it. + assert [c._closed for c in opened[:2] + opened[3:]] == [True] * 3 # pyright: ignore[reportPrivateUsage] # =========================================================================== @@ -400,83 +239,28 @@ def test_overrides_forwarded_to_connector(self): class TestStreamBehaviour: - def test_producer_is_none(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + def test_producer_is_none_and_upstreams_empty(self): + _make_table() + src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) assert src.producer is None - - def test_upstreams_is_empty(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) assert src.upstreams == () - def test_iter_data_yields_one_per_row(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - data = list(src.iter_data()) - assert len(data) == 3 - - def test_iter_data_tags_contain_pk(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - for tags, _ in src.iter_data(): - assert "session_id" in tags - - def test_output_schema_returns_two_schemas(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - result = src.output_schema() - assert len(result) == 2 - - def test_as_table_returns_pyarrow_table(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - t = src.as_table() - assert isinstance(t, pa.Table) - - def test_as_table_row_count_matches_source(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert src.as_table().num_rows == 3 - - def test_data_values_are_correct(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - firing_rates = sorted(pkt["firing_rate"] for _, pkt in src.iter_data()) - assert firing_rates == pytest.approx([0.1, 0.2, 0.3]) - - def test_tag_values_are_correct(self): - from orcapod.core.sources import SpiralDBTableSource + def test_iter_data_yields_one_pair_per_row(self): + _make_table() + pairs = list(SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).iter_data()) + assert [tag.as_dict()["session_id"] for tag, _ in pairs] == ["s1", "s2", "s3"] + assert [data.as_dict()["firing_rate"] for _, data in pairs] == pytest.approx( + [0.1, 0.2, 0.3] + ) - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - session_ids = sorted(tags["session_id"] for tags, _ in src.iter_data()) - assert session_ids == ["s1", "s2", "s3"] + def test_as_table_returns_the_rows(self): + _make_table() + table = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).as_table() + assert isinstance(table, pa.Table) + assert table.to_pydict() == { + "session_id": ["s1", "s2", "s3"], + "firing_rate": [0.1, 0.2, 0.3], + } # =========================================================================== @@ -485,74 +269,43 @@ def test_tag_values_are_correct(self): class TestDeterministicHashing: - def test_pipeline_hash_is_deterministic(self): - from orcapod.core.sources import SpiralDBTableSource - - c1 = _make_mock_connector() - c2 = _make_mock_connector() - with _patch_connector(c1): - src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - with _patch_connector(c2): - src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + def test_hashes_are_deterministic(self): + _make_table() + src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) assert src1.pipeline_hash() == src2.pipeline_hash() - - def test_content_hash_is_deterministic(self): - from orcapod.core.sources import SpiralDBTableSource - - c1 = _make_mock_connector() - c2 = _make_mock_connector() - with _patch_connector(c1): - src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - with _patch_connector(c2): - src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) assert src1.content_hash() == src2.content_hash() - def test_different_tag_columns_yields_different_pipeline_hash(self): - from orcapod.core.sources import SpiralDBTableSource - - # src1 uses PK (session_id) as tag; src2 uses firing_rate - batches = [ - pa.record_batch( - { - "session_id": pa.array(["s1", "s2"], type=pa.large_string()), - "firing_rate": pa.array([0.1, 0.2], type=pa.float64()), - "amplitude": pa.array([1.0, 2.0], type=pa.float64()), - } - ) - ] - c1 = _make_mock_connector(batches=batches) - c2 = _make_mock_connector(batches=batches) - with _patch_connector(c1): - src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - c2.iter_batches.return_value = iter(batches) - with _patch_connector(c2): - src2 = SpiralDBTableSource( - _PROJECT_ID, _TABLE_NAME, tag_columns=["firing_rate"] - ) + def test_different_tag_columns_yield_different_pipeline_hash(self): + _make_composite_pk_table() + src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=["probe_id"]) assert src1.pipeline_hash() != src2.pipeline_hash() def test_different_schemas_yield_different_pipeline_hash(self): """pipeline_hash is schema-only; different column schemas → different hash.""" - from orcapod.core.sources import SpiralDBTableSource - - # src1: tag=session_id (large_string), data=firing_rate (float64) - c1 = _make_mock_connector() - # src2: tag=session_id (large_string), data=neuron_count (int64) - batches2 = [ - pa.record_batch( + _make_table() + _make_table( + name="counts", + rows=pa.table( { "session_id": pa.array(["s1", "s2"], type=pa.large_string()), "neuron_count": pa.array([10, 20], type=pa.int64()), } - ) - ] - c2 = _make_mock_connector(batches=batches2) - with _patch_connector(c1): - src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - with _patch_connector(c2): - src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + ), + ) + src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + src2 = SpiralDBTableSource(_PROJECT_ID, "counts") assert src1.pipeline_hash() != src2.pipeline_hash() + def test_same_schema_in_two_tables_shares_the_pipeline_hash_only(self): + _make_table() + _make_table(name="other") + src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + src2 = SpiralDBTableSource(_PROJECT_ID, "other") + assert src1.pipeline_hash() == src2.pipeline_hash() + assert src1.content_hash() != src2.content_hash() + # =========================================================================== # 9. Config round-trip (to_config / from_config) @@ -560,161 +313,49 @@ def test_different_schemas_yield_different_pipeline_hash(self): class TestConfigSerialization: - def test_to_config_source_type(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert src.to_config()["source_type"] == "spiraldb_table" - - def test_to_config_has_no_connector_key(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert "connector" not in src.to_config() - - def test_to_config_has_project_id(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert src.to_config()["project_id"] == _PROJECT_ID - - def test_to_config_has_dataset(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, dataset="prod") - assert src.to_config()["dataset"] == "prod" - - def test_to_config_has_table_name(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert src.to_config()["table_name"] == _TABLE_NAME - - def test_to_config_has_tag_columns(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector(pk_columns=["session_id"]) - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert "session_id" in src.to_config()["tag_columns"] - - def test_to_config_has_overrides(self): - from orcapod.core.sources import SpiralDBTableSource - + def test_to_config_fields(self): + _make_table(dataset="prod") overrides = {"server.url": "http://api.spiraldb.dev"} - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, overrides=overrides) - assert src.to_config()["overrides"] == overrides - - def test_to_config_overrides_none_by_default(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - assert src.to_config()["overrides"] is None - - def test_to_config_has_identity_fields(self): - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) + src = SpiralDBTableSource( + _PROJECT_ID, _TABLE_NAME, dataset="prod", overrides=overrides + ) cfg = src.to_config() - assert "content_hash" in cfg - assert "pipeline_hash" in cfg - - def test_to_config_works_after_connector_is_closed(self): - """to_config() must not call the connector — it is closed after __init__. - - Regression test: the mock connector's to_config() raises RuntimeError - when closed (matching SpiralDBConnector's behaviour). If - SpiralDBTableSource.to_config() delegates to the connector, this test - will raise RuntimeError and fail. - """ - from orcapod.core.sources import SpiralDBTableSource - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - # Connector is closed at this point; calling to_config() must not - # raise. - assert connector._closed, "connector should be closed after __init__" - cfg = src.to_config() # must not raise RuntimeError assert cfg["source_type"] == "spiraldb_table" + assert "connector" not in cfg assert cfg["project_id"] == _PROJECT_ID - - def test_from_config_reconstructs_successfully(self): - from orcapod.core.sources import SpiralDBTableSource - - c1 = _make_mock_connector() - with _patch_connector(c1): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - cfg = src.to_config() - - c2 = _make_mock_connector() - with _patch_connector(c2): - src2 = SpiralDBTableSource.from_config(cfg) - assert src2.source_id == src.source_id + assert cfg["dataset"] == "prod" + assert cfg["table_name"] == _TABLE_NAME + assert cfg["table_id"] == src.table_id + assert cfg["tag_columns"] == ["session_id"] + assert cfg["overrides"] == overrides + assert cfg["source_id"] == src.source_id + for field in ("content_hash", "pipeline_hash", "tag_schema", "data_schema"): + assert field in cfg + + def test_to_config_defaults(self): + _make_table() + cfg = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).to_config() + assert cfg["overrides"] is None + assert cfg["where"] == {} + assert cfg["limit"] is None + assert cfg["asof"] is None + assert cfg["max_rows"] == 100_000 + assert cfg["batch_rows"] == 8_192 def test_from_config_hashes_match(self): - from orcapod.core.sources import SpiralDBTableSource - - c1 = _make_mock_connector() - with _patch_connector(c1): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - cfg = src.to_config() - - c2 = _make_mock_connector() - with _patch_connector(c2): - src2 = SpiralDBTableSource.from_config(cfg) - assert src2.content_hash() == src.content_hash() - assert src2.pipeline_hash() == src.pipeline_hash() - - def test_from_config_forwards_dataset(self): - """from_config must pass dataset back to the constructor.""" - from orcapod.core.sources import SpiralDBTableSource - - c1 = _make_mock_connector() - with _patch_connector(c1): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, dataset="prod") - cfg = src.to_config() - - assert cfg["dataset"] == "prod" - c2 = _make_mock_connector() - with patch( - "orcapod.core.sources.spiraldb_table_source.SpiralDBConnector", - return_value=c2, - ) as mock_cls: - SpiralDBTableSource.from_config(cfg) - mock_cls.assert_called_once_with( - project_id=_PROJECT_ID, dataset="prod", overrides=None - ) + _make_table(dataset="prod") + src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, dataset="prod") + src2 = SpiralDBTableSource.from_config(src.to_config()) + assert src.pipeline_hash() == src2.pipeline_hash() + assert src.content_hash() == src2.content_hash() + assert src2.as_table().num_rows == 3 def test_resolve_source_from_config_works(self): - from orcapod.core.sources import SpiralDBTableSource from orcapod.pipeline.serialization import resolve_source_from_config - c1 = _make_mock_connector() - with _patch_connector(c1): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - cfg = src.to_config() - - c2 = _make_mock_connector() - with _patch_connector(c2): - src2 = resolve_source_from_config(cfg) - assert isinstance(src2, SpiralDBTableSource) + _make_table() + cfg = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).to_config() + assert isinstance(resolve_source_from_config(cfg), SpiralDBTableSource) # =========================================================================== @@ -726,52 +367,21 @@ def test_resolve_source_from_config_works(self): class TestPipelineIntegration: def test_spiraldb_source_in_pipeline(self): """Verify SpiralDBTableSource drives a full pipeline end-to-end.""" - from orcapod.core.function_pod import FunctionPod - from orcapod.core.data_function import PythonDataFunction - from orcapod.core.sources import SpiralDBTableSource + from orcapod import PipelineJob, function_pod from orcapod.databases import InMemoryArrowDatabase - from orcapod.pipeline import PipelineJob - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - def double_rate(firing_rate: float) -> float: - return firing_rate * 2.0 - - connector = _make_mock_connector() - with _patch_connector(connector): - src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - - pf = PythonDataFunction(double_rate, output_keys="doubled") - pod = FunctionPod(pf) + @function_pod(output_keys="doubled") + def double_rate(firing_rate: float | None) -> float | None: + return None if firing_rate is None else firing_rate * 2.0 + _make_table() job = PipelineJob(name="spiraldb_integration", store=InMemoryArrowDatabase()) with job: - pod(src, label="doubler") - - from orcapod.pipeline.dag import OrcaDAG - - exec_dag: OrcaDAG = OrcaDAG() - for node in job._persistent_node_map.values(): - exec_dag.add_node(node) - for u_hash, v_hash in job._graph_edges: - if ( - u_hash in job._persistent_node_map - and v_hash in job._persistent_node_map - ): - exec_dag.add_edge( - job._persistent_node_map[u_hash], - job._persistent_node_map[v_hash], - ) - orch = SyncPipelineOrchestrator() - result = orch.run(exec_dag) - - fn_outputs = [ - v for k, v in result.node_outputs.items() if k.node_type == "function" - ] - assert len(fn_outputs) == 1 - assert len(fn_outputs[0]) == 3 - - doubled_values = sorted([pkt.as_dict()["doubled"] for _, pkt in fn_outputs[0]]) - assert doubled_values == pytest.approx([0.2, 0.4, 0.6]) - - tag_values = sorted([tags["session_id"] for tags, _ in fn_outputs[0]]) - assert tag_values == ["s1", "s2", "s3"] + double_rate.pod( + SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME), label="doubler" + ) + job.run() + + rows = job.nodes["doubler"].as_table().sort_by("session_id").to_pydict() + assert rows["session_id"] == ["s1", "s2", "s3"] + assert rows["doubled"] == pytest.approx([0.2, 0.4, 0.6]) diff --git a/tests/test_core/sources/test_spiraldb_table_source_fake.py b/tests/test_core/sources/test_spiraldb_table_source_fake.py index 47ea6840f..8def0ecfa 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_fake.py +++ b/tests/test_core/sources/test_spiraldb_table_source_fake.py @@ -261,10 +261,16 @@ def _identity(source: SpiralDBTableSource) -> tuple[str, str, str]: table.column("_source_sex")[0].as_py(), ) - def test_default_source_id_is_project_dataset_table(self, two_projects): + # The row token of subject "m1": the Arrow hash of its primary key (ITL-186). + _M1 = "arrow_v1:00000290f26c354e2b6a1683a500d4a3dd8dc0a10d8db76aa7d2c804fc908bd44f749e" + + def test_default_source_id_is_project_dataset_table_and_table_id( + self, two_projects + ): source = SpiralDBTableSource("edi-sample-a", "subject") - assert source.source_id == "edi-sample-a/default/subject" - assert self._identity(source)[2] == "edi-sample-a/default/subject::row_0::sex" + expected = f"edi-sample-a/default.subject@{source.table_id}" + assert source.source_id == expected + assert self._identity(source)[2] == f"{expected}::{self._M1}::sex" def test_same_table_in_two_projects_does_not_collide(self, two_projects): a = SpiralDBTableSource("edi-sample-a", "subject") @@ -278,6 +284,9 @@ def test_unrecorded_table_hashes_are_unchanged(self, two_projects): # column nullable, as SpiralDB reports it. The values were pinned at 49956a6a, # before the record existed, and re-recorded once for the v0.3 hash step # (ITL-967: starfix 0.5 and the hash label rule move every one of them). + # ITL-186 moved the record id and the row token only (from the row's position to + # the hash of its primary key); the token is the same with and without a + # nullability record, because key fields are hashed as non-nullable. source = SpiralDBTableSource("edi-sample-a", "unrecorded", source_id="subject") assert source.output_schema() == ( Schema({"subject_id": str | None}), @@ -285,8 +294,8 @@ def test_unrecorded_table_hashes_are_unchanged(self, two_projects): ) assert self._identity(source) == ( "semantic_v1:896d8f323df90ee26e6a38f1eb730be7a9de343fd1c852dbf0f7655b360bc7ff", - "18d1efa53fbc52b6a40f406eca35ca81", - "subject::row_0::sex", + "1493a9f4caa95b7ca33dcf61e15d5eca", + f"subject::{self._M1}::sex", ) assert source.pipeline_hash().to_string() == ( "semantic_v1:cc6c553ba26af325cd96432edb5bc4ea5453427594965cca7e85a66a420e87f1" @@ -306,8 +315,8 @@ def test_explicit_source_id_keeps_its_hashes(self, two_projects): source = SpiralDBTableSource("edi-sample-a", "subject", source_id="subject") assert self._identity(source) == ( "semantic_v1:a06a417294cabb0625e47a9357ebf9cc8183576b17498282901bd68d0f02e820", - "18d1efa53fbc52b6a40f406eca35ca81", - "subject::row_0::sex", + "1493a9f4caa95b7ca33dcf61e15d5eca", + f"subject::{self._M1}::sex", ) assert source.pipeline_hash().to_string() == ( "semantic_v1:2e5095e09a558132e652e106af1dca4abf8654285409fc4db63a66ddc63a9f94" From 83fca788f2869ba0484007d758a495a44518b885 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:23:16 -0800 Subject: [PATCH 05/17] wip(spiraldb): docs, changelog fragment, design issue, live tests (ITL-186) Co-Authored-By: Claude Fable 5.2 --- changelog.d/ITL-186.md | 54 +++++++ design_issues/ITL-186-SP4.md | 27 ++++ docs/concepts/sources.md | 78 ++++++++-- .../core/sources/spiraldb_table_source.py | 35 ++--- .../as_table_after_sync_vs_async_run.py | 52 +++++++ .../test_spiraldb_table_source_live.py | 143 ++++++++++++++++++ 6 files changed, 360 insertions(+), 29 deletions(-) create mode 100644 changelog.d/ITL-186.md create mode 100644 design_issues/ITL-186-SP4.md create mode 100644 superpowers/specs/itl-186-experiments/as_table_after_sync_vs_async_run.py create mode 100644 tests/test_core/sources/test_spiraldb_table_source_live.py diff --git a/changelog.d/ITL-186.md b/changelog.d/ITL-186.md new file mode 100644 index 000000000..61b525e95 --- /dev/null +++ b/changelog.d/ITL-186.md @@ -0,0 +1,54 @@ +### Breaking changes and migration + +#### `SpiralDBTableSource` reads no rows when it is built, and reads in batches (ITL-186) + +A Spiral source used to read its whole table when it was built, which made large tables +unusable (about 4.6 GB for an 8-million-row table). It is now built from the table's metadata +alone, in about a second, and reads rows only when asked: in batches of at most `batch_rows` +rows (default 8,192) for `iter_data()` and pipeline jobs, and all at once for `as_table()` and +the dataframe methods. See "Reading a Spiral table" in `docs/concepts/sources.md`. + +What changes for existing code: + +- **Default `source_id`.** It is now `/.
@`, for + example `edi-tutorial-v1-841226/default.session@table_Jo7J0lpkLcOM` (it was + `//
`). SpiralDB's own table id makes it unique: a table dropped + and created again under the same name is a different source. For a Spiral source built + without `source_id`, the `content_hash`, every record ID and every source-info token + change. `pipeline_hash` does not. +- **Record IDs come from the primary key.** A row's token is the Arrow hash of its key + columns (for example `arrow_v1:000002...`), no longer its position (`row_0`). Record IDs + and source-info tokens of every Spiral source change, also with an explicit `source_id`; + `content_hash` and `pipeline_hash` of a source with an explicit `source_id` do not. + Pipeline records stored earlier for a Spiral source are not found again: the rows are + recorded once more, and no result is recomputed, because function results are found by + the row's data. +- **`record_id_column` is removed** from `SpiralDBTableSource`: the primary key identifies + the row. +- **A table without a key schema is refused**, also when `tag_columns` is given. +- **`SpiralDBTableSource` is no longer a `DBTableSource`.** +- **A read is capped at `max_rows` rows** (default 100,000); a larger read raises + `SourceTooLargeError`. Pass `limit=`, `where=`, or `max_rows=None`. +- **A saved job** stores the table id. Loading it raises `ValueError` if the saved table name + now leads to a table with another id. Loading reads no rows. + +### Added + +- `SpiralDBTableSource(..., where={column: value}, limit=n, asof=version)`: read part of a + table. SpiralDB applies the restriction before any row is transferred. Restrictions are + not part of the source's identity (ITL-186). +- `SpiralDBTableSource(..., max_rows=, batch_rows=)`, the `row_count_estimate` and `table_id` + properties, and `orcapod.errors.SourceTooLargeError` (ITL-186). +- `SpiralDBConnector.scan_batches(table, where=, limit=, asof=)`, `get_arrow_schema`, + `get_spiral_table_id`, `get_table_version` and `get_row_count_estimate`: a restricted scan + and metadata-only reads (ITL-186). + +### Known limitations + +- A pipeline job keeps every row it processes in memory (about 5 KB per row and node), so a + Spiral table of millions of rows can be iterated but not yet run through a job; the + `max_rows` cap guards against it (ITL-186). +- A Spiral row's key is hashed with one call per row: about 0.2 ms per row, 18 s per 100,000 + rows (ITL-186). +- A Spiral source does not poll for new rows within a run and has no watermark: a new run + scans the (restricted) table again and computes only rows it has not seen (ITL-186). diff --git a/design_issues/ITL-186-SP4.md b/design_issues/ITL-186-SP4.md new file mode 100644 index 000000000..7787532b2 --- /dev/null +++ b/design_issues/ITL-186-SP4.md @@ -0,0 +1,27 @@ +## `src/orcapod/core/sources/spiraldb_table_source.py` — Spiral tables as a workshop source (ITL-926) + +### SP4 — A Spiral source read its whole table when built; large tables were unusable +**Status:** in progress +**Severity:** high +**Issue:** ITL-186 + +`SpiralDBTableSource` inherited `DBTableSource`'s constructor, which read every row and built +provenance for the whole table: about 4.6 GB for the 8,087,906-row `spike_event`. Design and +measurements: `superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md`. + +**Fix:** the source is built from metadata only (schema, key columns, SpiralDB's table id) and +reads rows in batches of at most `batch_rows` rows. The default `source_id` carries SpiralDB's +table id, record IDs come from the hash of the primary key, `where=` / `limit=` / `asof=` +restrict a read without entering identity, and `max_rows` caps a read. + +**Still open (not in the first pull request):** + +* A pipeline job keeps every row it processes (operator and function nodes, the orchestrator's + collectors, `Join`'s buffers), about 5 KB per row and node, so a table of millions of rows + cannot be run through a job (`superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md`). +* The row token needs one hasher call per row (183 µs); a vectorized per-row digest needs a + new hasher entry point. +* No watermark or polling for new rows within a run; no key ranges (pyspiral 0.16.33 cannot + build a key holding a timestamp); no column selection; no lazy frame with pushdown. +* `PostgreSQLTableSource` and `SQLiteTableSource` still read the whole table when built and + default their `source_id` to the bare table name. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 553bd9d44..24d8adb6e 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -95,21 +95,79 @@ Things to know about Spiral tables: that strings, binaries and lists become their large variants. Joining two Spiral tables on their shared key columns works; joining with another source requires the same Arrow types on both sides (`uint32` against `int64` is an error, not a conversion). -- **The whole table is read when the source is built.** Use tables of up to a few hundred - thousand rows. Very large tables (for example `spike_event`) need the streaming source that - is still being designed. - **Not supported:** columns of type `map`, which the Arrow hasher does not hash. Building the source raises `ValueError` naming each such column and its type; drop or convert those columns first. `fixed_size_list` columns are supported. Also not supported: time zones (SpiralDB stores timestamps without one) and Orcapod logical types such as `Path` (they come back as their storage type). -- **Source identity.** The default `source_id` is `//
`, for - example `edi-sample-v1-855632/default/session`, so tables with the same name in different - projects stay apart. -- **Record identity.** Without `record_id_column`, a row's record ID comes from its position - in the table, which SpiralDB orders by key, so a row inserted before existing rows changes - the IDs of the rows after it. Pass `record_id_column=` (a column whose value is unique per - row) to derive the IDs from that column's values instead, so they stay with their rows. +- **Source identity.** A Spiral source is identified by where its table is, never by its + rows. The default `source_id` is `/.
@`, for + example `edi-tutorial-v1-841226/default.session@table_Jo7J0lpkLcOM`. The table id is + SpiralDB's own, so tables with the same name in different projects stay apart, and a table + that is dropped and created again under the same name is a different source: nothing + recorded for the old table is reused. A renamed table also gets a new `source_id`. +- **Record identity.** A row is identified by its primary key: its record ID is derived from + the hash of its key columns. A row keeps its record ID when other rows are added, and has + the same one whichever read returned it. + +#### Reading a Spiral table + +Building a `SpiralDBTableSource` reads no rows. It reads the table's schema, key columns and +table id, which takes about a second whatever the table's size. `output_schema()`, `keys()`, +`content_hash()` and `pipeline_hash()` read no rows either. Rows are read when you ask for +them, and every read scans the table again: + +- `iter_data()` and a pipeline job read the table in batches of at most `batch_rows` rows + (default 8,192) and hold one batch at a time. +- `as_table()`, `as_df()`, `as_polars_df()`, `as_pandas_df()` and `as_lazy_frame()` read + the rows into memory and return them all. + +To read part of a table, say so when you build the source. SpiralDB applies both +restrictions before any row is transferred: + + +```python +first = SpiralDBTableSource("edi-tutorial-v1-841226", "spike_event", limit=1000) +unit = SpiralDBTableSource( + "edi-tutorial-v1-841226", "spike_event", where={"unit_id": 106} +) +print(first.row_count_estimate) # rows in the whole table, without reading any +``` + +- `limit=n` reads the first `n` rows in key order. +- `where={column: value, ...}` reads the rows in which every named column equals its value. +- `asof=` reads the table as of an earlier version. Without it, a source reads the version + the table had when the source was built, in every read. + +**The row cap.** One read returns at most `max_rows` rows (default 100,000). A read that +would return more raises `SourceTooLargeError`, which gives the row count and the ways out: +`limit=`, `where=`, or a larger `max_rows` (`max_rows=None` removes the cap). The cap is there +because a pipeline job keeps every row it processes in memory, about 5 KB per row for each +node, so 100,000 rows cost about 0.5 GB per node. A table of millions of rows can be iterated +with `max_rows=None`, but it cannot yet be run through a job on a laptop. + +**Restrictions are not part of identity.** `where`, `limit` and `asof` choose rows of the +same table, so two sources on one table have the same `source_id` and `content_hash` +whatever their restrictions, and a row's result is computed once and shared. One +consequence: with a shared store, reading a job's node back (`get_all_records()`, a loaded +job, and `as_table()` after an async run) returns every row that any job with the same +pipeline has read from that table, not only the rows of the latest restriction. The rows +sent downstream during a run are only the rows that run read. + +**Running again.** A second run scans the table again and computes nothing for the rows it +has already seen. Rows added to the Spiral table since the last run are new keys: they are +the only rows computed. + +**Known limitations.** + +- Each row's key is hashed with one call, about 0.2 ms per row (about 18 s per 100,000 + rows), in addition to about 0.1 ms per row to build the row. +- A job keeps its rows in memory (see the row cap above). +- Only equality on column values can be pushed down; there is no key range, no column + selection (use `SelectDataColumns` after the source), and no polling for new rows within + one run. +- A table that has only its key columns (for example one never written to) has no data + column and cannot be a source. #### Nullability of Spiral columns diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 1fb7afa29..2e2bc85f3 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -61,7 +61,7 @@ _DATE_KEY = "date" -def _where_to_config(where: Mapping[str, object]) -> dict[str, Any]: +def _locationto_config(where: Mapping[str, object]) -> dict[str, Any]: """Return ``where`` with its date and datetime values in a JSON form.""" out: dict[str, Any] = {} for column, value in where.items(): @@ -74,8 +74,8 @@ def _where_to_config(where: Mapping[str, object]) -> dict[str, Any]: return out -def _where_from_config(saved: Mapping[str, Any]) -> dict[str, object]: - """Return the ``where`` mapping that ``_where_to_config`` wrote.""" +def _locationfrom_config(saved: Mapping[str, Any]) -> dict[str, object]: + """Return the ``where`` mapping that ``_locationto_config`` wrote.""" out: dict[str, object] = {} for column, value in saved.items(): if isinstance(value, dict) and set(value) == {_DATETIME_KEY}: @@ -186,19 +186,19 @@ def __init__( self._max_rows = max_rows self._batch_rows = batch_rows self._given_asof = asof - where_ = f"SpiralDB project {project_id!r}, dataset {dataset!r}" + location = f"SpiralDB project {project_id!r}, dataset {dataset!r}" with self._connector() as connector: if table_name not in connector.get_table_names(): raise ValueError( - f"{where_}: Table {table_name!r} not found in database." + f"{location}: Table {table_name!r} not found in database." ) self._key_columns: tuple[str, ...] = tuple( connector.get_pk_columns(table_name) ) if not self._key_columns: raise ValueError( - f"{where_}: Table {table_name!r} has no primary key columns, so " + f"{location}: Table {table_name!r} has no primary key columns, so " "its rows cannot be identified." ) self._table_id = connector.get_spiral_table_id(table_name) @@ -217,7 +217,7 @@ def __init__( unknown = [c for c in self._where if c not in schema.names] if unknown: raise ValueError( - f"{where_}: where names columns that table {table_name!r} does not " + f"{location}: where names columns that table {table_name!r} does not " f"have: {unknown}. Columns: {schema.names}" ) if self._source_id is None: @@ -233,7 +233,7 @@ def __init__( system_tag_columns=system_tag_columns, ) except ValueError as exc: - raise ValueError(f"{where_}, table {table_name!r}: {exc}") from exc + raise ValueError(f"{location}, table {table_name!r}: {exc}") from exc self._stream = template.stream self._schema_hash = template.schema_hash self._tag_columns = template.tag_columns @@ -340,15 +340,12 @@ def _batch_streams(self) -> Iterator["ArrowTableStream"]: """ with self._connector() as connector: if self._max_rows is not None and not self._where: - # Nothing but limit bounds the read, so its size is known up front. - expected = ( - self._limit - if self._limit is not None and self._limit <= self._max_rows - else min( - connector.get_row_count_estimate(self._table_name), - self._limit if self._limit is not None else 2**63, - ) - ) + # Nothing but limit bounds the read, so its size is known up front: + # the limit, or the table's recorded row count when that is smaller. + expected = self._limit + if expected is None or expected > self._max_rows: + recorded = connector.get_row_count_estimate(self._table_name) + expected = recorded if expected is None else min(expected, recorded) if expected > self._max_rows: raise self._too_large(f"the read would return {expected:,} rows") seen = 0 @@ -436,7 +433,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "table_id": self._table_id, "tag_columns": list(self._tag_columns), "system_tag_columns": list(self._system_tag_columns), - "where": _where_to_config(self._where), + "where": _locationto_config(self._where), "limit": self._limit, "asof": self._given_asof, "max_rows": self._max_rows, @@ -468,7 +465,7 @@ def from_config( dataset=config.get("dataset", "default"), tag_columns=config.get("tag_columns"), system_tag_columns=config.get("system_tag_columns", ()), - where=_where_from_config(config.get("where") or {}), + where=_locationfrom_config(config.get("where") or {}), limit=config.get("limit"), asof=config.get("asof"), max_rows=config.get("max_rows", DEFAULT_MAX_ROWS), diff --git a/superpowers/specs/itl-186-experiments/as_table_after_sync_vs_async_run.py b/superpowers/specs/itl-186-experiments/as_table_after_sync_vs_async_run.py new file mode 100644 index 000000000..2e390fc84 --- /dev/null +++ b/superpowers/specs/itl-186-experiments/as_table_after_sync_vs_async_run.py @@ -0,0 +1,52 @@ +"""On main: ``node.as_table()`` after a run returns different rows on the two orchestrators. + +Run from the repository root: + + uv run python superpowers/specs/itl-186-experiments/as_table_after_sync_vs_async_run.py + +Two jobs with the same pipeline run one after the other on one store. Their sources have +the same schema and ``source_id`` (so the same content hash) and different rows: job A has +rows k=0..4, job B rows k=0..1. After B's run: + +* sync orchestrator: ``B.nodes["double"].as_table()`` returns B's 2 rows; +* async orchestrator: it returns 5 rows, every row stored for the node. + +``get_all_records()`` returns 5 rows on both. Found while measuring ITL-186 (decision D2); +not caused by it and not changed there. +""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.sources import DictSource + + +def run(keys: list[int], store: Path, use_async: bool) -> PipelineJob: + @function_pod(output_keys="w") + def double(v: int) -> int: + return 2 * v + + job = PipelineJob(name="job", store=DeltaTableDatabase(store)) + with job: + source = DictSource( + [{"k": k, "v": k} for k in keys], tag_columns=["k"], source_id="same" + ) + double.pod(source, label="double") + job.run(orchestrator=AsyncPipelineOrchestrator() if use_async else None) + return job + + +for use_async in (False, True): + store = Path(tempfile.mkdtemp()) / "store" + run([0, 1, 2, 3, 4], store, use_async) + node = run([0, 1], store, use_async).nodes["double"] + print( + f"{'async' if use_async else 'sync '}: B's source has 2 rows; " + f"as_table() -> {node.as_table().num_rows} rows, " + f"get_all_records() -> {node.get_all_records().num_rows} rows" + ) diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py new file mode 100644 index 000000000..da2d1e22b --- /dev/null +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -0,0 +1,143 @@ +"""SpiralDBTableSource against a live SpiralDB project (ITL-186). Opt-in. + +Skipped unless ``ORCAPOD_SPIRAL_LIVE_PROJECT`` names a project the saved ``spiral login`` +can read, with a ``default.spike_event`` table (the workshop project is +``edi-tutorial-v1-841226``). Every test is read-only except +``test_rows_appended_between_runs_are_the_only_ones_computed``, which also needs +``ORCAPOD_SPIRAL_LIVE_WRITE=1`` and writes only to a table it creates and drops in the +project's ``scratch`` dataset. + + ORCAPOD_SPIRAL_LIVE_PROJECT=edi-tutorial-v1-841226 \\ + uv run --extra spiraldb pytest tests/test_core/sources/test_spiraldb_table_source_live.py +""" + +from __future__ import annotations + +import os +import time +import uuid +from pathlib import Path + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import SourceTooLargeError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.sources import SpiralDBTableSource + +PROJECT = os.environ.get("ORCAPOD_SPIRAL_LIVE_PROJECT", "") +WRITE = os.environ.get("ORCAPOD_SPIRAL_LIVE_WRITE") == "1" + +pytestmark = pytest.mark.skipif( + not PROJECT, reason="Set ORCAPOD_SPIRAL_LIVE_PROJECT to run the live Spiral tests" +) + + +def test_building_a_source_is_quick_and_names_the_table(): + start = time.monotonic() + source = SpiralDBTableSource(PROJECT, "spike_event") + elapsed = time.monotonic() - start + + assert elapsed < 10 + assert source.source_id == f"{PROJECT}/default.spike_event@{source.table_id}" + assert source.table_id.startswith("table_") + assert "unit_id" in source.keys()[0] + assert source.row_count_estimate > 0 + + +def test_limit_reads_the_first_rows(): + source = SpiralDBTableSource(PROJECT, "spike_event", limit=1000) + + table = source.as_table(all_info=True) + + assert table.num_rows == 1000 + (record_ids,) = [ + table.column(name) + for name in table.column_names + if name.startswith("_tag_record_id::") + ] + assert len(set(record_ids.to_pylist())) == 1000 + + +def test_where_reads_only_matching_rows_with_the_same_record_ids(): + first = SpiralDBTableSource(PROJECT, "spike_event", limit=1).as_table(all_info=True) + unit_id = first.column("unit_id")[0].as_py() + source = SpiralDBTableSource( + PROJECT, "spike_event", where={"unit_id": unit_id}, max_rows=None + ) + + table = source.as_table(all_info=True) + + assert table.num_rows > 0 + assert set(table.column("unit_id").to_pylist()) == {unit_id} + record_id = next(n for n in table.column_names if n.startswith("_tag_record_id::")) + assert first.column(record_id)[0].as_py() in table.column(record_id).to_pylist() + + +def test_a_whole_table_over_the_cap_is_refused_without_reading(): + source = SpiralDBTableSource(PROJECT, "spike_event", max_rows=10) + + start = time.monotonic() + with pytest.raises(SourceTooLargeError, match="limit="): + source.as_table() + assert time.monotonic() - start < 10 + + +def test_an_async_job_over_a_limited_read(tmp_path: Path): + calls: list[float | None] = [] + + @function_pod(output_keys="doubled") + def double(spike_amplitude_uv: float | None) -> float | None: + calls.append(spike_amplitude_uv) + return None if spike_amplitude_uv is None else 2 * spike_amplitude_uv + + def run() -> PipelineJob: + job = PipelineJob(name="live", store=DeltaTableDatabase(tmp_path / "store")) + with job: + double.pod( + SpiralDBTableSource(PROJECT, "spike_event", limit=500), label="double" + ) + job.run(orchestrator=AsyncPipelineOrchestrator()) + return job + + assert run().nodes["double"].as_table().num_rows == 500 + assert 0 < len(calls) <= 500 + calls.clear() + run() + assert calls == [] + + +@pytest.mark.skipif(not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table") +def test_rows_appended_between_runs_are_the_only_ones_computed(tmp_path: Path): + import spiral # pyright: ignore[reportMissingImports] # spiraldb extra, absent in the type-check job + + name = f"itl186_live_{uuid.uuid4().hex[:8]}" + project = spiral.Spiral().project(PROJECT) + table = project.create_table(f"scratch.{name}", key_schema=[("k", pa.int64())]) + calls: list[int | None] = [] + + @function_pod(output_keys="w") + def double(v: int | None) -> int | None: + calls.append(v) + return None if v is None else 2 * v + + def run() -> None: + job = PipelineJob(name="append", store=DeltaTableDatabase(tmp_path / "store")) + with job: + double.pod( + SpiralDBTableSource(PROJECT, name, dataset="scratch"), label="double" + ) + job.run() + + try: + table.write(pa.table({"k": [2, 3], "v": [20, 30]})) + run() + assert sorted(calls) == [20, 30] + calls.clear() + table.write(pa.table({"k": [1], "v": [10]})) + run() + assert calls == [10] + finally: + project.drop_table(f"scratch.{name}") From 012a86eba8f7baf21f5a5e874188756f9d2b4d4b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:32:47 -0800 Subject: [PATCH 06/17] feat(spiraldb)!: SpiralDBTableSource reads no rows when built and streams in batches (ITL-186) The source is built from the table's metadata (schema, key columns, SpiralDB's table id) and reads rows only when asked, in batches of at most batch_rows. The default source id carries SpiralDB's table id, a row's record id comes from the Arrow hash of its primary key, where= / limit= / asof= restrict a read without entering identity, and max_rows caps a read. Co-Authored-By: Claude Fable 5.2 --- .../core/sources/spiraldb_table_source.py | 28 +++++++---- ...6-11-12-itl-186-streaming-spiral-source.md | 33 ++++++++----- .../d2_restriction_identity.py | 9 ++-- .../repros/h2_positional_record_ids.py | 12 ++--- .../test_spiraldb_table_source_live.py | 6 ++- .../test_spiraldb_table_source_streaming.py | 49 +++++++++++++------ 6 files changed, 89 insertions(+), 48 deletions(-) diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 2e2bc85f3..9100fa62b 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -30,12 +30,13 @@ import asyncio import datetime -from collections.abc import AsyncIterator, Collection, Iterator, Mapping +from collections.abc import AsyncIterator, Collection, Generator, Iterator, Mapping from contextlib import contextmanager from typing import TYPE_CHECKING, Any from orcapod.core.sources.base import RootSource from orcapod.core.sources.stream_builder import SourceStreamBuilder +from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases.spiraldb_connector import SpiralDBConnector from orcapod.errors import SourceTooLargeError from orcapod.types import ColumnConfig @@ -48,7 +49,6 @@ from orcapod import contexts from orcapod.config import OrcapodConfig - from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.protocols.core_protocols import DataProtocol, TagProtocol else: pa = LazyModule("pyarrow") @@ -329,7 +329,7 @@ def _key_tokens(self, table: "pa.Table") -> list[str]: for i in range(keys.num_rows) ] - def _batch_streams(self) -> Iterator["ArrowTableStream"]: + def _batch_streams(self) -> Generator["ArrowTableStream", None, None]: """Scan the table once and yield its rows as streams of at most ``batch_rows`` rows. Raises: @@ -395,7 +395,12 @@ async def async_iter_data( for pair in stream.iter_data(): yield pair finally: - streams.close() + try: + streams.close() + except ValueError: + # Cancelled while a batch was being read in the worker thread: the + # scan cannot be closed from here and ends with that read. + pass def as_table( self, @@ -404,13 +409,18 @@ def as_table( all_info: bool = False, ) -> "pa.Table": """Scan the table and return its rows as one PyArrow table (all held in memory).""" - tables = [ - stream.as_table(columns=columns, all_info=all_info) - for stream in self._batch_streams() - ] + # One stream over all the rows, so that a whole-table option (sorting by + # tags) applies across batches. + enriched = {"source": True, "system_tags": True} + tables = [stream.as_table(columns=enriched) for stream in self._batch_streams()] if not tables: return self._stream.as_table(columns=columns, all_info=all_info) - return pa.concat_tables(tables) + whole = ArrowTableStream( + table=pa.concat_tables(tables), + tag_columns=self._tag_columns, + system_tag_columns=self._system_tag_columns, + ) + return whole.as_table(columns=columns, all_info=all_info) # ------------------------------------------------------------------------- # Serialization diff --git a/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md b/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md index f5d93602e..79aa0cc6a 100644 --- a/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md +++ b/superpowers/plans/2026-11-12-itl-186-streaming-spiral-source.md @@ -16,12 +16,22 @@ pyspiral fake, which gains a scan log. ## State (keep current; a successor continues from here) * Branch `eywalker/itl-186-add-dynamic-mode-to-spiraldbsource`, from `origin/main` 25650c42. -* Nov 12: design sent to the coordinator. Approved by the coordinator: D4 (batches of 8,192), - D7 (no orchestrator / Join change), D8 (full re-scan), D9 (PR scope), D10 (tests). - **Waiting on the developer: D1, D2, D3, D5, D6.** No library code until the reply. -* Allowed meanwhile: Phase A below (fake + tests that do not depend on D1/D3/D5), local only. -* Live probe scripts: `.untracked/itl186_probe_{sizes,scan,lazy,ids}.py` (untracked). -* Finding passed to ITL-858: `superpowers/specs/2026-11-12-itl-186-what-a-job-keeps.md`. +* Decisions (Nov 12): D1 option A (source id `/.
@
`), + D2 OUT (restrictions not in the content hash; one function, `_restriction_identity`), + D3 row token = labelled Arrow hash of the primary key (key fields non-nullable), + D4 batches of 8,192, D5 `max_rows` cap 100,000, D6 one class without a mode, + D7 no orchestrator or Join change, D8 full re-scan, D9 scope, D10 tests. +* Done: Phase A and Phase B below. Fake extended; connector `scan_batches`, + `get_arrow_schema`, `get_spiral_table_id`, `get_table_version`, `get_row_count_estimate`; + `SourceStreamBuilder.attach_provenance`; the source; tests (`test_spiraldb_table_source.py` + rewritten on the fake, `..._streaming.py`, `..._live.py` opt-in, three pins in `..._fake.py`); + docs, changelog fragment, `design_issues/ITL-186-SP4.md`. +* Verified live, read-only, on `edi-tutorial-v1-841226`: all 26 tables build (about 1.5 s) + and read with `limit=50`; `spike_event` 63,349 rows iterate in 25 s; the opt-in live tests + pass, including the scratch-table append test. +* Waits (reported, not in this PR): bounded-memory jobs, vectorized row digest, watermark and + polling, key ranges, column selection, lazy frame, PostgreSQL / SQLite sources. +* Live probe scripts: `.untracked/itl186_*.py` (untracked). ## File structure @@ -135,9 +145,10 @@ class FakeScan: The two failing tests are marked `xfail(strict=True, reason="ITL-186")` in the commit so the suite stays green; the marks are removed in Phase B. -## Phase B (gated on D1, D2, D3, D5, D6; detailed after the reply) +## Phase B (done) -Written out task by task once the decisions are known, because the constructor signature, -the source id, the token format and the cap all depend on them. Order: connector metadata -and restricted scan; per-batch enrichment; the source; save/load; cap; async; docs, -changelog fragment, design-issue entry; live opt-in tests; full suite, lint, types; PR. +Order followed: connector metadata reads and restricted scan; `attach_provenance` extracted +from `SourceStreamBuilder.build` (byte-identical for every other source); the source; tests +shown failing against the old source (32 of 37) and passing against the new one; save and +load; cap; async; docs, changelog fragment, design-issue entry; live opt-in tests; full +suite, ruff, basedpyright. diff --git a/superpowers/specs/itl-186-experiments/d2_restriction_identity.py b/superpowers/specs/itl-186-experiments/d2_restriction_identity.py index 5347df4e3..03e588f29 100644 --- a/superpowers/specs/itl-186-experiments/d2_restriction_identity.py +++ b/superpowers/specs/itl-186-experiments/d2_restriction_identity.py @@ -16,9 +16,10 @@ import sys import tempfile from pathlib import Path -from typing import Any +from typing import Any, cast import pyarrow as pa +import pyarrow.compute as pc from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase @@ -37,7 +38,7 @@ class Restricted(ArrowTableSource): def __init__(self, keys: list[int], restriction: str, in_identity: bool) -> None: self._restriction = restriction self._in_identity = in_identity - rows = FULL.filter(pa.compute.is_in(FULL["k"], pa.array(keys, pa.int64()))) + rows = FULL.filter(pc.is_in(FULL["k"], pa.array(keys, pa.int64()))) super().__init__( rows, tag_columns=["k"], source_id=SOURCE_ID, record_id_column="k" ) @@ -60,7 +61,9 @@ def double(v: int) -> int: def keys_of(table: pa.Table | None) -> list[int] | None: - return None if table is None else sorted(table.column("k").to_pylist()) + if table is None: + return None + return sorted(cast("list[int]", table.column("k").to_pylist())) def report(name: str, job: PipelineJob, calls: list[int], store: Path) -> None: diff --git a/superpowers/specs/itl-926-live/repros/h2_positional_record_ids.py b/superpowers/specs/itl-926-live/repros/h2_positional_record_ids.py index 9ddd27b12..c8842f591 100644 --- a/superpowers/specs/itl-926-live/repros/h2_positional_record_ids.py +++ b/superpowers/specs/itl-926-live/repros/h2_positional_record_ids.py @@ -8,6 +8,10 @@ (ebf2511b): inserting a row whose key sorts first shifts every later row's provenance token (``row_0`` → ``row_1`` ...), so each gets a new record ID; ``record_id_column`` accepts one column only, and a non-unique one gives two rows the same token. + +Fixed by ITL-186: a Spiral row's token is the hash of its primary key, so the tokens printed +before and after the insert are now the same for the rows that were already there, and +``SpiralDBTableSource`` no longer takes ``record_id_column``. """ import sys @@ -38,11 +42,3 @@ def tokens(source: SpiralDBTableSource) -> list[Any]: print("before insert:", tokens(SpiralDBTableSource("p", "t"))) table.write(pa.table({"s": ["a"], "n": pa.array([0], pa.int32()), "v": [0.0]})) print("after insert: ", tokens(SpiralDBTableSource("p", "t"))) - print( - "record_id_column='s':", - tokens(SpiralDBTableSource("p", "t", record_id_column="s")), - ) - try: - SpiralDBTableSource("p", "t", record_id_column=["s", "n"]) # pyright: ignore[reportArgumentType] (deliberate) - except ValueError as exc: - print(f"composite record_id_column: ValueError: {exc}") diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index da2d1e22b..3df849c38 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -109,7 +109,9 @@ def run() -> PipelineJob: assert calls == [] -@pytest.mark.skipif(not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table") +@pytest.mark.skipif( + not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table" +) def test_rows_appended_between_runs_are_the_only_ones_computed(tmp_path: Path): import spiral # pyright: ignore[reportMissingImports] # spiraldb extra, absent in the type-check job @@ -134,7 +136,7 @@ def run() -> None: try: table.write(pa.table({"k": [2, 3], "v": [20, 30]})) run() - assert sorted(calls) == [20, 30] + assert set(calls) == {20, 30} and len(calls) == 2 calls.clear() table.write(pa.table({"k": [1], "v": [10]})) run() diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 1e763c7f4..1ce9566bc 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -9,6 +9,7 @@ import asyncio import datetime as dt from pathlib import Path +from typing import cast import pyarrow as pa import pytest @@ -55,16 +56,16 @@ def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): def _timestamps(source: SpiralDBTableSource) -> list[int]: - return source.as_table().column("timestamp").to_pylist() + return cast("list[int]", source.as_table().column("timestamp").to_pylist()) def _record_ids(source: SpiralDBTableSource) -> dict[int, bytes]: """Return each row's record ID by its ``timestamp`` key.""" table = source.as_table(all_info=True) (column,) = [n for n in table.column_names if n.startswith("_tag_record_id::")] - return dict( - zip(table.column("timestamp").to_pylist(), table.column(column).to_pylist()) - ) + timestamps = cast("list[int]", table.column("timestamp").to_pylist()) + record_ids = cast("list[bytes]", table.column(column).to_pylist()) + return dict(zip(timestamps, record_ids, strict=True)) class TestNoRowIsReadUntilAsked: @@ -148,10 +149,22 @@ def test_streamed_rows_equal_as_table_rows(self, project): k: table_row[k] for k in shared } + def test_sorting_by_tags_sorts_across_batches(self, project): + keys = pa.schema([pa.field("k", pa.int64())]) + project.create_table("default.groups", key_schema=keys).write( + pa.table({"k": [1, 2, 3, 4, 5], "g": [5, 4, 3, 2, 1], "v": [0.0] * 5}) + ) + source = SpiralDBTableSource(_PROJECT, "groups", tag_columns="g") + + table = source.as_table(columns={"sort_by_tags": True}) + + assert table.column("g").to_pylist() == [1, 2, 3, 4, 5] + assert table.column("k").to_pylist() == [5, 4, 3, 2, 1] + def test_async_iteration_yields_the_same_rows(self, project): source = SpiralDBTableSource(_PROJECT, "event") - async def collect() -> list[dict]: + async def collect() -> list[dict[str, object]]: return [ {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} async for tag, data in source.async_iter_data() @@ -196,10 +209,13 @@ def test_where_on_an_unknown_column_is_refused(self, project): with pytest.raises(ValueError, match="nope"): SpiralDBTableSource(_PROJECT, "event", where={"nope": 1}) - @pytest.mark.parametrize("name", ["limit", "max_rows", "batch_rows"]) - def test_sizes_must_be_positive(self, project, name): - with pytest.raises(ValueError, match=name): - SpiralDBTableSource(_PROJECT, "event", **{name: 0}) + def test_sizes_must_be_positive(self, project): + with pytest.raises(ValueError, match="limit"): + SpiralDBTableSource(_PROJECT, "event", limit=0) + with pytest.raises(ValueError, match="max_rows"): + SpiralDBTableSource(_PROJECT, "event", max_rows=0) + with pytest.raises(ValueError, match="batch_rows"): + SpiralDBTableSource(_PROJECT, "event", batch_rows=0) def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project): table = project.table("default.event") @@ -403,7 +419,9 @@ def test_loading_refuses_a_name_that_now_leads_to_another_table(self, project): class TestInAJob: - def _job(self, store: Path, calls: list[float], **source_kwargs) -> PipelineJob: + def _job( + self, store: Path, calls: list[float | None], max_rows: int | None = 100_000 + ) -> PipelineJob: @function_pod(output_keys="doubled") def double(amplitude: float | None) -> float | None: calls.append(amplitude) @@ -412,20 +430,21 @@ def double(amplitude: float | None) -> float | None: job = PipelineJob(name="events", store=DeltaTableDatabase(store)) with job: double.pod( - SpiralDBTableSource(_PROJECT, "event", **source_kwargs), label="double" + SpiralDBTableSource(_PROJECT, "event", max_rows=max_rows), + label="double", ) return job @pytest.mark.parametrize("orchestrator", [None, AsyncPipelineOrchestrator]) def test_run_rerun_and_an_appended_row(self, project, tmp_path, orchestrator): - def run(calls: list[float]) -> PipelineJob: + def run(calls: list[float | None]) -> PipelineJob: job = self._job(tmp_path / "store", calls) job.run(orchestrator=orchestrator() if orchestrator else None) return job - calls: list[float] = [] + calls: list[float | None] = [] first = run(calls) - assert sorted(calls) == [1.5, 2.5, 3.5, 4.5, 5.5] + assert set(calls) == {1.5, 2.5, 3.5, 4.5, 5.5} and len(calls) == _ROWS assert first.nodes["double"].as_table().num_rows == _ROWS calls.clear() @@ -439,7 +458,7 @@ def run(calls: list[float]) -> PipelineJob: assert third.nodes["double"].get_all_records().num_rows == _ROWS + 1 def test_a_job_over_the_cap_fails_before_computing(self, project, tmp_path): - calls: list[float] = [] + calls: list[float | None] = [] job = self._job(tmp_path / "store", calls, max_rows=3) with pytest.raises(SourceTooLargeError): From 3e5773ed13149c45c5a6adfda83857a81697fe86 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:49:35 -0800 Subject: [PATCH 07/17] feat(spiraldb): a displayed SpiralDBTableSource reads a 10-row head (ITL-186) StreamBase gains a private _preview hook whose default is today's behaviour; _repr_html_ and view() go through it and add its note to the header. SpiralDBTableSource reads at most 10 rows in one scan without the max_rows cap and names the table's recorded row count. A private class switch shows the schema only, with no scan. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-186.md | 4 + docs/concepts/sources.md | 9 ++ .../core/sources/spiraldb_table_source.py | 97 ++++++++++++++-- src/orcapod/core/streams/base.py | 47 ++++++-- .../2026-11-12-itl-186-notebook-preview.md | 65 +++++++++++ .../test_spiraldb_table_source_streaming.py | 107 ++++++++++++++++++ tests/test_core/streams/test_streams.py | 23 ++++ 7 files changed, 333 insertions(+), 19 deletions(-) create mode 100644 superpowers/plans/2026-11-12-itl-186-notebook-preview.md diff --git a/changelog.d/ITL-186.md b/changelog.d/ITL-186.md index 61b525e95..de6b334da 100644 --- a/changelog.d/ITL-186.md +++ b/changelog.d/ITL-186.md @@ -39,6 +39,10 @@ What changes for existing code: not part of the source's identity (ITL-186). - `SpiralDBTableSource(..., max_rows=, batch_rows=)`, the `row_count_estimate` and `table_id` properties, and `orcapod.errors.SourceTooLargeError` (ITL-186). +- A `SpiralDBTableSource` shown in a notebook (its HTML display, or `view()`) reads only its + first 10 rows, in one scan to which `max_rows` does not apply, and its header says how + many rows the table has, for example `first 10 rows of about 63,349`. Other streams + display as before (ITL-186). - `SpiralDBConnector.scan_batches(table, where=, limit=, asof=)`, `get_arrow_schema`, `get_spiral_table_id`, `get_table_version` and `get_row_count_estimate`: a restricted scan and metadata-only reads (ITL-186). diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 24d8adb6e..6851543eb 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -146,6 +146,15 @@ because a pipeline job keeps every row it processes in memory, about 5 KB per ro node, so 100,000 rows cost about 0.5 GB per node. A table of millions of rows can be iterated with `max_rows=None`, but it cannot yet be run through a job on a laptop. +**Looking at a table in a notebook.** A Spiral source shown at the end of a notebook cell +(or through `view()`) reads only its first 10 rows, in one scan to which the row cap does not +apply, so a table of any size can be looked at. The header line says what is shown, for +example `SpiralDBTableSource[spike_event] — first 10 rows of about 63,349` (the table's +recorded row count, read without scanning), `all 4 rows` when the source reads no more than +that, or `first 10 matching rows (where unit_id=106); the table has about 63,349 rows` with +`where=`. Tag columns come first, marked with `*`. To see more rows, use `as_df()` with a +`limit=` or `where=` source. + **Restrictions are not part of identity.** `where`, `limit` and `asof` choose rows of the same table, so two sources on one table have the same `source_id` and `content_hash` whatever their restrictions, and a row's result is computed once and shared. One diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 9100fa62b..77db2d80c 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -45,6 +45,7 @@ from orcapod.utils.schema_utils import _normalize_column_list if TYPE_CHECKING: + import polars as pl import pyarrow as pa from orcapod import contexts @@ -52,9 +53,12 @@ from orcapod.protocols.core_protocols import DataProtocol, TagProtocol else: pa = LazyModule("pyarrow") + pl = LazyModule("polars") DEFAULT_MAX_ROWS = 100_000 DEFAULT_BATCH_ROWS = 8_192 +# The most rows a notebook display of the source reads. +PREVIEW_ROWS = 10 # How a datetime or date value of ``where`` is written in a saved config. _DATETIME_KEY = "datetime" @@ -101,6 +105,10 @@ class SpiralDBTableSource(RootSource): ``as_table`` and the dataframe methods hold every row they return. Nothing is cached: every read scans again. + Display. Shown in a notebook (``_repr_html_``, ``view``), the source reads only its + first ``PREVIEW_ROWS`` rows (10), in one scan to which ``max_rows`` does not apply, + and the header says how many rows the table has. + Identity. The source is identified by where the table is, never by its rows. The default ``source_id`` is ``/.@
``, where the table id is SpiralDB's own: a table dropped and created again under the same @@ -146,6 +154,11 @@ class SpiralDBTableSource(RootSource): ``batch_rows`` is not a positive integer. """ + # Whether a notebook display reads the first rows. False shows the schema and the + # table's recorded row count only, with no scan (kept for the ITL-991 ruling on what + # a preview may read). + _preview_reads_rows: bool = True + def __init__( self, project_id: str, @@ -332,6 +345,25 @@ def _key_tokens(self, table: "pa.Table") -> list[str]: def _batch_streams(self) -> Generator["ArrowTableStream", None, None]: """Scan the table once and yield its rows as streams of at most ``batch_rows`` rows. + Raises: + SourceTooLargeError: If the read would return, or returns, more than + ``max_rows`` rows. + ValueError: If a batch's columns differ from the source's schema, or a + key column holds a null. + """ + return self._read_batches(limit=self._limit, max_rows=self._max_rows) + + def _read_batches( + self, *, limit: int | None, max_rows: int | None + ) -> Generator["ArrowTableStream", None, None]: + """Scan the table once, reading at most ``limit`` rows, in streams of ``batch_rows``. + + ``where`` and the pinned version apply as for every read. + + Args: + limit: Read at most this many rows, the first in key order. + max_rows: Raise when the read would return more rows; ``None`` for no cap. + Raises: SourceTooLargeError: If the read would return, or returns, more than ``max_rows`` rows. @@ -339,20 +371,20 @@ def _batch_streams(self) -> Generator["ArrowTableStream", None, None]: key column holds a null. """ with self._connector() as connector: - if self._max_rows is not None and not self._where: + if max_rows is not None and not self._where: # Nothing but limit bounds the read, so its size is known up front: # the limit, or the table's recorded row count when that is smaller. - expected = self._limit - if expected is None or expected > self._max_rows: + expected = limit + if expected is None or expected > max_rows: recorded = connector.get_row_count_estimate(self._table_name) expected = recorded if expected is None else min(expected, recorded) - if expected > self._max_rows: + if expected > max_rows: raise self._too_large(f"the read would return {expected:,} rows") seen = 0 for batch in connector.scan_batches( self._table_name, where=self._where or None, - limit=self._limit, + limit=limit, asof=self._asof, ): for start in range(0, batch.num_rows, self._batch_rows): @@ -366,7 +398,7 @@ def _batch_streams(self) -> Generator["ArrowTableStream", None, None]: f"{self._table_schema}." ) seen += rows.num_rows - if self._max_rows is not None and seen > self._max_rows: + if max_rows is not None and seen > max_rows: raise self._too_large("the read returned more rows") yield self._builder.attach_provenance( rows, @@ -409,10 +441,22 @@ def as_table( all_info: bool = False, ) -> "pa.Table": """Scan the table and return its rows as one PyArrow table (all held in memory).""" + return self._whole_table( + self._batch_streams(), columns=columns, all_info=all_info + ) + + def _whole_table( + self, + streams: Iterator["ArrowTableStream"], + *, + columns: ColumnConfig | dict[str, Any] | None, + all_info: bool, + ) -> "pa.Table": + """Return the rows of ``streams`` (batches of one read) as one table.""" # One stream over all the rows, so that a whole-table option (sorting by # tags) applies across batches. enriched = {"source": True, "system_tags": True} - tables = [stream.as_table(columns=enriched) for stream in self._batch_streams()] + tables = [stream.as_table(columns=enriched) for stream in streams] if not tables: return self._stream.as_table(columns=columns, all_info=all_info) whole = ArrowTableStream( @@ -422,6 +466,45 @@ def as_table( ) return whole.as_table(columns=columns, all_info=all_info) + def _preview( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "tuple[pl.DataFrame, str | None]": + """Return the first rows for a notebook display, read in one bounded scan. + + At most ``PREVIEW_ROWS`` rows are read (fewer when the source's ``limit`` is + smaller), with ``where`` and the pinned version applied and ``max_rows`` not, so + a table of any size displays. The note says whether these are all the rows the + source reads and, when not, the table's recorded row count (no scan). + """ + if not self._preview_reads_rows: + schema_only = self._stream.as_table(columns=columns, all_info=all_info) + return pl.DataFrame(schema_only), ( + f"rows not read; the table has about {self.row_count_estimate:,} rows" + ) + head_limit = ( + PREVIEW_ROWS if self._limit is None else min(PREVIEW_ROWS, self._limit) + ) + head = self._whole_table( + self._read_batches(limit=head_limit, max_rows=None), + columns=columns, + all_info=all_info, + ) + shown = head.num_rows + if shown < head_limit or head_limit == self._limit: + return pl.DataFrame(head), f"all {shown:,} row{'' if shown == 1 else 's'}" + recorded = self.row_count_estimate + if self._where: + condition = ", ".join(f"{c}={v!r}" for c, v in self._where.items()) + return pl.DataFrame(head), ( + f"first {shown:,} matching rows (where {condition}); the table has " + f"about {recorded:,} rows" + ) + total = recorded if self._limit is None else min(self._limit, recorded) + return pl.DataFrame(head), f"first {shown:,} rows of about {total:,}" + # ------------------------------------------------------------------------- # Serialization # ------------------------------------------------------------------------- diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index e6d05cebf..73028332f 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -440,8 +440,34 @@ def flow( """ return [e for e in self.iter_data()] + def _preview( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "tuple[pl.DataFrame, str | None]": + """Return the rows a notebook display of the stream shows, and a note on them. + + ``_repr_html_`` and ``view`` show what this returns. The default is every row of + the stream and no note. A stream whose rows are costly to read overrides it to + show fewer, and says so in the note. + + Args: + columns: Column configuration, as for ``as_table``. + all_info: Include all column groups, as for ``as_table``. + + Returns: + The rows to show and a note for the header line, or ``None`` for no note. + """ + return self.as_polars_df(columns=columns, all_info=all_info), None + + def _preview_header(self, note: str | None) -> str: + """Return the header line a notebook display puts above the rows.""" + header = f"{self.__class__.__name__}[{self.label}]" + return header if note is None else f"{header} — {note}" + def _repr_html_(self) -> str: - df = self.as_polars_df() + df, note = self._preview() # reorder columns new_column_order = [c for c in df.columns if c in self.keys()[0]] + [ c for c in df.columns if c not in self.keys()[0] @@ -450,7 +476,7 @@ def _repr_html_(self) -> str: tag_map = {t: f"*{t}" for t in self.keys()[0]} # TODO: construct repr html better df = df.rename(tag_map) - return f"{self.__class__.__name__}[{self.label}]\n" + df._repr_html_() + return self._preview_header(note) + "\n" + df._repr_html_() def view( self, @@ -458,23 +484,20 @@ def view( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "StreamView": - df = self.as_polars_df( - columns=columns, - all_info=all_info, - ) + df, note = self._preview(columns=columns, all_info=all_info) tag_map = {t: f"*{t}" for t in self.keys()[0]} # TODO: construct repr html better df = df.rename(tag_map) - return StreamView(self, df) + return StreamView(self, df, self._preview_header(note)) class StreamView: - def __init__(self, stream: StreamBase, view_df: "pl.DataFrame") -> None: + def __init__( + self, stream: StreamBase, view_df: "pl.DataFrame", header: str + ) -> None: self._stream = stream self._view_df = view_df + self._header = header def _repr_html_(self) -> str: - return ( - f"{self._stream.__class__.__name__}[{self._stream.label}]\n" - + self._view_df._repr_html_() - ) + return self._header + "\n" + self._view_df._repr_html_() diff --git a/superpowers/plans/2026-11-12-itl-186-notebook-preview.md b/superpowers/plans/2026-11-12-itl-186-notebook-preview.md new file mode 100644 index 000000000..e31d99070 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-186-notebook-preview.md @@ -0,0 +1,65 @@ +# ITL-186 Notebook Preview of SpiralDBTableSource Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Displaying a `SpiralDBTableSource` in a notebook (`_repr_html_`, `view()`) reads a +head of at most 10 rows through Spiral's limit, never trips `max_rows`, and says how many rows +the table has; every other stream renders exactly as before. + +**Architecture:** `StreamBase` gains a private hook `_preview(columns, all_info) -> (polars +frame, note | None)` whose default is today's behaviour (`as_polars_df`, no note). +`_repr_html_` and `view()` go through it and append the note to the header line. +`SpiralDBTableSource` overrides it: one scan with `limit=min(10, limit)`, no cap, and a +manifest row count only when the head is not the whole read. A private class switch +`_preview_reads_rows` (default `True`) turns the override into schema only, no scan +(fallback for ITL-991, coordinator condition 1). + +**Tech Stack:** Python 3.11+, pyarrow, polars, pyspiral fake, pytest via `uv run`. + +--- + +## State (keep current; a successor continues from here) + +* PR #333 open on `eywalker/itl-186-add-dynamic-mode-to-spiraldbsource`. +* Approach approved by the coordinator (Nov 12) with two conditions: (1) the no-scan + fallback switch, tested; (2) check large cells: done, Polars' HTML already shortens lists + (first 3, `…`, last) and strings (30 chars); no preview-only cell summary added. +* Then PART 2: CI green, CodeRabbit rounds, all threads replied, worker_done with the sha. + +## Note texts + +| Case | Header after `SpiralDBTableSource[
"``. label: Human-readable label for this source node. @@ -178,11 +211,11 @@ def __init__( overrides: dict[str, str] | None = None, ) -> None: for name, value in (("limit", limit), ("max_rows", max_rows)): - if value is not None and (not isinstance(value, int) or value < 1): + if value is not None and not _is_positive_int(value): raise ValueError( f"{name} must be a positive integer or None, got {value!r}" ) - if not isinstance(batch_rows, int) or batch_rows < 1: + if not _is_positive_int(batch_rows): raise ValueError( f"batch_rows must be a positive integer, got {batch_rows!r}" ) @@ -216,11 +249,13 @@ def __init__( ) self._table_id = connector.get_spiral_table_id(table_name) schema = connector.get_arrow_schema(table_name) - # One version for every read of this source object. A table that was - # never written has no version to pin. + # One version for every read of this source object. A given version is + # used as it is; a table that was never written has no version to pin. self._asof = ( - asof if asof is not None else connector.get_table_version(table_name) - ) or None + asof + if asof is not None + else (connector.get_table_version(table_name) or None) + ) resolved_tags = ( list(self._key_columns) @@ -345,6 +380,8 @@ def _key_tokens(self, table: "pa.Table") -> list[str]: def _batch_streams(self) -> Generator["ArrowTableStream", None, None]: """Scan the table once and yield its rows as streams of at most ``batch_rows`` rows. + Each SpiralDB batch is held while its slices are yielded. + Raises: SourceTooLargeError: If the read would return, or returns, more than ``max_rows`` rows. @@ -371,15 +408,23 @@ def _read_batches( key column holds a null. """ with self._connector() as connector: - if max_rows is not None and not self._where: - # Nothing but limit bounds the read, so its size is known up front: - # the limit, or the table's recorded row count when that is smaller. - expected = limit - if expected is None or expected > max_rows: - recorded = connector.get_row_count_estimate(self._table_name) - expected = recorded if expected is None else min(expected, recorded) - if expected > max_rows: - raise self._too_large(f"the read would return {expected:,} rows") + if ( + max_rows is not None + and not self._where + and (limit is None or limit > max_rows) + ): + # Nothing but limit bounds the read, and the limit does not keep it + # under the cap: the table's recorded row count decides, before any + # row is read (so no row reaches a function of a job that would fail). + # The count can be above the rows a scan returns, never below. + recorded = connector.get_row_count_estimate(self._table_name) + bound = recorded if limit is None else min(limit, recorded) + if bound > max_rows: + raise self._too_large( + f"the read may return up to {bound:,} rows: the table records " + f"about {recorded:,} rows (SpiralDB's count, which can include " + "deleted rows not yet compacted)" + ) seen = 0 for batch in connector.scan_batches( self._table_name, @@ -391,11 +436,14 @@ def _read_batches( rows = arrow_utils.drop_system_columns( pa.Table.from_batches([batch.slice(start, self._batch_rows)]) ) - if not rows.schema.equals(self._table_schema): + # Metadata too: field metadata is how a column's logical type is + # recorded, and the schema hash was computed at construction. + if not rows.schema.equals(self._table_schema, check_metadata=True): raise ValueError( - f"SpiralDB table {self._qualified!r} returned columns " - f"{rows.schema} but the source was built with " - f"{self._table_schema}." + f"SpiralDB table {self._qualified!r}: " + f"{_schema_difference(rows.schema, self._table_schema)}. " + "The table changed after the source was built; build the " + "source again to read it as it is now." ) seen += rows.num_rows if max_rows is not None and seen > max_rows: diff --git a/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md b/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md index 638f96fac..f0b326752 100644 --- a/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md +++ b/superpowers/specs/2026-11-12-itl-186-streaming-spiral-source-design.md @@ -94,8 +94,9 @@ Options for where the table id lives: * **C: readable source id, table id only in `identity_structure`.** Rejected: the content hash is unique, but record ids and tokens of a recreated table collide with the old table's. -Under A and B a table dropped and recreated under the same name is a different source and -nothing cached is reused. Under B only, a renamed table keeps its identity. +Under A and B a table dropped and recreated under the same name is a different source: its +pipeline records are entered again, and function results are still reused for rows whose +content is unchanged, because they are found by row content. Under B only, a renamed table keeps its identity. A `source_id=` given by the user is used as it is, as for every source. diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index 3df849c38..bab413208 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -3,8 +3,9 @@ Skipped unless ``ORCAPOD_SPIRAL_LIVE_PROJECT`` names a project the saved ``spiral login`` can read, with a ``default.spike_event`` table (the workshop project is ``edi-tutorial-v1-841226``). Every test is read-only except -``test_rows_appended_between_runs_are_the_only_ones_computed``, which also needs -``ORCAPOD_SPIRAL_LIVE_WRITE=1`` and writes only to a table it creates and drops in the +``test_rows_appended_between_runs_are_the_only_ones_computed`` and +``test_a_source_reads_the_version_it_was_built_at``, which also need +``ORCAPOD_SPIRAL_LIVE_WRITE=1`` and write only to a table each creates and drops in the project's ``scratch`` dataset. ORCAPOD_SPIRAL_LIVE_PROJECT=edi-tutorial-v1-841226 \\ @@ -143,3 +144,28 @@ def run() -> None: assert calls == [10] finally: project.drop_table(f"scratch.{name}") + + +@pytest.mark.skipif( + not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table" +) +def test_a_source_reads_the_version_it_was_built_at(): + import spiral # pyright: ignore[reportMissingImports] # spiraldb extra, absent in the type-check job + + name = f"itl186_asof_{uuid.uuid4().hex[:8]}" + project = spiral.Spiral().project(PROJECT) + table = project.create_table(f"scratch.{name}", key_schema=[("k", pa.int64())]) + + def keys(source: SpiralDBTableSource) -> list[int]: + return source.as_table().column("k").to_pylist() + + try: + table.write(pa.table({"k": [1, 2], "v": [10, 20]})) + before = SpiralDBTableSource(PROJECT, name, dataset="scratch") + table.write(pa.table({"k": [3], "v": [30]})) + + assert keys(before) == [1, 2] + assert [tag.as_dict()["k"] for tag, _ in before.iter_data()] == [1, 2] + assert keys(SpiralDBTableSource(PROJECT, name, dataset="scratch")) == [1, 2, 3] + finally: + project.drop_table(f"scratch.{name}") diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 41d4f1720..d3e481253 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -8,6 +8,7 @@ import asyncio import datetime as dt +import json from pathlib import Path from typing import cast @@ -16,6 +17,7 @@ from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase +from orcapod.databases.spiraldb_connector import _serialize_arrow_metadata from orcapod.errors import SourceTooLargeError from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.serialization import resolve_source_from_config @@ -108,6 +110,28 @@ def test_a_read_that_matches_nothing_has_the_schema_of_one_that_does(self, proje assert empty.num_rows == 0 assert empty.schema == rows.as_table(all_info=True).schema + def test_a_metadata_change_after_construction_is_refused(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + marked = _rows([1]).cast( + pa.schema( + [ + pa.field("subject_id", pa.string()), + pa.field("timestamp", pa.uint64()), + pa.field("amplitude", pa.float32(), metadata={"unit": "uV"}), + ] + ) + ) + # About the connector's non-public metadata encoding: it is how a table's + # field metadata is stored, and the test changes only that. + kv = _serialize_arrow_metadata(marked) # pyright: ignore[reportPrivateUsage] + assert kv is not None + project.table("default.event").set_metadata(kv) + + with pytest.raises( + ValueError, match="column 'amplitude'.*build the source again" + ): + source.as_table() + def test_a_table_with_key_columns_only_is_refused(self, project): project.create_table("default.never_written", key_schema=_KEYS) @@ -217,6 +241,11 @@ def test_sizes_must_be_positive(self, project): with pytest.raises(ValueError, match="batch_rows"): SpiralDBTableSource(_PROJECT, "event", batch_rows=0) + @pytest.mark.parametrize("name", ["limit", "max_rows", "batch_rows"]) + def test_a_bool_is_not_a_size(self, project, name): + with pytest.raises(ValueError, match=name): + SpiralDBTableSource(_PROJECT, "event", **{name: True}) + def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project): table = project.table("default.event") version = table.last_modified_at() @@ -236,6 +265,14 @@ def test_a_given_version_is_used(self, project): assert fake_pyspiral.scan_log[-1].kwargs["asof"] == 1 assert source.to_config()["asof"] == 1 + def test_a_given_version_zero_is_used(self, project): + source = SpiralDBTableSource(_PROJECT, "event", asof=0) + + source.as_table() + + assert fake_pyspiral.scan_log[-1].kwargs["asof"] == 0 + assert source.to_config()["asof"] == 0 + class TestRowCap: def test_a_read_over_the_cap_is_refused_before_scanning(self, project): @@ -246,7 +283,9 @@ def test_a_read_over_the_cap_is_refused_before_scanning(self, project): assert fake_pyspiral.scan_log == [] message = str(raised.value) - assert "5 rows" in message and "max_rows=3" in message + assert "up to 5 rows" in message and "max_rows=3" in message + assert "records about 5 rows" in message + assert "deleted rows not yet compacted" in message assert "limit=" in message and "where=" in message with pytest.raises(SourceTooLargeError): list(source.iter_data()) @@ -405,6 +444,19 @@ def test_a_date_and_time_in_where_survive_the_round_trip(self, project): assert loaded.as_table().num_rows == 1 + def test_bytes_in_where_survive_a_json_round_trip(self, project): + keys = pa.schema([pa.field("blob", pa.binary())]) + project.create_table("default.blobs", key_schema=keys).write( + pa.table({"blob": pa.array([b"\x00\xff", b"\x01"]), "n": [1, 2]}) + ) + source = SpiralDBTableSource(_PROJECT, "blobs", where={"blob": b"\x00\xff"}) + + saved = json.loads(json.dumps(source.to_config())) + loaded = SpiralDBTableSource.from_config(saved) + + assert loaded.to_config() == saved + assert loaded.as_table().column("n").to_pylist() == [1] + def test_loading_refuses_a_name_that_now_leads_to_another_table(self, project): config = SpiralDBTableSource(_PROJECT, "event").to_config() project.drop_table("default.event") From dba32fadc528bdf3bf5f44cf6f89f78ea07f31ed Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 14:58:49 -0800 Subject: [PATCH 09/17] test(spiraldb): type the bool-size and live asof tests (ITL-186) Co-Authored-By: Claude Opus 5.5 --- tests/test_core/sources/test_spiraldb_table_source_live.py | 3 ++- .../sources/test_spiraldb_table_source_streaming.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index bab413208..9530dabf9 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -18,6 +18,7 @@ import time import uuid from pathlib import Path +from typing import cast import pyarrow as pa import pytest @@ -157,7 +158,7 @@ def test_a_source_reads_the_version_it_was_built_at(): table = project.create_table(f"scratch.{name}", key_schema=[("k", pa.int64())]) def keys(source: SpiralDBTableSource) -> list[int]: - return source.as_table().column("k").to_pylist() + return cast("list[int]", source.as_table().column("k").to_pylist()) try: table.write(pa.table({"k": [1, 2], "v": [10, 20]})) diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index d3e481253..1b4b31765 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -10,7 +10,7 @@ import datetime as dt import json from pathlib import Path -from typing import cast +from typing import Any, cast import pyarrow as pa import pytest @@ -244,7 +244,9 @@ def test_sizes_must_be_positive(self, project): @pytest.mark.parametrize("name", ["limit", "max_rows", "batch_rows"]) def test_a_bool_is_not_a_size(self, project, name): with pytest.raises(ValueError, match=name): - SpiralDBTableSource(_PROJECT, "event", **{name: True}) + # A bool where an int is declared: the call a type checker would refuse. + sizes: dict[str, Any] = {name: True} + SpiralDBTableSource(_PROJECT, "event", **sizes) def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project): table = project.table("default.event") From 94980c1443f315cb1f4a212604dc3fceb5984960 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:06:43 -0800 Subject: [PATCH 10/17] refactor(spiraldb): the preview reads a head with its size as data (ITL-186) _head returns the rows and the row count (exact or the table's recorded count) as data and _head_note words them in one place, so ITL-991's public head() can take this over mechanically. Co-Authored-By: Claude Opus 5.5 --- .../core/sources/spiraldb_table_source.py | 84 ++++++++++++++----- .../2026-11-12-itl-186-notebook-preview.md | 10 ++- 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index fc957b606..6adc5f4af 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -32,7 +32,7 @@ import datetime from collections.abc import AsyncIterator, Collection, Generator, Iterator, Mapping from contextlib import contextmanager -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NamedTuple from orcapod.core.sources.base import RootSource from orcapod.core.sources.stream_builder import SourceStreamBuilder @@ -101,6 +101,25 @@ def _is_positive_int(value: object) -> bool: return isinstance(value, int) and not isinstance(value, bool) and value >= 1 +class _Head(NamedTuple): + """The first rows of a source, and what is known of how many rows it reads. + + Attributes: + rows: The rows read; zero rows (the source's columns) when ``rows_read`` is + false. + rows_read: Whether rows were read at all. + total_rows: The rows the source reads or, as an estimate, the table's recorded + row count. + total_is_estimate: Whether ``total_rows`` is the recorded row count rather than + an exact count. + """ + + rows: "pa.Table" + rows_read: bool + total_rows: int + total_is_estimate: bool + + def _schema_difference(found: "pa.Schema", expected: "pa.Schema") -> str: """Describe how a scanned batch's columns differ from the ones the source expects.""" if found.names != expected.names: @@ -514,44 +533,65 @@ def _whole_table( ) return whole.as_table(columns=columns, all_info=all_info) - def _preview( + def _head( self, + n: int, *, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> "tuple[pl.DataFrame, str | None]": - """Return the first rows for a notebook display, read in one bounded scan. + ) -> "_Head": + """Return the source's first ``n`` rows, read in one bounded scan, and its size. + + At most ``n`` rows are read (fewer when the source's ``limit`` is smaller), with + ``where`` and the pinned version applied and ``max_rows`` not, so a table of any + size can be looked at. When these are not all the rows the source reads, the + size is the table's recorded row count (no scan), an estimate. - At most ``PREVIEW_ROWS`` rows are read (fewer when the source's ``limit`` is - smaller), with ``where`` and the pinned version applied and ``max_rows`` not, so - a table of any size displays. The note says whether these are all the rows the - source reads and, when not, the table's recorded row count (no scan). + Args: + n: The most rows to read. + columns: Column configuration, as for ``as_table``. + all_info: Include all column groups, as for ``as_table``. """ if not self._preview_reads_rows: schema_only = self._stream.as_table(columns=columns, all_info=all_info) - return pl.DataFrame(schema_only), ( - f"rows not read; the table has about {self.row_count_estimate:,} rows" - ) - head_limit = ( - PREVIEW_ROWS if self._limit is None else min(PREVIEW_ROWS, self._limit) - ) - head = self._whole_table( + return _Head(schema_only, False, self.row_count_estimate, True) + head_limit = n if self._limit is None else min(n, self._limit) + rows = self._whole_table( self._read_batches(limit=head_limit, max_rows=None), columns=columns, all_info=all_info, ) - shown = head.num_rows - if shown < head_limit or head_limit == self._limit: - return pl.DataFrame(head), f"all {shown:,} row{'' if shown == 1 else 's'}" + if rows.num_rows < head_limit or head_limit == self._limit: + return _Head(rows, True, rows.num_rows, False) recorded = self.row_count_estimate + if self._limit is not None and not self._where: + recorded = min(self._limit, recorded) + return _Head(rows, True, recorded, True) + + def _head_note(self, head: "_Head") -> str: + """Return the words a notebook display puts after the header: the one place.""" + shown = head.rows.num_rows + if not head.rows_read: + return f"rows not read; the table has about {head.total_rows:,} rows" + if not head.total_is_estimate: + return f"all {shown:,} row{'' if shown == 1 else 's'}" if self._where: condition = ", ".join(f"{c}={v!r}" for c, v in self._where.items()) - return pl.DataFrame(head), ( + return ( f"first {shown:,} matching rows (where {condition}); the table has " - f"about {recorded:,} rows" + f"about {head.total_rows:,} rows" ) - total = recorded if self._limit is None else min(self._limit, recorded) - return pl.DataFrame(head), f"first {shown:,} rows of about {total:,}" + return f"first {shown:,} rows of about {head.total_rows:,}" + + def _preview( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "tuple[pl.DataFrame, str | None]": + """Return the first ``PREVIEW_ROWS`` rows for a notebook display, and the note.""" + head = self._head(PREVIEW_ROWS, columns=columns, all_info=all_info) + return pl.DataFrame(head.rows), self._head_note(head) # ------------------------------------------------------------------------- # Serialization diff --git a/superpowers/plans/2026-11-12-itl-186-notebook-preview.md b/superpowers/plans/2026-11-12-itl-186-notebook-preview.md index e31d99070..d05f90197 100644 --- a/superpowers/plans/2026-11-12-itl-186-notebook-preview.md +++ b/superpowers/plans/2026-11-12-itl-186-notebook-preview.md @@ -24,7 +24,15 @@ manifest row count only when the head is not the whole read. A private class swi * Approach approved by the coordinator (Nov 12) with two conditions: (1) the no-scan fallback switch, tested; (2) check large cells: done, Polars' HTML already shortens lists (first 3, `…`, last) and strings (30 chars); no preview-only cell summary added. -* Then PART 2: CI green, CodeRabbit rounds, all threads replied, worker_done with the sha. +* Part 1 done: 3e5773ed (CodeRabbit rate-limited on it). +* CodeRabbit round 1 (review of 012a86eb): 6 threads + nitpick + additional + linked-issues + check, plan approved by the coordinator; fixed in 3bca8ec9, merged origin/main (23bb59d2), + typed tests (dba32fad), pushed once. All 6 threads replied; top-level comment + issuecomment-6838421621 answers the body-only findings. PR body updated. +* Live (read-only + scratch tables created and dropped): 7 live tests pass. One earlier run + had a transient failure of `test_an_async_job_over_a_limited_read` that did not recur. +* Next: CI and CodeRabbit on dba32fad; further rounds; check every reviewThread has a reply; + worker_done with the reviewed sha. ## Note texts From bb8e4644357e1ed63aad7928af37c5f14fc85df7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:23:56 -0800 Subject: [PATCH 11/17] fix(spiraldb): count the pinned version for the cap; escape the display header (ITL-186) CodeRabbit round 2: get_row_count_estimate takes asof, and the cap pre-check and row_count_estimate count the version the source reads, so rows appended after construction no longer make a pinned source refuse a read. The notebook header is HTML-escaped. Docs: a source object keeps its version; source-info values carry the column suffix. Live tests state and check the table size they need; the connector test closes an unfinished read. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/sources.md | 6 +++-- .../core/sources/spiraldb_table_source.py | 13 +++++++---- src/orcapod/core/sources/stream_builder.py | 9 +++++--- src/orcapod/core/streams/base.py | 7 ++++-- src/orcapod/databases/spiraldb_connector.py | 6 +++-- .../sources/test_spiraldb_table_source.py | 10 +++++--- .../test_spiraldb_table_source_live.py | 23 +++++++++++++++++-- .../test_spiraldb_table_source_streaming.py | 14 +++++++++++ 8 files changed, 69 insertions(+), 19 deletions(-) diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 902fa7f80..e0866a6db 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -169,8 +169,10 @@ pipeline has read from that table, not only the rows of the latest restriction. sent downstream during a run are only the rows that run read. **Running again.** A second run scans the table again and computes nothing for the rows it -has already seen. Rows added to the Spiral table since the last run are new keys: they are -the only rows computed. +has already seen. A source object keeps reading the version its table had when it was +built, so rows added to the Spiral table afterwards are seen only by a new source: build +the source (and the job around it) again, and the added rows are new keys, the only rows +computed. **Known limitations.** diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 6adc5f4af..6031ec308 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -358,12 +358,13 @@ def _qualified(self) -> str: def row_count_estimate(self) -> int: """The whole table's row count as SpiralDB's manifests record it. - No row is read. ``where`` and ``limit`` are not taken into account, and the - count can be above what a scan returns when rows were deleted or overwritten - and not yet compacted. + The count is of the version this source reads (the one it was built at, or + the given ``asof``). No row is read. ``where`` and ``limit`` are not taken into + account, and the count can be above what a scan returns when rows were + deleted or overwritten and not yet compacted. """ with self._connector() as connector: - return connector.get_row_count_estimate(self._table_name) + return connector.get_row_count_estimate(self._table_name, asof=self._asof) def _too_large(self, found: str) -> SourceTooLargeError: """Return the error for a read that goes over ``max_rows``.""" @@ -436,7 +437,9 @@ def _read_batches( # under the cap: the table's recorded row count decides, before any # row is read (so no row reaches a function of a job that would fail). # The count can be above the rows a scan returns, never below. - recorded = connector.get_row_count_estimate(self._table_name) + recorded = connector.get_row_count_estimate( + self._table_name, asof=self._asof + ) bound = recorded if limit is None else min(limit, recorded) if bound > max_rows: raise self._too_large( diff --git a/src/orcapod/core/sources/stream_builder.py b/src/orcapod/core/sources/stream_builder.py index 5e36d0cd5..8018e73fa 100644 --- a/src/orcapod/core/sources/stream_builder.py +++ b/src/orcapod/core/sources/stream_builder.py @@ -306,9 +306,12 @@ def attach_provenance( ) -> ArrowTableStream: """Add source-info and system-tag columns to ``table`` and wrap it as a stream. - Each row's source-info value is ``::`` and its record ID is - the UUID v5 of the same two parts, so a row keeps its record ID wherever and - however often it is read, as long as its token is the same. + Each row's source-info prefix is ``::``: the value in a + source-info column is that prefix followed by ``::`` (for example + ``_source_amplitude`` holds ``::::amplitude``). The row's + record ID is the UUID v5 of ``source_id`` and ``token``, so a row keeps its + record ID wherever and however often it is read, as long as its token is the + same. Args: table: The rows, with nullable flags already set correctly. diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 73028332f..9805cf35f 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import html import logging from abc import abstractmethod from collections.abc import AsyncIterator, Collection, Iterator, Mapping @@ -462,9 +463,11 @@ def _preview( return self.as_polars_df(columns=columns, all_info=all_info), None def _preview_header(self, note: str | None) -> str: - """Return the header line a notebook display puts above the rows.""" + """Return the header line a notebook display puts above the rows, HTML-escaped.""" header = f"{self.__class__.__name__}[{self.label}]" - return header if note is None else f"{header} — {note}" + return html.escape( + header if note is None else f"{header} — {note}", quote=False + ) def _repr_html_(self) -> str: df, note = self._preview() diff --git a/src/orcapod/databases/spiraldb_connector.py b/src/orcapod/databases/spiraldb_connector.py index cb83e853c..346039ec2 100644 --- a/src/orcapod/databases/spiraldb_connector.py +++ b/src/orcapod/databases/spiraldb_connector.py @@ -711,7 +711,7 @@ def get_table_version(self, table_name: str) -> int: self._require_open() return int(self._project.table(self._table_id(table_name)).last_modified_at()) - def get_row_count_estimate(self, table_name: str) -> int: + def get_row_count_estimate(self, table_name: str, asof: int | None = None) -> int: """Return the table's row count as SpiralDB's manifests record it, without scanning. The count is what the manifests hold: it can be above the number of rows a @@ -719,13 +719,15 @@ def get_row_count_estimate(self, table_name: str) -> int: Args: table_name: Plain table name (no dataset prefix). + asof: Count the table as of this version (a ``get_table_version`` value, + as ``scan_batches`` takes). If ``None`` (default), the current version. Returns: The recorded row count. """ self._require_open() tbl = self._project.table(self._table_id(table_name)) - stats = self._spiral.core.manifest_stats(tbl.core.get_snapshot(asof=None)) + stats = self._spiral.core.manifest_stats(tbl.core.get_snapshot(asof=asof)) return int(stats.key_space_stats.total_rows) def delete_table(self, table_name: str) -> None: diff --git a/tests/test_core/sources/test_spiraldb_table_source.py b/tests/test_core/sources/test_spiraldb_table_source.py index 580950175..684e9f9c8 100644 --- a/tests/test_core/sources/test_spiraldb_table_source.py +++ b/tests/test_core/sources/test_spiraldb_table_source.py @@ -21,7 +21,9 @@ from __future__ import annotations +from collections.abc import Generator from pathlib import Path +from typing import Any, cast from unittest.mock import patch import pyarrow as pa @@ -224,13 +226,15 @@ def connect(**kwargs): with patch(_CONNECTOR, side_effect=connect): src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) src.as_table() - next(iter(src.iter_data())) # a read that is not finished + # iter_data is a generator function, so the read can be closed. + unfinished = cast("Generator[Any, None, None]", src.iter_data()) + next(unfinished) # a read that is not finished + unfinished.close() with pytest.raises(ValueError): SpiralDBTableSource(_PROJECT_ID, "no_such_table") - del src assert len(opened) == 4 # About the connector's own closed flag: nothing public reports it. - assert [c._closed for c in opened[:2] + opened[3:]] == [True] * 3 # pyright: ignore[reportPrivateUsage] + assert [c._closed for c in opened] == [True] * 4 # pyright: ignore[reportPrivateUsage] # =========================================================================== diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index 9530dabf9..f8920b1a3 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -1,8 +1,9 @@ """SpiralDBTableSource against a live SpiralDB project (ITL-186). Opt-in. Skipped unless ``ORCAPOD_SPIRAL_LIVE_PROJECT`` names a project the saved ``spiral login`` -can read, with a ``default.spike_event`` table (the workshop project is -``edi-tutorial-v1-841226``). Every test is read-only except +can read, with a ``default.spike_event`` table of more than 1,000 rows (the workshop +project is ``edi-tutorial-v1-841226``, 63,349 rows); the tests that need that many skip +when the table records fewer. Every test is read-only except ``test_rows_appended_between_runs_are_the_only_ones_computed`` and ``test_a_source_reads_the_version_it_was_built_at``, which also need ``ORCAPOD_SPIRAL_LIVE_WRITE=1`` and write only to a table each creates and drops in the @@ -36,6 +37,16 @@ not PROJECT, reason="Set ORCAPOD_SPIRAL_LIVE_PROJECT to run the live Spiral tests" ) +# The rows ``default.spike_event`` must record for the tests that read or refuse many. +_MIN_ROWS = 1_000 + + +def _require_a_large_table() -> None: + """Skip the calling test unless ``spike_event`` records more than ``_MIN_ROWS`` rows.""" + recorded = SpiralDBTableSource(PROJECT, "spike_event").row_count_estimate + if recorded <= _MIN_ROWS: + pytest.skip(f"spike_event records {recorded} rows; these tests need more") + def test_building_a_source_is_quick_and_names_the_table(): start = time.monotonic() @@ -50,6 +61,7 @@ def test_building_a_source_is_quick_and_names_the_table(): def test_limit_reads_the_first_rows(): + _require_a_large_table() source = SpiralDBTableSource(PROJECT, "spike_event", limit=1000) table = source.as_table(all_info=True) @@ -79,6 +91,7 @@ def test_where_reads_only_matching_rows_with_the_same_record_ids(): def test_a_whole_table_over_the_cap_is_refused_without_reading(): + _require_a_large_table() source = SpiralDBTableSource(PROJECT, "spike_event", max_rows=10) start = time.monotonic() @@ -168,5 +181,11 @@ def keys(source: SpiralDBTableSource) -> list[int]: assert keys(before) == [1, 2] assert [tag.as_dict()["k"] for tag, _ in before.iter_data()] == [1, 2] assert keys(SpiralDBTableSource(PROJECT, name, dataset="scratch")) == [1, 2, 3] + + # The cap counts the rows of the version the source reads, not the current one. + capped = SpiralDBTableSource(PROJECT, name, dataset="scratch", max_rows=3) + table.write(pa.table({"k": [4], "v": [40]})) + assert capped.row_count_estimate == 3 + assert keys(capped) == [1, 2, 3] finally: project.drop_table(f"scratch.{name}") diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 1b4b31765..00fb5bf5b 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -588,6 +588,20 @@ def test_where_is_applied_and_named(self, big): "the table has about 25 rows\n" ) in html + def test_the_header_is_escaped_in_both_displays(self, project): + """A ``where`` value with markup characters is shown as text, not markup.""" + project.create_table("default.marked", key_schema=_KEYS).write( + _rows(list(range(1, 26)), subject="") + ) + source = SpiralDBTableSource(_PROJECT, "marked", where={"subject_id": ""}) + + for shown in (source._repr_html_(), source.view()._repr_html_()): + header = shown.splitlines()[0] + assert header.endswith( + "first 10 matching rows (where subject_id='<b>'); " + "the table has about 25 rows" + ) + def test_view_reads_the_same_head(self, big): source = SpiralDBTableSource(_PROJECT, "big", label="big") From 848d90b2573082d5358a47bc6f88a8ac9d1022b7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:29:11 -0800 Subject: [PATCH 12/17] docs(spiraldb): docstrings for the Spiral source tests and the fake (ITL-186) Docstrings only (code ASTs unchanged), as CodeRabbit round 2 asked. Co-Authored-By: Claude Opus 5.5 --- .../sources/test_spiraldb_table_source.py | 44 ++++++++++ .../test_spiraldb_table_source_fake.py | 23 ++++++ .../test_spiraldb_table_source_live.py | 12 +++ .../test_spiraldb_table_source_streaming.py | 80 +++++++++++++++++++ tests/test_databases/fake_pyspiral.py | 38 +++++++++ 5 files changed, 197 insertions(+) diff --git a/tests/test_core/sources/test_spiraldb_table_source.py b/tests/test_core/sources/test_spiraldb_table_source.py index 684e9f9c8..3dcfc30db 100644 --- a/tests/test_core/sources/test_spiraldb_table_source.py +++ b/tests/test_core/sources/test_spiraldb_table_source.py @@ -85,18 +85,21 @@ def fake_spiral(tmp_path: Path): def test_import_from_core_sources(): + """``SpiralDBTableSource`` is importable from ``orcapod.core.sources``.""" from orcapod.core.sources import SpiralDBTableSource assert SpiralDBTableSource is not None def test_import_from_orcapod_sources(): + """``SpiralDBTableSource`` is importable from ``orcapod.sources``.""" from orcapod.sources import SpiralDBTableSource assert SpiralDBTableSource is not None def test_in_core_sources_all(): + """``SpiralDBTableSource`` is listed in ``orcapod.core.sources.__all__``.""" import orcapod.core.sources as m assert "SpiralDBTableSource" in m.__all__ @@ -108,19 +111,24 @@ def test_in_core_sources_all(): class TestProtocolConformance: + """A ``SpiralDBTableSource`` satisfies the source, stream and hashing protocols.""" + def test_is_source_protocol(self): + """The source satisfies ``SourceProtocol``.""" from orcapod.protocols.core_protocols import SourceProtocol _make_table() assert isinstance(SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME), SourceProtocol) def test_is_stream_protocol(self): + """The source satisfies ``StreamProtocol``.""" from orcapod.protocols.core_protocols import StreamProtocol _make_table() assert isinstance(SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME), StreamProtocol) def test_is_pipeline_element_protocol(self): + """The source satisfies ``PipelineElementProtocol``.""" from orcapod.protocols.hashing_protocols import PipelineElementProtocol _make_table() @@ -135,7 +143,10 @@ def test_is_pipeline_element_protocol(self): class TestPKAsDefaultTags: + """By default the primary key columns are the tag columns.""" + def test_single_pk_is_tag_column(self): + """A one-column primary key is the only tag; the other column is data.""" _make_table() tag_schema, data_schema = SpiralDBTableSource( _PROJECT_ID, _TABLE_NAME @@ -144,6 +155,7 @@ def test_single_pk_is_tag_column(self): assert list(data_schema) == ["firing_rate"] def test_composite_pk_all_columns_are_tags(self): + """Every column of a composite primary key is a tag.""" _make_composite_pk_table() tag_schema, data_schema = SpiralDBTableSource( _PROJECT_ID, _TABLE_NAME @@ -158,7 +170,10 @@ def test_composite_pk_all_columns_are_tags(self): class TestExplicitTagOverride: + """Given ``tag_columns`` replace the primary key as the tags.""" + def test_explicit_tag_columns_override_pk(self): + """Given ``tag_columns`` are the tags; key columns left out become data.""" _make_composite_pk_table() src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=["probe_id"]) tag_schema, data_schema = src.output_schema() @@ -166,11 +181,13 @@ def test_explicit_tag_columns_override_pk(self): assert set(data_schema) == {"session_id", "neuron_count"} def test_a_bare_string_names_one_tag_column(self): + """A bare string names a single tag column.""" _make_composite_pk_table() src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns="session_id") assert list(src.output_schema()[0]) == ["session_id"] def test_unknown_tag_column_raises(self): + """A ``tag_columns`` name the table lacks is refused.""" _make_table() with pytest.raises(ValueError, match="tag_columns not found"): SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=["nope"]) @@ -182,7 +199,10 @@ def test_unknown_tag_column_raises(self): class TestErrorCases: + """Construction refuses a table it cannot read as a source.""" + def test_missing_table_raises_value_error(self): + """A missing table is refused, naming the project and dataset.""" _make_table() with pytest.raises(ValueError, match="not found in database") as raised: SpiralDBTableSource(_PROJECT_ID, "no_such_table", dataset="default") @@ -204,7 +224,10 @@ def test_a_table_without_a_key_schema_raises(self, tag_columns): class TestConnector: + """The source forwards its arguments to the connector and closes it.""" + def test_dataset_and_overrides_forwarded_to_connector(self): + """``dataset`` and ``overrides`` are forwarded to ``SpiralDBConnector``.""" overrides = {"server.url": "http://api.spiraldb.dev"} _make_table(dataset="prod") with patch(_CONNECTOR, wraps=SpiralDBConnector) as connector_class: @@ -216,10 +239,12 @@ def test_dataset_and_overrides_forwarded_to_connector(self): ) def test_every_connector_is_closed(self): + """Every connector is closed: build, full read, unfinished read, error.""" _make_table() opened: list[SpiralDBConnector] = [] def connect(**kwargs): + """Open a real connector with ``kwargs`` and record it in ``opened``.""" opened.append(SpiralDBConnector(**kwargs)) return opened[-1] @@ -243,13 +268,17 @@ def connect(**kwargs): class TestStreamBehaviour: + """The source is a root stream whose rows are the table's.""" + def test_producer_is_none_and_upstreams_empty(self): + """A source has no producer and no upstreams.""" _make_table() src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) assert src.producer is None assert src.upstreams == () def test_iter_data_yields_one_pair_per_row(self): + """``iter_data`` yields one tag and data pair per row, in key order.""" _make_table() pairs = list(SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).iter_data()) assert [tag.as_dict()["session_id"] for tag, _ in pairs] == ["s1", "s2", "s3"] @@ -258,6 +287,7 @@ def test_iter_data_yields_one_pair_per_row(self): ) def test_as_table_returns_the_rows(self): + """``as_table`` returns the table's rows as a ``pa.Table``.""" _make_table() table = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).as_table() assert isinstance(table, pa.Table) @@ -273,7 +303,10 @@ def test_as_table_returns_the_rows(self): class TestDeterministicHashing: + """Hashes depend on the table and its schema, not on the source object.""" + def test_hashes_are_deterministic(self): + """Two sources of one table have equal pipeline and content hashes.""" _make_table() src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) @@ -281,6 +314,7 @@ def test_hashes_are_deterministic(self): assert src1.content_hash() == src2.content_hash() def test_different_tag_columns_yield_different_pipeline_hash(self): + """Other tag columns give another pipeline hash.""" _make_composite_pk_table() src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) src2 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, tag_columns=["probe_id"]) @@ -303,6 +337,7 @@ def test_different_schemas_yield_different_pipeline_hash(self): assert src1.pipeline_hash() != src2.pipeline_hash() def test_same_schema_in_two_tables_shares_the_pipeline_hash_only(self): + """Two tables of one schema share the pipeline hash, not the content hash.""" _make_table() _make_table(name="other") src1 = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) @@ -317,7 +352,10 @@ def test_same_schema_in_two_tables_shares_the_pipeline_hash_only(self): class TestConfigSerialization: + """``to_config`` and ``from_config`` round-trip a source.""" + def test_to_config_fields(self): + """``to_config`` records the table, its id, tags, overrides, id and hashes.""" _make_table(dataset="prod") overrides = {"server.url": "http://api.spiraldb.dev"} src = SpiralDBTableSource( @@ -337,6 +375,7 @@ def test_to_config_fields(self): assert field in cfg def test_to_config_defaults(self): + """``to_config`` records the default restrictions, cap and batch size.""" _make_table() cfg = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME).to_config() assert cfg["overrides"] is None @@ -347,6 +386,7 @@ def test_to_config_defaults(self): assert cfg["batch_rows"] == 8_192 def test_from_config_hashes_match(self): + """A source rebuilt by ``from_config`` keeps its hashes and reads the rows.""" _make_table(dataset="prod") src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME, dataset="prod") src2 = SpiralDBTableSource.from_config(src.to_config()) @@ -355,6 +395,7 @@ def test_from_config_hashes_match(self): assert src2.as_table().num_rows == 3 def test_resolve_source_from_config_works(self): + """``resolve_source_from_config`` rebuilds a ``SpiralDBTableSource``.""" from orcapod.pipeline.serialization import resolve_source_from_config _make_table() @@ -369,6 +410,8 @@ def test_resolve_source_from_config_works(self): @pytest.mark.integration class TestPipelineIntegration: + """A ``SpiralDBTableSource`` feeds a function pod in a ``PipelineJob``.""" + def test_spiraldb_source_in_pipeline(self): """Verify SpiralDBTableSource drives a full pipeline end-to-end.""" from orcapod import PipelineJob, function_pod @@ -376,6 +419,7 @@ def test_spiraldb_source_in_pipeline(self): @function_pod(output_keys="doubled") def double_rate(firing_rate: float | None) -> float | None: + """Return twice ``firing_rate``, or ``None`` for a null.""" return None if firing_rate is None else firing_rate * 2.0 _make_table() diff --git a/tests/test_core/sources/test_spiraldb_table_source_fake.py b/tests/test_core/sources/test_spiraldb_table_source_fake.py index 8def0ecfa..f4364331e 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_fake.py +++ b/tests/test_core/sources/test_spiraldb_table_source_fake.py @@ -87,7 +87,10 @@ def spiral_root(tmp_path: Path): class TestSchema: + """Spiral tables arrive with their nullability and Arrow types.""" + def test_nullable_spiral_columns_are_optional(self, spiral_root): + """Nullable Spiral columns arrive as ``T | None`` tags and data.""" tag, data = SpiralDBTableSource(_PROJECT, "session").output_schema() assert dict(tag) == { "subject_id": str | None, @@ -96,6 +99,7 @@ def test_nullable_spiral_columns_are_optional(self, spiral_root): assert dict(data) == {"booth": str | None, "date_of_record": dt.date | None} def test_arrow_types_are_kept(self, spiral_root): + """Arrow types are kept, only normalized to their large variants.""" table = SpiralDBTableSource(_PROJECT, "experiment").as_table() assert {f.name: f.type for f in table.schema} == { "subject_id": pa.large_string(), @@ -108,6 +112,7 @@ def test_arrow_types_are_kept(self, spiral_root): } def test_dataframe_view_has_tags_and_data_only(self, spiral_root): + """``as_df`` holds the tag and data columns only.""" frame = SpiralDBTableSource(_PROJECT, "session").as_df() assert frame.columns == [ "subject_id", @@ -117,8 +122,11 @@ def test_dataframe_view_has_tags_and_data_only(self, spiral_root): ] def test_plain_typed_pod_is_refused(self, spiral_root): + """A pod taking ``int`` is refused over a nullable column.""" + @function_pod(output_keys="n") def plain(total_trials: int) -> int: + """Return ``total_trials`` unchanged.""" return total_trials source = SpiralDBTableSource(_PROJECT, "experiment") @@ -128,7 +136,10 @@ def plain(total_trials: int) -> int: class TestErrors: + """A table that cannot be found is reported by name.""" + def test_missing_table_names_project_and_dataset(self, spiral_root): + """A missing table is refused, naming the table, project and dataset.""" with pytest.raises(ValueError) as info: SpiralDBTableSource(_PROJECT, "no_such_table", dataset="default") message = str(info.value) @@ -138,6 +149,8 @@ def test_missing_table_names_project_and_dataset(self, spiral_root): class TestWorkshopPath: + """The workshop job runs, reruns from its store, and saves and loads.""" + def _job(self, store: Path, calls: list[int]) -> PipelineJob: """Build the workshop job (Join, select, counting pod) over the Delta ``store``.""" @@ -145,6 +158,7 @@ def _job(self, store: Path, calls: list[int]) -> PipelineJob: def counted( correct_trials: int | None, total_trials: int | None ) -> float | None: + """Return the share of correct trials, or ``None``; counts the call.""" calls.append(1) if correct_trials is None or not total_trials: return None @@ -164,6 +178,7 @@ def counted( return job def test_run_rerun_save_load(self, spiral_root, tmp_path): + """Each row is computed once, a rerun computes nothing, and the job reloads.""" store = tmp_path / "store" calls: list[int] = [] job = self._job(store, calls) @@ -197,6 +212,7 @@ def test_run_rerun_save_load(self, spiral_root, tmp_path): def test_hashes_are_stable_across_processes(spiral_root): + """A second Python process computes the same content and pipeline hashes.""" source = SpiralDBTableSource(_PROJECT, "experiment") script = ( "import json\n" @@ -251,6 +267,7 @@ def two_projects(self, tmp_path: Path): @staticmethod def _identity(source: SpiralDBTableSource) -> tuple[str, str, str]: + """Return the content hash, first record id and first ``_source_sex`` token.""" table = source.as_table(all_info=True) record_id = next( c for c in table.column_names if c.startswith("_tag_record_id") @@ -267,12 +284,14 @@ def _identity(source: SpiralDBTableSource) -> tuple[str, str, str]: def test_default_source_id_is_project_dataset_table_and_table_id( self, two_projects ): + """The default source id is the project, dataset, table and table id.""" source = SpiralDBTableSource("edi-sample-a", "subject") expected = f"edi-sample-a/default.subject@{source.table_id}" assert source.source_id == expected assert self._identity(source)[2] == f"{expected}::{self._M1}::sex" def test_same_table_in_two_projects_does_not_collide(self, two_projects): + """One table in two projects shares the pipeline hash only.""" a = SpiralDBTableSource("edi-sample-a", "subject") b = SpiralDBTableSource("edi-sample-b", "subject") assert a.pipeline_hash() == b.pipeline_hash() # schema only @@ -287,6 +306,7 @@ def test_unrecorded_table_hashes_are_unchanged(self, two_projects): # ITL-186 moved the record id and the row token only (from the row's position to # the hash of its primary key); the token is the same with and without a # nullability record, because key fields are hashed as non-nullable. + """An unrecorded table reads all-optional, with its pinned hashes.""" source = SpiralDBTableSource("edi-sample-a", "unrecorded", source_id="subject") assert source.output_schema() == ( Schema({"subject_id": str | None}), @@ -312,6 +332,7 @@ def test_explicit_source_id_keeps_its_hashes(self, two_projects): # for the v0.3 hash step (ITL-967). The table was # written non-null; with the nullability record it reads back non-null, so the # hashes are those of the table as written. + """A recorded table with a given source id keeps its pinned hashes.""" source = SpiralDBTableSource("edi-sample-a", "subject", source_id="subject") assert self._identity(source) == ( "semantic_v1:a06a417294cabb0625e47a9357ebf9cc8183576b17498282901bd68d0f02e820", @@ -456,6 +477,7 @@ def test_plain_float_pod_is_refused_on_an_unrecorded_table(self, strict_root): @function_pod(output_keys="half") def halve(weight: float) -> float: + """Return half of ``weight``.""" return weight / 2 source = SpiralDBTableSource(_PROJECT, "session_unrecorded") @@ -478,6 +500,7 @@ def test_plain_typed_pod_runs_through_join_and_save_load( @function_pod(output_keys="score") def score(total_trials: int, weight: float) -> float: + """Return ``total_trials`` times ``weight``.""" return total_trials * weight store = tmp_path / "store" diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index f8920b1a3..abdb24428 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -49,6 +49,7 @@ def _require_a_large_table() -> None: def test_building_a_source_is_quick_and_names_the_table(): + """Building a source takes under 10 seconds and names the table and its id.""" start = time.monotonic() source = SpiralDBTableSource(PROJECT, "spike_event") elapsed = time.monotonic() - start @@ -61,6 +62,7 @@ def test_building_a_source_is_quick_and_names_the_table(): def test_limit_reads_the_first_rows(): + """``limit=1000`` reads 1,000 rows with distinct record ids.""" _require_a_large_table() source = SpiralDBTableSource(PROJECT, "spike_event", limit=1000) @@ -76,6 +78,7 @@ def test_limit_reads_the_first_rows(): def test_where_reads_only_matching_rows_with_the_same_record_ids(): + """``where`` reads only matching rows, each with its unfiltered record id.""" first = SpiralDBTableSource(PROJECT, "spike_event", limit=1).as_table(all_info=True) unit_id = first.column("unit_id")[0].as_py() source = SpiralDBTableSource( @@ -91,6 +94,7 @@ def test_where_reads_only_matching_rows_with_the_same_record_ids(): def test_a_whole_table_over_the_cap_is_refused_without_reading(): + """A table over ``max_rows`` is refused within 10 seconds, without a read.""" _require_a_large_table() source = SpiralDBTableSource(PROJECT, "spike_event", max_rows=10) @@ -101,14 +105,17 @@ def test_a_whole_table_over_the_cap_is_refused_without_reading(): def test_an_async_job_over_a_limited_read(tmp_path: Path): + """An async job computes a 500-row read once; a rerun computes nothing.""" calls: list[float | None] = [] @function_pod(output_keys="doubled") def double(spike_amplitude_uv: float | None) -> float | None: + """Return twice ``spike_amplitude_uv``, recording the call.""" calls.append(spike_amplitude_uv) return None if spike_amplitude_uv is None else 2 * spike_amplitude_uv def run() -> PipelineJob: + """Build the job over a 500-row read and run it asynchronously.""" job = PipelineJob(name="live", store=DeltaTableDatabase(tmp_path / "store")) with job: double.pod( @@ -128,6 +135,7 @@ def run() -> PipelineJob: not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table" ) def test_rows_appended_between_runs_are_the_only_ones_computed(tmp_path: Path): + """After rows are appended, a rerun computes only the new rows.""" import spiral # pyright: ignore[reportMissingImports] # spiraldb extra, absent in the type-check job name = f"itl186_live_{uuid.uuid4().hex[:8]}" @@ -137,10 +145,12 @@ def test_rows_appended_between_runs_are_the_only_ones_computed(tmp_path: Path): @function_pod(output_keys="w") def double(v: int | None) -> int | None: + """Return twice ``v``, recording the call.""" calls.append(v) return None if v is None else 2 * v def run() -> None: + """Build the job over the scratch table and run it.""" job = PipelineJob(name="append", store=DeltaTableDatabase(tmp_path / "store")) with job: double.pod( @@ -164,6 +174,7 @@ def run() -> None: not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table" ) def test_a_source_reads_the_version_it_was_built_at(): + """A source, and its row cap, use the table version current when it was built.""" import spiral # pyright: ignore[reportMissingImports] # spiraldb extra, absent in the type-check job name = f"itl186_asof_{uuid.uuid4().hex[:8]}" @@ -171,6 +182,7 @@ def test_a_source_reads_the_version_it_was_built_at(): table = project.create_table(f"scratch.{name}", key_schema=[("k", pa.int64())]) def keys(source: SpiralDBTableSource) -> list[int]: + """Return the ``k`` key of every row ``source`` reads.""" return cast("list[int]", source.as_table().column("k").to_pylist()) try: diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 00fb5bf5b..6cd8a92d6 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -58,6 +58,7 @@ def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): def _timestamps(source: SpiralDBTableSource) -> list[int]: + """Return the ``timestamp`` key of every row ``source`` reads, in order.""" return cast("list[int]", source.as_table().column("timestamp").to_pylist()) @@ -71,7 +72,10 @@ def _record_ids(source: SpiralDBTableSource) -> dict[int, bytes]: class TestNoRowIsReadUntilAsked: + """A source reads no rows until they are asked for, then reads in batches.""" + def test_construction_and_identity_read_no_rows(self, project): + """Building a source and reading its schema, hashes and config scan nothing.""" source = SpiralDBTableSource(_PROJECT, "event") source.output_schema(all_info=True) source.keys(all_info=True) @@ -83,6 +87,7 @@ def test_construction_and_identity_read_no_rows(self, project): assert fake_pyspiral.scan_log == [] def test_iter_data_reads_batches_only_as_rows_are_consumed(self, project): + """``iter_data`` takes a batch from the scan only as its rows are consumed.""" source = SpiralDBTableSource(_PROJECT, "event") rows = iter(source.iter_data()) @@ -93,6 +98,7 @@ def test_iter_data_reads_batches_only_as_rows_are_consumed(self, project): assert [scan.batches_read for scan in fake_pyspiral.scan_log] == [3] def test_batches_handed_on_hold_at_most_batch_rows(self, project, monkeypatch): + """Batches handed on hold at most ``batch_rows`` rows, whatever is scanned.""" monkeypatch.setattr(fake_pyspiral, "scan_batch_rows", 4) source = SpiralDBTableSource(_PROJECT, "event", batch_rows=3) @@ -102,6 +108,7 @@ def test_batches_handed_on_hold_at_most_batch_rows(self, project, monkeypatch): assert sizes == [3, 1, 1] def test_a_read_that_matches_nothing_has_the_schema_of_one_that_does(self, project): + """A read matching no row yields nothing, with the schema of a full read.""" nothing = SpiralDBTableSource(_PROJECT, "event", where={"subject_id": "nobody"}) rows = SpiralDBTableSource(_PROJECT, "event") @@ -111,6 +118,7 @@ def test_a_read_that_matches_nothing_has_the_schema_of_one_that_does(self, proje assert empty.schema == rows.as_table(all_info=True).schema def test_a_metadata_change_after_construction_is_refused(self, project): + """Field metadata changed after construction fails the read: rebuild needed.""" source = SpiralDBTableSource(_PROJECT, "event") marked = _rows([1]).cast( pa.schema( @@ -133,12 +141,14 @@ def test_a_metadata_change_after_construction_is_refused(self, project): source.as_table() def test_a_table_with_key_columns_only_is_refused(self, project): + """A table never written, holding key columns only, is refused.""" project.create_table("default.never_written", key_schema=_KEYS) with pytest.raises(ValueError, match="never_written.*data column"): SpiralDBTableSource(_PROJECT, "never_written") def test_schema_is_the_tables(self, project): + """Keys are the tags and other columns data, all optional, in table order.""" source = SpiralDBTableSource(_PROJECT, "event") assert source.keys() == (("subject_id", "timestamp"), ("amplitude",)) @@ -153,7 +163,10 @@ def test_schema_is_the_tables(self, project): class TestRows: + """Every way of reading a source yields the same rows.""" + def test_streamed_rows_equal_as_table_rows(self, project): + """``iter_data`` rows equal ``as_table`` rows, system columns included.""" source = SpiralDBTableSource(_PROJECT, "event") streamed = [ @@ -174,6 +187,7 @@ def test_streamed_rows_equal_as_table_rows(self, project): } def test_sorting_by_tags_sorts_across_batches(self, project): + """``sort_by_tags`` orders the whole table by tag, not each batch alone.""" keys = pa.schema([pa.field("k", pa.int64())]) project.create_table("default.groups", key_schema=keys).write( pa.table({"k": [1, 2, 3, 4, 5], "g": [5, 4, 3, 2, 1], "v": [0.0] * 5}) @@ -186,9 +200,11 @@ def test_sorting_by_tags_sorts_across_batches(self, project): assert table.column("k").to_pylist() == [5, 4, 3, 2, 1] def test_async_iteration_yields_the_same_rows(self, project): + """``async_iter_data`` yields exactly the rows ``iter_data`` yields.""" source = SpiralDBTableSource(_PROJECT, "event") async def collect() -> list[dict[str, object]]: + """Return every row ``async_iter_data`` yields, tags and data merged.""" return [ {**tag.as_dict(all_info=True), **data.as_dict(all_info=True)} async for tag, data in source.async_iter_data() @@ -201,6 +217,7 @@ async def collect() -> list[dict[str, object]]: assert asyncio.run(collect()) == expected def test_dataframe_methods_return_the_rows(self, project): + """``as_df``, ``as_lazy_frame`` and ``as_pandas_df`` return the rows read.""" source = SpiralDBTableSource(_PROJECT, "event", limit=2) assert source.as_df().get_column("timestamp").to_list() == [1, 2] @@ -209,19 +226,24 @@ def test_dataframe_methods_return_the_rows(self, project): class TestRestrictions: + """``where``, ``limit``, ``asof`` and the size arguments shape the read.""" + def test_limit_reads_the_first_rows_in_key_order(self, project): + """``limit`` reads the first rows in key order and is passed to the scan.""" source = SpiralDBTableSource(_PROJECT, "event", limit=2) assert _timestamps(source) == [1, 2] assert fake_pyspiral.scan_log[-1].kwargs["limit"] == 2 def test_where_reads_only_matching_rows(self, project): + """``where`` reads only the rows whose columns equal the given values.""" project.table("default.event").write(_rows([7, 8], subject="m2")) source = SpiralDBTableSource(_PROJECT, "event", where={"subject_id": "m2"}) assert _timestamps(source) == [7, 8] def test_where_and_limit_together(self, project): + """``where`` with ``limit`` reads the first matching rows.""" project.table("default.event").write(_rows([7, 8], subject="m2")) source = SpiralDBTableSource( _PROJECT, "event", where={"subject_id": "m2"}, limit=1 @@ -230,10 +252,12 @@ def test_where_and_limit_together(self, project): assert _timestamps(source) == [7] def test_where_on_an_unknown_column_is_refused(self, project): + """A ``where`` on a column the table lacks is refused at construction.""" with pytest.raises(ValueError, match="nope"): SpiralDBTableSource(_PROJECT, "event", where={"nope": 1}) def test_sizes_must_be_positive(self, project): + """A zero ``limit``, ``max_rows`` or ``batch_rows`` is refused.""" with pytest.raises(ValueError, match="limit"): SpiralDBTableSource(_PROJECT, "event", limit=0) with pytest.raises(ValueError, match="max_rows"): @@ -243,12 +267,14 @@ def test_sizes_must_be_positive(self, project): @pytest.mark.parametrize("name", ["limit", "max_rows", "batch_rows"]) def test_a_bool_is_not_a_size(self, project, name): + """``True`` is refused as ``limit``, ``max_rows`` or ``batch_rows``.""" with pytest.raises(ValueError, match=name): # A bool where an int is declared: the call a type checker would refuse. sizes: dict[str, Any] = {name: True} SpiralDBTableSource(_PROJECT, "event", **sizes) def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project): + """Every read scans the version current at construction, not a later write.""" table = project.table("default.event") version = table.last_modified_at() source = SpiralDBTableSource(_PROJECT, "event") @@ -260,6 +286,7 @@ def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project assert [scan.kwargs["asof"] for scan in fake_pyspiral.scan_log] == [version] * 2 def test_a_given_version_is_used(self, project): + """A given ``asof`` is passed to the scan and kept in the config.""" source = SpiralDBTableSource(_PROJECT, "event", asof=1) source.as_table() @@ -268,6 +295,7 @@ def test_a_given_version_is_used(self, project): assert source.to_config()["asof"] == 1 def test_a_given_version_zero_is_used(self, project): + """An ``asof`` of ``0`` is passed to the scan and saved, not taken as unset.""" source = SpiralDBTableSource(_PROJECT, "event", asof=0) source.as_table() @@ -277,7 +305,10 @@ def test_a_given_version_zero_is_used(self, project): class TestRowCap: + """``max_rows`` refuses a read that could exceed it.""" + def test_a_read_over_the_cap_is_refused_before_scanning(self, project): + """A read over ``max_rows`` fails before any scan, saying how to narrow it.""" source = SpiralDBTableSource(_PROJECT, "event", max_rows=3) with pytest.raises(SourceTooLargeError) as raised: @@ -293,17 +324,20 @@ def test_a_read_over_the_cap_is_refused_before_scanning(self, project): list(source.iter_data()) def test_a_limit_within_the_cap_is_read(self, project): + """A ``limit`` within ``max_rows`` lets a larger table be read.""" source = SpiralDBTableSource(_PROJECT, "event", max_rows=3, limit=3) assert _timestamps(source) == [1, 2, 3] def test_a_limit_over_the_cap_is_refused(self, project): + """A ``limit`` above ``max_rows`` is refused, naming the limit.""" source = SpiralDBTableSource(_PROJECT, "event", max_rows=3, limit=4) with pytest.raises(SourceTooLargeError, match="4 rows"): source.as_table() def test_a_filtered_read_over_the_cap_stops_while_reading(self, project): + """A ``where`` read that turns out over ``max_rows`` fails while reading.""" source = SpiralDBTableSource( _PROJECT, "event", max_rows=3, where={"subject_id": "m1"} ) @@ -312,6 +346,7 @@ def test_a_filtered_read_over_the_cap_stops_while_reading(self, project): list(source.iter_data()) def test_a_filtered_read_within_the_cap_is_read(self, project): + """A ``where`` read matching no more than ``max_rows`` rows is read.""" project.table("default.event").write(_rows([7, 8], subject="m2")) source = SpiralDBTableSource( _PROJECT, "event", max_rows=3, where={"subject_id": "m2"} @@ -320,13 +355,17 @@ def test_a_filtered_read_within_the_cap_is_read(self, project): assert _timestamps(source) == [7, 8] def test_no_cap(self, project): + """``max_rows=None`` reads the whole table.""" source = SpiralDBTableSource(_PROJECT, "event", max_rows=None) assert len(_timestamps(source)) == _ROWS class TestIdentity: + """A source's identity names its table, not the restrictions on the read.""" + def test_default_source_id_names_the_table_and_carries_its_spiral_id(self, project): + """The default source id is ``/.
@
``.""" source = SpiralDBTableSource(_PROJECT, "event") table_id = project.table("default.event").table_id @@ -334,11 +373,13 @@ def test_default_source_id_names_the_table_and_carries_its_spiral_id(self, proje assert source.source_id == f"{_PROJECT}/default.event@{table_id}" def test_a_given_source_id_is_used_as_it_is(self, project): + """A ``source_id`` given by the user is used unchanged.""" assert SpiralDBTableSource(_PROJECT, "event", source_id="mine").source_id == ( "mine" ) def test_a_recreated_table_is_a_different_source(self, project): + """A recreated table is a new source; only its pipeline hash is unchanged.""" before = SpiralDBTableSource(_PROJECT, "event") project.drop_table("default.event") project.create_table("default.event", key_schema=_KEYS).write( @@ -352,6 +393,7 @@ def test_a_recreated_table_is_a_different_source(self, project): assert _record_ids(after)[1] != _record_ids(before)[1] def test_restrictions_do_not_change_identity(self, project): + """Restrictions on the read leave the source id and both hashes unchanged.""" whole = SpiralDBTableSource(_PROJECT, "event") part = SpiralDBTableSource( _PROJECT, "event", where={"subject_id": "m1"}, limit=2, asof=1, max_rows=9 @@ -363,7 +405,10 @@ def test_restrictions_do_not_change_identity(self, project): class TestRecordIdentity: + """A row's record identity comes from its primary key.""" + def test_the_row_token_is_the_labelled_hash_of_the_primary_key(self, project): + """The source-info row token is the labelled Arrow hash of the primary key.""" source = SpiralDBTableSource(_PROJECT, "event", limit=1) key = pa.table( { @@ -385,6 +430,7 @@ def test_the_row_token_is_the_labelled_hash_of_the_primary_key(self, project): assert "::" not in token def test_a_row_has_the_same_record_id_in_every_read(self, project): + """A row has the same record id in a whole, a limited and a filtered read.""" whole = _record_ids(SpiralDBTableSource(_PROJECT, "event")) limited = _record_ids(SpiralDBTableSource(_PROJECT, "event", limit=2)) filtered = _record_ids( @@ -396,6 +442,7 @@ def test_a_row_has_the_same_record_id_in_every_read(self, project): assert filtered == {4: whole[4]} def test_a_row_keeps_its_record_id_when_an_earlier_key_is_added(self, project): + """Adding a row with an earlier key leaves the other record ids unchanged.""" before = _record_ids(SpiralDBTableSource(_PROJECT, "event")) project.table("default.event").write(_rows([0])) @@ -405,6 +452,7 @@ def test_a_row_keeps_its_record_id_when_an_earlier_key_is_added(self, project): assert {t: after[t] for t in before} == before def test_record_ids_do_not_depend_on_tag_columns(self, project): + """Choosing other tag columns leaves the record ids unchanged.""" by_key = _record_ids(SpiralDBTableSource(_PROJECT, "event")) by_subject = _record_ids( SpiralDBTableSource(_PROJECT, "event", tag_columns="subject_id") @@ -414,7 +462,10 @@ def test_record_ids_do_not_depend_on_tag_columns(self, project): class TestSaveAndLoad: + """A source saved to its config loads as the same source.""" + def test_config_round_trip_reads_no_rows(self, project): + """A config round trip keeps the arguments and hash and scans nothing.""" source = SpiralDBTableSource( _PROJECT, "event", @@ -435,6 +486,7 @@ def test_config_round_trip_reads_no_rows(self, project): assert _timestamps(loaded) == [2] def test_a_date_and_time_in_where_survive_the_round_trip(self, project): + """A datetime value in ``where`` survives the config round trip.""" keys = pa.schema([pa.field("start", pa.timestamp("us"))]) start = dt.datetime(2026, 3, 24, 17, 54, 17) project.create_table("default.session", key_schema=keys).write( @@ -447,6 +499,7 @@ def test_a_date_and_time_in_where_survive_the_round_trip(self, project): assert loaded.as_table().num_rows == 1 def test_bytes_in_where_survive_a_json_round_trip(self, project): + """A bytes value in ``where`` survives saving the config as JSON.""" keys = pa.schema([pa.field("blob", pa.binary())]) project.create_table("default.blobs", key_schema=keys).write( pa.table({"blob": pa.array([b"\x00\xff", b"\x01"]), "n": [1, 2]}) @@ -460,6 +513,7 @@ def test_bytes_in_where_survive_a_json_round_trip(self, project): assert loaded.as_table().column("n").to_pylist() == [1] def test_loading_refuses_a_name_that_now_leads_to_another_table(self, project): + """Loading refuses a name that now opens a recreated table, naming both ids.""" config = SpiralDBTableSource(_PROJECT, "event").to_config() project.drop_table("default.event") project.create_table("default.event", key_schema=_KEYS).write(_rows([1])) @@ -473,11 +527,22 @@ def test_loading_refuses_a_name_that_now_leads_to_another_table(self, project): class TestInAJob: + """A source in a ``PipelineJob`` has each row computed once.""" + def _job( self, store: Path, calls: list[float | None], max_rows: int | None = 100_000 ) -> PipelineJob: + """Build a job that doubles ``amplitude`` over the ``event`` table. + + Args: + store: The Delta store directory. + calls: Receives each amplitude the pod is called with. + max_rows: The source's row cap. + """ + @function_pod(output_keys="doubled") def double(amplitude: float | None) -> float | None: + """Return twice ``amplitude``, recording the call.""" calls.append(amplitude) return None if amplitude is None else 2 * amplitude @@ -491,7 +556,10 @@ def double(amplitude: float | None) -> float | None: @pytest.mark.parametrize("orchestrator", [None, AsyncPipelineOrchestrator]) def test_run_rerun_and_an_appended_row(self, project, tmp_path, orchestrator): + """A rerun computes nothing; then an appended row is the only one computed.""" + def run(calls: list[float | None]) -> PipelineJob: + """Build and run the job, recording the pod's calls in ``calls``.""" job = self._job(tmp_path / "store", calls) job.run(orchestrator=orchestrator() if orchestrator else None) return job @@ -512,6 +580,7 @@ def run(calls: list[float | None]) -> PipelineJob: assert third.nodes["double"].get_all_records().num_rows == _ROWS + 1 def test_a_job_over_the_cap_fails_before_computing(self, project, tmp_path): + """A job whose source is over its cap fails before the pod is called.""" calls: list[float | None] = [] job = self._job(tmp_path / "store", calls, max_rows=3) @@ -535,9 +604,11 @@ def big(self, project): @staticmethod def _scan_limits() -> list[object]: + """Return the ``limit`` of every scan opened, oldest first.""" return [scan.kwargs.get("limit") for scan in fake_pyspiral.scan_log] def test_display_reads_one_head_of_ten_rows(self, big): + """``_repr_html_`` reads one 10-row head, headed with the table's size.""" source = SpiralDBTableSource(_PROJECT, "big", label="big") html = source._repr_html_() @@ -548,6 +619,7 @@ def test_display_reads_one_head_of_ten_rows(self, big): assert "*subject_id" in html and "*timestamp" in html def test_a_table_above_the_cap_previews_without_raising(self, big): + """A source refused by its cap still previews a 10-row head.""" source = SpiralDBTableSource(_PROJECT, "big", max_rows=20) with pytest.raises(SourceTooLargeError): source.as_table() @@ -559,6 +631,7 @@ def test_a_table_above_the_cap_previews_without_raising(self, big): assert "first 10 rows of about 25" in html def test_a_small_table_shows_all_its_rows(self, project): + """A table of fewer than 10 rows is shown whole, saying so.""" html = SpiralDBTableSource(_PROJECT, "event")._repr_html_() assert self._scan_limits() == [10] @@ -566,18 +639,21 @@ def test_a_small_table_shows_all_its_rows(self, project): assert "shape: (5, 3)" in html def test_the_sources_own_limit_bounds_the_head(self, big): + """A ``limit`` below 10 bounds the head, shown as all rows.""" html = SpiralDBTableSource(_PROJECT, "big", limit=3)._repr_html_() assert self._scan_limits() == [3] assert " — all 3 rows\n" in html def test_a_limit_above_the_head_is_the_row_count_shown(self, big): + """A ``limit`` above 10 reads 10 rows and is shown as the row count.""" html = SpiralDBTableSource(_PROJECT, "big", limit=15)._repr_html_() assert self._scan_limits() == [10] assert " — first 10 rows of about 15\n" in html def test_where_is_applied_and_named(self, big): + """A ``where`` restricts the head and is named in the header.""" html = SpiralDBTableSource( _PROJECT, "big", where={"subject_id": "m1"} )._repr_html_() @@ -603,6 +679,7 @@ def test_the_header_is_escaped_in_both_displays(self, project): ) def test_view_reads_the_same_head(self, big): + """``view()`` reads the same 10-row head as the notebook display.""" source = SpiralDBTableSource(_PROJECT, "big", label="big") html = source.view()._repr_html_() @@ -612,12 +689,14 @@ def test_view_reads_the_same_head(self, big): assert "shape: (10, 3)" in html def test_view_with_all_info_shows_system_columns(self, big): + """``view(all_info=True)`` shows the system tag columns.""" html = SpiralDBTableSource(_PROJECT, "big").view(all_info=True)._repr_html_() assert self._scan_limits() == [10] assert "_tag_record_id::" in html def test_repr_reads_nothing(self, big): + """``repr`` is the class name and scans nothing.""" source = SpiralDBTableSource(_PROJECT, "big") assert repr(source) == "SpiralDBTableSource" @@ -627,6 +706,7 @@ def test_switched_off_preview_shows_the_schema_without_a_scan( self, big, monkeypatch ): # About the non-public switch: the fallback kept for the ITL-991 ruling. + """With the preview off, both displays show the empty schema, unscanned.""" monkeypatch.setattr(SpiralDBTableSource, "_preview_reads_rows", False) source = SpiralDBTableSource(_PROJECT, "big", label="big") diff --git a/tests/test_databases/fake_pyspiral.py b/tests/test_databases/fake_pyspiral.py index 531d6b271..e9b3b6130 100644 --- a/tests/test_databases/fake_pyspiral.py +++ b/tests/test_databases/fake_pyspiral.py @@ -360,6 +360,7 @@ class FakeSchema: """What ``Table.schema()`` returns: a wrapper whose ``to_arrow()`` gives the schema.""" def __init__(self, schema: pa.Schema) -> None: + """Wrap ``schema``.""" self._schema = schema def to_arrow(self) -> pa.Schema: @@ -371,6 +372,7 @@ class FakeExpr: """The result of ``Table.select()``: names the table to scan.""" def __init__(self, table: FakeTable) -> None: + """Name ``table`` as the table to scan.""" self.table = table @@ -378,12 +380,15 @@ class FakePredicate: """A ``where`` expression: every named column equals its value.""" def __init__(self, equalities: list[tuple[str, Any]]) -> None: + """Hold the ``(column, value)`` pairs a row must all match.""" self.equalities = equalities def __and__(self, other: FakePredicate) -> FakePredicate: + """Return a predicate requiring this one's equalities and ``other``'s.""" return FakePredicate(self.equalities + other.equalities) def __repr__(self) -> str: + """Return the equalities as ``name == value`` terms joined by ``&``.""" return " & ".join(f"{name} == {value!r}" for name, value in self.equalities) @@ -391,9 +396,11 @@ class FakeColumn: """What ``table[name]`` returns: compare it with ``==`` to get a predicate.""" def __init__(self, name: str) -> None: + """Name the column ``name``.""" self._name = name def __eq__(self, value: object) -> FakePredicate: # pyright: ignore[reportIncompatibleMethodOverride] # as pyspiral's Expr does + """Return the predicate that this column equals ``value``.""" return FakePredicate([(self._name, value)]) __hash__ = None # pyright: ignore[reportAssignmentType] # an expression, not a value @@ -409,12 +416,21 @@ def __init__( where: FakePredicate | None, limit: int | None, ) -> None: + """Prepare a scan of ``table``; no row is read until rows are asked for. + + Args: + table: The table to scan. + record: The ``scan_log`` entry that counts the batches handed out. + where: The equalities a row must match, or ``None`` for every row. + limit: The most rows to return, or ``None`` for no limit. + """ self._table = table self._record = record self._where = where self._limit = limit def _rows(self) -> pa.Table: + """Return the rows matching ``where``, cut to ``limit``, in key order.""" rows = self._table._read_all() for name, value in self._where.equalities if self._where else []: rows = rows.filter( @@ -434,6 +450,7 @@ def to_record_batches(self, **kwargs: Any) -> pa.RecordBatchReader: def batches() -> Iterator[pa.RecordBatch]: # SpiralDB yields more than one batch for larger tables; split to exercise that. + """Yield ``rows`` in batches, counting each in the scan record.""" for batch in rows.to_batches(max_chunksize=scan_batch_rows): self._record.batches_read += 1 yield batch @@ -454,6 +471,7 @@ class FakeTable: """ def __init__(self, store: _Store, project_id: str, identifier: str) -> None: + """Open table ``identifier`` of ``project_id`` in ``store``.""" if not store.exists(project_id, identifier): raise ValueError(f"Table not found: {project_id}/{identifier}") self._store = store @@ -631,6 +649,7 @@ class FakeProject: """What ``Spiral.project(id)`` returns: list, open, create and drop tables.""" def __init__(self, store: _Store, project_id: str) -> None: + """Bind project ``project_id`` in ``store``.""" self._store = store self._project_id = project_id @@ -655,6 +674,23 @@ def create_table( root_uri: Any = None, exist_ok: bool = False, ) -> FakeTable: + """Create the table ``".
"`` with ``key_schema`` and open it. + + Key columns are stored the way SpiralDB stores them, nullable under + ``server_nullability`` however they were declared. + + Args: + identifier: ``".
"``. + key_schema: An Arrow schema, or ``(name, type)`` pairs as pyspiral accepts. + root_uri: Accepted for signature compatibility; unused. + exist_ok: Open an existing table instead of raising. + + Returns: + The new table, or with ``exist_ok`` the existing one. + + Raises: + ValueError: If the table exists and ``exist_ok`` is false. + """ if self._store.exists(self._project_id, identifier): if not exist_ok: raise ValueError(f"Table already exists: {identifier}") @@ -697,6 +733,7 @@ class FakeSpiral: def __init__( self, *, config: Any = None, overrides: dict[str, str] | None = None ) -> None: + """Open the store under ``FAKE_SPIRAL_ROOT`` and record ``overrides``.""" root = os.environ.get(_ROOT_ENV) if not root: raise RuntimeError(f"{_ROOT_ENV} is not set; call fake_pyspiral.install()") @@ -735,6 +772,7 @@ def core(self) -> Any: """The client's low-level handle; only ``manifest_stats`` is modelled.""" def manifest_stats(snapshot: FakeTable) -> Any: + """Return stats whose ``key_space_stats.total_rows`` counts rows.""" rows = snapshot._read_all().num_rows return types.SimpleNamespace( key_space_stats=types.SimpleNamespace(total_rows=rows) From 68f8fd10c99ebf2d2c9819479864e6122cd1e610 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:08:59 -0800 Subject: [PATCH 13/17] test(spiraldb): failing tests for tags_as_data on SpiralDBTableSource (ITL-996) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-12-itl-996-spiral-tags-as-data.md | 76 ++++ ...test_spiraldb_table_source_tags_as_data.py | 353 ++++++++++++++++++ 2 files changed, 429 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md create mode 100644 tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py diff --git a/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md b/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md new file mode 100644 index 000000000..9002511e0 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md @@ -0,0 +1,76 @@ +# ITL-996: SpiralDBTableSource `tags_as_data` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this +> plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `SpiralDBTableSource(project, table, tags_as_data={"timestamp": "spike_time_us"})` +also offers named key columns as data columns, copied per batch in Arrow. + +**Architecture:** The copy is one function on the class, applied to the zero-row template at +construction (so the schema is known without reading) and to each slice in `_read_batches` +after the key-token null check, before `attach_provenance`. Validation reuses +`derived_source._validate_tags_as_data` plus a key-column rule. Identity: the sorted mapping is +appended to `identity_structure` only when given. + +**Base:** branch `eywalker/itl-996-spiraldbtablesource-copy-key-columns-into` = `origin/main` +290889e8 + PR #333 (`848d90b2`, merged locally, fast-forward). Do NOT push until #333 is on +main; then merge main and push once. + +## Decisions (coordinator approved, November 12, 2026) + +* (a) Signature `tags_as_data: Mapping[str, str] | None = None`. Copy field: + `pa.field(name, key_field.type, nullable=False, metadata=key_field.metadata)`. Copies follow + the table's data columns, in mapping order. +* (b) Errors: `InputValidationError` from `_validate_tags_as_data` (not a tag column; target + already a column; target repeated; reserved prefix) plus "not a key column"; the + constructor prefixes `project/dataset/table`. +* (c) `identity_structure` gains `("tags_as_data", tuple(sorted(items)))` only when given. + Absent: structure exactly as today. pipeline hash stays schema-only. PR body has a + "What enters identity" heading with the measurement. +* (d) `to_config` writes `tags_as_data` (dict or None); `from_config` reads it. +* (e) Docs use the caveat: `group_by(["unit_id"])` alone sorts members by + spike_extraction_hash, subject, session, recording, probe, then timestamp, so time order + holds only within one session/probe; group on the full unit key. + +## Measurements (prototype, scratch only) + +Fake: 0 scans at construction; source_id, record ids, `_tag_source_id` values and +`_source_amplitude` tokens equal with and without; content/pipeline hash differ; tag datagram +hash equal, data datagram hash differs; system-tag column names differ (schema hash). +Live `edi-tutorial-v1-841226` `spike_event`, `limit=20000`: 32 units, every `spike_time_us` +list ascending after `group_by(all keys but timestamp).drop_tag_columns("timestamp")`. + +## Files + +* Modify `src/orcapod/core/sources/spiraldb_table_source.py` +* Create `tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py` +* Modify `tests/test_core/sources/test_spiraldb_table_source_live.py` (one opt-in test) +* Modify `docs/concepts/sources.md` (one short section) +* Create `changelog.d/ITL-996.md`, `design_issues/ITL-996-SP5.md` + +## Tasks + +- [ ] 1. Failing tests on the fake (new test file): schema at construction with 0 scans; copy + type/non-null/metadata/values across batches (`scan_batch_rows=2`); `iter_data`, + `async_iter_data` and `as_table` agree; each error names the column; identity (source_id, + record ids, tokens equal; content and pipeline hash differ; two mappings with one schema + differ in content hash, equal pipeline hash; absent option identical to a source built + without the argument); copy token `::::`; data hash moves, tag + hash does not; `to_config`/`from_config` and job save/load; preview shows the copy; + `limit`/`where`/`max_rows`; workshop order with rows written out of order over 2 subjects. + Run: `uv run pytest tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py` → FAIL + (unexpected keyword `tags_as_data`). Commit WIP. +- [ ] 2. Implement: constructor arg, validation, `_with_tag_copies(table)`, template, read path, + identity, config, docstrings. Run the new file and all `test_spiraldb_table_source*` → PASS. + Commit. +- [ ] 3. Opt-in live test (read-only, `limit=20000`): workshop case ascending per unit. Run with + `ORCAPOD_SPIRAL_LIVE_PROJECT=edi-tutorial-v1-841226 uv run --extra spiraldb pytest ... -k tags_as_data`. +- [ ] 4. Docs section in `docs/concepts/sources.md` (convenience methods, caveat (e)); run + `uv run pytest tests/test_docs`. Changelog fragment + `--check`; design-issue file + `--check`. +- [ ] 5. ruff check/format, basedpyright (exit code), full `uv run pytest tests/`. Commit. +- [ ] 6. Wait for #333 on main; merge origin/main; rerun suite; push; PR "Fixes ITL-996"; + `orca linear attach`, `orca worktree set --pr`; CodeRabbit rounds; worker_done. + +## Progress notes + +(Successor: read this section first.) diff --git a/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py b/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py new file mode 100644 index 000000000..cc368ddaf --- /dev/null +++ b/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py @@ -0,0 +1,353 @@ +"""SpiralDBTableSource copies named key columns into data as rows are read (ITL-996). + +Runs against the disk-backed pyspiral fake, whose ``scan_log`` records every scan a source +opens. The ``spike`` table mirrors the EDI ``spike_event`` table: its key holds +``timestamp`` before ``unit_id``, and it is read 2 rows per batch. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any, cast + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import InputValidationError, SourceTooLargeError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.serialization import resolve_source_from_config +from orcapod.sources import SpiralDBTableSource +from tests.test_databases import fake_pyspiral + +_PROJECT = "edi-sample" +_KEYS = pa.schema( + [ + pa.field("subject_id", pa.string()), + pa.field("timestamp", pa.uint64()), + pa.field("unit_id", pa.uint64()), + ] +) +# (subject, timestamp, unit), written out of time order. +_SPIKES = [ + ("m1", 50, 2), + ("m1", 10, 1), + ("m1", 40, 1), + ("m1", 20, 2), + ("m1", 30, 1), + ("m2", 15, 1), + ("m2", 5, 1), +] +_COPY = {"timestamp": "spike_time_us"} + + +def _spike_rows(spikes: list[tuple[str, int, int]]) -> pa.Table: + """Return all-nullable ``spike`` rows, one per ``(subject, timestamp, unit)``.""" + table = pa.table( + { + "subject_id": [s for s, _, _ in spikes], + "timestamp": pa.array([t for _, t, _ in spikes], pa.uint64()), + "unit_id": pa.array([u for _, _, u in spikes], pa.uint64()), + "amplitude": pa.array([t + 0.5 for _, t, _ in spikes], pa.float32()), + } + ) + return table.cast(pa.schema([pa.field(f.name, f.type, True) for f in table.schema])) + + +@pytest.fixture() +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Provide a fake project holding the 7-row ``spike`` table, read 2 rows per batch.""" + monkeypatch.setattr(fake_pyspiral, "scan_batch_rows", 2) + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project(_PROJECT) + project.create_table("default.spike", key_schema=_KEYS).write( + _spike_rows(_SPIKES) + ) + fake_pyspiral.scan_log.clear() + yield project + fake_pyspiral.scan_log.clear() + + +def _source(**kwargs: Any) -> SpiralDBTableSource: + """Return a source over the ``spike`` table.""" + return SpiralDBTableSource(_PROJECT, "spike", **kwargs) + + +def _column(name: str, table: pa.Table) -> list[Any]: + """Return the values of column ``name`` of ``table``.""" + return table.column(name).to_pylist() + + +def _system_column(prefix: str, table: pa.Table) -> str: + """Return the name of the one column of ``table`` that starts with ``prefix``.""" + (name,) = [n for n in table.column_names if n.startswith(prefix)] + return name + + +class TestTheCopy: + """Each named key column is also a data column under the given name.""" + + def test_the_schema_has_the_copy_and_construction_reads_no_rows(self, project): + """The copy is a data column after the table's own, known without a scan.""" + source = _source(tags_as_data=_COPY) + + tag_keys, data_keys = source.keys() + _, data_schema = source.output_schema() + + assert fake_pyspiral.scan_log == [] + assert tag_keys == ("subject_id", "timestamp", "unit_id") + assert data_keys == ("amplitude", "spike_time_us") + assert data_schema["spike_time_us"] is int + + def test_the_copy_has_the_key_type_not_nullable_and_the_key_values(self, project): + """The copy is the key column's Arrow type, non-nullable, with its values.""" + table = _source(tags_as_data=_COPY).as_table() + + field = table.schema.field("spike_time_us") + assert field.type == pa.uint64() + assert not field.nullable + assert _column("spike_time_us", table) == _column("timestamp", table) + assert _column("timestamp", table) == [10, 20, 30, 40, 50, 5, 15] + + def test_copies_follow_in_mapping_order(self, project): + """Several copies follow the table's data columns in the mapping's order.""" + source = _source(tags_as_data={"unit_id": "unit", "subject_id": "subject"}) + + assert source.keys()[1] == ("amplitude", "unit", "subject") + table = source.as_table() + assert table.schema.field("subject").type == pa.large_string() + assert _column("subject", table) == _column("subject_id", table) + + def test_iteration_and_as_table_agree(self, project): + """``iter_data``, ``async_iter_data`` and ``as_table`` give the same copies.""" + source = _source(tags_as_data=_COPY) + + async def collect() -> list[tuple[int, int]]: + """Return ``(timestamp, copy)`` of every row the async iteration yields.""" + return [ + (cast(int, tag["timestamp"]), cast(int, data["spike_time_us"])) + async for tag, data in source.async_iter_data() + ] + + synced = [ + (cast(int, tag["timestamp"]), cast(int, data["spike_time_us"])) + for tag, data in source.iter_data() + ] + table = source.as_table() + whole = list( + zip(_column("timestamp", table), _column("spike_time_us", table), strict=True) + ) + + assert synced == asyncio.run(collect()) == whole + assert all(t == c for t, c in synced) + + def test_the_copy_is_offered_to_a_function_pod(self, project): + """A function pod gets the copy as an argument: the point of the option.""" + + @function_pod(output_keys="ms") + def to_ms(spike_time_us: int) -> float: + """Return the spike time in milliseconds.""" + return spike_time_us / 1000 + + stream = to_ms.pod(_source(tags_as_data=_COPY, limit=2)) + + assert _column("ms", stream.as_table()) == [0.01, 0.02] + + def test_limit_where_and_max_rows_apply_unchanged(self, project): + """``limit``, ``where`` and ``max_rows`` work as without the option.""" + limited = _source(tags_as_data=_COPY, limit=2).as_table() + filtered = _source(tags_as_data=_COPY, where={"subject_id": "m2"}).as_table() + + assert _column("spike_time_us", limited) == [10, 20] + assert _column("spike_time_us", filtered) == [5, 15] + with pytest.raises(SourceTooLargeError): + _source(tags_as_data=_COPY, max_rows=3).as_table() + + def test_the_notebook_preview_shows_the_copy(self, project): + """The bounded preview reads one head and shows the copied column.""" + html = _source(tags_as_data=_COPY)._repr_html_() + + assert [scan.kwargs.get("limit") for scan in fake_pyspiral.scan_log] == [10] + assert "spike_time_us" in html + assert "shape: (7, 5)" in html + + +class TestRefusedMappings: + """A mapping that cannot be applied is refused at construction, naming the column.""" + + @pytest.mark.parametrize( + ("mapping", "named"), + [ + ({"missing": "copy"}, "'missing' is not a column"), + ({"amplitude": "copy"}, "'amplitude' is a data column"), + ({"timestamp": "amplitude"}, "'amplitude', which is already a column"), + ({"timestamp": "unit_id"}, "'unit_id', which is already a column"), + ({"timestamp": "t", "unit_id": "t"}, "'t' is given to more than one tag"), + ({"timestamp": "__t"}, "'__t'"), + ], + ) + def test_refused(self, project, mapping, named): + """Each refusal names the table and the column.""" + with pytest.raises(InputValidationError, match=named) as raised: + _source(tags_as_data=mapping) + + assert "'spike'" in str(raised.value) + assert fake_pyspiral.scan_log == [] + + def test_a_tag_that_is_not_a_key_column_is_refused(self, project): + """Only a key column, which is never null, can be copied.""" + with pytest.raises(InputValidationError, match="'amplitude' is not a key"): + _source(tag_columns=["subject_id", "amplitude"], tags_as_data={"amplitude": "a"}) + + def test_a_key_column_that_is_not_a_tag_is_refused(self, project): + """A key column the source offers as data already is not a tag to copy.""" + with pytest.raises(InputValidationError, match="'timestamp' is a data column"): + _source(tag_columns=["subject_id", "unit_id"], tags_as_data=_COPY) + + +class TestIdentity: + """The copies are data columns, so they are in the schema and in identity.""" + + def test_source_id_record_ids_and_tokens_do_not_move(self, project): + """The source id, each row's record id and its other tokens are unchanged.""" + plain = _source() + copied = _source(tags_as_data=_COPY) + before = plain.as_table(all_info=True) + after = copied.as_table(all_info=True) + + assert copied.source_id == plain.source_id + for prefix in ("_tag_record_id::", "_tag_source_id::"): + assert _column(_system_column(prefix, after), after) == _column( + _system_column(prefix, before), before + ) + assert _column("_source_amplitude", after) == _column( + "_source_amplitude", before + ) + + def test_the_copys_token_has_the_form_of_every_data_column(self, project): + """A copy's source-info value is ``::::``.""" + table = _source(tags_as_data=_COPY).as_table(columns={"source": True}) + + for amplitude, copy in zip( + _column("_source_amplitude", table), + _column("_source_spike_time_us", table), + strict=True, + ): + assert copy == amplitude.removesuffix("amplitude") + "spike_time_us" + + def test_content_and_pipeline_hash_move_with_the_schema(self, project): + """The schema gains a column, so both hashes differ from the plain source.""" + plain = _source() + copied = _source(tags_as_data=_COPY) + + assert copied.content_hash() != plain.content_hash() + assert copied.pipeline_hash() != plain.pipeline_hash() + + def test_two_mappings_with_one_schema_are_different_sources(self, project): + """The mapping is in the content hash; the pipeline hash is schema-only.""" + by_time = _source(tags_as_data={"timestamp": "copy"}) + by_unit = _source(tags_as_data={"unit_id": "copy"}) + + assert by_time.output_schema() == by_unit.output_schema() + assert by_time.content_hash() != by_unit.content_hash() + assert by_time.pipeline_hash() == by_unit.pipeline_hash() + + def test_no_mapping_is_the_source_without_the_option(self, project): + """``None`` and an omitted argument give the same identity as before.""" + plain = _source() + for same in (_source(tags_as_data=None), _source(tags_as_data={})): + assert same.content_hash() == plain.content_hash() + assert same.pipeline_hash() == plain.pipeline_hash() + assert same.keys() == plain.keys() + + def test_a_rows_data_hash_moves_and_its_tag_hash_does_not(self, project): + """The row has one more data value; its tag is the same.""" + ((plain_tag, plain_data), *_) = _source().iter_data() + ((tag, data), *_) = _source(tags_as_data=_COPY).iter_data() + + assert tag.content_hash() == plain_tag.content_hash() + assert data.content_hash() != plain_data.content_hash() + + +class TestSaveAndLoad: + """The mapping is saved and loaded with the source.""" + + def test_config_round_trip(self, project): + """A config round trip keeps the mapping and the content hash, scanning nothing.""" + source = _source(tags_as_data={"timestamp": "t", "unit_id": "u"}) + config = source.to_config() + + loaded = resolve_source_from_config(json.loads(json.dumps(config))) + + assert fake_pyspiral.scan_log == [] + assert config["tags_as_data"] == {"timestamp": "t", "unit_id": "u"} + assert isinstance(loaded, SpiralDBTableSource) + assert loaded.to_config() == config + assert loaded.content_hash() == source.content_hash() + assert loaded.keys() == source.keys() + + def test_a_config_without_the_mapping_loads_the_plain_source(self, project): + """A config with no ``tags_as_data`` entry loads with no copies.""" + config = _source().to_config() + assert config["tags_as_data"] is None + del config["tags_as_data"] + + loaded = resolve_source_from_config(config) + + assert loaded.content_hash() == _source().content_hash() + + @pytest.mark.parametrize("orchestrator", [None, AsyncPipelineOrchestrator]) + def test_a_saved_and_loaded_job_computes_nothing_again( + self, project, tmp_path, orchestrator + ): + """A job saved after its run and loaded again finds every result stored.""" + calls: list[int] = [] + + @function_pod(output_keys="ms") + def to_ms(spike_time_us: int) -> float: + """Return the spike time in milliseconds, recording the call.""" + calls.append(spike_time_us) + return spike_time_us / 1000 + + job = PipelineJob(name="spikes", store=DeltaTableDatabase(tmp_path / "store")) + with job: + to_ms.pod(_source(tags_as_data=_COPY), label="to_ms") + job.run(orchestrator=orchestrator() if orchestrator else None) + assert sorted(calls) == sorted(t for _, t, _ in _SPIKES) + path = tmp_path / "job.json" + job.save(str(path)) + assert '"tags_as_data": {"timestamp": "spike_time_us"}' in path.read_text() + + calls.clear() + loaded = PipelineJob.load(str(path)) + loaded.run(orchestrator=orchestrator() if orchestrator else None) + + assert calls == [] + + +class TestTheWorkshopCase: + """Grouping on the unit key gives each unit's spike times in time order.""" + + def test_group_by_the_unit_key_orders_spike_times_by_time(self, project): + """The only tag left to order members is the time, so the lists are in time order.""" + spikes = _source(tags_as_data=_COPY) + + per_unit = spikes.group_by(["subject_id", "unit_id"]).drop_tag_columns( + "timestamp" + ) + + assert per_unit.keys() == (("subject_id", "unit_id"), ("amplitude", "spike_time_us")) + rows = per_unit.as_table().to_pylist() + assert [(r["subject_id"], r["unit_id"], r["spike_time_us"]) for r in rows] == [ + ("m1", 1, [10, 30, 40]), + ("m1", 2, [20, 50]), + ("m2", 1, [5, 15]), + ] + assert [r["amplitude"] for r in rows] == [ + [10.5, 30.5, 40.5], + [20.5, 50.5], + [5.5, 15.5], + ] From 0483972fb360d970e893b6aebb39251b5bf7f266 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:11:32 -0800 Subject: [PATCH 14/17] feat(spiraldb): tags_as_data copies key columns into data as rows are read (ITL-996) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/sources/derived_source.py | 20 ++-- .../core/sources/spiraldb_table_source.py | 110 ++++++++++++++++-- ...test_spiraldb_table_source_tags_as_data.py | 53 +++++++-- 3 files changed, 153 insertions(+), 30 deletions(-) diff --git a/src/orcapod/core/sources/derived_source.py b/src/orcapod/core/sources/derived_source.py index 8ec06e038..1f170f6c9 100644 --- a/src/orcapod/core/sources/derived_source.py +++ b/src/orcapod/core/sources/derived_source.py @@ -58,20 +58,22 @@ def _derivation_hash( ) -def _validate_tags_as_data( +def validate_tags_as_data( tags_as_data: Mapping[str, str], tag_keys: tuple[str, ...], data_keys: tuple[str, ...], ) -> None: - """Check a ``tags_as_data`` mapping against the origin's columns. + """Check a ``tags_as_data`` mapping against the columns it copies from. + + The one rule for ``DerivedSource`` and ``SpiralDBTableSource``. Args: - tags_as_data: Maps a tag column of the origin to the name of its data copy. - tag_keys: The origin's user tag columns. - data_keys: The origin's data columns. + tags_as_data: Maps a tag column to the name of its data copy. + tag_keys: The user tag columns of the stream copied from. + data_keys: The data columns of the stream copied from. Raises: - InputValidationError: If a key is not a tag column of the origin, or a new + InputValidationError: If a key is not a tag column, or a new name is already a column, is used twice, or starts with the prefix reserved for system columns. """ @@ -80,13 +82,13 @@ def _validate_tags_as_data( if tag not in tag_keys: kind = "a data column" if tag in data_keys else "not a column" raise InputValidationError( - f"tags_as_data: {tag!r} is {kind} of the origin, not a tag column. " + f"tags_as_data: {tag!r} is {kind}, not a tag column. " f"Tag columns: {list(tag_keys)}." ) if name in tag_keys or name in data_keys: raise InputValidationError( f"tags_as_data: cannot copy tag {tag!r} to {name!r}, which is " - "already a column of the origin. Choose a new name." + "already a column. Choose a new name." ) if name in seen: raise InputValidationError( @@ -233,7 +235,7 @@ def __init__( else: self._tags_as_data = dict(tags_as_data) tag_keys, data_keys = origin.keys() - _validate_tags_as_data(self._tags_as_data, tag_keys, data_keys) + validate_tags_as_data(self._tags_as_data, tag_keys, data_keys) kwargs.setdefault("data_context", origin.data_context_key) self._explicit_source_id = source_id diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 6031ec308..f043175f3 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -20,6 +20,11 @@ "my-project-123456", "spike_event", where={"unit_id": 106} ) + # The spike time is a key column (a tag); also offer it as data + source = SpiralDBTableSource( + "my-project-123456", "spike_event", tags_as_data={"timestamp": "spike_time_us"} + ) + # Explicit dataset, tag columns overridden source = SpiralDBTableSource( "my-project-123456", "spike_event", dataset="prod", tag_columns=["unit_id"] @@ -35,10 +40,11 @@ from typing import TYPE_CHECKING, Any, NamedTuple from orcapod.core.sources.base import RootSource +from orcapod.core.sources.derived_source import validate_tags_as_data from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases.spiraldb_connector import SpiralDBConnector -from orcapod.errors import SourceTooLargeError +from orcapod.errors import InputValidationError, SourceTooLargeError from orcapod.types import ColumnConfig from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -165,7 +171,18 @@ class SpiralDBTableSource(RootSource): the table id is SpiralDB's own: a table dropped and created again under the same name is a different source, and a renamed table gets a new source id too. ``where``, ``limit`` and ``asof`` choose which rows are read and are not part of - the source's identity. + the source's identity. ``tags_as_data`` is: it decides what a data column holds, + so its sorted items enter the content hash (when it is given), and its copies + are columns of the schema, so they enter both the content and the pipeline hash. + The source id and every row's record ID do not depend on it. + + Key columns as data. A function pod never sees tags, so a key it needs (a + spike's time) is offered as data too with ``tags_as_data={"timestamp": + "spike_time_us"}``. The tag keeps its name; the copy is a data column of the + given name, after the table's own data columns in mapping order, with the key + column's Arrow type and field metadata, non-nullable (a key is never null). It is + copied in Arrow as each batch is read, and its source-info value has the form of + every data column's, ``::::``. Record identity. A row is identified by its primary key: the row's token is the data context's Arrow hash of its key columns (in key order, as non-nullable @@ -192,6 +209,9 @@ class SpiralDBTableSource(RootSource): memory, about 5 KB per row and node. batch_rows: The most rows handed on at once while iterating (default 8,192). A SpiralDB batch larger than this is held while its slices are handed on. + tags_as_data: Maps a key column that is a tag column of this source to the + name of a data column that receives a copy of it, as in + ``DerivedSource``. ``None`` (default) or an empty mapping copies nothing. source_id: Canonical source name for the registry and provenance tokens. Defaults to ``"/.@
"``. label: Human-readable label for this source node. @@ -204,6 +224,9 @@ class SpiralDBTableSource(RootSource): ValueError: If the table is not found, has no key schema, a tag column or a ``where`` column is not in the table, or ``limit``, ``max_rows`` or ``batch_rows`` is not a positive integer. + InputValidationError: If ``tags_as_data`` names a column that is not a tag + column or not a key column, or a new name that is already a column, is + given twice, or is reserved for system columns. """ # Whether a notebook display reads the first rows. False shows the schema and the @@ -223,6 +246,7 @@ def __init__( asof: int | None = None, max_rows: int | None = DEFAULT_MAX_ROWS, batch_rows: int = DEFAULT_BATCH_ROWS, + tags_as_data: Mapping[str, str] | None = None, source_id: str | None = None, label: str | None = None, data_context: "str | contexts.DataContext | None" = None, @@ -251,6 +275,7 @@ def __init__( self._max_rows = max_rows self._batch_rows = batch_rows self._given_asof = asof + self._tags_as_data: dict[str, str] = dict(tags_as_data) if tags_as_data else {} location = f"SpiralDB project {project_id!r}, dataset {dataset!r}" with self._connector() as connector: @@ -301,13 +326,63 @@ def __init__( ) except ValueError as exc: raise ValueError(f"{location}, table {table_name!r}: {exc}") from exc + self._table_schema: pa.Schema = arrow_utils.drop_system_columns( + schema.empty_table() + ).schema + if self._tags_as_data: + try: + self._check_tags_as_data(template.stream.keys()) + except InputValidationError as exc: + raise InputValidationError( + f"{location}, table {table_name!r}: {exc}" + ) from exc + # The copies are columns of the schema: build the template with them. + template = self._builder.build( + self._with_tag_copies(self._table_schema.empty_table()), + tag_columns=template.tag_columns, + source_id=self._source_id, + system_tag_columns=template.system_tag_columns, + ) self._stream = template.stream self._schema_hash = template.schema_hash self._tag_columns = template.tag_columns self._system_tag_columns = template.system_tag_columns - self._table_schema: pa.Schema = arrow_utils.drop_system_columns( - schema.empty_table() - ).schema + + def _check_tags_as_data( + self, keys: tuple[tuple[str, ...], tuple[str, ...]] + ) -> None: + """Check ``tags_as_data`` against the source's tag and data columns. + + Args: + keys: The source's tag and data columns, without the copies. + + Raises: + InputValidationError: If the mapping fails ``DerivedSource``'s rule, or a + tag it names is not a key column of the table. + """ + tag_keys, data_keys = keys + validate_tags_as_data(self._tags_as_data, tag_keys, data_keys) + for tag in self._tags_as_data: + if tag not in self._key_columns: + raise InputValidationError( + f"tags_as_data: {tag!r} is not a key column, so it may hold a " + f"null. Key columns, which can be copied: {list(self._key_columns)}." + ) + + def _with_tag_copies(self, table: "pa.Table") -> "pa.Table": + """Return ``table`` with each ``tags_as_data`` copy appended, in mapping order. + + A copy is the key column's own Arrow column under a field of the new name, + with the key's type and field metadata and declared non-nullable (a key is + never null; a read checks it before copying). + """ + for tag, name in self._tags_as_data.items(): + key = table.schema.field(tag) + table = table.append_column( + pa.field(name, key.type, nullable=False, metadata=key.metadata), + table.column(tag), + ) + return table # ------------------------------------------------------------------------- # Identity @@ -327,10 +402,20 @@ def _restriction_identity(self) -> None: return None def identity_structure(self) -> Any: - """Identity from the class, the schema and the source id (the table's locator).""" - base = super().identity_structure() + """Identity from the class, the schema and the source id (the table's locator). + + With ``tags_as_data`` given, its sorted items follow: two mappings can give + the same schema (two key columns of one type copied to one name) while the + copy holds different values. Without it the structure is the plain one. + """ + extra: list[Any] = [] restriction = self._restriction_identity() - return base if restriction is None else (*base, restriction) + if restriction is not None: + extra.append(restriction) + if self._tags_as_data: + extra.append(("tags_as_data", tuple(sorted(self._tags_as_data.items())))) + base = super().identity_structure() + return (*base, *extra) if extra else base # ------------------------------------------------------------------------- # Reading @@ -470,13 +555,16 @@ def _read_batches( seen += rows.num_rows if max_rows is not None and seen > max_rows: raise self._too_large("the read returned more rows") + # Tokens first: they refuse a null key before it is copied into + # a non-nullable column. + tokens = self._key_tokens(rows) yield self._builder.attach_provenance( - rows, + self._with_tag_copies(rows), tag_columns=self._tag_columns, system_tag_columns=self._system_tag_columns, schema_hash=self._schema_hash, source_id=self.source_id, - provenance_tokens=self._key_tokens(rows), + provenance_tokens=tokens, ) def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: @@ -622,6 +710,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "asof": self._given_asof, "max_rows": self._max_rows, "batch_rows": self._batch_rows, + "tags_as_data": dict(self._tags_as_data) or None, "source_id": self.source_id, **self._identity_config(), } @@ -654,6 +743,7 @@ def from_config( asof=config.get("asof"), max_rows=config.get("max_rows", DEFAULT_MAX_ROWS), batch_rows=config.get("batch_rows", DEFAULT_BATCH_ROWS), + tags_as_data=config.get("tags_as_data"), source_id=config.get("source_id"), label=config.get("label"), data_context=config.get("data_context"), diff --git a/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py b/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py index cc368ddaf..a137c3dbf 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py +++ b/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py @@ -87,6 +87,16 @@ def _system_column(prefix: str, table: pa.Table) -> str: return name +def _spiral_configs(saved: object) -> list[dict[str, Any]]: + """Return every saved Spiral source config found anywhere in a saved job.""" + if isinstance(saved, dict): + found = [saved] if saved.get("source_type") == "spiraldb_table" else [] + return found + [c for v in saved.values() for c in _spiral_configs(v)] + if isinstance(saved, list): + return [c for v in saved for c in _spiral_configs(v)] + return [] + + class TestTheCopy: """Each named key column is also a data column under the given name.""" @@ -138,7 +148,11 @@ async def collect() -> list[tuple[int, int]]: ] table = source.as_table() whole = list( - zip(_column("timestamp", table), _column("spike_time_us", table), strict=True) + zip( + _column("timestamp", table), + _column("spike_time_us", table), + strict=True, + ) ) assert synced == asyncio.run(collect()) == whole @@ -200,7 +214,9 @@ def test_refused(self, project, mapping, named): def test_a_tag_that_is_not_a_key_column_is_refused(self, project): """Only a key column, which is never null, can be copied.""" with pytest.raises(InputValidationError, match="'amplitude' is not a key"): - _source(tag_columns=["subject_id", "amplitude"], tags_as_data={"amplitude": "a"}) + _source( + tag_columns=["subject_id", "amplitude"], tags_as_data={"amplitude": "a"} + ) def test_a_key_column_that_is_not_a_tag_is_refused(self, project): """A key column the source offers as data already is not a tag to copy.""" @@ -300,10 +316,10 @@ def test_a_config_without_the_mapping_loads_the_plain_source(self, project): assert loaded.content_hash() == _source().content_hash() @pytest.mark.parametrize("orchestrator", [None, AsyncPipelineOrchestrator]) - def test_a_saved_and_loaded_job_computes_nothing_again( + def test_a_saved_and_loaded_job_reads_its_results( self, project, tmp_path, orchestrator ): - """A job saved after its run and loaded again finds every result stored.""" + """A job saved after its run keeps the mapping and, loaded, reads its results.""" calls: list[int] = [] @function_pod(output_keys="ms") @@ -312,19 +328,31 @@ def to_ms(spike_time_us: int) -> float: calls.append(spike_time_us) return spike_time_us / 1000 - job = PipelineJob(name="spikes", store=DeltaTableDatabase(tmp_path / "store")) - with job: - to_ms.pod(_source(tags_as_data=_COPY), label="to_ms") + def build() -> PipelineJob: + """Build the job over the source with the copy.""" + job = PipelineJob( + name="spikes", store=DeltaTableDatabase(tmp_path / "store") + ) + with job: + to_ms.pod(_source(tags_as_data=_COPY), label="to_ms") + return job + + job = build() job.run(orchestrator=orchestrator() if orchestrator else None) assert sorted(calls) == sorted(t for _, t, _ in _SPIKES) path = tmp_path / "job.json" job.save(str(path)) - assert '"tags_as_data": {"timestamp": "spike_time_us"}' in path.read_text() + (saved,) = _spiral_configs(json.loads(path.read_text())) + assert saved["tags_as_data"] == _COPY - calls.clear() loaded = PipelineJob.load(str(path)) - loaded.run(orchestrator=orchestrator() if orchestrator else None) + stored = loaded.nodes["to_ms"].get_all_records() + assert stored is not None + assert sorted(_column("ms", stored)) == sorted(t / 1000 for _, t, _ in _SPIKES) + # The same job built again finds every row's result. + calls.clear() + build().run(orchestrator=orchestrator() if orchestrator else None) assert calls == [] @@ -339,7 +367,10 @@ def test_group_by_the_unit_key_orders_spike_times_by_time(self, project): "timestamp" ) - assert per_unit.keys() == (("subject_id", "unit_id"), ("amplitude", "spike_time_us")) + assert per_unit.keys() == ( + ("subject_id", "unit_id"), + ("amplitude", "spike_time_us"), + ) rows = per_unit.as_table().to_pylist() assert [(r["subject_id"], r["unit_id"], r["spike_time_us"]) for r in rows] == [ ("m1", 1, [10, 30, 40]), From 82000e1779295b2940a16186e24f533d14224be0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:12:59 -0800 Subject: [PATCH 15/17] docs(spiraldb): key columns as data; changelog fragment and design issue (ITL-996) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-996.md | 14 ++++++++ design_issues/ITL-996-SP5.md | 19 +++++++++++ docs/concepts/sources.md | 34 +++++++++++++++++++ .../test_spiraldb_table_source_live.py | 25 ++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 changelog.d/ITL-996.md create mode 100644 design_issues/ITL-996-SP5.md diff --git a/changelog.d/ITL-996.md b/changelog.d/ITL-996.md new file mode 100644 index 000000000..7bae46664 --- /dev/null +++ b/changelog.d/ITL-996.md @@ -0,0 +1,14 @@ +### Added + +- `SpiralDBTableSource(..., tags_as_data={"timestamp": "spike_time_us"})` also offers named + key columns as data columns, copied in Arrow as each batch is read, so a function pod can + use a key such as a spike's time (ITL-996). The tag keeps its name; the copy has the key + column's Arrow type and field metadata and is never null. The source still reads no rows + when built, and `limit=`, `where=`, `max_rows` and the notebook preview work as before. The + copies are part of the schema, so the `content_hash` and `pipeline_hash` of a source built + with the option differ from the same source without it; the mapping itself is also part of + the `content_hash`. The `source_id`, record IDs and source-info tokens are unchanged, and + without the option nothing changes. A mapping that names a column that is not a key tag + column, or a new name that is already a column, repeated or reserved, raises + `InputValidationError` when the source is built. `DerivedSource`'s `tags_as_data` errors no + longer say "of the origin". See "Key columns as data" in `docs/concepts/sources.md`. diff --git a/design_issues/ITL-996-SP5.md b/design_issues/ITL-996-SP5.md new file mode 100644 index 000000000..ffe4aaf1a --- /dev/null +++ b/design_issues/ITL-996-SP5.md @@ -0,0 +1,19 @@ +## `src/orcapod/core/sources/spiraldb_table_source.py` — Spiral tables as a workshop source (ITL-926) + +### SP5 — A key a function pod needed could only be copied into data by reading the whole table +**Status:** resolved +**Severity:** medium +**Issue:** ITL-996 + +In the EDI tables every time series has `timestamp` in its key, so the time of a spike is a +tag, and a function pod never sees tags. Copying it into data took +`DerivedSource(src, tags_as_data=...)`, which reads the whole origin when it is built (7 to 8 +seconds per source, measured) and is no longer a Spiral source, so it loses `limit=`, +`where=` and the bounded notebook preview. + +**Fix:** `SpiralDBTableSource(..., tags_as_data={key column: new name})` copies the named key +columns into data in Arrow as each batch is read. The copy has the key column's type and field +metadata, non-nullable. Validation is `DerivedSource`'s rule (`validate_tags_as_data`) plus +"the tag must be a key column". The copies are in the schema (content and pipeline hash), the +sorted mapping is in the content hash when given, and the source id and record IDs do not +change. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index e0866a6db..4e68997c6 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -174,6 +174,40 @@ built, so rows added to the Spiral table afterwards are seen only by a new sourc the source (and the job around it) again, and the added rows are new keys, the only rows computed. +**Key columns as data.** A function pod never receives a tag, and in a time-series table the +time is part of the key, so it is a tag. `tags_as_data` also offers named key columns as data +columns, copied as each batch is read (so the source still reads no rows when built, and +`limit=`, `where=`, `max_rows` and the notebook preview work as before). The tag keeps its +name; you name the copy, which comes after the table's data columns, with the key column's +type, never null: + + +```python +spikes = SpiralDBTableSource( + "edi-tutorial-v1-841226", "spike_event", tags_as_data={"timestamp": "spike_time_us"} +) +unit_key = [ + "spike_extraction_hash", "subject_id", "session_start_datetime", + "recording_index", "probe_index", "unit_id", +] +per_unit = spikes.group_by(unit_key).drop_tag_columns("timestamp") +print(per_unit.keys()) +# (('spike_extraction_hash', 'subject_id', 'session_start_datetime', 'recording_index', 'probe_index', 'unit_id'), ('spike_amplitude_uv', 'spike_time_us')) +``` + +`per_unit` has one row per unit, and its `spike_time_us` list is in time order: `group_by` +orders the members of a group by the tags outside the group key, and on the full unit key +the only such tag is `timestamp`. `group_by(["unit_id"])` alone sorts by +`spike_extraction_hash`, subject, session, recording, probe, then `timestamp`, so time order +holds only within one session and probe; group on the full unit key. + +The copies are data columns, so they are part of the schema, the `content_hash` and the +`pipeline_hash`; the mapping itself is part of the `content_hash` too, because two mappings +can give the same schema while the copy holds different values. The `source_id` and every +row's record id are the same as without the option. Compared with +`DerivedSource(spikes, tags_as_data=...)`, which reads the whole table when it is built, +the option keeps a Spiral source. + **Known limitations.** - Each row's key is hashed with one call, about 0.2 ms per row (about 18 s per 100,000 diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index abdb24428..330112657 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -131,6 +131,31 @@ def run() -> PipelineJob: assert calls == [] +def test_tags_as_data_gives_each_unit_its_spike_times_in_time_order(): + """Grouped on the unit key, each unit's copied spike times are in time order (ITL-996). + + ``timestamp`` is the only key column outside the unit key, so it alone orders the + members of a group. + """ + _require_a_large_table() + spikes = SpiralDBTableSource( + PROJECT, + "spike_event", + tags_as_data={"timestamp": "spike_time_us"}, + limit=20_000, + ) + unit_key = [c for c in spikes.keys()[0] if c != "timestamp"] + + per_unit = spikes.group_by(unit_key).drop_tag_columns("timestamp") + + assert "spike_time_us" in per_unit.keys()[1] + rows = per_unit.as_table().to_pylist() + assert len(rows) > 1 + assert sum(len(r["spike_time_us"]) for r in rows) == 20_000 + for row in rows: + assert row["spike_time_us"] == sorted(row["spike_time_us"]) + + @pytest.mark.skipif( not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table" ) From a1796df091f270883e32d6663d109c37f92dea21 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:20:39 -0800 Subject: [PATCH 16/17] docs(plan): ITL-996 progress notes Co-Authored-By: Claude Opus 5.5 --- superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md b/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md index 9002511e0..69bfb16ea 100644 --- a/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md +++ b/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md @@ -74,3 +74,8 @@ list ascending after `group_by(all keys but timestamp).drop_tag_columns("timesta ## Progress notes (Successor: read this section first.) + +* 2026-11-12: tasks 1 to 5 done (commits 68f8fd10, 0483972f, 82000e17); #333 merged into + main, origin/main merged (conflicts in three files resolved to the branch's side: main's + copies equal 848d90b2). Full suite green. PR #336 opened, attached to ITL-996, worktree + linked. Next: CodeRabbit rounds (read review BODIES too), CI green, worker_done. From 158a661a48e484d21252e38615ad8fa9ca4d8669 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:34:39 -0800 Subject: [PATCH 17/17] fix(spiraldb): tags_as_data is the last parameter; precise DerivedSource read wording; live total from the read (ITL-996) Addresses CodeRabbit round 1 on PR #336. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-996-SP5.md | 6 +++--- docs/concepts/sources.md | 5 +++-- src/orcapod/core/sources/spiraldb_table_source.py | 8 ++++---- .../test_core/sources/test_spiraldb_table_source_live.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/design_issues/ITL-996-SP5.md b/design_issues/ITL-996-SP5.md index ffe4aaf1a..c14819687 100644 --- a/design_issues/ITL-996-SP5.md +++ b/design_issues/ITL-996-SP5.md @@ -7,9 +7,9 @@ In the EDI tables every time series has `timestamp` in its key, so the time of a spike is a tag, and a function pod never sees tags. Copying it into data took -`DerivedSource(src, tags_as_data=...)`, which reads the whole origin when it is built (7 to 8 -seconds per source, measured) and is no longer a Spiral source, so it loses `limit=`, -`where=` and the bounded notebook preview. +`DerivedSource(src, tags_as_data=...)`, which reads the whole origin as soon as its schema or +rows are asked for, as building a job does (7 to 8 seconds per source, measured), and is no +longer a Spiral source, so it loses `limit=`, `where=` and the bounded notebook preview. **Fix:** `SpiralDBTableSource(..., tags_as_data={key column: new name})` copies the named key columns into data in Arrow as each batch is read. The copy has the key column's type and field diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 4e68997c6..52c596b5e 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -205,8 +205,9 @@ The copies are data columns, so they are part of the schema, the `content_hash` `pipeline_hash`; the mapping itself is part of the `content_hash` too, because two mappings can give the same schema while the copy holds different values. The `source_id` and every row's record id are the same as without the option. Compared with -`DerivedSource(spikes, tags_as_data=...)`, which reads the whole table when it is built, -the option keeps a Spiral source. +`DerivedSource(spikes, tags_as_data=...)`, which reads the whole table as soon as its schema +or rows are asked for (building a job asks for the schema), the option keeps a Spiral source, +whose schema is known without a scan. **Known limitations.** diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index f043175f3..054601174 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -209,9 +209,6 @@ class SpiralDBTableSource(RootSource): memory, about 5 KB per row and node. batch_rows: The most rows handed on at once while iterating (default 8,192). A SpiralDB batch larger than this is held while its slices are handed on. - tags_as_data: Maps a key column that is a tag column of this source to the - name of a data column that receives a copy of it, as in - ``DerivedSource``. ``None`` (default) or an empty mapping copies nothing. source_id: Canonical source name for the registry and provenance tokens. Defaults to ``"/.@
"``. label: Human-readable label for this source node. @@ -219,6 +216,9 @@ class SpiralDBTableSource(RootSource): config: OrcaPod configuration. overrides: Optional pyspiral client config overrides passed through to ``SpiralDBConnector``, e.g. ``{"server.url": "http://api.spiraldb.dev"}``. + tags_as_data: Maps a key column that is a tag column of this source to the + name of a data column that receives a copy of it, as in + ``DerivedSource``. ``None`` (default) or an empty mapping copies nothing. Raises: ValueError: If the table is not found, has no key schema, a tag column or a @@ -246,12 +246,12 @@ def __init__( asof: int | None = None, max_rows: int | None = DEFAULT_MAX_ROWS, batch_rows: int = DEFAULT_BATCH_ROWS, - tags_as_data: Mapping[str, str] | None = None, source_id: str | None = None, label: str | None = None, data_context: "str | contexts.DataContext | None" = None, config: "OrcapodConfig | None" = None, overrides: dict[str, str] | None = None, + tags_as_data: Mapping[str, str] | None = None, ) -> None: for name, value in (("limit", limit), ("max_rows", max_rows)): if value is not None and not _is_positive_int(value): diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index 330112657..ad128b44b 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -151,7 +151,7 @@ def test_tags_as_data_gives_each_unit_its_spike_times_in_time_order(): assert "spike_time_us" in per_unit.keys()[1] rows = per_unit.as_table().to_pylist() assert len(rows) > 1 - assert sum(len(r["spike_time_us"]) for r in rows) == 20_000 + assert sum(len(r["spike_time_us"]) for r in rows) == spikes.as_table().num_rows for row in rows: assert row["spike_time_us"] == sorted(row["spike_time_us"])