diff --git a/changelog.d/ITL-996.md b/changelog.d/ITL-996.md new file mode 100644 index 000000000..7bae46664 --- /dev/null +++ b/changelog.d/ITL-996.md @@ -0,0 +1,14 @@ +### Added + +- `SpiralDBTableSource(..., tags_as_data={"timestamp": "spike_time_us"})` also offers named + key columns as data columns, copied in Arrow as each batch is read, so a function pod can + use a key such as a spike's time (ITL-996). The tag keeps its name; the copy has the key + column's Arrow type and field metadata and is never null. The source still reads no rows + when built, and `limit=`, `where=`, `max_rows` and the notebook preview work as before. The + copies are part of the schema, so the `content_hash` and `pipeline_hash` of a source built + with the option differ from the same source without it; the mapping itself is also part of + the `content_hash`. The `source_id`, record IDs and source-info tokens are unchanged, and + without the option nothing changes. A mapping that names a column that is not a key tag + column, or a new name that is already a column, repeated or reserved, raises + `InputValidationError` when the source is built. `DerivedSource`'s `tags_as_data` errors no + longer say "of the origin". See "Key columns as data" in `docs/concepts/sources.md`. diff --git a/design_issues/ITL-996-SP5.md b/design_issues/ITL-996-SP5.md new file mode 100644 index 000000000..c14819687 --- /dev/null +++ b/design_issues/ITL-996-SP5.md @@ -0,0 +1,19 @@ +## `src/orcapod/core/sources/spiraldb_table_source.py` — Spiral tables as a workshop source (ITL-926) + +### SP5 — A key a function pod needed could only be copied into data by reading the whole table +**Status:** resolved +**Severity:** medium +**Issue:** ITL-996 + +In the EDI tables every time series has `timestamp` in its key, so the time of a spike is a +tag, and a function pod never sees tags. Copying it into data took +`DerivedSource(src, tags_as_data=...)`, which reads the whole origin as soon as its schema or +rows are asked for, as building a job does (7 to 8 seconds per source, measured), and is no +longer a Spiral source, so it loses `limit=`, `where=` and the bounded notebook preview. + +**Fix:** `SpiralDBTableSource(..., tags_as_data={key column: new name})` copies the named key +columns into data in Arrow as each batch is read. The copy has the key column's type and field +metadata, non-nullable. Validation is `DerivedSource`'s rule (`validate_tags_as_data`) plus +"the tag must be a key column". The copies are in the schema (content and pipeline hash), the +sorted mapping is in the content hash when given, and the source id and record IDs do not +change. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index e0866a6db..52c596b5e 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -174,6 +174,41 @@ built, so rows added to the Spiral table afterwards are seen only by a new sourc the source (and the job around it) again, and the added rows are new keys, the only rows computed. +**Key columns as data.** A function pod never receives a tag, and in a time-series table the +time is part of the key, so it is a tag. `tags_as_data` also offers named key columns as data +columns, copied as each batch is read (so the source still reads no rows when built, and +`limit=`, `where=`, `max_rows` and the notebook preview work as before). The tag keeps its +name; you name the copy, which comes after the table's data columns, with the key column's +type, never null: + + +```python +spikes = SpiralDBTableSource( + "edi-tutorial-v1-841226", "spike_event", tags_as_data={"timestamp": "spike_time_us"} +) +unit_key = [ + "spike_extraction_hash", "subject_id", "session_start_datetime", + "recording_index", "probe_index", "unit_id", +] +per_unit = spikes.group_by(unit_key).drop_tag_columns("timestamp") +print(per_unit.keys()) +# (('spike_extraction_hash', 'subject_id', 'session_start_datetime', 'recording_index', 'probe_index', 'unit_id'), ('spike_amplitude_uv', 'spike_time_us')) +``` + +`per_unit` has one row per unit, and its `spike_time_us` list is in time order: `group_by` +orders the members of a group by the tags outside the group key, and on the full unit key +the only such tag is `timestamp`. `group_by(["unit_id"])` alone sorts by +`spike_extraction_hash`, subject, session, recording, probe, then `timestamp`, so time order +holds only within one session and probe; group on the full unit key. + +The copies are data columns, so they are part of the schema, the `content_hash` and the +`pipeline_hash`; the mapping itself is part of the `content_hash` too, because two mappings +can give the same schema while the copy holds different values. The `source_id` and every +row's record id are the same as without the option. Compared with +`DerivedSource(spikes, tags_as_data=...)`, which reads the whole table as soon as its schema +or rows are asked for (building a job asks for the schema), the option keeps a Spiral source, +whose schema is known without a scan. + **Known limitations.** - Each row's key is hashed with one call, about 0.2 ms per row (about 18 s per 100,000 diff --git a/src/orcapod/core/sources/derived_source.py b/src/orcapod/core/sources/derived_source.py index 8ec06e038..1f170f6c9 100644 --- a/src/orcapod/core/sources/derived_source.py +++ b/src/orcapod/core/sources/derived_source.py @@ -58,20 +58,22 @@ def _derivation_hash( ) -def _validate_tags_as_data( +def validate_tags_as_data( tags_as_data: Mapping[str, str], tag_keys: tuple[str, ...], data_keys: tuple[str, ...], ) -> None: - """Check a ``tags_as_data`` mapping against the origin's columns. + """Check a ``tags_as_data`` mapping against the columns it copies from. + + The one rule for ``DerivedSource`` and ``SpiralDBTableSource``. Args: - tags_as_data: Maps a tag column of the origin to the name of its data copy. - tag_keys: The origin's user tag columns. - data_keys: The origin's data columns. + tags_as_data: Maps a tag column to the name of its data copy. + tag_keys: The user tag columns of the stream copied from. + data_keys: The data columns of the stream copied from. Raises: - InputValidationError: If a key is not a tag column of the origin, or a new + InputValidationError: If a key is not a tag column, or a new name is already a column, is used twice, or starts with the prefix reserved for system columns. """ @@ -80,13 +82,13 @@ def _validate_tags_as_data( if tag not in tag_keys: kind = "a data column" if tag in data_keys else "not a column" raise InputValidationError( - f"tags_as_data: {tag!r} is {kind} of the origin, not a tag column. " + f"tags_as_data: {tag!r} is {kind}, not a tag column. " f"Tag columns: {list(tag_keys)}." ) if name in tag_keys or name in data_keys: raise InputValidationError( f"tags_as_data: cannot copy tag {tag!r} to {name!r}, which is " - "already a column of the origin. Choose a new name." + "already a column. Choose a new name." ) if name in seen: raise InputValidationError( @@ -233,7 +235,7 @@ def __init__( else: self._tags_as_data = dict(tags_as_data) tag_keys, data_keys = origin.keys() - _validate_tags_as_data(self._tags_as_data, tag_keys, data_keys) + validate_tags_as_data(self._tags_as_data, tag_keys, data_keys) kwargs.setdefault("data_context", origin.data_context_key) self._explicit_source_id = source_id diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 6031ec308..054601174 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -20,6 +20,11 @@ "my-project-123456", "spike_event", where={"unit_id": 106} ) + # The spike time is a key column (a tag); also offer it as data + source = SpiralDBTableSource( + "my-project-123456", "spike_event", tags_as_data={"timestamp": "spike_time_us"} + ) + # Explicit dataset, tag columns overridden source = SpiralDBTableSource( "my-project-123456", "spike_event", dataset="prod", tag_columns=["unit_id"] @@ -35,10 +40,11 @@ from typing import TYPE_CHECKING, Any, NamedTuple from orcapod.core.sources.base import RootSource +from orcapod.core.sources.derived_source import validate_tags_as_data from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases.spiraldb_connector import SpiralDBConnector -from orcapod.errors import SourceTooLargeError +from orcapod.errors import InputValidationError, SourceTooLargeError from orcapod.types import ColumnConfig from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -165,7 +171,18 @@ class SpiralDBTableSource(RootSource): the table id is SpiralDB's own: a table dropped and created again under the same name is a different source, and a renamed table gets a new source id too. ``where``, ``limit`` and ``asof`` choose which rows are read and are not part of - the source's identity. + the source's identity. ``tags_as_data`` is: it decides what a data column holds, + so its sorted items enter the content hash (when it is given), and its copies + are columns of the schema, so they enter both the content and the pipeline hash. + The source id and every row's record ID do not depend on it. + + Key columns as data. A function pod never sees tags, so a key it needs (a + spike's time) is offered as data too with ``tags_as_data={"timestamp": + "spike_time_us"}``. The tag keeps its name; the copy is a data column of the + given name, after the table's own data columns in mapping order, with the key + column's Arrow type and field metadata, non-nullable (a key is never null). It is + copied in Arrow as each batch is read, and its source-info value has the form of + every data column's, ``::::``. Record identity. A row is identified by its primary key: the row's token is the data context's Arrow hash of its key columns (in key order, as non-nullable @@ -199,11 +216,17 @@ class SpiralDBTableSource(RootSource): config: OrcaPod configuration. overrides: Optional pyspiral client config overrides passed through to ``SpiralDBConnector``, e.g. ``{"server.url": "http://api.spiraldb.dev"}``. + tags_as_data: Maps a key column that is a tag column of this source to the + name of a data column that receives a copy of it, as in + ``DerivedSource``. ``None`` (default) or an empty mapping copies nothing. Raises: ValueError: If the table is not found, has no key schema, a tag column or a ``where`` column is not in the table, or ``limit``, ``max_rows`` or ``batch_rows`` is not a positive integer. + InputValidationError: If ``tags_as_data`` names a column that is not a tag + column or not a key column, or a new name that is already a column, is + given twice, or is reserved for system columns. """ # Whether a notebook display reads the first rows. False shows the schema and the @@ -228,6 +251,7 @@ def __init__( data_context: "str | contexts.DataContext | None" = None, config: "OrcapodConfig | None" = None, overrides: dict[str, str] | None = None, + tags_as_data: Mapping[str, str] | None = None, ) -> None: for name, value in (("limit", limit), ("max_rows", max_rows)): if value is not None and not _is_positive_int(value): @@ -251,6 +275,7 @@ def __init__( self._max_rows = max_rows self._batch_rows = batch_rows self._given_asof = asof + self._tags_as_data: dict[str, str] = dict(tags_as_data) if tags_as_data else {} location = f"SpiralDB project {project_id!r}, dataset {dataset!r}" with self._connector() as connector: @@ -301,13 +326,63 @@ def __init__( ) except ValueError as exc: raise ValueError(f"{location}, table {table_name!r}: {exc}") from exc + self._table_schema: pa.Schema = arrow_utils.drop_system_columns( + schema.empty_table() + ).schema + if self._tags_as_data: + try: + self._check_tags_as_data(template.stream.keys()) + except InputValidationError as exc: + raise InputValidationError( + f"{location}, table {table_name!r}: {exc}" + ) from exc + # The copies are columns of the schema: build the template with them. + template = self._builder.build( + self._with_tag_copies(self._table_schema.empty_table()), + tag_columns=template.tag_columns, + source_id=self._source_id, + system_tag_columns=template.system_tag_columns, + ) self._stream = template.stream self._schema_hash = template.schema_hash self._tag_columns = template.tag_columns self._system_tag_columns = template.system_tag_columns - self._table_schema: pa.Schema = arrow_utils.drop_system_columns( - schema.empty_table() - ).schema + + def _check_tags_as_data( + self, keys: tuple[tuple[str, ...], tuple[str, ...]] + ) -> None: + """Check ``tags_as_data`` against the source's tag and data columns. + + Args: + keys: The source's tag and data columns, without the copies. + + Raises: + InputValidationError: If the mapping fails ``DerivedSource``'s rule, or a + tag it names is not a key column of the table. + """ + tag_keys, data_keys = keys + validate_tags_as_data(self._tags_as_data, tag_keys, data_keys) + for tag in self._tags_as_data: + if tag not in self._key_columns: + raise InputValidationError( + f"tags_as_data: {tag!r} is not a key column, so it may hold a " + f"null. Key columns, which can be copied: {list(self._key_columns)}." + ) + + def _with_tag_copies(self, table: "pa.Table") -> "pa.Table": + """Return ``table`` with each ``tags_as_data`` copy appended, in mapping order. + + A copy is the key column's own Arrow column under a field of the new name, + with the key's type and field metadata and declared non-nullable (a key is + never null; a read checks it before copying). + """ + for tag, name in self._tags_as_data.items(): + key = table.schema.field(tag) + table = table.append_column( + pa.field(name, key.type, nullable=False, metadata=key.metadata), + table.column(tag), + ) + return table # ------------------------------------------------------------------------- # Identity @@ -327,10 +402,20 @@ def _restriction_identity(self) -> None: return None def identity_structure(self) -> Any: - """Identity from the class, the schema and the source id (the table's locator).""" - base = super().identity_structure() + """Identity from the class, the schema and the source id (the table's locator). + + With ``tags_as_data`` given, its sorted items follow: two mappings can give + the same schema (two key columns of one type copied to one name) while the + copy holds different values. Without it the structure is the plain one. + """ + extra: list[Any] = [] restriction = self._restriction_identity() - return base if restriction is None else (*base, restriction) + if restriction is not None: + extra.append(restriction) + if self._tags_as_data: + extra.append(("tags_as_data", tuple(sorted(self._tags_as_data.items())))) + base = super().identity_structure() + return (*base, *extra) if extra else base # ------------------------------------------------------------------------- # Reading @@ -470,13 +555,16 @@ def _read_batches( seen += rows.num_rows if max_rows is not None and seen > max_rows: raise self._too_large("the read returned more rows") + # Tokens first: they refuse a null key before it is copied into + # a non-nullable column. + tokens = self._key_tokens(rows) yield self._builder.attach_provenance( - rows, + self._with_tag_copies(rows), tag_columns=self._tag_columns, system_tag_columns=self._system_tag_columns, schema_hash=self._schema_hash, source_id=self.source_id, - provenance_tokens=self._key_tokens(rows), + provenance_tokens=tokens, ) def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: @@ -622,6 +710,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "asof": self._given_asof, "max_rows": self._max_rows, "batch_rows": self._batch_rows, + "tags_as_data": dict(self._tags_as_data) or None, "source_id": self.source_id, **self._identity_config(), } @@ -654,6 +743,7 @@ def from_config( asof=config.get("asof"), max_rows=config.get("max_rows", DEFAULT_MAX_ROWS), batch_rows=config.get("batch_rows", DEFAULT_BATCH_ROWS), + tags_as_data=config.get("tags_as_data"), source_id=config.get("source_id"), label=config.get("label"), data_context=config.get("data_context"), diff --git a/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md b/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md new file mode 100644 index 000000000..69bfb16ea --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-996-spiral-tags-as-data.md @@ -0,0 +1,81 @@ +# ITL-996: SpiralDBTableSource `tags_as_data` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this +> plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `SpiralDBTableSource(project, table, tags_as_data={"timestamp": "spike_time_us"})` +also offers named key columns as data columns, copied per batch in Arrow. + +**Architecture:** The copy is one function on the class, applied to the zero-row template at +construction (so the schema is known without reading) and to each slice in `_read_batches` +after the key-token null check, before `attach_provenance`. Validation reuses +`derived_source._validate_tags_as_data` plus a key-column rule. Identity: the sorted mapping is +appended to `identity_structure` only when given. + +**Base:** branch `eywalker/itl-996-spiraldbtablesource-copy-key-columns-into` = `origin/main` +290889e8 + PR #333 (`848d90b2`, merged locally, fast-forward). Do NOT push until #333 is on +main; then merge main and push once. + +## Decisions (coordinator approved, November 12, 2026) + +* (a) Signature `tags_as_data: Mapping[str, str] | None = None`. Copy field: + `pa.field(name, key_field.type, nullable=False, metadata=key_field.metadata)`. Copies follow + the table's data columns, in mapping order. +* (b) Errors: `InputValidationError` from `_validate_tags_as_data` (not a tag column; target + already a column; target repeated; reserved prefix) plus "not a key column"; the + constructor prefixes `project/dataset/table`. +* (c) `identity_structure` gains `("tags_as_data", tuple(sorted(items)))` only when given. + Absent: structure exactly as today. pipeline hash stays schema-only. PR body has a + "What enters identity" heading with the measurement. +* (d) `to_config` writes `tags_as_data` (dict or None); `from_config` reads it. +* (e) Docs use the caveat: `group_by(["unit_id"])` alone sorts members by + spike_extraction_hash, subject, session, recording, probe, then timestamp, so time order + holds only within one session/probe; group on the full unit key. + +## Measurements (prototype, scratch only) + +Fake: 0 scans at construction; source_id, record ids, `_tag_source_id` values and +`_source_amplitude` tokens equal with and without; content/pipeline hash differ; tag datagram +hash equal, data datagram hash differs; system-tag column names differ (schema hash). +Live `edi-tutorial-v1-841226` `spike_event`, `limit=20000`: 32 units, every `spike_time_us` +list ascending after `group_by(all keys but timestamp).drop_tag_columns("timestamp")`. + +## Files + +* Modify `src/orcapod/core/sources/spiraldb_table_source.py` +* Create `tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py` +* Modify `tests/test_core/sources/test_spiraldb_table_source_live.py` (one opt-in test) +* Modify `docs/concepts/sources.md` (one short section) +* Create `changelog.d/ITL-996.md`, `design_issues/ITL-996-SP5.md` + +## Tasks + +- [ ] 1. Failing tests on the fake (new test file): schema at construction with 0 scans; copy + type/non-null/metadata/values across batches (`scan_batch_rows=2`); `iter_data`, + `async_iter_data` and `as_table` agree; each error names the column; identity (source_id, + record ids, tokens equal; content and pipeline hash differ; two mappings with one schema + differ in content hash, equal pipeline hash; absent option identical to a source built + without the argument); copy token `::::`; data hash moves, tag + hash does not; `to_config`/`from_config` and job save/load; preview shows the copy; + `limit`/`where`/`max_rows`; workshop order with rows written out of order over 2 subjects. + Run: `uv run pytest tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py` → FAIL + (unexpected keyword `tags_as_data`). Commit WIP. +- [ ] 2. Implement: constructor arg, validation, `_with_tag_copies(table)`, template, read path, + identity, config, docstrings. Run the new file and all `test_spiraldb_table_source*` → PASS. + Commit. +- [ ] 3. Opt-in live test (read-only, `limit=20000`): workshop case ascending per unit. Run with + `ORCAPOD_SPIRAL_LIVE_PROJECT=edi-tutorial-v1-841226 uv run --extra spiraldb pytest ... -k tags_as_data`. +- [ ] 4. Docs section in `docs/concepts/sources.md` (convenience methods, caveat (e)); run + `uv run pytest tests/test_docs`. Changelog fragment + `--check`; design-issue file + `--check`. +- [ ] 5. ruff check/format, basedpyright (exit code), full `uv run pytest tests/`. Commit. +- [ ] 6. Wait for #333 on main; merge origin/main; rerun suite; push; PR "Fixes ITL-996"; + `orca linear attach`, `orca worktree set --pr`; CodeRabbit rounds; worker_done. + +## Progress notes + +(Successor: read this section first.) + +* 2026-11-12: tasks 1 to 5 done (commits 68f8fd10, 0483972f, 82000e17); #333 merged into + main, origin/main merged (conflicts in three files resolved to the branch's side: main's + copies equal 848d90b2). Full suite green. PR #336 opened, attached to ITL-996, worktree + linked. Next: CodeRabbit rounds (read review BODIES too), CI green, worker_done. diff --git a/tests/test_core/sources/test_spiraldb_table_source_live.py b/tests/test_core/sources/test_spiraldb_table_source_live.py index abdb24428..ad128b44b 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_live.py +++ b/tests/test_core/sources/test_spiraldb_table_source_live.py @@ -131,6 +131,31 @@ def run() -> PipelineJob: assert calls == [] +def test_tags_as_data_gives_each_unit_its_spike_times_in_time_order(): + """Grouped on the unit key, each unit's copied spike times are in time order (ITL-996). + + ``timestamp`` is the only key column outside the unit key, so it alone orders the + members of a group. + """ + _require_a_large_table() + spikes = SpiralDBTableSource( + PROJECT, + "spike_event", + tags_as_data={"timestamp": "spike_time_us"}, + limit=20_000, + ) + unit_key = [c for c in spikes.keys()[0] if c != "timestamp"] + + per_unit = spikes.group_by(unit_key).drop_tag_columns("timestamp") + + assert "spike_time_us" in per_unit.keys()[1] + rows = per_unit.as_table().to_pylist() + assert len(rows) > 1 + assert sum(len(r["spike_time_us"]) for r in rows) == spikes.as_table().num_rows + for row in rows: + assert row["spike_time_us"] == sorted(row["spike_time_us"]) + + @pytest.mark.skipif( not WRITE, reason="Set ORCAPOD_SPIRAL_LIVE_WRITE=1 to write a scratch table" ) diff --git a/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py b/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py new file mode 100644 index 000000000..a137c3dbf --- /dev/null +++ b/tests/test_core/sources/test_spiraldb_table_source_tags_as_data.py @@ -0,0 +1,384 @@ +"""SpiralDBTableSource copies named key columns into data as rows are read (ITL-996). + +Runs against the disk-backed pyspiral fake, whose ``scan_log`` records every scan a source +opens. The ``spike`` table mirrors the EDI ``spike_event`` table: its key holds +``timestamp`` before ``unit_id``, and it is read 2 rows per batch. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any, cast + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import InputValidationError, SourceTooLargeError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.serialization import resolve_source_from_config +from orcapod.sources import SpiralDBTableSource +from tests.test_databases import fake_pyspiral + +_PROJECT = "edi-sample" +_KEYS = pa.schema( + [ + pa.field("subject_id", pa.string()), + pa.field("timestamp", pa.uint64()), + pa.field("unit_id", pa.uint64()), + ] +) +# (subject, timestamp, unit), written out of time order. +_SPIKES = [ + ("m1", 50, 2), + ("m1", 10, 1), + ("m1", 40, 1), + ("m1", 20, 2), + ("m1", 30, 1), + ("m2", 15, 1), + ("m2", 5, 1), +] +_COPY = {"timestamp": "spike_time_us"} + + +def _spike_rows(spikes: list[tuple[str, int, int]]) -> pa.Table: + """Return all-nullable ``spike`` rows, one per ``(subject, timestamp, unit)``.""" + table = pa.table( + { + "subject_id": [s for s, _, _ in spikes], + "timestamp": pa.array([t for _, t, _ in spikes], pa.uint64()), + "unit_id": pa.array([u for _, _, u in spikes], pa.uint64()), + "amplitude": pa.array([t + 0.5 for _, t, _ in spikes], pa.float32()), + } + ) + return table.cast(pa.schema([pa.field(f.name, f.type, True) for f in table.schema])) + + +@pytest.fixture() +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Provide a fake project holding the 7-row ``spike`` table, read 2 rows per batch.""" + monkeypatch.setattr(fake_pyspiral, "scan_batch_rows", 2) + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project(_PROJECT) + project.create_table("default.spike", key_schema=_KEYS).write( + _spike_rows(_SPIKES) + ) + fake_pyspiral.scan_log.clear() + yield project + fake_pyspiral.scan_log.clear() + + +def _source(**kwargs: Any) -> SpiralDBTableSource: + """Return a source over the ``spike`` table.""" + return SpiralDBTableSource(_PROJECT, "spike", **kwargs) + + +def _column(name: str, table: pa.Table) -> list[Any]: + """Return the values of column ``name`` of ``table``.""" + return table.column(name).to_pylist() + + +def _system_column(prefix: str, table: pa.Table) -> str: + """Return the name of the one column of ``table`` that starts with ``prefix``.""" + (name,) = [n for n in table.column_names if n.startswith(prefix)] + return name + + +def _spiral_configs(saved: object) -> list[dict[str, Any]]: + """Return every saved Spiral source config found anywhere in a saved job.""" + if isinstance(saved, dict): + found = [saved] if saved.get("source_type") == "spiraldb_table" else [] + return found + [c for v in saved.values() for c in _spiral_configs(v)] + if isinstance(saved, list): + return [c for v in saved for c in _spiral_configs(v)] + return [] + + +class TestTheCopy: + """Each named key column is also a data column under the given name.""" + + def test_the_schema_has_the_copy_and_construction_reads_no_rows(self, project): + """The copy is a data column after the table's own, known without a scan.""" + source = _source(tags_as_data=_COPY) + + tag_keys, data_keys = source.keys() + _, data_schema = source.output_schema() + + assert fake_pyspiral.scan_log == [] + assert tag_keys == ("subject_id", "timestamp", "unit_id") + assert data_keys == ("amplitude", "spike_time_us") + assert data_schema["spike_time_us"] is int + + def test_the_copy_has_the_key_type_not_nullable_and_the_key_values(self, project): + """The copy is the key column's Arrow type, non-nullable, with its values.""" + table = _source(tags_as_data=_COPY).as_table() + + field = table.schema.field("spike_time_us") + assert field.type == pa.uint64() + assert not field.nullable + assert _column("spike_time_us", table) == _column("timestamp", table) + assert _column("timestamp", table) == [10, 20, 30, 40, 50, 5, 15] + + def test_copies_follow_in_mapping_order(self, project): + """Several copies follow the table's data columns in the mapping's order.""" + source = _source(tags_as_data={"unit_id": "unit", "subject_id": "subject"}) + + assert source.keys()[1] == ("amplitude", "unit", "subject") + table = source.as_table() + assert table.schema.field("subject").type == pa.large_string() + assert _column("subject", table) == _column("subject_id", table) + + def test_iteration_and_as_table_agree(self, project): + """``iter_data``, ``async_iter_data`` and ``as_table`` give the same copies.""" + source = _source(tags_as_data=_COPY) + + async def collect() -> list[tuple[int, int]]: + """Return ``(timestamp, copy)`` of every row the async iteration yields.""" + return [ + (cast(int, tag["timestamp"]), cast(int, data["spike_time_us"])) + async for tag, data in source.async_iter_data() + ] + + synced = [ + (cast(int, tag["timestamp"]), cast(int, data["spike_time_us"])) + for tag, data in source.iter_data() + ] + table = source.as_table() + whole = list( + zip( + _column("timestamp", table), + _column("spike_time_us", table), + strict=True, + ) + ) + + assert synced == asyncio.run(collect()) == whole + assert all(t == c for t, c in synced) + + def test_the_copy_is_offered_to_a_function_pod(self, project): + """A function pod gets the copy as an argument: the point of the option.""" + + @function_pod(output_keys="ms") + def to_ms(spike_time_us: int) -> float: + """Return the spike time in milliseconds.""" + return spike_time_us / 1000 + + stream = to_ms.pod(_source(tags_as_data=_COPY, limit=2)) + + assert _column("ms", stream.as_table()) == [0.01, 0.02] + + def test_limit_where_and_max_rows_apply_unchanged(self, project): + """``limit``, ``where`` and ``max_rows`` work as without the option.""" + limited = _source(tags_as_data=_COPY, limit=2).as_table() + filtered = _source(tags_as_data=_COPY, where={"subject_id": "m2"}).as_table() + + assert _column("spike_time_us", limited) == [10, 20] + assert _column("spike_time_us", filtered) == [5, 15] + with pytest.raises(SourceTooLargeError): + _source(tags_as_data=_COPY, max_rows=3).as_table() + + def test_the_notebook_preview_shows_the_copy(self, project): + """The bounded preview reads one head and shows the copied column.""" + html = _source(tags_as_data=_COPY)._repr_html_() + + assert [scan.kwargs.get("limit") for scan in fake_pyspiral.scan_log] == [10] + assert "spike_time_us" in html + assert "shape: (7, 5)" in html + + +class TestRefusedMappings: + """A mapping that cannot be applied is refused at construction, naming the column.""" + + @pytest.mark.parametrize( + ("mapping", "named"), + [ + ({"missing": "copy"}, "'missing' is not a column"), + ({"amplitude": "copy"}, "'amplitude' is a data column"), + ({"timestamp": "amplitude"}, "'amplitude', which is already a column"), + ({"timestamp": "unit_id"}, "'unit_id', which is already a column"), + ({"timestamp": "t", "unit_id": "t"}, "'t' is given to more than one tag"), + ({"timestamp": "__t"}, "'__t'"), + ], + ) + def test_refused(self, project, mapping, named): + """Each refusal names the table and the column.""" + with pytest.raises(InputValidationError, match=named) as raised: + _source(tags_as_data=mapping) + + assert "'spike'" in str(raised.value) + assert fake_pyspiral.scan_log == [] + + def test_a_tag_that_is_not_a_key_column_is_refused(self, project): + """Only a key column, which is never null, can be copied.""" + with pytest.raises(InputValidationError, match="'amplitude' is not a key"): + _source( + tag_columns=["subject_id", "amplitude"], tags_as_data={"amplitude": "a"} + ) + + def test_a_key_column_that_is_not_a_tag_is_refused(self, project): + """A key column the source offers as data already is not a tag to copy.""" + with pytest.raises(InputValidationError, match="'timestamp' is a data column"): + _source(tag_columns=["subject_id", "unit_id"], tags_as_data=_COPY) + + +class TestIdentity: + """The copies are data columns, so they are in the schema and in identity.""" + + def test_source_id_record_ids_and_tokens_do_not_move(self, project): + """The source id, each row's record id and its other tokens are unchanged.""" + plain = _source() + copied = _source(tags_as_data=_COPY) + before = plain.as_table(all_info=True) + after = copied.as_table(all_info=True) + + assert copied.source_id == plain.source_id + for prefix in ("_tag_record_id::", "_tag_source_id::"): + assert _column(_system_column(prefix, after), after) == _column( + _system_column(prefix, before), before + ) + assert _column("_source_amplitude", after) == _column( + "_source_amplitude", before + ) + + def test_the_copys_token_has_the_form_of_every_data_column(self, project): + """A copy's source-info value is ``::::``.""" + table = _source(tags_as_data=_COPY).as_table(columns={"source": True}) + + for amplitude, copy in zip( + _column("_source_amplitude", table), + _column("_source_spike_time_us", table), + strict=True, + ): + assert copy == amplitude.removesuffix("amplitude") + "spike_time_us" + + def test_content_and_pipeline_hash_move_with_the_schema(self, project): + """The schema gains a column, so both hashes differ from the plain source.""" + plain = _source() + copied = _source(tags_as_data=_COPY) + + assert copied.content_hash() != plain.content_hash() + assert copied.pipeline_hash() != plain.pipeline_hash() + + def test_two_mappings_with_one_schema_are_different_sources(self, project): + """The mapping is in the content hash; the pipeline hash is schema-only.""" + by_time = _source(tags_as_data={"timestamp": "copy"}) + by_unit = _source(tags_as_data={"unit_id": "copy"}) + + assert by_time.output_schema() == by_unit.output_schema() + assert by_time.content_hash() != by_unit.content_hash() + assert by_time.pipeline_hash() == by_unit.pipeline_hash() + + def test_no_mapping_is_the_source_without_the_option(self, project): + """``None`` and an omitted argument give the same identity as before.""" + plain = _source() + for same in (_source(tags_as_data=None), _source(tags_as_data={})): + assert same.content_hash() == plain.content_hash() + assert same.pipeline_hash() == plain.pipeline_hash() + assert same.keys() == plain.keys() + + def test_a_rows_data_hash_moves_and_its_tag_hash_does_not(self, project): + """The row has one more data value; its tag is the same.""" + ((plain_tag, plain_data), *_) = _source().iter_data() + ((tag, data), *_) = _source(tags_as_data=_COPY).iter_data() + + assert tag.content_hash() == plain_tag.content_hash() + assert data.content_hash() != plain_data.content_hash() + + +class TestSaveAndLoad: + """The mapping is saved and loaded with the source.""" + + def test_config_round_trip(self, project): + """A config round trip keeps the mapping and the content hash, scanning nothing.""" + source = _source(tags_as_data={"timestamp": "t", "unit_id": "u"}) + config = source.to_config() + + loaded = resolve_source_from_config(json.loads(json.dumps(config))) + + assert fake_pyspiral.scan_log == [] + assert config["tags_as_data"] == {"timestamp": "t", "unit_id": "u"} + assert isinstance(loaded, SpiralDBTableSource) + assert loaded.to_config() == config + assert loaded.content_hash() == source.content_hash() + assert loaded.keys() == source.keys() + + def test_a_config_without_the_mapping_loads_the_plain_source(self, project): + """A config with no ``tags_as_data`` entry loads with no copies.""" + config = _source().to_config() + assert config["tags_as_data"] is None + del config["tags_as_data"] + + loaded = resolve_source_from_config(config) + + assert loaded.content_hash() == _source().content_hash() + + @pytest.mark.parametrize("orchestrator", [None, AsyncPipelineOrchestrator]) + def test_a_saved_and_loaded_job_reads_its_results( + self, project, tmp_path, orchestrator + ): + """A job saved after its run keeps the mapping and, loaded, reads its results.""" + calls: list[int] = [] + + @function_pod(output_keys="ms") + def to_ms(spike_time_us: int) -> float: + """Return the spike time in milliseconds, recording the call.""" + calls.append(spike_time_us) + return spike_time_us / 1000 + + def build() -> PipelineJob: + """Build the job over the source with the copy.""" + job = PipelineJob( + name="spikes", store=DeltaTableDatabase(tmp_path / "store") + ) + with job: + to_ms.pod(_source(tags_as_data=_COPY), label="to_ms") + return job + + job = build() + job.run(orchestrator=orchestrator() if orchestrator else None) + assert sorted(calls) == sorted(t for _, t, _ in _SPIKES) + path = tmp_path / "job.json" + job.save(str(path)) + (saved,) = _spiral_configs(json.loads(path.read_text())) + assert saved["tags_as_data"] == _COPY + + loaded = PipelineJob.load(str(path)) + stored = loaded.nodes["to_ms"].get_all_records() + assert stored is not None + assert sorted(_column("ms", stored)) == sorted(t / 1000 for _, t, _ in _SPIKES) + + # The same job built again finds every row's result. + calls.clear() + build().run(orchestrator=orchestrator() if orchestrator else None) + assert calls == [] + + +class TestTheWorkshopCase: + """Grouping on the unit key gives each unit's spike times in time order.""" + + def test_group_by_the_unit_key_orders_spike_times_by_time(self, project): + """The only tag left to order members is the time, so the lists are in time order.""" + spikes = _source(tags_as_data=_COPY) + + per_unit = spikes.group_by(["subject_id", "unit_id"]).drop_tag_columns( + "timestamp" + ) + + assert per_unit.keys() == ( + ("subject_id", "unit_id"), + ("amplitude", "spike_time_us"), + ) + rows = per_unit.as_table().to_pylist() + assert [(r["subject_id"], r["unit_id"], r["spike_time_us"]) for r in rows] == [ + ("m1", 1, [10, 30, 40]), + ("m1", 2, [20, 50]), + ("m2", 1, [5, 15]), + ] + assert [r["amplitude"] for r in rows] == [ + [10.5, 30.5, 40.5], + [20.5, 50.5], + [5.5, 15.5], + ]