From fc22de6a952c850199ae8aa7aa7fa3e654e408fa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 21:07:15 -0800 Subject: [PATCH 01/72] test(referential_tags): failing EDI-shaped end-to-end test (ITL-1023) Specification section 15.7: two time series sharing five keys, a third table keyed on a subset, GroupBy, automatic join and select, a function reading the timestamp lists and a key by name; function node on the in-memory and Delta stores, sync and async; second run computes nothing. Also adds the slice's implementation plan. Refs ITL-1023 Co-Authored-By: Claude Opus 5.5 --- ...6-11-12-itl-1023-referential-tags-slice.md | 141 ++++++++++++++++ tests/test_core/referential_tags/__init__.py | 0 .../referential_tags/test_edi_end_to_end.py | 151 ++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md create mode 100644 tests/test_core/referential_tags/__init__.py create mode 100644 tests/test_core/referential_tags/test_edi_end_to_end.py diff --git a/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md b/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md new file mode 100644 index 000000000..f4c0e5b73 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md @@ -0,0 +1,141 @@ +# ITL-1023 Referential tags workshop slice — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. A successor picking this up +> after a crash: read the "Status log" at the bottom first. + +**Goal:** The thinnest working path of referential tags (specification "Orcapod v0.4 — +Referential Tags", attached to ITL-1004) on branch +`eywalker/itl-1023-workshop-slice-referential-tags-working`, usable in the workshop of +November 17, 2026: sources (referential by default), function pods, `Join`, `GroupBy`, +select/drop, row-preserving operators, function-node storage. Everything else raises +"not supported with referential tags yet (ITL-xxxx)". + +**Architecture:** A referential tag `x` is ONE physical column `x` in the stream's single Arrow +table whose name is in both role lists (spec 4.3 as code: binding = `tag_keys ∩ data_keys`). +No change to `types.Schema` or to the `(tag_schema, data_schema)` tuple: `x` appears in both +halves of `output_schema()` / `keys()` at graph-build time. `ArrowTableStream(table, +tag_columns, reference_tags=...)` makes data = non-tags ∪ references. Sources default every +declared tag to referential; `materialized_tags=[...]` (approved name, coordinator Nov 12) +keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference exists. + +**Tech Stack:** Python 3.11-3.13, PyArrow, Polars, pytest, uv. + +--- + +## Decisions (go into the PR body's DECISIONS section) + +1. Representation: name intersection, one physical column, no Schema change (audit, firm). +2. Option name `materialized_tags: Collection[str]` on sources; subset of `tag_columns`; unknown + name → `ValueError`. Builder (`SourceStreamBuilder.build`) takes explicit `reference_tags` + (default `()` = plain); sources compute `reference_tags = tag_columns - materialized_tags`. +3. ListSource / DerivedSource / CachedSource tags stay plain (computed values, or stored records). +4. Function pod output data = (input data columns that are referential tags, passed through + unchanged with their source info) + function outputs. Spec 6.1 "tags and bindings pass + through"; Orcapod pods drop input data otherwise, so the pass-through is what keeps the + binding valid. The result store keeps only function outputs (result identity unchanged, + spec 12.4); on read from the node's store the referenced column comes back from the stored + tag column. Output name equal to any tag name → `InputValidationError` at build time. +5. Join merged source-info (spec 9.4): the merged column's `_source_x` keeps the tokens of + every referential input, as `large_list` in canonical input order (a list + input contributes its elements). Plain+referential keeps the referential side's column as is. +6. Materialization diagnostic: module-level `orcapod.config`-free setting + `orcapod.core.referential_tags.reference_materialization = "warn" | "silent"`, default + `"warn"`, emitted with `warnings.warn(..., ReferenceMaterializationWarning)` once per + structural operation at graph build. Never changes a result or a hash. +7. Unsupported operators: one guard in the operator base (`StaticOutputOperatorPod.validate_inputs`) + raises `NotImplementedError(": not supported with referential tags yet (ITL-nnnn)")` + unless the class declares `SUPPORTS_REFERENTIAL_TAGS = True`. MapTags/MapData raise only + when a rename touches a referenced name. +8. Identity: `ArrowTableStream.identity_structure` (no producer) adds `("reference_tags", refs)` + only when non-empty. Everything else gets it through the schema halves. Identities of + referential streams are NOT frozen by this slice (docs say so). + +## Files + +- Create `src/orcapod/core/referential_tags.py`: `reference_tags(tag_schema_or_keys, data_...)`, + `validate_tag_bindings(...)` (shared namespace validator, spec 4/13.4), + `require_referential_support(op, streams)` guard, materialization setting + warning class. +- Modify `src/orcapod/core/streams/arrow_table_stream.py`: `reference_tags` arg, data columns, + identity term, `iter_data`. +- Modify `src/orcapod/core/sources/stream_builder.py` (+ `arrow_table_source.py`, + `dict_source.py`, `csv_source.py`, `data_frame_source.py`, `delta_table_source.py`, + `db_table_source.py`, `spiraldb_table_source.py`, `polling_source.py`): default referential, + `materialized_tags` option, to_config. +- Modify `src/orcapod/core/operators/static_output_pod.py` (guard, `_rows_to_table` dedupe), + `pipeline/sync_orchestrator.py` (dedupe, pass references). +- Modify operators: `join.py`, `group_by.py`, `column_selection.py`, `filters.py`, + `drop_nulls.py`, `semijoin.py`, `pick.py`, `index.py`, `mappers.py`; guard others. +- Modify `src/orcapod/core/function_pod.py` + `core/nodes/function_node.py`: pass-through, + collision check, table building dedupe, stored-row stream references. +- Tests: `tests/test_core/referential_tags/` (new): `test_edi_end_to_end.py` (step 0), + `test_schema_rules.py`, `test_sources.py`, `test_function_pods.py`, `test_join.py`, + `test_group_by.py`, `test_select_drop.py`, `test_row_preserving.py`, `test_not_yet.py`, + `test_hashes.py`; additions to `tests/test_core/operators/test_async_sync_equivalence.py`. +- Golden fixtures: `tests/test_hashing/golden/cases.py` switch to `materialized_tags=` — values + must stay byte-identical (run golden test before and after). +- `changelog.d/ITL-1023.md`, `design_issues/ITL-1023-RT1.md` (slice limitations), + `docs/concepts/referential-tags.md` (short; states identities not frozen). + +## Tasks (in the order of the brief; keep every step green; push after each) + +### Task 0: failing end-to-end EDI-shaped test +- [ ] `tests/test_core/referential_tags/test_edi_end_to_end.py`: spikes (k1..k5, spike_time, + amplitude) and gaze (k1..k5, gaze_time, gx) as `DictSource` with tag_columns keys + time + (referential by default); `trials` keyed on (k1, k2) with explicit `materialized_tags`; + `GroupBy(k1..k5)` on both; `pod(spikes_g, gaze_g, trials)` (automatic join + select) with + `def align(spike_time: list[int], gaze_time: list[int], k1: str) -> int`; `PipelineJob` on + `InMemoryArrowDatabase` and on `DeltaTableDatabase(tmp_path)`; sync and async orchestrators; + assert values, that `k1` is a tag and data, call counter = n groups on run 1 and 0 on run 2. +- [ ] Run → fails (`materialized_tags` unknown / tags not data). Commit, push, open draft PR + (Refs ITL-1023), send progress message. + +### Task 1: representation + shared validator +- [ ] Unit tests: ArrowTableStream with `reference_tags` → keys/schema both halves, `as_table` + has the column once, rows carry it in Tag and Data, unknown reference → ValueError. +- [ ] Implement in `arrow_table_stream.py` and `referential_tags.py`. + +### Task 2: sources +- [ ] Tests: DictSource default referential; `materialized_tags` gives v0.3 schema/hashes + equal to the builder's plain form; unknown name raises. +- [ ] Implement builder `reference_tags`, each source's option, to_config. + +### Task 3: hashes +- [ ] Golden fixtures switch to `materialized_tags=` (all sites with tag_columns in cases.py); + run golden suite: no value moves. +- [ ] Identity term in ArrowTableStream; test referential vs plain over one table differ; + spec 12.4 test: function not reading `x` has equal input data hash / result store hit + across plain vs referential source. + +### Task 4: function pods +- [ ] Tests: read referenced column by name; tags pass through and `x` stays referential in + output; output name == tag name raises (referential and materialized); auto-select drops + `x` → plain tag + warning; `silent` → no warning, same hashes. +- [ ] Implement pass-through in `_FunctionPodBase` (one helper, used by sync/async/node paths), + `output_schema`, collision check, table dedupe in `FunctionPodStream._table_from_pairs` and + `FunctionNodeBase._table_from_pairs`. + +### Task 5: Join +- [ ] Tests: plain+plain unchanged; plain+ref and ref+ref → one data column, tag referential; + other collisions raise (data/data, mat tag vs data); type mismatch raises; async == sync. +- [ ] Implement `_joined_user_schemas`, `static_process` (source-info merge), async merge. + +### Task 6: GroupBy +- [ ] Tests: key ref stays scalar both roles; non-key list in both with same order; async == sync. +- [ ] Implement (column appears once in table; pass `reference_tags`). + +### Task 7: row-preserving + select/drop + guards +- [ ] Filters, SemiJoin, DropNulls, Pick, Index: pass references; Select/Drop data + materializes; Select/Drop tag leaves data. Guard Batch, MergeJoin, renames touching refs, + OperatorNode storage, DerivedSource, Pipeline save. + +### Task 8: node storage +- [ ] FunctionNode stores/reads back with bindings (in-memory + Delta), head() shows `x` once, + second run computes nothing (covered by Task 0 test + unit tests). + +### Finish +- [ ] Full suite on 3.11/3.12/3.13, ruff, format, basedpyright; changelog fragment; design + issue entry; docs page; merge origin/main; PR body with DECISIONS; worker_done. + +## Status log +- 2026-11-12: plan written; branch created from origin/main f51ce67c; option name approved. diff --git a/tests/test_core/referential_tags/__init__.py b/tests/test_core/referential_tags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_core/referential_tags/test_edi_end_to_end.py b/tests/test_core/referential_tags/test_edi_end_to_end.py new file mode 100644 index 000000000..6141c7bd8 --- /dev/null +++ b/tests/test_core/referential_tags/test_edi_end_to_end.py @@ -0,0 +1,151 @@ +"""An EDI-shaped pipeline on referential tags, end to end (ITL-1023). + +Specification "Orcapod v0.4 — Referential Tags" (ITL-1004), section 15.7: two time series +(spikes, gaze) share five key columns and each has its own timestamp; a third table is keyed +on a subset of the keys. Both time series are grouped by the shared keys, the three are +joined, and a function reads the timestamp lists and a key column by name. A source's +declared tags are referential by default (developer, November 12, 2026), so the keys and +timestamps are data as well as tags, with no ``tags_as_data`` copy and no hand-written drop +before the join. The third table keeps its tags plain (``materialized_tags``), so the join +meets plain with referential (section 9.2) and referential with referential (section 9.3). + +The pipeline runs through a function node on the in-memory store and on Delta, with the +sync and the async orchestrator; a second run on the same store computes nothing. +""" + +from __future__ import annotations + +import warnings +from collections.abc import Callable +from pathlib import Path + +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol + +KEYS = ["lab", "subject", "session", "probe", "unit"] + +SPIKES = [ + { + "lab": "a", + "subject": "m1", + "session": "s1", + "probe": "p0", + "unit": unit, + "spike_time": t, + "amplitude": 0.5 + unit, + } + for unit in (1, 2) + for t in ((30, 10, 20) if unit == 1 else (15, 5)) +] + +GAZE = [ + { + "lab": "a", + "subject": "m1", + "session": "s1", + "probe": "p0", + "unit": unit, + "gaze_time": t, + "gaze_x": 0.1 * t, + } + for unit in (1, 2) + for t in (12, 2) +] + +TRIALS = [{"lab": "a", "subject": "m1", "condition": "dark"}] + +CALLS: list[int] = [] + + +@op.function_pod(output_keys=["summary"]) +def align(spike_time: list[int], gaze_time: list[int], unit: int) -> str: + """Summarize one unit's spike and gaze timestamps; reads the key ``unit`` as data.""" + CALLS.append(unit) + return f"unit {unit}: spikes {spike_time} gaze {gaze_time}" + + +def _build(job: PipelineJob) -> None: + """Record the EDI-shaped pipeline in ``job``.""" + spikes = op.sources.DictSource( + SPIKES, tag_columns=[*KEYS, "spike_time"], source_id="spikes" + ) + gaze = op.sources.DictSource( + GAZE, tag_columns=[*KEYS, "gaze_time"], source_id="gaze" + ) + trials = op.sources.DictSource( + TRIALS, + tag_columns=["lab", "subject"], + materialized_tags=["lab", "subject"], + source_id="trials", + ) + with job: + spikes_by_unit = spikes.group_by(KEYS) + gaze_by_unit = gaze.group_by(KEYS) + # Automatic join and automatic select of the pod call (ITL-807). + align.pod(spikes_by_unit, gaze_by_unit, trials, label="align") + + +def _in_memory(_: Path) -> ArrowDatabaseProtocol: + return InMemoryArrowDatabase() + + +def _delta(tmp_path: Path) -> ArrowDatabaseProtocol: + return DeltaTableDatabase(tmp_path / "store") + + +@pytest.mark.parametrize("make_store", [_in_memory, _delta], ids=["memory", "delta"]) +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +def test_edi_shaped_pipeline_runs_and_second_run_computes_nothing( + tmp_path: Path, + make_store: Callable[[Path], ArrowDatabaseProtocol], + orchestrator: type[SyncPipelineOrchestrator | AsyncPipelineOrchestrator], +) -> None: + store = make_store(tmp_path) + CALLS.clear() + with warnings.catch_warnings(): + # The pod call's automatic select materializes the keys it drops (section 7). + warnings.simplefilter("ignore") + job = PipelineJob(store=store) + _build(job) + job.run(orchestrator=orchestrator()) + + assert sorted(CALLS) == [1, 2] + node = job.nodes["align"] + tag_keys, data_keys = node.keys() + # ``unit`` was read by the function and stays a referential tag: tag and data. + assert "unit" in tag_keys and "unit" in data_keys + # Keys the function did not read were dropped from data by the automatic select + # and stay as plain tags (section 7). + assert "session" in tag_keys and "session" not in data_keys + table = node.as_table() + # A referential tag is stored and shown once. + assert table.column_names.count("unit") == 1 + rows = sorted(table.to_pylist(), key=lambda r: r["unit"]) + assert [r["summary"] for r in rows] == [ + "unit 1: spikes [10, 20, 30] gaze [2, 12]", + "unit 2: spikes [5, 15] gaze [2, 12]", + ] + assert node.head().table.column_names.count("unit") == 1 + + # Second run on the same store: everything is found, nothing is computed. + CALLS.clear() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + job2 = PipelineJob(store=store) + _build(job2) + job2.run(orchestrator=orchestrator()) + assert CALLS == [] + rows2 = sorted(job2.nodes["align"].as_table().to_pylist(), key=lambda r: r["unit"]) + assert [r["summary"] for r in rows2] == [r["summary"] for r in rows] From 02b6235893f804f518a23260aaac3858802c5d87 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 21:15:26 -0800 Subject: [PATCH 02/72] feat(referential_tags): WIP representation, sources, select/drop, Join, GroupBy, pod checks (ITL-1023) A referential tag is one column named in both the tag and the data role (specification section 4.3). Sources make declared tags referential by default; tag_only_columns keeps chosen tags plain (v0.3 behaviour). Golden fixtures use tag_only_columns=True and no golden value moves. Operators that do not handle references yet raise. Refs ITL-1023 Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 36 ++- src/orcapod/core/nodes/operator_node.py | 18 +- src/orcapod/core/operators/batch.py | 5 +- .../core/operators/column_selection.py | 62 ++++- src/orcapod/core/operators/drop_nulls.py | 16 +- src/orcapod/core/operators/filters.py | 7 +- src/orcapod/core/operators/group_by.py | 15 +- src/orcapod/core/operators/join.py | 170 +++++++++++-- src/orcapod/core/operators/merge_join.py | 5 +- src/orcapod/core/operators/semijoin.py | 7 +- .../core/operators/static_output_pod.py | 58 ++++- src/orcapod/core/referential_tags.py | 237 ++++++++++++++++++ .../core/sources/arrow_table_source.py | 7 + src/orcapod/core/sources/base.py | 13 + src/orcapod/core/sources/csv_source.py | 8 + src/orcapod/core/sources/data_frame_source.py | 7 + src/orcapod/core/sources/db_table_source.py | 12 + .../core/sources/delta_table_source.py | 8 + src/orcapod/core/sources/dict_source.py | 7 + .../core/sources/spiraldb_table_source.py | 16 ++ src/orcapod/core/sources/stream_builder.py | 30 ++- .../core/streams/arrow_table_stream.py | 51 +++- src/orcapod/pipeline/sync_orchestrator.py | 7 +- ...6-11-12-itl-1023-referential-tags-slice.md | 34 +-- .../referential_tags/test_edi_end_to_end.py | 16 +- tests/test_hashing/golden/cases.py | 49 +++- 26 files changed, 810 insertions(+), 91 deletions(-) create mode 100644 src/orcapod/core/referential_tags.py diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 2768b1e53..844ff79b6 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -13,6 +13,7 @@ from orcapod.config import OrcapodConfig from orcapod.core.base import TraceableBase from orcapod.core.data_function import CachedDataFunction, PythonDataFunction +from orcapod.core.referential_tags import materialization_diagnostic_off from orcapod.core.streams.base import ( StreamBase, check_head_size, @@ -26,7 +27,7 @@ StreamHead, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.errors import StoreError, TypeIdentityError +from orcapod.errors import InputValidationError, StoreError, TypeIdentityError from orcapod.utils.store_errors import keep_store_error, note_callback_failure from orcapod.protocols.core_protocols import ( ArgumentGroup, @@ -724,7 +725,10 @@ def __call__( self.label, dropped, ) - input_stream = SelectDataColumns(columns=kept)(input_stream) + # The select is Orcapod's own step: the referential tags it turns into + # plain tags are not reported (ITL-1023). + with materialization_diagnostic_off(): + input_stream = SelectDataColumns(columns=kept)(input_stream) return self.process(input_stream, label=label) def argument_symmetry(self, streams: Collection[StreamProtocol]) -> ArgumentGroup: @@ -741,11 +745,37 @@ def output_schema( ) # validate that incoming_data_schema is valid self._validate_input_schema(incoming_data_schema) + self._check_output_names(tag_schema) # The output schema of the FunctionPodProtocol is determined by the data function # TODO: handle and extend to include additional columns # Namely, the source columns + # A pod's output data is the function's outputs only, so every referential tag + # of the input is a plain tag of the output, with the same values (developer, + # November 12, 2026, ITL-1023). return tag_schema, self.data_function.output_data_schema + def _check_output_names(self, tag_names: Collection[str]) -> None: + """Raise if a function output is named like a tag of the input (section 6.2). + + A function produces new values; it never replaces, rebinds or removes a tag, + referential or plain. Checked when the pipeline is built. + + Args: + tag_names: The tag columns of the pod's input. + + Raises: + InputValidationError: If an output name equals a tag name. + """ + clashes = [ + name for name in self.data_function.output_data_schema if name in tag_names + ] + if clashes: + raise InputValidationError( + f"Function output {clashes} conflicts with existing tag(s) of the " + "same name. Resolve the tag binding (drop or rename the tag) before " + "producing a data column with the same name." + ) + class FunctionPod(_FunctionPodBase): def __init__( @@ -834,7 +864,9 @@ def process( input_stream = self.handle_input_streams(*streams) # perform input stream schema validation + tag_keys, _ = input_stream.keys() self._validate_input_schema(input_stream.output_schema()[1]) + self._check_output_names(tag_keys) self.tracker_manager.record_function_pod_invocation( self, input_stream, label=label ) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 9f22703ec..127a1a88f 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from orcapod import contexts +from orcapod.core.referential_tags import not_supported_yet, stream_references from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod @@ -928,7 +929,13 @@ def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: # ------------------------------------------------------------------ def _store_output_stream(self, stream: StreamProtocol) -> None: - """Materialize stream and store in the pipeline database with per-row dedup.""" + """Materialize stream and store in the pipeline database with per-row dedup. + + Raises: + NotImplementedError: If the stream has a referential tag (ITL-1019). + """ + if stream_references(stream): + raise not_supported_yet("Operator node storage", "ITL-1019") pipeline_database = self._pipeline_database if pipeline_database is None: raise RuntimeError( @@ -1112,6 +1119,8 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": if cols_to_drop: records_table = records_table.drop(cols_to_drop) tag_keys = self.keys()[0] + if records is not None and stream_references(self): + raise not_supported_yet("Operator node storage", "ITL-1019") # Stored rows are this node's: give them its data context, not the default. return ArrowTableStream( records_table, tag_columns=tag_keys, data_context=self.data_context @@ -1225,6 +1234,8 @@ def _replay_from_cache(self) -> None: records = records.drop(cols_to_drop) tag_keys = self.keys()[0] + if stream_references(self): + raise not_supported_yet("Operator node storage", "ITL-1019") self._cached_output_stream = ArrowTableStream( records, tag_columns=tag_keys, data_context=self.data_context ) @@ -1314,7 +1325,9 @@ def as_table( return pa.table({}) empty_records = self._make_empty_table() tag_keys = self.keys()[0] - empty_stream = ArrowTableStream(empty_records, tag_columns=tag_keys) + empty_stream = ArrowTableStream( + empty_records, tag_columns=tag_keys, reference_tags=stream_references(self) + ) return empty_stream.as_table(columns=columns, all_info=all_info) def head( @@ -1582,6 +1595,7 @@ async def forward() -> None: self._make_empty_table(), tag_columns=self.keys()[0], data_context=self.data_context, + reference_tags=stream_references(self), ) self._update_modified_time() diff --git a/src/orcapod/core/operators/batch.py b/src/orcapod/core/operators/batch.py index 2fc0fe32b..f968e48c9 100644 --- a/src/orcapod/core/operators/batch.py +++ b/src/orcapod/core/operators/batch.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Sequence -from typing import TYPE_CHECKING, Any +from typing import Any, ClassVar, TYPE_CHECKING from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator @@ -27,6 +27,9 @@ class Batch(UnaryOperator): Base class for all operators. """ + # Referential tags (ITL-1023): not handled yet. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = "ITL-1017" + def __init__(self, batch_size: int = 0, drop_partial_batch: bool = False, **kwargs): if batch_size < 0: raise ValueError("Batch size must be non-negative.") diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index f3ed67c56..8023198e7 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -1,9 +1,10 @@ import logging from collections.abc import Collection, Mapping, Sequence -from typing import TYPE_CHECKING, Any +from typing import Any, ClassVar, TYPE_CHECKING from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator +from orcapod.core.referential_tags import stream_references, warn_materialized from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol @@ -39,6 +40,9 @@ class SelectTagColumns(UnaryOperator): Operator that selects specified columns from a stream. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__(self, columns: str | Collection[str], strict: bool = True, **kwargs): self.columns = _canonical_columns(columns) self.strict = strict @@ -71,11 +75,17 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: columns={"source": True, "system_tags": True, "sort_by_tags": False} ) - modified_table = table.drop_columns(list(tags_to_drop)) + # A dropped referential tag leaves its column as plain data (specification + # section 10, "Drop tag": remove the tag, retain data independently). + references = stream_references(stream) + modified_table = table.drop_columns( + [c for c in tags_to_drop if c not in references] + ) return ArrowTableStream( modified_table, tag_columns=new_tag_columns, + reference_tags=[c for c in references if c in new_tag_columns], ) def validate_unary_input(self, stream: StreamProtocol) -> None: @@ -160,6 +170,9 @@ class SelectDataColumns(UnaryOperator): Operator that selects specified columns from a stream. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__(self, columns: str | Collection[str], strict: bool = True, **kwargs): self.columns = _canonical_columns(columns) self.strict = strict @@ -195,13 +208,18 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: associated_source_fields = [ f"{constants.SOURCE_PREFIX}{c}" for c in data_columns_to_drop ] - data_columns_to_drop.extend(associated_source_fields) - - modified_table = table.drop_columns(data_columns_to_drop) + # A referential tag whose data column is removed keeps its column as a plain + # tag with the same values (specification section 7.2). + references = stream_references(stream) + modified_table = table.drop_columns( + [c for c in data_columns_to_drop if c not in references] + + associated_source_fields + ) return ArrowTableStream( modified_table, tag_columns=tag_columns, + reference_tags=[c for c in references if c not in data_columns_to_drop], ) def validate_unary_input(self, stream: StreamProtocol) -> None: @@ -217,6 +235,10 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: raise InputValidationError( f"Missing data columns: {missing_columns}. Make sure all specified columns to select are present or use strict=False to ignore missing columns" ) + warn_materialized( + "SelectDataColumns", + [c for c in stream_references(stream) if c not in self.columns], + ) def unary_output_schema( self, @@ -288,6 +310,9 @@ class DropTagColumns(UnaryOperator): Operator that drops specified columns from a stream. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__(self, columns: str | Collection[str], strict: bool = True, **kwargs): self.columns = _canonical_columns(columns) self.strict = strict @@ -323,11 +348,17 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: columns={"source": True, "system_tags": True, "sort_by_tags": False} ) - modified_table = table.drop_columns(list(columns_to_drop)) + # A dropped referential tag leaves its column as plain data (specification + # section 10, "Drop tag": remove the tag, retain data independently). + references = stream_references(stream) + modified_table = table.drop_columns( + [c for c in columns_to_drop if c not in references] + ) return ArrowTableStream( modified_table, tag_columns=new_tag_columns, + reference_tags=[c for c in references if c in new_tag_columns], ) def validate_unary_input(self, stream: StreamProtocol) -> None: @@ -415,6 +446,9 @@ class DropDataColumns(UnaryOperator): Operator that drops specified columns from a stream. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__(self, columns: str | Collection[str], strict: bool = True, **kwargs): self.columns = _canonical_columns(columns) self.strict = strict @@ -448,17 +482,21 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: associated_source_columns = [ f"{constants.SOURCE_PREFIX}{c}" for c in columns_to_drop ] - columns_to_drop.extend(associated_source_columns) - table = stream.as_table( columns={"source": True, "system_tags": True, "sort_by_tags": False} ) - - modified_table = table.drop_columns(columns_to_drop) + # A referential tag whose data column is removed keeps its column as a plain + # tag with the same values (specification section 7.2). + references = stream_references(stream) + modified_table = table.drop_columns( + [c for c in columns_to_drop if c not in references] + + associated_source_columns + ) return ArrowTableStream( modified_table, tag_columns=tag_columns, + reference_tags=[c for c in references if c not in columns_to_drop], ) def validate_unary_input(self, stream: StreamProtocol) -> None: @@ -473,6 +511,10 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: raise InputValidationError( f"Missing data columns: {missing_columns}. Make sure all specified columns to drop are present or use strict=False to ignore missing columns" ) + warn_materialized( + "DropDataColumns", + [c for c in stream_references(stream) if c in self.columns], + ) def unary_output_schema( self, diff --git a/src/orcapod/core/operators/drop_nulls.py b/src/orcapod/core/operators/drop_nulls.py index 437fbfcff..39d01645f 100644 --- a/src/orcapod/core/operators/drop_nulls.py +++ b/src/orcapod/core/operators/drop_nulls.py @@ -5,8 +5,9 @@ import types import typing from collections.abc import Collection, Sequence -from typing import TYPE_CHECKING, Any +from typing import Any, ClassVar, TYPE_CHECKING +from orcapod.core.referential_tags import stream_references from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.datagrams import Data, Tag from orcapod.core.operators.base import UnaryOperator @@ -92,6 +93,9 @@ class DropNulls(UnaryOperator): ValueError: If ``columns`` is an empty collection. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__( self, columns: str | Collection[str] | None = None, **kwargs: Any ) -> None: @@ -154,10 +158,13 @@ def _target_columns( """ if self.columns is not None: return self.columns + # A referential tag is one column in both lists: named once. return tuple( - name - for name in (*tag_columns, *data_columns) - if not name.startswith(constants.META_PREFIX) + dict.fromkeys( + name + for name in (*tag_columns, *data_columns) + if not name.startswith(constants.META_PREFIX) + ) ) # ------------------------------------------------------------------ @@ -306,6 +313,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: return ArrowTableStream( self._narrow_table(table, self._target_columns(tag_columns, data_columns)), tag_columns=tag_columns, + reference_tags=stream_references(stream), data_context=stream.data_context, # pyright: ignore[reportAttributeAccessIssue] # StreamProtocol omits data_context, which every concrete stream provides (ITL-844) ) diff --git a/src/orcapod/core/operators/filters.py b/src/orcapod/core/operators/filters.py index 51f96afa0..156649444 100644 --- a/src/orcapod/core/operators/filters.py +++ b/src/orcapod/core/operators/filters.py @@ -1,7 +1,8 @@ import logging from collections.abc import Collection, Iterable, Mapping -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import Any, ClassVar, TYPE_CHECKING, TypeAlias +from orcapod.core.referential_tags import stream_references from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams import ArrowTableStream from orcapod.protocols.core_protocols import StreamProtocol @@ -29,6 +30,9 @@ class PolarsFilter(UnaryOperator): Operator that applies Polars filtering to a stream """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__( self, predicates: Collection[ @@ -80,6 +84,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: return ArrowTableStream( filtered_table, tag_columns=stream.keys()[0], + reference_tags=stream_references(stream), ) def validate_unary_input(self, stream: StreamProtocol) -> None: diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index cbe21cbc7..54a854fa3 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -3,9 +3,10 @@ from __future__ import annotations from collections.abc import Collection -from typing import TYPE_CHECKING, Any, get_origin +from typing import Any, ClassVar, TYPE_CHECKING, get_origin from orcapod.core.operators.base import UnaryOperator +from orcapod.core.referential_tags import stream_references from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import StreamProtocol @@ -94,6 +95,9 @@ class GroupBy(UnaryOperator): duplicates, and must all be scalar tag columns of the input stream. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__(self, by: Collection[str], **kwargs: Any) -> None: by_tuple = tuple(by) if not by_tuple: @@ -255,10 +259,14 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: # Every input tag stays a tag (ITL-955): the non-key ones are the # list-valued member columns built above. Declaring only `self.by` # here would let them fall into the data group. + # A referential tag is one column, so its tag and data roles get the same + # treatment by construction: a key stays scalar in both, a non-key becomes + # the same list in both, in the same member order (specification section 8). return ArrowTableStream( grouped_table, tag_columns=tag_columns, data_context=stream.data_context, + reference_tags=stream_references(stream), ) # ------------------------------------------------------------------ @@ -299,6 +307,9 @@ def unary_output_schema( # A non-key tag stays a tag, list-valued (ITL-955). out_tag_types[name] = list[col_type] - out_data_types = {name: list[t] for name, t in data_types.items()} + # A referential group key is scalar as data too (specification section 8.1). + out_data_types = { + name: t if name in self.by else list[t] for name, t in data_types.items() + } return Schema(out_tag_types), Schema(out_data_types) diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index b6ac78d5b..52f692fea 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -1,12 +1,18 @@ import asyncio from collections.abc import Callable, Collection, Mapping, Sequence -from typing import TYPE_CHECKING, Any +from typing import Any, ClassVar, TYPE_CHECKING from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import ( NonZeroInputOperator, require_shared_arrow_type, ) +from orcapod.core.referential_tags import ( + check_namespaces, + merged_source_tokens, + reference_names, + stream_references, +) from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import ( @@ -28,7 +34,15 @@ pl = LazyModule("polars") +# Temporary name of the right input's source info of a merged referential column +# during the join (``static_process``); never in an output. +_MERGED_SOURCE_PREFIX = "__join_merged_source__" + + class Join(NonZeroInputOperator): + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + @property def kernel_id(self) -> tuple[str, ...]: """ @@ -94,30 +108,76 @@ def _joined_user_schemas(self, *streams: StreamProtocol) -> tuple[Schema, Schema Args: *streams: Two or more input streams. + Referential tags (specification section 9): a shared tag ``x`` that is + referential in either input is referential in the output. When it is + referential in both, their data columns ``x`` are merged into one: the join + equated the tags, so the values are equal (section 9.3). Every other data + column shared by two inputs is still an error, and so is a plain tag of one + input named like a data column of another (section 4.3). + Returns: The ``(tag_schema, data_schema)`` of the joined user columns. Raises: - InputValidationError: If two inputs share a data column. + InputValidationError: If two inputs share a data column that is not the + target of a referential tag both of them join on, a merged column's + types differ, or a plain tag would share a name with a data column. """ ordered_streams = self.order_input_streams(*streams) stream = ordered_streams[0] tag_schema, data_schema = stream.output_schema() data_schema = self._without_meta_fields(data_schema) + references = set(reference_names(tag_schema, data_schema)) for other_stream in ordered_streams[1:]: other_tag_schema, other_data_schema = other_stream.output_schema() other_data_schema = self._without_meta_fields(other_data_schema) - tag_schema = schema_utils.union_schemas(tag_schema, other_tag_schema) - intersection_data_schema = schema_utils.intersection_schemas( - data_schema, other_data_schema + other_references = set(reference_names(other_tag_schema, other_data_schema)) + merged = self._merged_references( + tag_schema, references, other_tag_schema, other_references ) - data_schema = schema_utils.union_schemas(data_schema, other_data_schema) - if intersection_data_schema: + shared_data = set(data_schema) & set(other_data_schema) + if shared_data - merged: raise InputValidationError( - f"Datas should not have overlapping keys, but {data_schema.keys()} found in {stream} and {other_stream}." + f"Datas should not have overlapping keys, but {sorted(shared_data - merged)} found in {stream} and {other_stream}." ) + for name in sorted(merged): + if data_schema[name] != other_data_schema[name]: + raise InputValidationError( + f"Join: referential tag {name!r} has type " + f"{data_schema[name]} in one input and " + f"{other_data_schema[name]} in another" + ) + tag_schema = schema_utils.union_schemas(tag_schema, other_tag_schema) + data_schema = schema_utils.union_schemas(data_schema, other_data_schema) + references |= other_references + check_namespaces(tag_schema, data_schema, references) return tag_schema, data_schema + @staticmethod + def _merged_references( + tags: Collection[str], + references: Collection[str], + other_tags: Collection[str], + other_references: Collection[str], + ) -> set[str]: + """Return the data columns two join inputs merge into one (section 9.5). + + A data column is merged when it is the target of a referential tag in both + inputs and that tag is a join key (a tag of both inputs). ``Join`` joins on + every shared tag, so the second condition holds for every shared reference. + + Args: + tags: The first input's tag names. + references: The first input's referential tags. + other_tags: The second input's tag names. + other_references: The second input's referential tags. + + Returns: + The names of the merged data columns. + """ + keys = set(tags) & set(other_tags) + return keys & set(references) & set(other_references) + def _predict_system_tag_schema(self, *streams: StreamProtocol) -> Schema: """Predict the system tag columns that the join would produce. @@ -168,6 +228,7 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: stream = ordered_streams[0] tag_keys, _ = [set(k) for k in stream.keys()] + references = set(stream_references(stream)) table = self._without_meta( stream.as_table(columns={"source": True, "system_tags": True}) ) @@ -200,6 +261,21 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: next_ref_schema = next_table.schema common_tag_keys = tag_keys.intersection(next_tag_keys) + next_references = set(stream_references(next_stream)) + merged = self._merged_references( + tag_keys, references, next_tag_keys, next_references + ) + # The merged columns are join keys, so the join keeps one of each. Their + # source info is kept from both sides: the right side's is set aside + # under a temporary name and merged after the join (section 9.4). + right_source = { + f"{constants.SOURCE_PREFIX}{name}": f"{_MERGED_SOURCE_PREFIX}{name}" + for name in sorted(merged) + } + if right_source: + next_table = next_table.rename_columns( + [right_source.get(c, c) for c in next_table.column_names] + ) # A shared tag key must have one Arrow type in both inputs (up to # layout); otherwise Polars would join and keep one side's type. for key in sorted(common_tag_keys): @@ -222,8 +298,11 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: table = arrow_utils.restore_schema_nullability( table, table_ref_schema, next_ref_schema ) + for left_name, right_name in right_source.items(): + table = self._merge_source_columns(table, left_name, right_name) tag_keys.update(next_tag_keys) + references |= next_references # reorder columns to bring tag columns to the front # TODO: come up with a better algorithm @@ -238,7 +317,8 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: # keys from ``tag_columns``, so both must follow it. No hash or record # ID depends on column order. tag_order = list(tag_schema.keys()) - data_order = list(data_schema.keys()) + # A referential tag's column is laid out once, among the tags. + data_order = [name for name in data_schema if name not in tag_schema] listed = {*tag_order, *data_order} result_table = table.select( tag_order @@ -248,8 +328,40 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: return ArrowTableStream( result_table, tag_columns=tuple(tag_order), + reference_tags=reference_names(tag_schema, data_schema), ) + @staticmethod + def _merge_source_columns( + table: "pa.Table", left_name: str, right_name: str + ) -> "pa.Table": + """Replace two source-info columns of one merged data column by their merge. + + Args: + table: The joined table. + left_name: The left side's source-info column; replaced in place. + right_name: The right side's source-info column; removed. + + Returns: + The table with ``left_name`` holding both sides' tokens + (``merged_source_tokens``) as ``large_list``. + """ + merged = pa.array( + [ + merged_source_tokens(left, right) + for left, right in zip( + table.column(left_name).to_pylist(), + table.column(right_name).to_pylist(), + ) + ], + type=pa.large_list(pa.large_string()), + ) + index = table.schema.get_field_index(left_name) + table = table.set_column( + index, pa.field(left_name, merged.type, nullable=True), merged + ) + return table.drop_columns([right_name]) + @staticmethod def _without_meta(table: "pa.Table") -> "pa.Table": """Drop the meta (``__``-prefixed) columns from an input table. @@ -756,16 +868,25 @@ def _build_tag( return Tag(table, system_tags=system_tags, data_context=data_context_key) @staticmethod - def _merge_data(left: DataProtocol, right: DataProtocol) -> DataProtocol: + def _merge_data( + left_tag: TagProtocol, + left: DataProtocol, + right_tag: TagProtocol, + right: DataProtocol, + ) -> DataProtocol: """Merge two data datagrams as Arrow, left columns first. Every column keeps the Arrow type it already has, and the source info of both sides is carried over. Neither side's meta columns are, as in - ``static_process`` (``_without_meta``). + ``static_process`` (``_without_meta``). A data column that is the target + of a referential tag on both sides, joined on, is kept once, with the + source info of both (``merged_source_tokens``, as ``static_process``). Args: + left_tag: The left row's tag. left: The canonically earlier data (in a staggered join, the result of the joins so far). + right_tag: The right row's tag. right: The later data. Returns: @@ -773,18 +894,33 @@ def _merge_data(left: DataProtocol, right: DataProtocol) -> DataProtocol: Raises: InputValidationError: If both sides have a data column of the same - name. + name that is not such a reference. """ from orcapod.core.datagrams import Data left_table, right_table = left.as_table(), right.as_table() - if overlap := set(left_table.column_names) & set(right_table.column_names): + merged = Join._merged_references( + left_tag.keys(), + reference_names(left_tag.keys(), left_table.column_names), + right_tag.keys(), + reference_names(right_tag.keys(), right_table.column_names), + ) + overlap = set(left_table.column_names) & set(right_table.column_names) + if overlap - merged: raise InputValidationError( - f"Join: inputs share data columns {sorted(overlap)}." + f"Join: inputs share data columns {sorted(overlap - merged)}." ) + source_info = {**left.source_info(), **right.source_info()} + if merged: + right_table = right_table.drop_columns(sorted(merged)) + left_info, right_info = left.source_info(), right.source_info() + for name in merged: + source_info[name] = merged_source_tokens( + left_info.get(name), right_info.get(name) + ) return Data( arrow_utils.hstack_tables(left_table, right_table), - source_info={**left.source_info(), **right.source_info()}, + source_info=source_info, data_context=left.data_context_key, ) @@ -820,7 +956,7 @@ def _merge_pair_rename( merged_sys, left_tag.data_context_key, ) - return merged_tag, Join._merge_data(left_pkt, right_pkt) + return merged_tag, Join._merge_data(left_tag, left_pkt, right_tag, right_pkt) @staticmethod def _merge_pair_passthrough( @@ -847,7 +983,7 @@ def _merge_pair_passthrough( merged_sys, left_tag.data_context_key, ) - return merged_tag, Join._merge_data(left_pkt, right_pkt) + return merged_tag, Join._merge_data(left_tag, left_pkt, right_tag, right_pkt) @staticmethod def _sort_merged_system_tags(merged_sys: dict) -> dict: diff --git a/src/orcapod/core/operators/merge_join.py b/src/orcapod/core/operators/merge_join.py index a5fabc65b..066c5f5ce 100644 --- a/src/orcapod/core/operators/merge_join.py +++ b/src/orcapod/core/operators/merge_join.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from typing import TYPE_CHECKING, Any +from typing import Any, ClassVar, TYPE_CHECKING from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type from orcapod.core.streams import ArrowTableStream @@ -151,6 +151,9 @@ class MergeJoin(BinaryOperator): MergeJoin(B, A), achieved by sorting merged values and system tag values. """ + # Referential tags (ITL-1023): not handled yet. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = "ITL-1018" + @property def kernel_id(self) -> tuple[str, ...]: return (f"{self.__class__.__name__}",) diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index 532363b23..da0a6a454 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,7 +1,8 @@ import math from collections.abc import Sequence -from typing import TYPE_CHECKING, Any, cast +from typing import Any, ClassVar, TYPE_CHECKING, cast +from orcapod.core.referential_tags import stream_references from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type from orcapod.core.streams import ArrowTableStream @@ -62,6 +63,9 @@ class SemiJoin(BinaryOperator): The output stream preserves the schema of the left stream exactly. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def binary_static_process( self, left_stream: StreamProtocol, right_stream: StreamProtocol ) -> StreamProtocol: @@ -130,6 +134,7 @@ def binary_static_process( return ArrowTableStream( semi_joined_table, tag_columns=tuple(left_tag_schema.keys()), + reference_tags=stream_references(left_stream), ) def binary_output_schema( diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 34ffa81db..102c9ffe7 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -5,12 +5,17 @@ from abc import abstractmethod from collections.abc import Collection, Iterator, Sequence from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from orcapod.channels import ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig from orcapod.contexts import DataContext from orcapod.core.base import TraceableBase +from orcapod.core.referential_tags import ( + not_supported_yet, + reference_names, + stream_references, +) from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ( StreamBase, @@ -53,12 +58,32 @@ class StaticOutputOperatorPod(TraceableBase): tracked by the tracker manager. """ + # The Linear issue that will make this operator handle referential tags, or + # ``None`` when it handles them (ITL-1023). An operator that declares nothing + # does not handle them yet: it raises instead of giving a wrong answer. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = "ITL-1022" + def __init__( self, tracker_manager: TrackerManagerProtocol | None = None, **kwargs ) -> None: self.tracker_manager = tracker_manager or DEFAULT_TRACKER_MANAGER super().__init__(**kwargs) + def check_reference_support(self, *streams: StreamProtocol) -> None: + """Raise if a referential tag reaches an operator that does not handle it yet. + + Args: + *streams: The input streams. + + Raises: + NotImplementedError: If ``REFERENTIAL_TAGS_ISSUE`` is set and an input + has a referential tag. + """ + if self.REFERENTIAL_TAGS_ISSUE is None: + return + if any(stream_references(stream) for stream in streams): + raise not_supported_yet(type(self).__name__, self.REFERENTIAL_TAGS_ISSUE) + def pipeline_identity_structure(self) -> Any: """Return the pipeline identity, which defaults to content identity for operators.""" return self.identity_structure() @@ -156,6 +181,7 @@ def process( logger.debug(f"Invoking kernel {self} on streams: {streams}") # perform input stream validation + self.check_reference_support(*streams) self.validate_inputs(*streams) self.tracker_manager.record_operator_pod_invocation( self, upstreams=streams, label=label @@ -207,7 +233,7 @@ def __call__(self, *streams: StreamProtocol, **kwargs) -> DynamicPodStream: @staticmethod def _rows_to_table( rows: Sequence[tuple[TagProtocol, DataProtocol]], - ) -> tuple[pa.Table, tuple[str, ...]]: + ) -> tuple[pa.Table, tuple[str, ...], tuple[str, ...]]: """Concatenate (Tag, Data) pairs into one table, keeping per-row source info. Each row contributes its own system-tag and ``_source_*`` columns, so @@ -221,8 +247,12 @@ def _rows_to_table( Args: rows: Non-empty sequence of (Tag, Data) pairs. + A referential tag (a name in both the tag and the data of a row) is one + column: it is taken from the tags, and its source info from the data. + Returns: - The concatenated table and the user tag column names. + The concatenated table, the user tag column names and the referential + tags. Raises: ValueError: If ``rows`` is empty. @@ -233,8 +263,12 @@ def _rows_to_table( if not rows: raise ValueError("Cannot materialize an empty list of rows into a stream") + first_tag, first_data = rows[0] + references = reference_names(first_tag.keys(), first_data.keys()) tag_tables = [tag.as_table(columns={"system_tags": True}) for tag, _ in rows] data_tables = [data.as_table(columns={"source": True}) for _, data in rows] + if references: + data_tables = [t.drop_columns(list(references)) for t in data_tables] # Rows may list the same columns in a different order (cached vs # freshly computed), so match columns by name. @@ -250,7 +284,7 @@ def _rows_to_table( combined_data = pa.Table.from_arrays(combined_data.columns, schema=data_schema) full_table = arrow_utils.hstack_tables(combined_tags, combined_data) - return full_table, tuple(rows[0][0].keys()) + return full_table, tuple(first_tag.keys()), references @staticmethod def _materialize_to_stream( @@ -274,10 +308,10 @@ def _materialize_to_stream( Raises: ValueError: If ``rows`` is empty. """ - table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) + table, tag_keys, references = StaticOutputOperatorPod._rows_to_table(rows) # No ``source_info`` argument: the table's own ``_source_*`` columns # pass through, one value per row. - return ArrowTableStream(table, tag_columns=tag_keys) + return ArrowTableStream(table, tag_columns=tag_keys, reference_tags=references) @staticmethod def _materialize_input_stream( @@ -302,8 +336,13 @@ def _materialize_input_stream( Raises: ValueError: If ``rows`` is empty. """ - table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) - return _RowsOfStream(table, tag_columns=tag_keys, pipeline_hash=pipeline_hash) + table, tag_keys, references = StaticOutputOperatorPod._rows_to_table(rows) + return _RowsOfStream( + table, + tag_columns=tag_keys, + pipeline_hash=pipeline_hash, + reference_tags=references, + ) async def async_execute( self, @@ -373,8 +412,9 @@ def __init__( *, tag_columns: Collection[str], pipeline_hash: ContentHash, + reference_tags: Collection[str] = (), ) -> None: - super().__init__(table, tag_columns=tag_columns) + super().__init__(table, tag_columns=tag_columns, reference_tags=reference_tags) self._input_pipeline_hash: ContentHash = pipeline_hash def pipeline_hash( diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py new file mode 100644 index 000000000..ac9fe72f8 --- /dev/null +++ b/src/orcapod/core/referential_tags.py @@ -0,0 +1,237 @@ +"""Referential tags: the rules every stream and operator shares (ITL-1023). + +Specification "Orcapod v0.4 — Referential Tags" (attached to ITL-1004). A tag is either +materialized (it carries its own value) or referential (a handle to the data column of the +same name). In a stream a referential tag ``x`` is one physical column ``x`` whose name is +both a tag key and a data key, so the binding is the name intersection of the two halves of +``output_schema()`` / ``keys()`` (section 4.3: "the presence of a tag and data column with +the same name must therefore be sufficient to identify the tag as referential"). No other +schema field records it. + +This module holds what more than one operator needs: finding the references of a stream, +the namespace validator (sections 4 and 13.4), and the diagnostic emitted when a structural +operator turns a reference into a plain tag (section 7). + +The identities of referential streams are not frozen yet: on the integration branch of +ITL-1023 they may still change. Plain (``tag_only_columns``) streams keep their v0.3 +identities exactly. +""" + +from __future__ import annotations + +import warnings +from collections.abc import Iterable, Iterator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Literal + +from orcapod.errors import InputValidationError + +# Diagnostic emitted when a structural operator materializes a reference (section 7.4). +# ``"warn"`` emits a ``ReferenceMaterializationWarning``; ``"silent"`` emits nothing. The +# setting never changes a result, a schema or a hash. A module-level default for now. +reference_materialization: Literal["warn", "silent"] = "warn" + + +class ReferenceMaterializationWarning(UserWarning): + """A structural operator turned a referential tag into a plain tag (section 7).""" + + +# Set while Orcapod itself adds a structural step the user did not write (the select +# of a function pod call): its materializations are not reported. +_diagnostic_off: ContextVar[bool] = ContextVar( + "reference_materialization_off", default=False +) + + +@contextmanager +def materialization_diagnostic_off() -> Iterator[None]: + """Build the steps in this block without the materialization diagnostic. + + Used for steps Orcapod adds by itself, such as the automatic select of a function + pod call, which would otherwise report on every call. The steps and their results + are exactly the same. + + Yields: + Nothing. + """ + token = _diagnostic_off.set(True) + try: + yield + finally: + _diagnostic_off.reset(token) + + +def reference_names( + tag_names: Iterable[str], data_names: Iterable[str] +) -> tuple[str, ...]: + """Return the referential tags: the tag names that are also data names, in tag order. + + Args: + tag_names: The tag column names (system tags may be included; they never + match a data name). + data_names: The data column names. + + Returns: + The names in both, in the order of ``tag_names``. + """ + data = set(data_names) + return tuple(name for name in tag_names if name in data) + + +def stream_references(stream: Any) -> tuple[str, ...]: # pyright: ignore[reportExplicitAny] # any object with keys(): a stream, source or node + """Return the referential tags of ``stream``, known from its keys at graph-build time. + + Args: + stream: A stream (anything with ``keys()`` returning tag and data names). + + Returns: + The names that are both tag keys and data keys, in tag order. + """ + tag_keys, data_keys = stream.keys() + return reference_names(tag_keys, data_keys) + + +def check_namespaces( + tag_schema: Mapping[str, Any], # pyright: ignore[reportExplicitAny] # Python types of any kind + data_schema: Mapping[str, Any], # pyright: ignore[reportExplicitAny] # Python types of any kind + references: Iterable[str], +) -> None: + """Check the column namespaces of an output schema (sections 4.1 to 4.4). + + A name in both halves must be one of ``references`` (a materialized tag and a data + column never share a name), and a reference has one type in both halves. + + Args: + tag_schema: The output tag schema. + data_schema: The output data schema. + references: The names the producer keeps as referential tags. + + Raises: + InputValidationError: If a tag and a data column share a name that is not a + reference, or a reference's tag and data types differ. + """ + refs = set(references) + for name in reference_names(tag_schema, data_schema): + if name not in refs: + raise InputValidationError( + f"tag {name!r} and data column {name!r} would share a name without " + "being one referential column: a plain tag and a data column cannot " + "share a name" + ) + if tag_schema[name] != data_schema[name]: + raise InputValidationError( + f"referential tag {name!r} has type {tag_schema[name]} as a tag and " + f"{data_schema[name]} as data" + ) + + +def warn_materialized(operation: str, names: Iterable[str]) -> None: + """Report that ``operation`` turned the referential tags ``names`` into plain tags. + + Its job is the diagnostic of section 7.4, so it takes the operation's name. Emits + nothing when ``names`` is empty or ``reference_materialization`` is ``"silent"``. + + Args: + operation: The structural operation, for the message. + names: The tags that became plain. + """ + names = tuple(names) + if not names or reference_materialization == "silent" or _diagnostic_off.get(): + return + warnings.warn( + f"{operation}: the data column of referential tag(s) {list(names)} was removed; " + "they are now plain tags with the same values (set " + "orcapod.core.referential_tags.reference_materialization = 'silent' to hide this)", + ReferenceMaterializationWarning, + stacklevel=3, + ) + + +def not_supported_yet(operation: str, issue: str) -> NotImplementedError: + """Return the error for an operation that does not handle referential tags yet. + + Args: + operation: The operation's name. + issue: The Linear issue that will add the support. + + Returns: + The error to raise. + """ + return NotImplementedError( + f"{operation}: not supported with referential tags yet ({issue}). Build the " + "source with tag_only_columns=[...] to keep those tags plain." + ) + + +TagOnlyColumns = str | Iterable[str] | bool +"""The ``tag_only_columns`` option of a source: names, or ``True`` for every tag.""" + + +def source_reference_tags( + tag_columns: Iterable[str], tag_only_columns: TagOnlyColumns +) -> tuple[str, ...]: + """Return a source's referential tags: its declared tags minus the tag-only ones. + + Every declared tag of a source is referential by default (developer, November 12, + 2026): it is also a data column of the same name. ``tag_only_columns`` names the tags + kept plain, exactly as in v0.3 (tag only, not data); ``True`` keeps all of them plain. + + Args: + tag_columns: The source's declared tag columns. + tag_only_columns: The tags kept plain: one name, several, ``True`` for all of + them, or ``False`` / empty for none. + + Returns: + The referential tags, in ``tag_columns`` order. + + Raises: + ValueError: If ``tag_only_columns`` names a column that is not in + ``tag_columns``. + """ + tags = tuple(tag_columns) + if tag_only_columns is True: + return () + if tag_only_columns is False: + plain: set[str] = set() + elif isinstance(tag_only_columns, str): + plain = {tag_only_columns} + else: + plain = set(tag_only_columns) + if unknown := plain - set(tags): + raise ValueError( + f"tag_only_columns {sorted(unknown)} are not in tag_columns {list(tags)}" + ) + return tuple(name for name in tags if name not in plain) + + +def merged_source_tokens(left: object, right: object) -> list[str | None]: + """Return the source info of a data column merged by a join, keeping both inputs'. + + Section 9.4: a join that merges the data columns of two referential tags keeps the + provenance of both. The merged column's source info is the list of both inputs' + tokens, left (canonically first) then right. A token that is already a list (an + earlier join, or a ``GroupBy`` key, whose source info lists its members) gives its + elements, so the result is always a flat list of tokens. + + Args: + left: The left input's source-info value: a token, ``None`` or a list of them. + right: The right input's source-info value, likewise. + + Returns: + The flat list of tokens. + + Raises: + NotImplementedError: If a value is a list nested more than one level + (ITL-1016). + """ + merged: list[str | None] = [] + for value in (left, right): + items = value if isinstance(value, list) else [value] + for item in items: # pyright: ignore[reportUnknownVariableType] # a token or a list of tokens + if isinstance(item, list): + raise not_supported_yet( + "Join of nested source info of a referential tag", "ITL-1016" + ) + merged.append(item) # pyright: ignore[reportUnknownArgumentType] # a token + return merged diff --git a/src/orcapod/core/sources/arrow_table_source.py b/src/orcapod/core/sources/arrow_table_source.py index b573d76f9..890fbfd9f 100644 --- a/src/orcapod/core/sources/arrow_table_source.py +++ b/src/orcapod/core/sources/arrow_table_source.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.utils import arrow_utils from orcapod.utils.schema_utils import _normalize_column_list @@ -79,6 +80,7 @@ def __init__( system_tag_columns: Collection[str] = (), record_id_column: str | None = None, infer_nullable: bool = False, + tag_only_columns: TagOnlyColumns = (), **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -96,6 +98,7 @@ def __init__( result = builder.build( table, tag_columns=tag_columns, + reference_tags=source_reference_tags(tag_columns, tag_only_columns), source_id=self._source_id, source_kind=self.SOURCE_KIND, record_id_column=record_id_column, @@ -106,6 +109,9 @@ def __init__( self._schema_hash = result.schema_hash self._table_hash = result.table_hash self._tag_columns = result.tag_columns + self._tag_only_columns = tuple( + c for c in result.tag_columns if c not in result.reference_tags + ) self._system_tag_columns = result.system_tag_columns self._record_id_column = record_id_column @@ -117,6 +123,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: return { "source_type": self.SOURCE_KIND, "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "source_id": self.source_id, **self._identity_config(), } diff --git a/src/orcapod/core/sources/base.py b/src/orcapod/core/sources/base.py index 195473241..8554807c4 100644 --- a/src/orcapod/core/sources/base.py +++ b/src/orcapod/core/sources/base.py @@ -129,6 +129,19 @@ def _identity_config(self) -> dict[str, Any]: "data_schema": serialize_schema(data_schema, type_converter), } + def _tag_only_config(self) -> dict[str, Any]: + """Return ``tag_only_columns`` for ``to_config``, when any tag is referential. + + A source whose tags are all plain writes nothing, so its config is the one + v0.3 wrote; a config without the key reads back with every tag plain + (``tag_only_columns=True``), as v0.3 configs mean. + """ + tag_columns = tuple(getattr(self, "_tag_columns", ())) + tag_only = tuple(getattr(self, "_tag_only_columns", tag_columns)) + if tag_only == tag_columns: + return {} + return {"tag_only_columns": list(tag_only)} + def pipeline_identity_structure(self) -> Any: """Return (tag_schema, data_schema) as the pipeline identity for this source. Schema-only: no data content is included, so sources with diff --git a/src/orcapod/core/sources/csv_source.py b/src/orcapod/core/sources/csv_source.py index ba110effd..645c24c9d 100644 --- a/src/orcapod/core/sources/csv_source.py +++ b/src/orcapod/core/sources/csv_source.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.utils import arrow_utils from orcapod.utils.schema_utils import _normalize_column_list @@ -31,6 +32,7 @@ def __init__( record_id_column: str | None = None, source_id: str | None = None, schema: pa.Schema | None = None, + tag_only_columns: TagOnlyColumns = (), **kwargs: Any, ) -> None: import pyarrow.csv as pa_csv @@ -51,6 +53,7 @@ def __init__( result = builder.build( table, tag_columns=tag_columns, + reference_tags=source_reference_tags(tag_columns, tag_only_columns), source_id=self._source_id, record_id_column=record_id_column, system_tag_columns=system_tag_columns, @@ -58,6 +61,9 @@ def __init__( self._stream = result.stream self._tag_columns = result.tag_columns + self._tag_only_columns = tuple( + c for c in result.tag_columns if c not in result.reference_tags + ) self._system_tag_columns = result.system_tag_columns self._record_id_column = record_id_column if self._source_id is None: @@ -69,6 +75,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "source_type": "csv", "file_path": self._file_path, "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "system_tag_columns": list(self._system_tag_columns), "record_id_column": self._record_id_column, "source_id": self.source_id, @@ -81,6 +88,7 @@ def from_config(cls, config: dict[str, Any], db_registry=None) -> CSVSource: return cls( file_path=config["file_path"], tag_columns=config.get("tag_columns", ()), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/data_frame_source.py b/src/orcapod/core/sources/data_frame_source.py index 00268bf19..c5b9af61e 100644 --- a/src/orcapod/core/sources/data_frame_source.py +++ b/src/orcapod/core/sources/data_frame_source.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, ClassVar from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.utils import arrow_utils, polars_data_utils from orcapod.utils.schema_utils import _normalize_column_list @@ -38,6 +39,7 @@ def __init__( system_tag_columns: Collection[str] = (), source_id: str | None = None, schema: pa.Schema | None = None, + tag_only_columns: TagOnlyColumns = (), **kwargs: Any, ) -> None: super().__init__(source_id=source_id, **kwargs) @@ -77,6 +79,7 @@ def __init__( result = builder.build( arrow_table, tag_columns=tag_columns, + reference_tags=source_reference_tags(tag_columns, tag_only_columns), source_id=self._source_id, source_kind=self.SOURCE_KIND, system_tag_columns=system_tag_columns, @@ -84,6 +87,9 @@ def __init__( self._stream = result.stream self._tag_columns = result.tag_columns + self._tag_only_columns = tuple( + c for c in result.tag_columns if c not in result.reference_tags + ) if self._source_id is None: self._source_id = result.source_id @@ -92,6 +98,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: return { "source_type": self.SOURCE_KIND, "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "source_id": self.source_id, **self._identity_config(), } diff --git a/src/orcapod/core/sources/db_table_source.py b/src/orcapod/core/sources/db_table_source.py index d0897b484..2ca1e7971 100644 --- a/src/orcapod/core/sources/db_table_source.py +++ b/src/orcapod/core/sources/db_table_source.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -56,6 +57,9 @@ class DBTableSource(RootSource): label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: Orcapod configuration (controls hash character counts, etc.). + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Every other tag column is referential: + also a data column of the same name (ITL-1023). Raises: ValueError: If the table is not found, has no PK columns and none are @@ -73,6 +77,7 @@ def __init__( label: str | None = None, data_context: str | contexts.DataContext | None = None, config: OrcapodConfig | None = None, + tag_only_columns: TagOnlyColumns = (), *, _query: str | None = None, ) -> None: @@ -139,6 +144,9 @@ def __init__( result = builder.build( table, tag_columns=resolved_tag_columns, + reference_tags=source_reference_tags( + resolved_tag_columns, tag_only_columns + ), source_id=self._source_id, record_id_column=record_id_column, system_tag_columns=system_tag_columns, @@ -146,6 +154,9 @@ def __init__( self._stream = result.stream self._tag_columns = result.tag_columns + self._tag_only_columns = tuple( + c for c in result.tag_columns if c not in result.reference_tags + ) self._system_tag_columns = result.system_tag_columns if self._source_id is None: self._source_id = result.source_id @@ -157,6 +168,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "connector": self._connector.to_config(), "table_name": self._table_name, "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "system_tag_columns": list(self._system_tag_columns), "record_id_column": self._record_id_column, "source_id": self.source_id, diff --git a/src/orcapod/core/sources/delta_table_source.py b/src/orcapod/core/sources/delta_table_source.py index 86a187932..8ca4e2d86 100644 --- a/src/orcapod/core/sources/delta_table_source.py +++ b/src/orcapod/core/sources/delta_table_source.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.databases.storage_utils import delta_to_pyarrow_dataset from orcapod.types import PathLike @@ -34,6 +35,7 @@ def __init__( system_tag_columns: Collection[str] = (), record_id_column: str | None = None, source_id: str | None = None, + tag_only_columns: TagOnlyColumns = (), **kwargs: Any, ) -> None: resolved = Path(delta_table_path).resolve() @@ -60,6 +62,7 @@ def __init__( result = builder.build( table, tag_columns=tag_columns, + reference_tags=source_reference_tags(tag_columns, tag_only_columns), source_id=self._source_id, record_id_column=record_id_column, system_tag_columns=system_tag_columns, @@ -67,6 +70,9 @@ def __init__( self._stream = result.stream self._tag_columns = result.tag_columns + self._tag_only_columns = tuple( + c for c in result.tag_columns if c not in result.reference_tags + ) self._system_tag_columns = result.system_tag_columns self._record_id_column = record_id_column if self._source_id is None: @@ -78,6 +84,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "source_type": "delta_table", "delta_table_path": str(self._delta_table_path), "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "system_tag_columns": list(self._system_tag_columns), "record_id_column": self._record_id_column, "source_id": self.source_id, @@ -92,6 +99,7 @@ def from_config( return cls( delta_table_path=config["delta_table_path"], tag_columns=config.get("tag_columns", ()), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/dict_source.py b/src/orcapod/core/sources/dict_source.py index 9cb7cb2bc..2f02fe734 100644 --- a/src/orcapod/core/sources/dict_source.py +++ b/src/orcapod/core/sources/dict_source.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, ClassVar from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.types import DataValue, SchemaLike from orcapod.utils import arrow_utils @@ -34,6 +35,7 @@ def __init__( system_tag_columns: Collection[str] = (), data_schema: SchemaLike | pa.Schema | None = None, source_id: str | None = None, + tag_only_columns: TagOnlyColumns = (), **kwargs: Any, ) -> None: super().__init__(source_id=source_id, **kwargs) @@ -63,6 +65,7 @@ def __init__( result = builder.build( arrow_table, tag_columns=tag_columns, + reference_tags=source_reference_tags(tag_columns, tag_only_columns), source_id=self._source_id, source_kind=self.SOURCE_KIND, system_tag_columns=system_tag_columns, @@ -70,6 +73,9 @@ def __init__( self._stream = result.stream self._tag_columns = result.tag_columns + self._tag_only_columns = tuple( + c for c in result.tag_columns if c not in result.reference_tags + ) if self._source_id is None: self._source_id = result.source_id @@ -78,6 +84,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: return { "source_type": self.SOURCE_KIND, "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "source_id": self.source_id, **self._identity_config(), } diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index eaf1abf8d..55a693016 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -41,6 +41,7 @@ from typing import TYPE_CHECKING, Any, NamedTuple from orcapod.core.sources.base import RootSource +from orcapod.core.referential_tags import TagOnlyColumns, source_reference_tags from orcapod.core.sources.derived_source import validate_tags_as_data from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -219,6 +220,9 @@ class SpiralDBTableSource(RootSource): config: OrcaPod configuration. overrides: Optional pyspiral client config overrides passed through to ``SpiralDBConnector``, e.g. ``{"server.url": "http://api.spiraldb.dev"}``. + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Every other tag column is referential: + also a data column of the same name (ITL-1023). tags_as_data: Maps a key column that is a tag column of this source to the name of a data column that receives a copy of it, as in ``DerivedSource``. ``None`` (default) or an empty mapping copies nothing. @@ -255,6 +259,7 @@ def __init__( config: "OrcapodConfig | None" = None, overrides: dict[str, str] | None = None, tags_as_data: Mapping[str, str] | None = None, + tag_only_columns: TagOnlyColumns = (), ) -> None: for name, value in (("limit", limit), ("max_rows", max_rows)): if value is not None and not _is_positive_int(value): @@ -317,6 +322,8 @@ def __init__( ) if self._source_id is None: self._source_id = f"{project_id}/{dataset}.{table_name}@{self._table_id}" + # Every declared tag is referential unless kept tag-only (ITL-1023). + self._reference_tags = source_reference_tags(resolved_tags, tag_only_columns) # A zero-row stream with the table's columns answers every schema question. self._builder = SourceStreamBuilder(self.data_context, self.orcapod_config) @@ -326,6 +333,7 @@ def __init__( tag_columns=resolved_tags, source_id=self._source_id, system_tag_columns=system_tag_columns, + reference_tags=self._reference_tags, ) except ValueError as exc: raise ValueError(f"{location}, table {table_name!r}: {exc}") from exc @@ -345,10 +353,14 @@ def __init__( tag_columns=template.tag_columns, source_id=self._source_id, system_tag_columns=template.system_tag_columns, + reference_tags=self._reference_tags, ) self._stream = template.stream self._schema_hash = template.schema_hash self._tag_columns = template.tag_columns + self._tag_only_columns = tuple( + c for c in self._tag_columns if c not in self._reference_tags + ) self._system_tag_columns = template.system_tag_columns def _check_tags_as_data( @@ -568,6 +580,7 @@ def _read_batches( schema_hash=self._schema_hash, source_id=self.source_id, provenance_tokens=tokens, + reference_tags=self._reference_tags, ) def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: @@ -624,6 +637,7 @@ def _whole_table( table=pa.concat_tables(tables), tag_columns=self._tag_columns, system_tag_columns=self._system_tag_columns, + reference_tags=self._reference_tags, ) return whole.as_table(columns=columns, all_info=all_info) @@ -741,6 +755,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "table_name": self._table_name, "table_id": self._table_id, "tag_columns": list(self._tag_columns), + **self._tag_only_config(), "system_tag_columns": list(self._system_tag_columns), "where": _locationto_config(self._where), "limit": self._limit, @@ -774,6 +789,7 @@ def from_config( table_name=config["table_name"], dataset=config.get("dataset", "default"), tag_columns=config.get("tag_columns"), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), where=_locationfrom_config(config.get("where") or {}), limit=config.get("limit"), diff --git a/src/orcapod/core/sources/stream_builder.py b/src/orcapod/core/sources/stream_builder.py index 8018e73fa..79da06233 100644 --- a/src/orcapod/core/sources/stream_builder.py +++ b/src/orcapod/core/sources/stream_builder.py @@ -126,6 +126,7 @@ class SourceStreamResult: source_id: str tag_columns: tuple[str, ...] system_tag_columns: tuple[str, ...] + reference_tags: tuple[str, ...] = () class SourceStreamBuilder: @@ -148,6 +149,7 @@ def build( record_id_column: str | None = None, system_tag_columns: Collection[str] = (), source_kind: str | None = None, + reference_tags: Collection[str] = (), ) -> SourceStreamResult: """Run the full enrichment pipeline. @@ -174,6 +176,10 @@ def build( source_kind: Kind prefix of the default source id, for example ``dict``. Required when ``source_id`` is not given; not used otherwise. + reference_tags: The tag columns that are referential: also data columns + of the same name, stored once (ITL-1023). Empty (the builder's + default) gives plain tags, exactly as in v0.3; sources pass their + declared tags minus ``tag_only_columns``. Returns: SourceStreamResult with enriched stream and metadata. @@ -185,6 +191,18 @@ def build( """ tag_columns_tuple = tuple(tag_columns) system_tag_columns_tuple = tuple(system_tag_columns) + reference_tags_tuple = tuple( + c for c in tag_columns_tuple if c in reference_tags + ) + if unknown := set(reference_tags) - set(tag_columns_tuple): + raise ValueError( + f"reference_tags {sorted(unknown)} are not in tag_columns " + f"{list(tag_columns_tuple)}" + ) + # Tag-only columns: the tags that are not also data. + tag_only_tuple = tuple( + c for c in tag_columns_tuple if c not in reference_tags_tuple + ) # 1. Drop system columns from raw input. table = arrow_utils.drop_system_columns(table) @@ -217,7 +235,7 @@ def build( # set them correctly before calling build(). non_sys = arrow_utils.drop_system_columns(table) tag_schema = non_sys.select(list(tag_columns_tuple)).schema - data_schema = non_sys.drop(list(tag_columns_tuple)).schema + data_schema = non_sys.drop(list(tag_only_tuple)).schema tag_python = self._data_context.type_converter.arrow_schema_to_python_schema( tag_schema ) @@ -283,6 +301,7 @@ def build( schema_hash=schema_hash, source_id=source_id, provenance_tokens=provenance_tokens, + reference_tags=reference_tags_tuple, ) return SourceStreamResult( @@ -292,6 +311,7 @@ def build( source_id=source_id, tag_columns=tag_columns_tuple, system_tag_columns=system_tag_columns_tuple, + reference_tags=reference_tags_tuple, ) def attach_provenance( @@ -303,6 +323,7 @@ def attach_provenance( schema_hash: str, source_id: str, provenance_tokens: Sequence[str], + reference_tags: Sequence[str] = (), ) -> ArrowTableStream: """Add source-info and system-tag columns to ``table`` and wrap it as a stream. @@ -320,6 +341,8 @@ def attach_provenance( schema_hash: The source's schema hash, which names its system-tag columns. source_id: The canonical source name. provenance_tokens: One token per row, identifying the row within the source. + reference_tags: The tag columns that are also data columns (referential); + they get source info like any data column. Returns: The stream over the enriched table. @@ -334,7 +357,9 @@ def attach_provenance( _make_record_id(source_id, token).bytes for token in provenance_tokens ] table = arrow_utils.add_source_info( - table, source_info, exclude_columns=tag_columns + table, + source_info, + exclude_columns=[c for c in tag_columns if c not in reference_tags], ) table = arrow_utils.add_system_tag_columns( table, @@ -347,4 +372,5 @@ def attach_provenance( table=table, tag_columns=tag_columns, system_tag_columns=system_tag_columns, + reference_tags=reference_tags, ) diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 721f59e36..32e909350 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -49,8 +49,27 @@ def __init__( source_info: dict[str, SourceInfoValue] | None = None, producer: PodProtocol | None = None, upstreams: tuple[StreamProtocol, ...] = (), + reference_tags: Collection[str] = (), **kwargs, ) -> None: + """Wrap ``table`` as a stream. + + Args: + table: The rows: tag, data, system-tag and source-info columns. + tag_columns: The user tag columns. + system_tag_columns: Columns to rename into system tags. + source_info: Source-info tokens to add, by column. + producer: The pod that produced the stream, if any. + upstreams: The streams the producer read. + reference_tags: The tag columns that are referential: each is also a data + column of the same name, stored once (specification "Referential + Tags", section 4). Must be a subset of ``tag_columns``. + **kwargs: Passed to ``StreamBase``. + + Raises: + ValueError: If a tag column or a reference tag is not a tag column of + the table, or the stream has no data column. + """ super().__init__(**kwargs) self._producer = producer @@ -104,15 +123,25 @@ def __init__( raise ValueError( f"Specified tag columns {delta} are not present in the table." ) + if unknown := set(reference_tags) - set(self._tag_columns): + raise ValueError( + f"Reference tags {sorted(unknown)} are not tag columns of the stream." + ) + # A referential tag is one column in both roles: a tag, and a data column of + # the same name (tag order). + self._reference_tags = tuple( + c for c in self._tag_columns if c in set(reference_tags) + ) + tag_only = tuple( + c for c in self._all_tag_columns if c not in self._reference_tags + ) table, prefix_tables = arrow_utils.prepare_prefixed_columns( table, prefix_info, - exclude_columns=self._all_tag_columns, + exclude_columns=tag_only, ) # now table should only contain tag columns and data columns - self._data_columns = tuple( - c for c in table.column_names if c not in self._all_tag_columns - ) + self._data_columns = tuple(c for c in table.column_names if c not in tag_only) self._table = table self._source_info_table = prefix_tables[constants.SOURCE_PREFIX] self._data_context_table = data_context_table @@ -161,8 +190,17 @@ def identity_structure(self) -> Any: ) ), self._tag_columns, + ) + ( + # Only when there is one: a plain stream keeps its v0.3 identity, and a + # referential and a plain stream over one table differ (ITL-1023). + (("reference_tags", self._reference_tags),) if self._reference_tags else () ) + @property + def reference_tags(self) -> tuple[str, ...]: + """The referential tags: tag columns that are also data columns (tag order).""" + return self._reference_tags + def pipeline_identity_structure(self) -> Any: if self._producer is None or not isinstance( self._producer, PipelineElementProtocol @@ -374,7 +412,10 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, Data]]: yield from self._cached_elements def __repr__(self) -> str: + refs = ( + f", reference_tags={self._reference_tags}" if self._reference_tags else "" + ) return ( f"{self.__class__.__name__}(table={self._table.column_names}, " - f"tag_columns={self._tag_columns})" + f"tag_columns={self._tag_columns}{refs})" ) diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index 981706352..a8c52d3ec 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -201,6 +201,7 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An Returns: An ArrowTableStream. """ + from orcapod.core.referential_tags import stream_references from orcapod.core.streams.arrow_table_stream import ArrowTableStream if not buf: @@ -218,12 +219,15 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An tag_columns=tag_keys, producer=upstream_node.producer, upstreams=upstream_node.upstreams, + reference_tags=stream_references(upstream_node), ) from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod # Each row keeps its own system tags and source info. - full_table, user_tag_keys = StaticOutputOperatorPod._rows_to_table(buf) + full_table, user_tag_keys, references = StaticOutputOperatorPod._rows_to_table( + buf + ) # Pass the upstream node's producer and upstreams so the # materialized stream inherits the correct identity_structure @@ -238,6 +242,7 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An tag_columns=user_tag_keys, producer=producer, upstreams=upstreams, + reference_tags=references, ) @staticmethod diff --git a/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md b/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md index f4c0e5b73..3d787c27c 100644 --- a/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md +++ b/superpowers/plans/2026-11-12-itl-1023-referential-tags-slice.md @@ -16,7 +16,7 @@ table whose name is in both role lists (spec 4.3 as code: binding = `tag_keys No change to `types.Schema` or to the `(tag_schema, data_schema)` tuple: `x` appears in both halves of `output_schema()` / `keys()` at graph-build time. `ArrowTableStream(table, tag_columns, reference_tags=...)` makes data = non-tags ∪ references. Sources default every -declared tag to referential; `materialized_tags=[...]` (approved name, coordinator Nov 12) +declared tag to referential; `tag_only_columns=[...]` (approved name, coordinator Nov 12) keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference exists. **Tech Stack:** Python 3.11-3.13, PyArrow, Polars, pytest, uv. @@ -26,16 +26,16 @@ keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference ex ## Decisions (go into the PR body's DECISIONS section) 1. Representation: name intersection, one physical column, no Schema change (audit, firm). -2. Option name `materialized_tags: Collection[str]` on sources; subset of `tag_columns`; unknown +2. Option name `tag_only_columns: Collection[str]` on sources (developer's choice, Nov 12); subset of `tag_columns`; unknown name → `ValueError`. Builder (`SourceStreamBuilder.build`) takes explicit `reference_tags` - (default `()` = plain); sources compute `reference_tags = tag_columns - materialized_tags`. + (default `()` = plain); sources compute `reference_tags = tag_columns - tag_only_columns`. 3. ListSource / DerivedSource / CachedSource tags stay plain (computed values, or stored records). -4. Function pod output data = (input data columns that are referential tags, passed through - unchanged with their source info) + function outputs. Spec 6.1 "tags and bindings pass - through"; Orcapod pods drop input data otherwise, so the pass-through is what keeps the - binding valid. The result store keeps only function outputs (result identity unchanged, - spec 12.4); on read from the node's store the referenced column comes back from the stored - tag column. Output name equal to any tag name → `InputValidationError` at build time. +4. Pod boundary (developer decision, November 12, 2026, replaces my first plan): a pod's output + data stays the function's outputs; every referential tag of the input becomes a PLAIN tag + (same value and type) in the pod's output schema. No diagnostic at a pod boundary + (coordinator). Output name equal to any tag name raises at build time. No change to what a + node stores or to function-result identity. The 'fall-through / append' convenience is a + separate issue, out of scope. 5. Join merged source-info (spec 9.4): the merged column's `_source_x` keeps the tokens of every referential input, as `large_list` in canonical input order (a list input contributes its elements). Plain+referential keeps the referential side's column as is. @@ -61,7 +61,7 @@ keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference ex - Modify `src/orcapod/core/sources/stream_builder.py` (+ `arrow_table_source.py`, `dict_source.py`, `csv_source.py`, `data_frame_source.py`, `delta_table_source.py`, `db_table_source.py`, `spiraldb_table_source.py`, `polling_source.py`): default referential, - `materialized_tags` option, to_config. + `tag_only_columns` option, to_config. - Modify `src/orcapod/core/operators/static_output_pod.py` (guard, `_rows_to_table` dedupe), `pipeline/sync_orchestrator.py` (dedupe, pass references). - Modify operators: `join.py`, `group_by.py`, `column_selection.py`, `filters.py`, @@ -72,7 +72,7 @@ keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference ex `test_schema_rules.py`, `test_sources.py`, `test_function_pods.py`, `test_join.py`, `test_group_by.py`, `test_select_drop.py`, `test_row_preserving.py`, `test_not_yet.py`, `test_hashes.py`; additions to `tests/test_core/operators/test_async_sync_equivalence.py`. -- Golden fixtures: `tests/test_hashing/golden/cases.py` switch to `materialized_tags=` — values +- Golden fixtures: `tests/test_hashing/golden/cases.py` switch to `tag_only_columns=` — values must stay byte-identical (run golden test before and after). - `changelog.d/ITL-1023.md`, `design_issues/ITL-1023-RT1.md` (slice limitations), `docs/concepts/referential-tags.md` (short; states identities not frozen). @@ -82,12 +82,12 @@ keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference ex ### Task 0: failing end-to-end EDI-shaped test - [ ] `tests/test_core/referential_tags/test_edi_end_to_end.py`: spikes (k1..k5, spike_time, amplitude) and gaze (k1..k5, gaze_time, gx) as `DictSource` with tag_columns keys + time - (referential by default); `trials` keyed on (k1, k2) with explicit `materialized_tags`; + (referential by default); `trials` keyed on (k1, k2) with explicit `tag_only_columns`; `GroupBy(k1..k5)` on both; `pod(spikes_g, gaze_g, trials)` (automatic join + select) with `def align(spike_time: list[int], gaze_time: list[int], k1: str) -> int`; `PipelineJob` on `InMemoryArrowDatabase` and on `DeltaTableDatabase(tmp_path)`; sync and async orchestrators; assert values, that `k1` is a tag and data, call counter = n groups on run 1 and 0 on run 2. -- [ ] Run → fails (`materialized_tags` unknown / tags not data). Commit, push, open draft PR +- [ ] Run → fails (`tag_only_columns` unknown / tags not data). Commit, push, open draft PR (Refs ITL-1023), send progress message. ### Task 1: representation + shared validator @@ -96,12 +96,12 @@ keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference ex - [ ] Implement in `arrow_table_stream.py` and `referential_tags.py`. ### Task 2: sources -- [ ] Tests: DictSource default referential; `materialized_tags` gives v0.3 schema/hashes +- [ ] Tests: DictSource default referential; `tag_only_columns` gives v0.3 schema/hashes equal to the builder's plain form; unknown name raises. - [ ] Implement builder `reference_tags`, each source's option, to_config. ### Task 3: hashes -- [ ] Golden fixtures switch to `materialized_tags=` (all sites with tag_columns in cases.py); +- [ ] Golden fixtures switch to `tag_only_columns=` (all sites with tag_columns in cases.py); run golden suite: no value moves. - [ ] Identity term in ArrowTableStream; test referential vs plain over one table differ; spec 12.4 test: function not reading `x` has equal input data hash / result store hit @@ -138,4 +138,6 @@ keeps chosen tags plain, exactly v0.3. Hash terms enter only when a reference ex issue entry; docs page; merge origin/main; PR body with DECISIONS; worker_done. ## Status log -- 2026-11-12: plan written; branch created from origin/main f51ce67c; option name approved. +- 2026-11-12: plan written; branch created from origin/main f51ce67c. +- 2026-11-12: developer: option is `tag_only_columns`; references become plain tags at a pod boundary. +- Task 0 done: fc22de6a, draft PR #342. diff --git a/tests/test_core/referential_tags/test_edi_end_to_end.py b/tests/test_core/referential_tags/test_edi_end_to_end.py index 6141c7bd8..8161f5344 100644 --- a/tests/test_core/referential_tags/test_edi_end_to_end.py +++ b/tests/test_core/referential_tags/test_edi_end_to_end.py @@ -6,7 +6,7 @@ joined, and a function reads the timestamp lists and a key column by name. A source's declared tags are referential by default (developer, November 12, 2026), so the keys and timestamps are data as well as tags, with no ``tags_as_data`` copy and no hand-written drop -before the join. The third table keeps its tags plain (``materialized_tags``), so the join +before the join. The third table keeps its tags plain (``tag_only_columns``), so the join meets plain with referential (section 9.2) and referential with referential (section 9.3). The pipeline runs through a function node on the in-memory store and on Delta, with the @@ -83,7 +83,7 @@ def _build(job: PipelineJob) -> None: trials = op.sources.DictSource( TRIALS, tag_columns=["lab", "subject"], - materialized_tags=["lab", "subject"], + tag_only_columns=["lab", "subject"], source_id="trials", ) with job: @@ -124,13 +124,13 @@ def test_edi_shaped_pipeline_runs_and_second_run_computes_nothing( assert sorted(CALLS) == [1, 2] node = job.nodes["align"] tag_keys, data_keys = node.keys() - # ``unit`` was read by the function and stays a referential tag: tag and data. - assert "unit" in tag_keys and "unit" in data_keys - # Keys the function did not read were dropped from data by the automatic select - # and stay as plain tags (section 7). - assert "session" in tag_keys and "session" not in data_keys + # A pod's output data is the function's outputs: at the pod boundary every + # referential tag becomes a plain tag with the same value (developer, November 12, + # 2026). ``unit`` was read by the function; ``session`` was dropped by the automatic + # select (section 7). Both are plain tags of the output. + assert set(KEYS) <= set(tag_keys) + assert data_keys == ("summary",) table = node.as_table() - # A referential tag is stored and shown once. assert table.column_names.count("unit") == 1 rows = sorted(table.to_pylist(), key=lambda r: r["unit"]) assert [r["summary"] for r in rows] == [ diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 07cb0bddc..5fe55b1bb 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -618,6 +618,7 @@ def schema_derived_source_tags_as_data() -> dict[str, Any]: origin = ArrowTableSource( _builder_table(), tag_columns=["subject"], + tag_only_columns=True, source_id="golden_source", data_context=_ctx(), ) @@ -944,7 +945,10 @@ def run() -> dict[str, Any]: from orcapod.sources import DictSource source = DictSource( - data=_datagram_rows()[name], tag_columns=["k"], source_id=f"golden_{name}" + data=_datagram_rows()[name], + tag_columns=["k"], + tag_only_columns=True, + source_id=f"golden_{name}", ) return stream_summary(source) @@ -967,7 +971,9 @@ def _arrow_source(column: pa.Array[Any], nullable: bool = False) -> Any: ] ), ) - return ArrowTableSource(table, tag_columns=["k"], source_id="golden_arrow") + return ArrowTableSource( + table, tag_columns=["k"], tag_only_columns=True, source_id="golden_arrow" + ) @case("datagram", "dict_order") @@ -991,6 +997,7 @@ def datagram_dict_order() -> dict[str, Any]: source = DictSource( data=[{"k": f"r{i}", "m": v} for i, v in enumerate(values)], tag_columns=["k"], + tag_only_columns=True, data_schema={"k": str, "m": value_type}, source_id=f"golden_dict_{name}", ) @@ -1056,20 +1063,33 @@ def golden_adjust(age: int, weight: float, score: float) -> float: def _subjects() -> Any: from orcapod.sources import DictSource - return DictSource(data=SUBJECT_ROWS, tag_columns=["subject"], source_id="subjects") + return DictSource( + data=SUBJECT_ROWS, + tag_columns=["subject"], + tag_only_columns=True, + source_id="subjects", + ) def _scans() -> Any: from orcapod.sources import DictSource - return DictSource(data=SCAN_ROWS, tag_columns=["subject"], source_id="scans") + return DictSource( + data=SCAN_ROWS, + tag_columns=["subject"], + tag_only_columns=True, + source_id="scans", + ) def _grouped() -> Any: from orcapod.sources import DictSource return DictSource( - data=GROUPED_ROWS, tag_columns=["cohort", "subject"], source_id="cohorts" + data=GROUPED_ROWS, + tag_columns=["cohort", "subject"], + tag_only_columns=True, + source_id="cohorts", ) @@ -1126,6 +1146,7 @@ def pipeline_merge_join() -> dict[str, Any]: right = DictSource( data=[{"subject": "s1", "weight": 30.0}, {"subject": "s3", "weight": 31.5}], tag_columns=["subject"], + tag_only_columns=True, source_id="reweigh", ) return stream_summary(MergeJoin()(left, right)) @@ -1163,6 +1184,7 @@ def pipeline_index_pick() -> dict[str, Any]: {"subject": "s2", "values": [4, 5], "meta": {"k": 20, "j": 21}}, ], tag_columns=["subject"], + tag_only_columns=True, source_id="lists", ) out = _prefixed("index", stream_summary(Index(column="values", i=0)(source))) @@ -1370,6 +1392,7 @@ def datagram_polars_data_hash() -> dict[str, Any]: source = DictSource( [{"k": "a", "v": value}], tag_columns=["k"], + tag_only_columns=True, data_schema={"k": str, "v": type(value)}, ) ((_, data),) = list(source.iter_data()) @@ -1413,6 +1436,7 @@ def run() -> dict[str, Any]: source = DictSource( data=_polars_rows()[name], tag_columns=["k"], + tag_only_columns=True, data_schema={"k": str, "c": column_type}, source_id=f"golden_{name}", ) @@ -1598,7 +1622,10 @@ def _nullable_readings() -> Any: ), ) return ArrowTableSource( - table, tag_columns=["k", "site"], source_id="golden_nullable" + table, + tag_columns=["k", "site"], + tag_only_columns=True, + source_id="golden_nullable", ) @@ -1694,11 +1721,15 @@ def pipeline_default_source_ids() -> dict[str, Any]: ) sources: dict[str, Any] = { - "arrow_table": ArrowTableSource(_builder_table(), tag_columns=["subject"]), - "dict": DictSource(data=SUBJECT_ROWS, tag_columns=["subject"]), + "arrow_table": ArrowTableSource( + _builder_table(), tag_columns=["subject"], tag_only_columns=True + ), + "dict": DictSource( + data=SUBJECT_ROWS, tag_columns=["subject"], tag_only_columns=True + ), "list": ListSource("weight", [20.5, 22.25, 19.0]), "data_frame": DataFrameSource( - pl.DataFrame(SUBJECT_ROWS), tag_columns=["subject"] + pl.DataFrame(SUBJECT_ROWS), tag_columns=["subject"], tag_only_columns=True ), } out: dict[str, Any] = {} From c3c5e995ef5eaadc46abc40e92dd3228c6764ce1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 21:21:31 -0800 Subject: [PATCH 03/72] feat(referential_tags): row-preserving ops, renames, guards, async Join parity; tests (ITL-1023) Filters, DropNulls, SemiJoin, Pick and Index keep references; MapTags and MapData work when they leave references alone; Batch, MergeJoin, operator node storage, DerivedSource and Pipeline.save raise 'not supported with referential tags yet'. The async Join merges a referenced data column as the sync path does (section 9). Referential cases join the async/sync equivalence matrix; its existing sources keep plain tags. Refs ITL-1023 Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/index.py | 21 +- src/orcapod/core/operators/join.py | 43 ++- src/orcapod/core/operators/mappers.py | 74 +++- src/orcapod/core/operators/pick.py | 21 +- src/orcapod/core/sources/cached_source.py | 6 +- src/orcapod/core/sources/derived_source.py | 3 + src/orcapod/pipeline/graph.py | 7 + .../operators/test_async_sync_equivalence.py | 122 ++++++- .../referential_tags/test_function_pods.py | 125 +++++++ .../test_model_and_sources.py | 121 +++++++ .../referential_tags/test_operators.py | 326 ++++++++++++++++++ 11 files changed, 844 insertions(+), 25 deletions(-) create mode 100644 tests/test_core/referential_tags/test_function_pods.py create mode 100644 tests/test_core/referential_tags/test_model_and_sources.py create mode 100644 tests/test_core/referential_tags/test_operators.py diff --git a/src/orcapod/core/operators/index.py b/src/orcapod/core/operators/index.py index 23c09b676..9ee5ebf23 100644 --- a/src/orcapod/core/operators/index.py +++ b/src/orcapod/core/operators/index.py @@ -5,8 +5,9 @@ import logging import typing from collections.abc import Sequence -from typing import TYPE_CHECKING, Any, cast +from typing import Any, ClassVar, TYPE_CHECKING, cast +from orcapod.core.referential_tags import stream_references from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -48,6 +49,9 @@ class Index(UnaryOperator): See ITL-439. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__( self, column: str, @@ -133,8 +137,20 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: NotImplementedError: If the column holds an extension type whose ``index_element`` is not yet implemented. """ - _, data_schema = stream.output_schema() + tag_schema, data_schema = stream.output_schema() data_columns = list(data_schema.keys()) + # Referential tags (ITL-1023): the output is a new data column, never a tag's + # value replaced in place or a name shared with a tag (section 4). + if self.out is None and self.column in tag_schema: + raise InputValidationError( + f"Index: column {self.column!r} is a referential tag; replacing it " + "in place would change the tag's value. Give out=." + ) + if self.out is not None and self.out in tag_schema: + raise InputValidationError( + f"Index: out column {self.out!r} is the name of a tag; a data " + "column cannot share a name with a tag it does not back." + ) if self.column not in data_columns: raise InputValidationError( @@ -366,6 +382,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: output_schema.empty_table(), tag_columns=tag_columns, data_context=stream.data_context, + reference_tags=stream_references(stream), ) return self._materialize_to_stream(out_rows) diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index 52f692fea..e6c9a046a 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -257,9 +257,6 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: 0, COMMON_JOIN_KEY, pa.array([0] * len(next_table), type=pa.int8()) ) - # Polars sets every field nullable; capture the schema before the join. - next_ref_schema = next_table.schema - common_tag_keys = tag_keys.intersection(next_tag_keys) next_references = set(stream_references(next_stream)) merged = self._merged_references( @@ -276,6 +273,8 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: next_table = next_table.rename_columns( [right_source.get(c, c) for c in next_table.column_names] ) + # Polars sets every field nullable; capture the schema before the join. + next_ref_schema = next_table.schema # A shared tag key must have one Arrow type in both inputs (up to # layout); otherwise Polars would join and keep one side's type. for key in sorted(common_tag_keys): @@ -912,14 +911,48 @@ def _merge_data( ) source_info = {**left.source_info(), **right.source_info()} if merged: - right_table = right_table.drop_columns(sorted(merged)) left_info, right_info = left.source_info(), right.source_info() for name in merged: source_info[name] = merged_source_tokens( left_info.get(name), right_info.get(name) ) + # Column order as ``static_process`` lays it out: the referential columns + # first, in the merged tag order, then each side's other data columns. + tag_order = [*left_tag.keys()] + tag_order += [name for name in right_tag.keys() if name not in tag_order] + referential = [ + name + for name in tag_order + if name in left_table.column_names or name in right_table.column_names + ] + if not referential: + return Data( + arrow_utils.hstack_tables(left_table, right_table), + source_info=source_info, + data_context=left.data_context_key, + ) + left_rest = left_table.drop_columns( + [c for c in referential if c in left_table.column_names] + ) + right_rest = right_table.drop_columns( + [c for c in referential if c in right_table.column_names] + ) + references = pa.table( + { + name: ( + left_table if name in left_table.column_names else right_table + ).column(name) + for name in referential + }, + schema=pa.schema( + ( + left_table if name in left_table.column_names else right_table + ).schema.field(name) + for name in referential + ), + ) return Data( - arrow_utils.hstack_tables(left_table, right_table), + arrow_utils.hstack_tables(references, left_rest, right_rest), source_info=source_info, data_context=left.data_context_key, ) diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 48db5c234..b62d5ca75 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -1,8 +1,9 @@ from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator +from orcapod.core.referential_tags import not_supported_yet, stream_references from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol @@ -30,6 +31,38 @@ def __init__( self.drop_unmapped = drop_unmapped super().__init__(**kwargs) + # Referential tags (ITL-1023): handled when the renaming leaves them alone. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + + def check_reference_support(self, *streams: StreamProtocol) -> None: + """Raise if the renaming would touch a referential tag (not supported yet). + + Renaming a referential tag or its data column, or dropping it as unmapped, + is a rule the specification leaves open (section 10.1, ITL-1018). A renaming + that leaves every referential tag alone is supported. + + Args: + *streams: The input stream. + + Raises: + NotImplementedError: If a referential tag is renamed, renamed to, or + dropped as unmapped. + """ + for stream in streams: + references = stream_references(stream) + touched = [ + name + for name in references + if name in self.name_map + or name in self.name_map.values() + or self.drop_unmapped + ] + if touched: + raise not_supported_yet( + f"{type(self).__name__} renaming referential tag(s) {touched}", + "ITL-1018", + ) + def to_config(self) -> dict[str, Any]: """Serialize this MapData operator to a config dict. @@ -80,7 +113,11 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: if self.drop_unmapped and unmapped_columns: renamed_table = renamed_table.drop_columns(list(unmapped_columns)) - return ArrowTableStream(renamed_table, tag_columns=tag_columns) + return ArrowTableStream( + renamed_table, + tag_columns=tag_columns, + reference_tags=stream_references(stream), + ) def validate_unary_input(self, stream: StreamProtocol) -> None: # verify that renamed value does NOT collide with other columns @@ -182,6 +219,38 @@ def __init__( self.drop_unmapped = drop_unmapped super().__init__(**kwargs) + # Referential tags (ITL-1023): handled when the renaming leaves them alone. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + + def check_reference_support(self, *streams: StreamProtocol) -> None: + """Raise if the renaming would touch a referential tag (not supported yet). + + Renaming a referential tag or its data column, or dropping it as unmapped, + is a rule the specification leaves open (section 10.1, ITL-1018). A renaming + that leaves every referential tag alone is supported. + + Args: + *streams: The input stream. + + Raises: + NotImplementedError: If a referential tag is renamed, renamed to, or + dropped as unmapped. + """ + for stream in streams: + references = stream_references(stream) + touched = [ + name + for name in references + if name in self.name_map + or name in self.name_map.values() + or self.drop_unmapped + ] + if touched: + raise not_supported_yet( + f"{type(self).__name__} renaming referential tag(s) {touched}", + "ITL-1018", + ) + def to_config(self) -> dict[str, Any]: """Serialize this MapTags operator to a config dict. @@ -226,6 +295,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: return ArrowTableStream( renamed_table, tag_columns=new_tag_columns, + reference_tags=stream_references(stream), ) def validate_unary_input(self, stream: StreamProtocol) -> None: diff --git a/src/orcapod/core/operators/pick.py b/src/orcapod/core/operators/pick.py index b69f071dc..b5b8fbf78 100644 --- a/src/orcapod/core/operators/pick.py +++ b/src/orcapod/core/operators/pick.py @@ -5,8 +5,9 @@ import logging import typing from collections.abc import Sequence -from typing import TYPE_CHECKING, Any, cast +from typing import Any, ClassVar, TYPE_CHECKING, cast +from orcapod.core.referential_tags import stream_references from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -50,6 +51,9 @@ class Pick(UnaryOperator): functional output semantics. See ITL-439. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__( self, column: str, @@ -140,8 +144,20 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: pick (including a logical type without ``pick_field``, such as ``numpy.ndarray``). """ - _, data_schema = stream.output_schema() + tag_schema, data_schema = stream.output_schema() data_columns = list(data_schema.keys()) + # Referential tags (ITL-1023): the output is a new data column, never a tag's + # value replaced in place or a name shared with a tag (section 4). + if self.out is None and self.column in tag_schema: + raise InputValidationError( + f"Pick: column {self.column!r} is a referential tag; replacing it " + "in place would change the tag's value. Give out=." + ) + if self.out is not None and self.out in tag_schema: + raise InputValidationError( + f"Pick: out column {self.out!r} is the name of a tag; a data " + "column cannot share a name with a tag it does not back." + ) if self.column not in data_columns: raise InputValidationError( @@ -380,6 +396,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: output_schema.empty_table(), tag_columns=tag_columns, data_context=stream.data_context, + reference_tags=stream_references(stream), ) return self._materialize_to_stream(out_rows) diff --git a/src/orcapod/core/sources/cached_source.py b/src/orcapod/core/sources/cached_source.py index 4e3847172..896b8c1b0 100644 --- a/src/orcapod/core/sources/cached_source.py +++ b/src/orcapod/core/sources/cached_source.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from orcapod import contexts +from orcapod.core.referential_tags import stream_references from orcapod.config import OrcapodConfig from orcapod.core.sources.base import RootSource from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -360,7 +361,10 @@ def _build_merged_stream(self) -> ArrowTableStream: tag_keys = self._source.keys()[0] # Cached rows are this source's: give them its data context, not the default. return ArrowTableStream( - all_records, tag_columns=tag_keys, data_context=self.data_context + all_records, + tag_columns=tag_keys, + data_context=self.data_context, + reference_tags=stream_references(self._source), ) def _empty_table(self) -> pa.Table: diff --git a/src/orcapod/core/sources/derived_source.py b/src/orcapod/core/sources/derived_source.py index 26c25f02a..90e93b88d 100644 --- a/src/orcapod/core/sources/derived_source.py +++ b/src/orcapod/core/sources/derived_source.py @@ -4,6 +4,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, ClassVar +from orcapod.core.referential_tags import not_supported_yet, stream_references from orcapod.core.sources.base import RootSource from orcapod.core.sources.source_ids import hash_source_id from orcapod.core.sources.stream_builder import SourceStreamBuilder @@ -232,6 +233,8 @@ def __init__( record_id_column: str | None = None, **kwargs: Any, ) -> None: + if stream_references(origin): + raise not_supported_yet("DerivedSource", "ITL-1019") if tags_as_data is None: if isinstance(origin, RootSource): raise TypeError( diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 3bc2f386d..d76b884ef 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -146,8 +146,15 @@ def save(self, path: str | Path) -> None: from orcapod.core.nodes import OperatorNode, FunctionNode from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass + from orcapod.core.referential_tags import ( + not_supported_yet, + stream_references, + ) + nodes: dict[str, Any] = {} for content_hash_str, node in self._persistent_node_map.items(): + if stream_references(node): + raise not_supported_yet("Saving a pipeline", "ITL-1019") tag_schema, data_schema = node.output_schema() try: type_converter = node.data_context.type_converter diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index ed95210d5..c5da5c92b 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -92,16 +92,24 @@ def _source( tags: Sequence[str], source_id: str, nullable: Sequence[str] = (), + referential: bool = False, ) -> ArrowTableSource: - """Build a source; every column not listed in ``nullable`` is non-nullable.""" + """Build a source; every column not listed in ``nullable`` is non-nullable. + + Its tags are plain (``tag_only_columns``), as in v0.3, unless ``referential`` + (ITL-1023): then every tag is also a data column of the same name. + """ table = pa.table(list(columns.values()), names=list(columns)) schema = pa.schema([f.with_nullable(f.name in nullable) for f in table.schema]) return ArrowTableSource( - table.cast(schema), tag_columns=list(tags), source_id=source_id + table.cast(schema), + tag_columns=list(tags), + source_id=source_id, + tag_only_columns=not referential, ) -def source_a() -> ArrowTableSource: +def source_a(referential: bool = False) -> ArrowTableSource: """Tags ``k: int16`` (non-nullable) and ``ta: int8``; many narrow data columns.""" return _source( { @@ -123,10 +131,11 @@ def source_a() -> ArrowTableSource: tags=("k", "ta"), source_id="a", nullable=("nul", "s"), + referential=referential, ) -def source_b(nested: bool = True) -> ArrowTableSource: +def source_b(nested: bool = True, referential: bool = False) -> ArrowTableSource: """Tag ``k: int16``; scalar data columns, plus list columns when ``nested``.""" columns: dict[str, Column] = { "k": pa.array([1, 2, 3], pa.int16()), @@ -138,10 +147,16 @@ def source_b(nested: bool = True) -> ArrowTableSource: if nested: columns["ll"] = pa.array([[0.5], [1.5, 2.5], []], pa.large_list(pa.float32())) columns["lnn"] = pa.array([[1], [2], [3]], NON_NULL_ITEMS) - return _source(columns, tags=("k",), source_id="b", nullable=("f32",)) + return _source( + columns, + tags=("k",), + source_id="b", + nullable=("f32",), + referential=referential, + ) -def source_c() -> ArrowTableSource: +def source_c(referential: bool = False) -> ArrowTableSource: """Tags ``k: int16`` and nullable ``tc: uint8``; a third input for the staggered path.""" return _source( { @@ -153,6 +168,7 @@ def source_c() -> ArrowTableSource: tags=("k", "tc"), source_id="c", nullable=("tc", "c8"), + referential=referential, ) @@ -209,10 +225,15 @@ def _row_key(tag: TagProtocol) -> str: def _row_schema(tag: TagProtocol, data: DataProtocol) -> pa.Schema: - """The row's Arrow schema: user and system tags, then data.""" - return arrow_utils.join_arrow_schemas( - tag.arrow_schema(columns={"system_tags": True}), data.arrow_schema() - ) + """The row's Arrow schema: user and system tags, then data. + + A referential tag (ITL-1023) is one column, named among the tags and the data: + it is listed once, among the tags. + """ + tag_schema = tag.arrow_schema(columns={"system_tags": True}) + data_schema = data.arrow_schema() + data_schema = pa.schema(f for f in data_schema if f.name not in tag_schema.names) + return arrow_utils.join_arrow_schemas(tag_schema, data_schema) def _paired(expected: list[Row], actual: list[Row]) -> list[tuple[Row, Row]]: @@ -322,6 +343,71 @@ class Case: ), # No names: every tag and data column, among them struct, logical and list ones. Case("drop_nulls_all", lambda: DropNulls(), lambda: (source_a(),)), + # Referential tags (ITL-1023): every tag of these sources is also data. + Case( + "ref_join_2", + Join, + lambda: (source_a(referential=True), source_b(referential=True)), + ), + Case( + "ref_join_2_plain_left", + Join, + lambda: (source_a(), source_b(referential=True)), + ), + Case( + "ref_join_2_plain_right", + Join, + lambda: (source_a(referential=True), source_b()), + ), + Case( + "ref_join_3", + Join, + lambda: ( + source_a(referential=True), + source_b(referential=True), + source_c(referential=True), + ), + ), + Case( + "ref_semijoin", + SemiJoin, + lambda: (source_a(referential=True), source_b(referential=True)), + ), + Case( + "ref_group_by_key", + lambda: GroupBy(["ta"]), + lambda: (source_a(referential=True),), + ), + Case( + "ref_polars_filter", + lambda: PolarsFilter(constraints={"ta": 5}), + lambda: (source_a(referential=True),), + ), + Case( + "ref_select_data_columns", + lambda: SelectDataColumns(["i8", "ta"]), + lambda: (source_a(referential=True),), + ), + Case( + "ref_drop_tag_columns", + lambda: DropTagColumns(["ta"]), + lambda: (source_a(referential=True),), + ), + Case( + "ref_drop_data_columns", + lambda: DropDataColumns(["k"]), + lambda: (source_a(referential=True),), + ), + Case( + "ref_index", + lambda: Index("l8", 0, out="first"), + lambda: (source_a(referential=True),), + ), + Case( + "ref_drop_nulls_all", + lambda: DropNulls(), + lambda: (source_a(referential=True),), + ), ] CASE = pytest.mark.parametrize("case", CASES, ids=[c.label for c in CASES]) @@ -652,6 +738,8 @@ def test_drop_nulls_without_tag_columns() -> None: PROVENANCE_XFAIL = { "select_data_columns": _DROPPED_SOURCE_INFO, "drop_data_columns": _DROPPED_SOURCE_INFO, + "ref_select_data_columns": _DROPPED_SOURCE_INFO, + "ref_drop_data_columns": _DROPPED_SOURCE_INFO, } PROVENANCE_CASES = [ pytest.param( @@ -741,8 +829,9 @@ def _with_meta(source: StreamProtocol, meta: dict[str, Column]) -> ArrowTableStr table = source.as_table(columns={"source": True, "system_tags": True}) for name, column in meta.items(): table = table.append_column(name, column) - tags = source.keys()[0] - return ArrowTableStream(table, tag_columns=tags) + tags, data = source.keys() + references = [name for name in tags if name in data] + return ArrowTableStream(table, tag_columns=tags, reference_tags=references) def _meta_columns(prefix: str, n_rows: int) -> dict[str, Column]: @@ -796,6 +885,10 @@ def make_sources() -> tuple[StreamProtocol, ...]: "select_data_columns": _SYNC_DROPS_META, "index": _SYNC_DROPS_META, "pick": _SYNC_DROPS_META, + "ref_group_by_key": _MATERIALIZE_DROPS_META, + "ref_polars_filter": _MATERIALIZE_DROPS_META, + "ref_select_data_columns": _SYNC_DROPS_META, + "ref_index": _SYNC_DROPS_META, } META_CASES = [ pytest.param( @@ -879,7 +972,10 @@ def _narrow_source(column: str, source_id: str) -> ArrowTableSource: table = pa.table( {"k": pa.array([1, 2], pa.int16()), column: pa.array([3, 4], pa.int8())} ) - return ArrowTableSource(table, tag_columns=["k"], source_id=source_id) + # Its hashes are pinned below: plain tags, as in v0.3 (ITL-1023). + return ArrowTableSource( + table, tag_columns=["k"], source_id=source_id, tag_only_columns=True + ) # Recorded from the sync path on main (ab544982) before the ITL-837 fix. The sync diff --git a/tests/test_core/referential_tags/test_function_pods.py b/tests/test_core/referential_tags/test_function_pods.py new file mode 100644 index 000000000..077a55631 --- /dev/null +++ b/tests/test_core/referential_tags/test_function_pods.py @@ -0,0 +1,125 @@ +"""Referential tags at function pods (ITL-1023). + +Specification "Orcapod v0.4 — Referential Tags" (ITL-1004), sections 5, 6, 12.4 and +tests 15.2 and 15.6. A function reads a referenced column through the ordinary data +binding. A pod's output data is the function's outputs, so every referential tag of the +input is a plain tag of the output, with the same values (developer, November 12, 2026). +An output named like a tag is an error when the pipeline is built. +""" + +from __future__ import annotations + +import warnings + +import pytest + +import orcapod as op +from orcapod.core.operators import SelectDataColumns +from orcapod.core.referential_tags import ( + ReferenceMaterializationWarning, + stream_references, +) +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import PipelineJob + +ROWS = [{"t": 10, "amp": 0.5}, {"t": 20, "amp": 0.25}] + + +def _source(**kwargs: object) -> op.sources.DictSource: + return op.sources.DictSource(ROWS, tag_columns=["t"], source_id="spikes", **kwargs) + + +def test_function_reads_a_referenced_column_by_name() -> None: + @op.function_pod(output_keys=["scaled"]) + def scale(t: int, amp: float) -> float: + return t * amp + + out = scale.pod(_source()) + assert sorted(r["scaled"] for r in out.as_table().to_pylist()) == [5.0, 5.0] + + +def test_tags_pass_through_and_become_plain_at_the_pod() -> None: + @op.function_pod(output_keys=["double"]) + def double(t: int) -> int: + return 2 * t + + out = double.pod(_source()) + assert out.keys() == (("t",), ("double",)) + assert stream_references(out) == () + rows = sorted(out.as_table().to_pylist(), key=lambda r: r["t"]) + assert rows == [{"t": 10, "double": 20}, {"t": 20, "double": 40}] + + +@pytest.mark.parametrize("tag_only", [(), True], ids=["referential", "plain"]) +def test_output_named_like_a_tag_raises_when_built(tag_only: object) -> None: + @op.function_pod(output_keys=["t"]) + def clobber(amp: float) -> int: + return 0 + + with pytest.raises(InputValidationError, match="conflicts with existing tag"): + clobber.pod(_source(tag_only_columns=tag_only)) + + +def test_ordinary_data_replacement_is_unchanged() -> None: + @op.function_pod(output_keys=["amp"]) + def halve(amp: float) -> float: + return amp / 2 + + out = halve.pod(_source()) + assert out.keys() == (("t",), ("amp",)) + + +def test_automatic_select_materializes_without_a_diagnostic() -> None: + @op.function_pod(output_keys=["half"]) + def half(amp: float) -> float: + return amp / 2 + + with warnings.catch_warnings(): + warnings.simplefilter("error", ReferenceMaterializationWarning) + out = half.pod(_source()) + # The same pipeline as with the select written by hand (ITL-807). + with pytest.warns(ReferenceMaterializationWarning): + by_hand = half.pod(SelectDataColumns(["amp"])(_source())) + assert out.content_hash() == by_hand.content_hash() + assert out.pipeline_hash() == by_hand.pipeline_hash() + + +CALLS: list[float] = [] + + +@op.function_pod(output_keys=["half"]) +def half_amp(amp: float) -> float: + """Reads only ``amp``: never the referenced ``t``.""" + CALLS.append(amp) + return amp / 2 + + +def test_function_not_reading_the_reference_keeps_its_result_identity() -> None: + """Section 12.4: retagging the source does not force the function to run again.""" + store = InMemoryArrowDatabase() + CALLS.clear() + plain_job = PipelineJob(store=store) + with plain_job: + half_amp.pod(_source(tag_only_columns=True), label="half") + plain_job.run() + assert sorted(CALLS) == [0.25, 0.5] + + # The same source with a referential tag: a new pipeline (new records), but the + # function's inputs are the same data, so every result is found in the store. + CALLS.clear() + referential_job = PipelineJob(store=store) + with referential_job: + half_amp.pod(_source(), label="half") + referential_job.run() + assert CALLS == [] + plain_rows = plain_job.nodes["half"].as_table().to_pylist() + referential_rows = referential_job.nodes["half"].as_table().to_pylist() + assert sorted(plain_rows, key=lambda r: r["t"]) == sorted( + referential_rows, key=lambda r: r["t"] + ) + # The pipelines differ: the binding is part of pipeline structure (section 12.3). + assert ( + plain_job.nodes["half"].pipeline_hash() + != referential_job.nodes["half"].pipeline_hash() + ) diff --git a/tests/test_core/referential_tags/test_model_and_sources.py b/tests/test_core/referential_tags/test_model_and_sources.py new file mode 100644 index 000000000..01e7beec2 --- /dev/null +++ b/tests/test_core/referential_tags/test_model_and_sources.py @@ -0,0 +1,121 @@ +"""Referential tags: the stream model and sources (ITL-1023). + +Specification "Orcapod v0.4 — Referential Tags" (ITL-1004), sections 4 (namespaces), +11 (sources) and 15.1 (schema tests). A referential tag ``x`` is one column whose name is +both a tag key and a data key. A source's declared tags are referential by default +(developer, November 12, 2026); ``tag_only_columns`` keeps chosen tags plain, as in v0.3. +""" + +from __future__ import annotations + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.core.referential_tags import check_namespaces, stream_references +from orcapod.core.sources import ArrowTableSource +from orcapod.core.streams import ArrowTableStream +from orcapod.errors import InputValidationError + +ROWS = [{"x": 1, "y": 0.5}, {"x": 2, "y": 0.75}] + + +def _source(**kwargs: object) -> op.sources.DictSource: + return op.sources.DictSource(ROWS, tag_columns=["x"], source_id="s", **kwargs) + + +def test_declared_tag_is_referential_by_default() -> None: + source = _source() + assert source.keys() == (("x",), ("x", "y")) + tag_schema, data_schema = source.output_schema() + assert tag_schema == {"x": int} and data_schema == {"x": int, "y": float} + assert stream_references(source) == ("x",) + + +def test_referential_column_is_stored_once_and_seen_through_both_roles() -> None: + source = _source() + table = source.as_table(columns={"source": True}) + assert table.column_names.count("x") == 1 + # As data, the column has source info like any data column. + assert "_source_x" in table.column_names + ((tag, data), _) = list(source.iter_data()) + assert tag.as_dict() == {"x": 1} + assert data.as_dict() == {"x": 1, "y": 0.5} + # Section 4.4: the same Arrow type in both roles. + assert tag.arrow_schema().field("x") == data.arrow_schema().field("x") + + +@pytest.mark.parametrize("tag_only", [["x"], "x", True]) +def test_tag_only_columns_keep_the_v03_shape(tag_only: object) -> None: + source = _source(tag_only_columns=tag_only) + assert source.keys() == (("x",), ("y",)) + assert stream_references(source) == () + assert "_source_x" not in source.as_table(columns={"source": True}).column_names + + +def test_tag_only_columns_must_be_declared_tags() -> None: + with pytest.raises(ValueError, match="tag_only_columns"): + _source(tag_only_columns=["y"]) + + +def test_referential_and_plain_sources_have_different_identities() -> None: + referential, plain = _source(), _source(tag_only_columns=True) + # The source id is the source's name: the same. + assert referential.source_id == plain.source_id + assert referential.content_hash() != plain.content_hash() + assert referential.pipeline_hash() != plain.pipeline_hash() + + +def test_arrow_table_stream_identity_has_the_references_only_when_present() -> None: + table = pa.table({"x": [1, 2], "y": [0.5, 0.75]}) + plain = ArrowTableStream(table, tag_columns=["x"]) + referential = ArrowTableStream(table, tag_columns=["x"], reference_tags=["x"]) + assert plain.keys() == (("x",), ("y",)) + assert referential.keys() == (("x",), ("x", "y")) + assert plain.content_hash() != referential.content_hash() + # Without references the identity structure is the v0.3 one. + assert plain.identity_structure()[-1] == ("x",) + + +def test_reference_must_be_a_tag_column() -> None: + table = pa.table({"x": [1], "y": [0.5]}) + with pytest.raises(ValueError, match="Reference tags"): + ArrowTableStream(table, tag_columns=["x"], reference_tags=["y"]) + + +def test_namespace_validator() -> None: + check_namespaces({"x": int}, {"x": int, "y": float}, references=["x"]) + with pytest.raises(InputValidationError, match="plain tag and a data column"): + check_namespaces({"x": int}, {"x": int}, references=[]) + with pytest.raises(InputValidationError, match="as a tag"): + check_namespaces({"x": int}, {"x": str}, references=["x"]) + + +def test_arrow_table_source_default_and_tag_only() -> None: + table = pa.table({"x": pa.array([1, 2], pa.int8()), "y": [0.5, 0.75]}) + referential = ArrowTableSource(table, tag_columns=["x"]) + plain = ArrowTableSource(table, tag_columns=["x"], tag_only_columns=True) + assert referential.keys() == (("x",), ("x", "y")) + assert plain.keys() == (("x",), ("y",)) + # The referenced column keeps its exact Arrow type (section 4.4). + ((_, data), *_) = list(referential.iter_data()) + assert data.arrow_schema().field("x").type == pa.int8() + + +def test_to_config_writes_tag_only_columns_only_for_referential_sources( + tmp_path: object, +) -> None: + from pathlib import Path + + path = Path(str(tmp_path)) / "rows.csv" + path.write_text("x,y\n1,0.5\n2,0.75\n") + plain = op.sources.CSVSource(str(path), tag_columns=["x"], tag_only_columns=True) + referential = op.sources.CSVSource(str(path), tag_columns=["x"]) + assert "tag_only_columns" not in plain.to_config() + assert referential.to_config()["tag_only_columns"] == [] + # A v0.3 config (no key) reads back with every tag plain. + assert op.sources.CSVSource.from_config(plain.to_config()).keys() == plain.keys() + assert ( + op.sources.CSVSource.from_config(referential.to_config()).keys() + == referential.keys() + ) diff --git a/tests/test_core/referential_tags/test_operators.py b/tests/test_core/referential_tags/test_operators.py new file mode 100644 index 000000000..b8d6c2fdd --- /dev/null +++ b/tests/test_core/referential_tags/test_operators.py @@ -0,0 +1,326 @@ +"""Referential tags through operators (ITL-1023). + +Specification "Orcapod v0.4 — Referential Tags" (ITL-1004): ``Join`` section 9 and +tests 15.4; ``GroupBy`` section 8 and tests 15.3; select and drop sections 7 and 10 and +tests 15.5; row-preserving operators section 10. Operators the slice does not cover raise +"not supported with referential tags yet" instead of giving a wrong answer. + +Every operator is driven through its public call, ``op(stream)``, and read through the +stream's public methods. +""" + +from __future__ import annotations + +import warnings + +import pytest + +import orcapod as op +from orcapod.core import referential_tags +from orcapod.core.operators import ( + Batch, + DropDataColumns, + DropNulls, + DropTagColumns, + GroupBy, + Index, + Join, + MapData, + MapTags, + MergeJoin, + Pick, + PolarsFilter, + SelectDataColumns, + SelectTagColumns, + SemiJoin, +) +from orcapod.core.referential_tags import ( + ReferenceMaterializationWarning, + stream_references, +) +from orcapod.errors import InputValidationError + + +def _ref(rows: list[dict[str, object]], tags: list[str], sid: str, **kw: object): + return op.sources.DictSource(rows, tag_columns=tags, source_id=sid, **kw) + + +def _a(**kw: object): + return _ref([{"x": 1, "a": 0.5}, {"x": 2, "a": 0.6}], ["x"], "A", **kw) + + +def _b(**kw: object): + return _ref([{"x": 1, "b": 10}, {"x": 3, "b": 30}], ["x"], "B", **kw) + + +def _rows(stream) -> list[dict[str, object]]: + return sorted(stream.as_table().to_pylist(), key=lambda r: repr(sorted(r.items()))) + + +# ── Join (section 9) ──────────────────────────────────────────────────────── + + +def test_join_plain_with_plain_is_unchanged() -> None: + joined = Join()(_a(tag_only_columns=True), _b(tag_only_columns=True)) + assert joined.keys() == (("x",), ("a", "b")) or joined.keys() == ( + ("x",), + ("b", "a"), + ) + assert stream_references(joined) == () + assert _rows(joined) == [{"x": 1, "a": 0.5, "b": 10}] + + +@pytest.mark.parametrize("plain_side", ["a", "b"]) +def test_join_plain_with_referential_gives_a_referential_tag(plain_side: str) -> None: + a = _a(tag_only_columns=True) if plain_side == "a" else _a() + b = _b(tag_only_columns=True) if plain_side == "b" else _b() + joined = Join()(a, b) + tags, data = joined.keys() + assert tags == ("x",) and set(data) == {"x", "a", "b"} + assert stream_references(joined) == ("x",) + table = joined.as_table(columns={"source": True}) + assert table.column_names.count("x") == 1 + # The data column comes from the referential side, with its source info. + side = "B" if plain_side == "a" else "A" + assert table.column("_source_x").to_pylist() == [f"{side}::row_0::x"] + + +def test_join_referential_with_referential_merges_one_data_column() -> None: + joined = Join()(_a(), _b()) + assert stream_references(joined) == ("x",) + table = joined.as_table(columns={"source": True}) + assert table.column_names.count("x") == 1 + assert [{k: r[k] for k in ("x", "a", "b")} for r in table.to_pylist()] == [ + {"x": 1, "a": 0.5, "b": 10} + ] + # Section 9.4: both inputs' provenance is kept, in canonical input order. + (tokens,) = table.column("_source_x").to_pylist() + assert sorted(tokens) == ["A::row_0::x", "B::row_0::x"] + + +def test_join_is_commutative_with_references() -> None: + assert ( + Join()(_a(), _b()).as_table(columns={"source": True}).to_pylist() + == Join()(_b(), _a()).as_table(columns={"source": True}).to_pylist() + ) + + +def test_join_rejects_other_data_collisions() -> None: + other = _ref([{"k": 1, "a": 0.9}], ["k"], "C") # data ``a`` again, not a reference + with pytest.raises(InputValidationError, match="overlapping"): + Join()(_a(), other) + # Data-only ``x`` on one side, referential ``x`` on the other: not equated by a tag. + data_only = _ref([{"k": 1, "x": 1}], ["k"], "D") + with pytest.raises(InputValidationError): + Join()(_a(), data_only) + + +def test_join_rejects_a_plain_tag_named_like_a_data_column() -> None: + plain_tag = _ref([{"x": 1, "c": 2}], ["x"], "P", tag_only_columns=True) + data_only = _ref([{"k": 1, "x": 1}], ["k"], "D") + with pytest.raises(InputValidationError): + Join()(plain_tag, data_only) + + +def test_join_rejects_disagreeing_reference_types() -> None: + as_text = _ref([{"x": "1", "b": 10}], ["x"], "T") + with pytest.raises(InputValidationError): + Join()(_a(), as_text) + + +def test_join_many_to_many_keeps_row_alignment() -> None: + left = _ref( + [{"x": 1, "u": 1, "a": 0.1}, {"x": 1, "u": 2, "a": 0.2}], ["x", "u"], "L" + ) + right = _ref([{"x": 1, "v": 1, "b": 1}, {"x": 1, "v": 2, "b": 2}], ["x", "v"], "R") + joined = Join()(left, right) + rows = joined.as_table().to_pylist() + assert len(rows) == 4 + assert all(r["x"] == 1 for r in rows) + for tag, data in joined.iter_data(): + assert tag["x"] == data["x"] + + +# ── GroupBy (section 8) ───────────────────────────────────────────────────── + + +GROUP_ROWS = [ + {"s": "s1", "x": 20, "y": 5}, + {"s": "s1", "x": 10, "y": 4}, + {"s": "s2", "x": 5, "y": 1}, +] + + +def test_group_by_referential_key_stays_scalar_in_both_roles() -> None: + grouped = GroupBy(["x"])(_ref(GROUP_ROWS, ["s", "x"], "G")) + tag_schema, data_schema = grouped.output_schema() + assert tag_schema["x"] is int and data_schema["x"] is int + assert stream_references(grouped) == ("s", "x") + for tag, data in grouped.iter_data(): + assert tag["x"] == data["x"] and isinstance(data["x"], int) + + +def test_group_by_referential_non_key_is_the_same_list_in_both_roles() -> None: + grouped = GroupBy(["s"])(_ref(GROUP_ROWS, ["s", "x"], "G")) + tag_schema, data_schema = grouped.output_schema() + assert tag_schema["x"] == list[int] and data_schema["x"] == list[int] + rows = {tag["s"]: (tag, data) for tag, data in grouped.iter_data()} + tag, data = rows["s1"] + # Section 8.3: the same member order in both roles, and the data follows it. + assert tag["x"] == data["x"] == [10, 20] + assert data["y"] == [4, 5] + assert grouped.as_table().column_names.count("x") == 1 + + +def test_group_by_prediction_matches_the_result() -> None: + source = _ref(GROUP_ROWS, ["s", "x"], "G") + for by in (["s"], ["x"]): + group_by = GroupBy(by) + # Schema prediction checked against static_process (CLAUDE.md standing rule). + assert ( + group_by.output_schema(source) + == group_by.static_process(source).output_schema() + ) + + +# ── select and drop (sections 7 and 10) ───────────────────────────────────── + + +def test_selecting_away_the_referenced_data_leaves_a_plain_tag() -> None: + source = _ref([{"x": 1, "y": 2}], ["x"], "S") + with pytest.warns(ReferenceMaterializationWarning): + selected = SelectDataColumns(["y"])(source) + assert selected.keys() == (("x",), ("y",)) + assert stream_references(selected) == () + assert selected.as_table().to_pylist() == [{"x": 1, "y": 2}] + assert "_source_x" not in selected.as_table(columns={"source": True}).column_names + + +def test_dropping_the_referenced_data_leaves_a_plain_tag() -> None: + source = _ref([{"x": 1, "y": 2}], ["x"], "S") + with pytest.warns(ReferenceMaterializationWarning): + dropped = DropDataColumns(["x"])(source) + assert dropped.keys() == (("x",), ("y",)) + + +def test_silent_materialization_changes_nothing_but_the_warning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = _ref([{"x": 1, "y": 2}], ["x"], "S") + with pytest.warns(ReferenceMaterializationWarning): + warned = SelectDataColumns(["y"])(source) + monkeypatch.setattr(referential_tags, "reference_materialization", "silent") + with warnings.catch_warnings(): + warnings.simplefilter("error") + silent = SelectDataColumns(["y"])(source) + assert silent.content_hash() == warned.content_hash() + assert silent.pipeline_hash() == warned.pipeline_hash() + assert silent.as_table().to_pylist() == warned.as_table().to_pylist() + + +@pytest.mark.parametrize( + "drop", [lambda: DropTagColumns(["x"]), lambda: SelectTagColumns(["k"])] +) +def test_removing_a_referential_tag_keeps_its_data(drop) -> None: + source = _ref([{"k": 1, "x": 1, "y": 2}], ["k", "x"], "S", tag_only_columns=["k"]) + result = drop()(source) + assert result.keys() == (("k",), ("x", "y")) + assert stream_references(result) == () + ((tag, data),) = list(result.iter_data()) + assert tag.as_dict() == {"k": 1} + assert data.as_dict() == {"x": 1, "y": 2} + assert data.source_info()["x"] == "S::row_0::x" + assert data.source_info()["x"] == "S::row_0::x" + + +# ── row-preserving operators (section 10) ─────────────────────────────────── + + +def _rows_source(): + return _ref( + [ + {"x": 1, "y": None, "l": [1, 2], "d": {"q": 1}}, + {"x": 2, "y": 3, "l": [3], "d": {"q": 2}}, + ], + ["x"], + "R", + ) + + +@pytest.mark.parametrize( + "make_op", + [ + lambda: PolarsFilter(constraints={"x": 2}), + lambda: DropNulls(), + lambda: DropNulls(["y"]), + lambda: Index("l", 0, out="first"), + lambda: Pick("d", "q", out="q"), + ], + ids=["filter", "drop_nulls_all", "drop_nulls", "index", "pick"], +) +def test_row_preserving_operators_keep_references(make_op) -> None: + result = make_op()(_rows_source()) + assert stream_references(result) == ("x",) + assert result.as_table().column_names.count("x") == 1 + for tag, data in result.iter_data(): + assert tag["x"] == data["x"] + + +def test_semijoin_keeps_the_left_references() -> None: + result = SemiJoin()(_a(), _b()) + assert stream_references(result) == ("x",) + assert [r["x"] for r in result.as_table().to_pylist()] == [1] + + +@pytest.mark.parametrize("make_op", [lambda: Index("x", 0), lambda: Pick("x", "q")]) +def test_in_place_extraction_of_a_referenced_column_raises(make_op) -> None: + source = _ref([{"x": [1], "y": 2}], ["x"], "R") + with pytest.raises(InputValidationError, match="referential tag"): + make_op()(source) + + +def test_extraction_named_like_a_tag_raises() -> None: + source = _ref([{"k": 1, "l": [1]}], ["k"], "R") + with pytest.raises(InputValidationError, match="tag"): + Index("l", 0, out="k")(source) + + +# ── not supported yet: raise, never a wrong answer ────────────────────────── + + +@pytest.mark.parametrize( + "call, issue", + [ + (lambda: Batch(batch_size=2)(_a()), "ITL-1017"), + (lambda: MergeJoin()(_a(), _b()), "ITL-1018"), + (lambda: MapTags({"x": "z"})(_a()), "ITL-1018"), + (lambda: MapData({"x": "z"})(_a()), "ITL-1018"), + (lambda: MapData({"a": "z"}, drop_unmapped=True)(_a()), "ITL-1018"), + ], + ids=["batch", "merge_join", "map_tags", "map_data", "map_data_drop_unmapped"], +) +def test_uncovered_operators_raise(call, issue: str) -> None: + with pytest.raises(NotImplementedError, match=issue): + call() + + +def test_renames_that_leave_references_alone_work() -> None: + renamed = MapData({"a": "z"})(_a()) + assert renamed.keys() == (("x",), ("x", "z")) + assert stream_references(renamed) == ("x",) + retagged = MapTags({"k": "kk"})( + _ref([{"k": 1, "x": 1, "y": 2}], ["k", "x"], "M", tag_only_columns=["k"]) + ) + assert retagged.keys() == (("kk", "x"), ("x", "y")) + assert stream_references(retagged) == ("x",) + + +def test_uncovered_operators_still_work_on_plain_tags() -> None: + plain = _a(tag_only_columns=True) + assert Batch(batch_size=2)(plain).keys() == (("x",), ("a",)) + assert MapTags({"x": "z"})(plain).keys() == (("z",), ("a",)) + + +def test_derived_source_over_a_referential_stream_raises() -> None: + with pytest.raises(NotImplementedError, match="ITL-1019"): + op.sources.DerivedSource(origin=_a(), tags_as_data={"x": "x_copy"}) From c42eece82e06a6bd3e4f4a1c480b9622a0ac1478 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 21:24:51 -0800 Subject: [PATCH 04/72] feat(referential_tags): renames per developer decision; EDI test on the real timestamp shape (ITL-1023) MapTags renames a referential tag in both roles (the reference survives); tag_only=True renames the tag only and leaves a plain tag. MapData is always a data-only move: the old referential tag stays as a plain tag. The EDI test now has a key column 'timestamp' in both series, renamed with map_tags. Refs ITL-1023 Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/mappers.py | 186 ++++++++++-------- src/orcapod/core/streams/base.py | 5 +- .../operators/test_async_sync_equivalence.py | 15 ++ .../referential_tags/test_edi_end_to_end.py | 53 +++-- .../referential_tags/test_operators.py | 62 +++++- 5 files changed, 223 insertions(+), 98 deletions(-) diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index b62d5ca75..63d816735 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -3,7 +3,7 @@ from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator -from orcapod.core.referential_tags import not_supported_yet, stream_references +from orcapod.core.referential_tags import stream_references from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol @@ -22,8 +22,15 @@ class MapData(UnaryOperator): Operator that maps data in a stream using a user-defined function. The function is applied to each data in the stream, and the resulting data are returned as a new stream. + + Renaming data is always a data-only move (developer, November 12, 2026, + ITL-1023): renaming the data column of a referential tag ``x`` gives the new + data column and leaves ``x`` a plain tag with the same values. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__( self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs ): @@ -31,38 +38,6 @@ def __init__( self.drop_unmapped = drop_unmapped super().__init__(**kwargs) - # Referential tags (ITL-1023): handled when the renaming leaves them alone. - REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None - - def check_reference_support(self, *streams: StreamProtocol) -> None: - """Raise if the renaming would touch a referential tag (not supported yet). - - Renaming a referential tag or its data column, or dropping it as unmapped, - is a rule the specification leaves open (section 10.1, ITL-1018). A renaming - that leaves every referential tag alone is supported. - - Args: - *streams: The input stream. - - Raises: - NotImplementedError: If a referential tag is renamed, renamed to, or - dropped as unmapped. - """ - for stream in streams: - references = stream_references(stream) - touched = [ - name - for name in references - if name in self.name_map - or name in self.name_map.values() - or self.drop_unmapped - ] - if touched: - raise not_supported_yet( - f"{type(self).__name__} renaming referential tag(s) {touched}", - "ITL-1018", - ) - def to_config(self) -> dict[str, Any]: """Serialize this MapData operator to a config dict. @@ -89,11 +64,16 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: columns={"source": True, "system_tags": True, "sort_by_tags": False} ) - name_map = { - c: c - for c in table.column_names - if c not in data_columns and not c.startswith("") - } + # A referenced column keeps its tag under its own name, now plain; the + # renamed data column is a copy beside it (a data-only move). + references = stream_references(stream) + moved = {c: self.name_map[c] for c in references if c in self.name_map} + for old, new in moved.items(): + index = table.schema.get_field_index(old) + table = table.add_column( + index + 1, table.schema.field(old).with_name(new), table.column(old) + ) + name_map = { tc: tc for tc in table.column_names @@ -101,7 +81,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: } # no renaming on tag columns for c in data_columns: if c in self.name_map: - name_map[c] = self.name_map[c] + name_map[c] = c if c in moved else self.name_map[c] name_map[f"{constants.SOURCE_PREFIX}{c}"] = ( f"{constants.SOURCE_PREFIX}{self.name_map[c]}" ) @@ -111,12 +91,24 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: renamed_table = table.rename_columns(name_map) if self.drop_unmapped and unmapped_columns: - renamed_table = renamed_table.drop_columns(list(unmapped_columns)) + # An unmapped referential tag loses its data role and stays a plain tag. + renamed_table = renamed_table.drop_columns( + [c for c in unmapped_columns if c not in references] + + [ + f"{constants.SOURCE_PREFIX}{c}" + for c in unmapped_columns + if c in references + ] + ) return ArrowTableStream( renamed_table, tag_columns=tag_columns, - reference_tags=stream_references(stream), + reference_tags=[ + c + for c in references + if c not in moved and not (self.drop_unmapped and c in unmapped_columns) + ], ) def validate_unary_input(self, stream: StreamProtocol) -> None: @@ -210,47 +202,35 @@ class MapTags(UnaryOperator): Operator that maps tags in a stream using a user-defined function. The function is applied to each tag in the stream, and the resulting tags are returned as a new stream. + + Referential tags (developer, November 12, 2026, ITL-1023): renaming a + referential tag renames its one column in both roles, so the reference + survives under the new name. With ``tag_only=True`` only the tag is renamed: + the data column keeps its name and the renamed tag is a plain tag with the + same values. A plain tag is renamed as before. A referential tag dropped as + unmapped keeps its data column. + + Args: + name_map: Old tag name to new tag name. + drop_unmapped: Drop the tags ``name_map`` does not name. + tag_only: Rename referential tags in the tag role only. """ + # Referential tags (ITL-1023): handled. + REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None + def __init__( - self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs + self, + name_map: Mapping[str, str], + drop_unmapped: bool = False, + tag_only: bool = False, + **kwargs, ): self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped + self.tag_only = tag_only super().__init__(**kwargs) - # Referential tags (ITL-1023): handled when the renaming leaves them alone. - REFERENTIAL_TAGS_ISSUE: ClassVar[str | None] = None - - def check_reference_support(self, *streams: StreamProtocol) -> None: - """Raise if the renaming would touch a referential tag (not supported yet). - - Renaming a referential tag or its data column, or dropping it as unmapped, - is a rule the specification leaves open (section 10.1, ITL-1018). A renaming - that leaves every referential tag alone is supported. - - Args: - *streams: The input stream. - - Raises: - NotImplementedError: If a referential tag is renamed, renamed to, or - dropped as unmapped. - """ - for stream in streams: - references = stream_references(stream) - touched = [ - name - for name in references - if name in self.name_map - or name in self.name_map.values() - or self.drop_unmapped - ] - if touched: - raise not_supported_yet( - f"{type(self).__name__} renaming referential tag(s) {touched}", - "ITL-1018", - ) - def to_config(self) -> dict[str, Any]: """Serialize this MapTags operator to a config dict. @@ -263,11 +243,14 @@ def to_config(self) -> dict[str, Any]: "name_map": dict(self.name_map), "drop_unmapped": self.drop_unmapped, } + if self.tag_only: + config["config"]["tag_only"] = True return config def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: tag_columns, data_columns = stream.keys() missing_tags = set(tag_columns) - set(self.name_map.keys()) + references = stream_references(stream) if not any(n in tag_columns for n in self.name_map): # nothing to rename in the tags, return stream as is @@ -283,19 +266,46 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: if tc in self.name_map or not self.drop_unmapped } # rename the tag as necessary new_tag_columns = list(name_map.values()) + renamed_refs = [c for c in references if c in self.name_map] + if self.tag_only: + # The renamed tag is a plain copy beside the data column, which keeps + # its name and its source info. + for old in renamed_refs: + index = table.schema.get_field_index(old) + new = self.name_map[old] + table = table.add_column( + index + 1, table.schema.field(old).with_name(new), table.column(old) + ) + name_map[new] = new + name_map[old] = old + else: + # One column in both roles: its data source info follows the name. + for old in renamed_refs: + name_map[f"{constants.SOURCE_PREFIX}{old}"] = ( + f"{constants.SOURCE_PREFIX}{self.name_map[old]}" + ) for c in data_columns: - name_map[c] = c # no renaming on data columns + name_map.setdefault(c, c) # no renaming on other data columns renamed_table = table.rename_columns(name_map) if missing_tags and self.drop_unmapped: - # drop any tags that are not in the name map - renamed_table = renamed_table.drop_columns(list(missing_tags)) - + # drop any tags that are not in the name map; a referential one keeps + # its column as data. + renamed_table = renamed_table.drop_columns( + [c for c in missing_tags if c not in references] + ) + + if self.tag_only: + new_references = [c for c in references if c not in self.name_map] + else: + new_references = [self.name_map.get(c, c) for c in references] + if self.drop_unmapped: + new_references = [c for c in new_references if c in new_tag_columns] return ArrowTableStream( renamed_table, tag_columns=new_tag_columns, - reference_tags=stream_references(stream), + reference_tags=new_references, ) def validate_unary_input(self, stream: StreamProtocol) -> None: @@ -339,6 +349,15 @@ def unary_output_schema( for k, v in tag_schema.items() if k in self.name_map or not self.drop_unmapped } + if not self.tag_only: + # A referential tag's data column is renamed with it (one column). + references = set(stream_references(stream)) + data_schema = Schema( + { + self.name_map.get(k, k) if k in references else k: v + for k, v in data_schema.items() + } + ) return Schema(new_tag_schema), data_schema @@ -375,13 +394,20 @@ async def async_execute( new_tag = tag.rename(rename_map) if unmapped: new_tag = new_tag.drop(*unmapped) + # A referential tag's data column is renamed with it, unless + # only the tag is renamed (ITL-1023). + data_keys = data.keys() + data_map = {k: v for k, v in rename_map.items() if k in data_keys} + if data_map and not self.tag_only: + data = data.rename(data_map) await output.send((new_tag, data)) finally: await output.close() def identity_structure(self) -> Any: + # ``tag_only`` enters only when set, so a plain renaming keeps its identity. return ( self.__class__.__name__, self.name_map, self.drop_unmapped, - ) + ) + ((("tag_only", True),) if self.tag_only else ()) diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index af13d5540..b201d35e1 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -171,14 +171,17 @@ def map_tags( name_map: Mapping[str, str], drop_unmapped: bool = True, label: str | None = None, + tag_only: bool = False, ) -> StreamBase: """ Maps the tags in this stream according to the provided name_map. If drop_unmapped is True, any tags that are not in the name_map will be dropped. + A referential tag is renamed in both roles unless tag_only is True; see + ``MapTags``. """ from orcapod.core.operators import MapTags - return MapTags(name_map, drop_unmapped)(self, label=label) + return MapTags(name_map, drop_unmapped, tag_only=tag_only)(self, label=label) def map_data( self, diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index c5da5c92b..e9633cb0b 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -408,6 +408,21 @@ class Case: lambda: DropNulls(), lambda: (source_a(referential=True),), ), + Case( + "ref_map_tags", + lambda: MapTags({"ta": "tb"}), + lambda: (source_a(referential=True),), + ), + Case( + "ref_map_tags_tag_only", + lambda: MapTags({"ta": "tb"}, tag_only=True), + lambda: (source_a(referential=True),), + ), + Case( + "ref_map_data", + lambda: MapData({"ta": "tb"}), + lambda: (source_a(referential=True),), + ), ] CASE = pytest.mark.parametrize("case", CASES, ids=[c.label for c in CASES]) diff --git a/tests/test_core/referential_tags/test_edi_end_to_end.py b/tests/test_core/referential_tags/test_edi_end_to_end.py index 8161f5344..dc4675d82 100644 --- a/tests/test_core/referential_tags/test_edi_end_to_end.py +++ b/tests/test_core/referential_tags/test_edi_end_to_end.py @@ -1,9 +1,11 @@ """An EDI-shaped pipeline on referential tags, end to end (ITL-1023). Specification "Orcapod v0.4 — Referential Tags" (ITL-1004), section 15.7: two time series -(spikes, gaze) share five key columns and each has its own timestamp; a third table is keyed -on a subset of the keys. Both time series are grouped by the shared keys, the three are -joined, and a function reads the timestamp lists and a key column by name. A source's +(spikes, gaze) share five key columns and each has a key column ``timestamp`` (as the real +EDI tables do); a third table is keyed on a subset of the keys. Both time series are grouped +by the shared keys, each ``timestamp`` is renamed in both roles with ``map_tags`` (developer, +November 12, 2026) so the join does not equate them, the three are joined, and a function +reads the timestamp lists and a key column by name. A source's declared tags are referential by default (developer, November 12, 2026), so the keys and timestamps are data as well as tags, with no ``tags_as_data`` copy and no hand-written drop before the join. The third table keeps its tags plain (``tag_only_columns``), so the join @@ -17,11 +19,13 @@ import warnings from collections.abc import Callable +from typing import Any from pathlib import Path import pytest import orcapod as op +from orcapod.core.operators import Join from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.pipeline import ( AsyncPipelineOrchestrator, @@ -39,7 +43,7 @@ "session": "s1", "probe": "p0", "unit": unit, - "spike_time": t, + "timestamp": t, "amplitude": 0.5 + unit, } for unit in (1, 2) @@ -53,7 +57,7 @@ "session": "s1", "probe": "p0", "unit": unit, - "gaze_time": t, + "timestamp": t, "gaze_x": 0.1 * t, } for unit in (1, 2) @@ -72,13 +76,13 @@ def align(spike_time: list[int], gaze_time: list[int], unit: int) -> str: return f"unit {unit}: spikes {spike_time} gaze {gaze_time}" -def _build(job: PipelineJob) -> None: - """Record the EDI-shaped pipeline in ``job``.""" +def _streams() -> tuple[Any, Any, Any]: + """Return the grouped spikes, the grouped gaze and the trials table.""" spikes = op.sources.DictSource( - SPIKES, tag_columns=[*KEYS, "spike_time"], source_id="spikes" + SPIKES, tag_columns=[*KEYS, "timestamp"], source_id="spikes" ) gaze = op.sources.DictSource( - GAZE, tag_columns=[*KEYS, "gaze_time"], source_id="gaze" + GAZE, tag_columns=[*KEYS, "timestamp"], source_id="gaze" ) trials = op.sources.DictSource( TRIALS, @@ -86,11 +90,36 @@ def _build(job: PipelineJob) -> None: tag_only_columns=["lab", "subject"], source_id="trials", ) + # A referential tag renamed with map_tags is renamed in both roles. + spikes_by_unit = spikes.group_by(KEYS).map_tags( + {"timestamp": "spike_time"}, drop_unmapped=False + ) + gaze_by_unit = gaze.group_by(KEYS).map_tags( + {"timestamp": "gaze_time"}, drop_unmapped=False + ) + return spikes_by_unit, gaze_by_unit, trials + + +def _build(job: PipelineJob) -> None: + """Record the EDI-shaped pipeline in ``job``.""" with job: - spikes_by_unit = spikes.group_by(KEYS) - gaze_by_unit = gaze.group_by(KEYS) # Automatic join and automatic select of the pod call (ITL-807). - align.pod(spikes_by_unit, gaze_by_unit, trials, label="align") + align.pod(*_streams(), label="align") + + +def test_the_join_merges_the_shared_keys_into_one_data_column_each() -> None: + joined = Join()(*_streams()) + tags, data = joined.keys() + # Referential with referential (the five keys) and plain with referential + # (``lab`` and ``subject`` of the trials table): one data column each. + for key in KEYS: + assert key in tags and data.count(key) == 1 + assert {"spike_time", "gaze_time"} <= set(tags) & set(data) + assert joined.as_table().column_names.count("unit") == 1 + rows = {tag["unit"]: data for tag, data in joined.iter_data()} + assert rows[1]["spike_time"] == [10, 20, 30] + assert rows[1]["gaze_time"] == [2, 12] + assert rows[2]["condition"] == "dark" def _in_memory(_: Path) -> ArrowDatabaseProtocol: diff --git a/tests/test_core/referential_tags/test_operators.py b/tests/test_core/referential_tags/test_operators.py index b8d6c2fdd..8ae1e264f 100644 --- a/tests/test_core/referential_tags/test_operators.py +++ b/tests/test_core/referential_tags/test_operators.py @@ -293,17 +293,17 @@ def test_extraction_named_like_a_tag_raises() -> None: [ (lambda: Batch(batch_size=2)(_a()), "ITL-1017"), (lambda: MergeJoin()(_a(), _b()), "ITL-1018"), - (lambda: MapTags({"x": "z"})(_a()), "ITL-1018"), - (lambda: MapData({"x": "z"})(_a()), "ITL-1018"), - (lambda: MapData({"a": "z"}, drop_unmapped=True)(_a()), "ITL-1018"), ], - ids=["batch", "merge_join", "map_tags", "map_data", "map_data_drop_unmapped"], + ids=["batch", "merge_join"], ) def test_uncovered_operators_raise(call, issue: str) -> None: with pytest.raises(NotImplementedError, match=issue): call() +# ── renames (developer decision, November 12, 2026) ───────────────────────── + + def test_renames_that_leave_references_alone_work() -> None: renamed = MapData({"a": "z"})(_a()) assert renamed.keys() == (("x",), ("x", "z")) @@ -315,10 +315,62 @@ def test_renames_that_leave_references_alone_work() -> None: assert stream_references(retagged) == ("x",) +def test_map_tags_renames_a_referential_tag_in_both_roles() -> None: + renamed = MapTags({"x": "z"})(_a()) + assert renamed.keys() == (("z",), ("z", "a")) + assert stream_references(renamed) == ("z",) + table = renamed.as_table(columns={"source": True}) + assert "x" not in table.column_names and table.column_names.count("z") == 1 + assert table.column("_source_z").to_pylist() == ["A::row_0::x", "A::row_1::x"] + # The prediction agrees with the result (checked against static_process). + op_ = MapTags({"x": "z"}) + assert op_.output_schema(_a()) == op_.static_process(_a()).output_schema() + + +def test_map_tags_tag_only_leaves_a_plain_tag_and_the_data_column() -> None: + renamed = MapTags({"x": "z"}, tag_only=True)(_a()) + assert renamed.keys() == (("z",), ("x", "a")) + assert stream_references(renamed) == () + for tag, data in renamed.iter_data(): + assert tag["z"] == data["x"] + # ``tag_only`` enters the identity only when set. + assert ( + MapTags({"x": "z"}).content_hash() + != MapTags({"x": "z"}, tag_only=True).content_hash() + ) + + +def test_map_data_is_a_data_only_move() -> None: + renamed = MapData({"x": "z"})(_a()) + assert renamed.keys() == (("x",), ("z", "a")) + assert stream_references(renamed) == () + for tag, data in renamed.iter_data(): + assert tag["x"] == data["z"] + assert renamed.as_table(columns={"source": True}).column( + "_source_z" + ).to_pylist() == ["A::row_0::x", "A::row_1::x"] + + +def test_map_data_dropping_an_unmapped_reference_leaves_a_plain_tag() -> None: + renamed = MapData({"a": "z"}, drop_unmapped=True)(_a()) + assert renamed.keys() == (("x",), ("z",)) + + +def test_map_tags_dropping_an_unmapped_reference_keeps_its_data() -> None: + source = _ref([{"k": 1, "x": 1, "y": 2}], ["k", "x"], "M") + renamed = MapTags({"k": "kk"}, drop_unmapped=True)(source) + assert renamed.keys() == (("kk",), ("kk", "x", "y")) + assert stream_references(renamed) == ("kk",) + + +def test_map_tags_on_plain_tags_is_unchanged() -> None: + plain = _a(tag_only_columns=True) + assert MapTags({"x": "z"})(plain).keys() == (("z",), ("a",)) + + def test_uncovered_operators_still_work_on_plain_tags() -> None: plain = _a(tag_only_columns=True) assert Batch(batch_size=2)(plain).keys() == (("x",), ("a",)) - assert MapTags({"x": "z"})(plain).keys() == (("z",), ("a",)) def test_derived_source_over_a_referential_stream_raises() -> None: From 853da828555cbe3c189524ebc254941df596edd0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 21:46:00 -0800 Subject: [PATCH 05/72] test(referential_tags): existing tests pin plain tags where they assert v0.3 shapes and hashes (ITL-1023) Existing tests built sources with tag columns and asserted v0.3 schemas, hashes or behaviour of operators that do not take references yet; they now pass tag_only_columns=True, which is exactly the v0.3 source. Doc pages that explain the v0.3 tag/data split do the same; a new concepts page documents referential tags. The materialization warning is reported once per operator (a job validates operators again). SQLite and PostgreSQL sources forward tag_only_columns. Adds the changelog fragment and design issue RT1. Refs ITL-1023 Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1023.md | 29 +++ design_issues/ITL-1023-RT1.md | 8 + docs/concepts/function-pods.md | 28 +-- docs/concepts/operators.md | 24 +-- docs/concepts/referential-tags.md | 107 ++++++++++ docs/concepts/sources.md | 6 +- docs/concepts/streams.md | 8 +- docs/getting-started.md | 4 +- docs/index.md | 2 +- docs/reference/hashing.md | 10 +- examples/save_and_load_pipelines.py | 2 + mkdocs.yml | 1 + .../core/operators/column_selection.py | 4 +- src/orcapod/core/operators/join.py | 18 +- src/orcapod/core/referential_tags.py | 38 +++- .../core/sources/postgresql_table_source.py | 7 + .../core/sources/sqlite_table_source.py | 7 + .../core/streams/arrow_table_stream.py | 2 +- tests/test_arrays/test_array_operators.py | 7 +- .../test_arrays/test_array_pipeline_stores.py | 4 + tests/test_arrays/test_array_value_format.py | 74 +++++-- .../test_cached_function_pod_job.py | 12 +- .../test_function_node_caching.py | 6 +- .../test_function_pod_auto_select.py | 37 +++- .../function_pod/test_function_pod_ctx_arg.py | 1 + .../test_node_config_on_job_node.py | 4 +- .../test_node_content_hash_redundancy.py | 16 +- .../function_pod/test_output_conformance.py | 6 +- .../test_polars_dataframe_values.py | 8 +- .../test_function_job_node_async_execute.py | 4 +- .../nodes/test_function_node_empty_data.py | 16 +- .../nodes/test_function_node_fetch_joined.py | 8 +- .../nodes/test_function_node_get_cached.py | 8 +- .../nodes/test_function_node_iteration.py | 2 +- tests/test_core/nodes/test_node_execute.py | 12 +- tests/test_core/nodes/test_source_node.py | 40 +++- .../operators/test_aggregated_list_types.py | 2 +- .../operators/test_chained_aggregation.py | 4 +- tests/test_core/operators/test_drop_nulls.py | 6 +- .../test_core/operators/test_empty_inputs.py | 2 +- tests/test_core/operators/test_group_by.py | 40 +++- .../operators/test_index_pick_arrow_types.py | 2 +- .../operators/test_join_tag_order.py | 7 +- .../operators/test_materialize_source_info.py | 9 +- tests/test_core/operators/test_merge_join.py | 14 ++ .../operators/test_merge_join_arrow_types.py | 8 +- tests/test_core/operators/test_operators.py | 39 +++- .../test_side_effect_function_pod.py | 5 +- .../side_effect_pod/test_side_effect_pod.py | 1 + .../test_core/sources/test_db_table_source.py | 67 ++++-- .../test_derived_source_tags_as_data.py | 33 ++- .../sources/test_postgresql_table_source.py | 83 ++++---- .../test_source_protocol_conformance.py | 12 +- .../sources/test_spiraldb_table_source.py | 101 ++++++--- .../test_spiraldb_table_source_fake.py | 68 ++++-- .../test_spiraldb_table_source_streaming.py | 197 ++++++++++++------ ...test_spiraldb_table_source_tags_as_data.py | 3 +- .../sources/test_sqlite_table_source.py | 97 +++++---- .../test_dynamic_pod_stream_staleness.py | 4 +- .../streams/test_head_and_preview.py | 29 ++- tests/test_core/test_caching_integration.py | 130 ++++++++---- tests/test_core/test_newcomer_papercuts.py | 15 +- tests/test_core/test_table_scope.py | 6 +- .../test_delta_arrow_type_roundtrip.py | 12 +- tests/test_hashing/test_dict_order_hash.py | 15 +- tests/test_hashing/test_hash_label_rule.py | 2 + .../test_polars_dataframe_hashing.py | 14 +- .../test_hashing/test_wrong_hash_findings.py | 22 +- .../loaded_job/save_job_script.py | 12 +- .../loaded_job/test_loaded_job_readback.py | 27 ++- .../loaded_job/test_loaded_logical_schemas.py | 14 +- tests/test_pipeline/test_aggregation_job.py | 4 + tests/test_pipeline/test_dag.py | 2 + tests/test_pipeline/test_drop_nulls_job.py | 4 +- .../test_pipeline/test_hash_step_combined.py | 14 +- .../test_invocation_transitions.py | 4 +- .../test_pipeline/test_job_batched_commits.py | 15 +- .../test_job_run_in_event_loop.py | 20 +- tests/test_pipeline/test_job_run_summary.py | 8 +- tests/test_pipeline/test_node_descriptors.py | 17 +- tests/test_pipeline/test_node_head.py | 8 +- tests/test_pipeline/test_node_protocols.py | 28 ++- .../test_operator_node_async_readback.py | 16 +- tests/test_pipeline/test_orchestrator.py | 4 +- tests/test_pipeline/test_pipeline.py | 5 +- .../test_pipeline_base_recording.py | 9 +- tests/test_pipeline/test_pipeline_job.py | 6 +- tests/test_pipeline/test_pod_invocation.py | 4 +- tests/test_pipeline/test_serialization.py | 10 +- .../test_store_errors_together.py | 1 + .../test_pipeline/test_store_write_failure.py | 16 +- .../test_pipeline/test_stored_data_context.py | 45 +++- tests/test_pipeline/test_sync_orchestrator.py | 4 +- tests/test_protocols/test_node_protocols.py | 4 +- .../test_struct_layout_identity.py | 4 +- 95 files changed, 1424 insertions(+), 500 deletions(-) create mode 100644 changelog.d/ITL-1023.md create mode 100644 design_issues/ITL-1023-RT1.md create mode 100644 docs/concepts/referential-tags.md diff --git a/changelog.d/ITL-1023.md b/changelog.d/ITL-1023.md new file mode 100644 index 000000000..3329eee59 --- /dev/null +++ b/changelog.d/ITL-1023.md @@ -0,0 +1,29 @@ +### Breaking changes and migration + +- A source's declared tag columns are now referential by default (ITL-1023, specification + "Orcapod v0.4 — Referential Tags"): each is also a data column of the same name, stored + once, so a function can read it and joins and groups use it as a tag. A script run + unchanged gets new schemas and hashes for such sources. Pass `tag_only_columns=[...]` (or + `tag_only_columns=True`) to keep tags plain, exactly as before; every existing golden hash + is unchanged under that option. + +### Added + +- `tag_only_columns` on `DictSource`, `ArrowTableSource`, `CSVSource`, `DataFrameSource`, + `DeltaTableSource`, the database sources and `SpiralDBTableSource`. +- `Join` merges the data columns of a referential tag both inputs join on into one column; + plain with referential gives a referential tag. `GroupBy` keeps a referential key scalar + in both roles and makes a non-key the same list in both roles. +- `MapTags` renames a referential tag in both roles; `tag_only=True` renames the tag only. + `MapData` is a data-only move. +- Removing the data column of a referential tag (`SelectDataColumns`, `DropDataColumns`) + leaves a plain tag and emits a `ReferenceMaterializationWarning` + (`orcapod.core.referential_tags.reference_materialization = "silent"` hides it). +- A function output named like a tag of its input is an error when the pipeline is built. + +### Known limitations + +- `Batch`, `MergeJoin`, operator-node result storage, `DerivedSource` over a referential + stream and saving a pipeline or job raise "not supported with referential tags yet". + `PollingSource` keeps its tags plain. The identities of referential streams may still + change. diff --git a/design_issues/ITL-1023-RT1.md b/design_issues/ITL-1023-RT1.md new file mode 100644 index 000000000..b791c065d --- /dev/null +++ b/design_issues/ITL-1023-RT1.md @@ -0,0 +1,8 @@ +## `src/orcapod/core/referential_tags.py` — referential tags workshop slice (ITL-1023) + +### RT1 — Referential tags are supported by part of the library only +**Status:** open +**Severity:** medium +**Issue:** ITL-1023 + +The workshop slice of referential tags (specification attached to ITL-1004) covers sources, function pods, `Join`, `GroupBy`, select and drop, renames, filters, `DropNulls`, `SemiJoin`, `Pick` and `Index`. Everything else raises "not supported with referential tags yet" when a referential tag reaches it, instead of giving a wrong answer: `Batch` (ITL-1017), `MergeJoin` (ITL-1018), storing an operator node's results in `LOG` or `REPLAY` mode, `DerivedSource` over a referential stream and saving a pipeline or job (ITL-1019), and any operator that does not declare `REFERENTIAL_TAGS_ISSUE = None` (ITL-1022). `PollingSource` keeps its tags plain. The identities of referential streams are not frozen yet. The project's issues (ITL-1009 onward) land the full design and close this entry. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2c5da734e..a586bc04e 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -68,7 +68,7 @@ source = DictSource( {"subject_id": "mouse_01", "weight": 25.3, "height": 0.12}, {"subject_id": "mouse_02", "weight": 22.1, "height": 0.10}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) # Apply the function pod to the source stream @@ -108,7 +108,7 @@ wide = DictSource( {"subject_id": "mouse_01", "weight": 25.3, "height": 0.12, "age": 12}, {"subject_id": "mouse_02", "weight": 22.1, "height": 0.10, "age": 8}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) result = compute_bmi.pod(wide) # 'age' is not a parameter: it is dropped @@ -200,7 +200,7 @@ from orcapod.errors import OutputTypeError def halve(weight: float) -> int: return weight / 2 # a float, although the function says it returns an int -weights = DictSource(data=[{"subject_id": "mouse_01", "weight": 25.3}], tag_columns=["subject_id"]) +weights = DictSource(data=[{"subject_id": "mouse_01", "weight": 25.3}], tag_columns=["subject_id"], tag_only_columns=True) try: list(halve.pod(weights).iter_data()) @@ -313,7 +313,7 @@ db = InMemoryArrowDatabase() job = PipelineJob(name="bmi_demo", store=db) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") compute_bmi.pod(source, label="bmi") job.run() @@ -347,7 +347,7 @@ mice.append({"subject_id": "mouse_03", "weight": 27.8, "height": 0.11}) job = PipelineJob(name="bmi_demo", store=db) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") compute_bmi.pod(source, label="bmi") job.run() @@ -413,7 +413,7 @@ from orcapod.pipeline import AsyncPipelineOrchestrator job = PipelineJob(name="bmi_async", store=InMemoryArrowDatabase()) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") compute_bmi.pod(source, label="bmi") job.run(orchestrator=AsyncPipelineOrchestrator()) @@ -452,7 +452,7 @@ import asyncio job = PipelineJob(name="bmi_awaited", store=InMemoryArrowDatabase()) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") compute_bmi.pod(source, label="bmi") @@ -477,7 +477,7 @@ Running the same job again computes nothing new, and the summary says so: ```python job = PipelineJob(name="bmi_demo", store=db) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") compute_bmi.pod(source, label="bmi") summary = job.run() @@ -499,7 +499,7 @@ def careless_bmi(weight: float, height: float) -> float: job = PipelineJob(name="bmi_typo", store=InMemoryArrowDatabase()) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") careless_bmi.pod(source, label="bmi") summary = job.run() @@ -665,7 +665,7 @@ workdir = Path(tempfile.mkdtemp()) # any directory you keep job = PipelineJob(name="bmi_demo", store=DeltaTableDatabase(workdir / "store")) with job: - source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + source = DictSource(data=mice, tag_columns=["subject_id"], tag_only_columns=True, source_id="mice") compute_bmi.pod(source, label="bmi") job.run() @@ -817,7 +817,7 @@ def count_rows(readings: pl.DataFrame) -> int: recordings = DictSource( data=[{"session": "s1", "n_samples": 2}, {"session": "s2", "n_samples": 3}], - tag_columns=["session"], + tag_columns=["session"], tag_only_columns=True, ) readings = load_readings.pod(recordings) for tag, data in readings.iter_data(): @@ -842,14 +842,14 @@ weights = DictSource( {"subject_id": "mouse_01", "weight": 25.3}, {"subject_id": "mouse_02", "weight": 22.1}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) heights = DictSource( data=[ {"subject_id": "mouse_01", "height": 0.12}, {"subject_id": "mouse_02", "height": 0.10}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) result = compute_bmi.pod(weights, heights) @@ -873,7 +873,7 @@ Two consequences: ```python chosen = DictSource( data=[{"subject_id": "mouse_02", "cohort": "b"}], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) result = compute_bmi.pod(weights, heights, chosen) diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index cd89f7fe5..30c85744e 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -76,7 +76,7 @@ subjects = DictSource( {"subject_id": "mouse_01", "age": 12}, {"subject_id": "mouse_02", "age": 8}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) measurements = DictSource( @@ -84,7 +84,7 @@ measurements = DictSource( {"subject_id": "mouse_01", "weight": 25.3}, {"subject_id": "mouse_02", "weight": 22.1}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) join = Join() @@ -110,14 +110,14 @@ scale_a = DictSource( {"subject_id": "mouse_01", "weight": 25.3}, {"subject_id": "mouse_02", "weight": 22.1}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) scale_b = DictSource( data=[ {"subject_id": "mouse_01", "weight": 24.9}, {"subject_id": "mouse_02", "weight": 22.6}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) merged = MergeJoin()(scale_a, scale_b) @@ -147,14 +147,14 @@ all_subjects = DictSource( {"subject_id": "mouse_02", "age": 8}, {"subject_id": "mouse_03", "age": 15}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) weighed = DictSource( data=[ {"subject_id": "mouse_01", "weight": 25.3}, {"subject_id": "mouse_02", "weight": 22.1}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) only_weighed = SemiJoin()(all_subjects, weighed) @@ -178,7 +178,7 @@ source = DictSource( {"subject_id": "mouse_01", "age": 12}, {"subject_id": "mouse_02", "age": 8}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) batch = Batch() @@ -214,7 +214,7 @@ source = DictSource( {"subject_id": "mouse_01", "probe": "imec1", "spike_count": 980}, {"subject_id": "mouse_02", "probe": "imec0", "spike_count": 1450}, ], - tag_columns=["subject_id", "probe"], + tag_columns=["subject_id", "probe"], tag_only_columns=True, ) grouped = GroupBy(by=["subject_id"])(source) @@ -282,7 +282,7 @@ source = DictSource( {"subject_id": "mouse_02", "age": 8, "weight": 22.1}, {"subject_id": "mouse_03", "age": 15, "weight": 27.8}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) select = SelectDataColumns(columns=["weight"]) @@ -401,7 +401,7 @@ recordings = ArrowTableSource( ] ), ), - tag_columns=["session"], + tag_columns=["session"], tag_only_columns=True, ) print(recordings.output_schema()[1]) # Schema({'start_ms': int | None}) @@ -475,7 +475,7 @@ visits = ArrowTableSource( "notes": pa.array([None, "noisy", None, "ok"], pa.large_string()), } ), - tag_columns=["session"], + tag_columns=["session"], tag_only_columns=True, ) print(visits.output_schema()) # (Schema({'session': str | None}), Schema({'start_ms': int | None, 'notes': str | None})) @@ -580,7 +580,7 @@ def counts(values): "count": pa.array(values, type=pa.int8()), } ) - return ArrowTableSource(table, tag_columns=["subject_id"]) + return ArrowTableSource(table, tag_columns=["subject_id"], tag_only_columns=True) merged = MergeJoin()(counts([1, 2]), counts([3, 4])) print(merged.as_table().schema.field("count").type) diff --git a/docs/concepts/referential-tags.md b/docs/concepts/referential-tags.md new file mode 100644 index 000000000..eafaff0ff --- /dev/null +++ b/docs/concepts/referential-tags.md @@ -0,0 +1,107 @@ +# Referential Tags + +!!! warning "Integration branch (ITL-1023)" + Referential tags are the first slice of Orcapod v0.4, on an integration branch. The + identities (content and pipeline hashes) of streams with referential tags are **not + frozen yet** and may change before the release. Streams whose tags are all plain + (`tag_only_columns`) keep their v0.3 identities exactly. + +A tag is either **plain** (it carries its own value, as every tag did in v0.3) or +**referential**: a handle to the data column of the **same name**. A referential tag is +one column seen through both roles, so a key column can be joined and grouped on as a tag +and read by a function as data, without a copy. + +## Sources: declared tags are referential by default + +Every tag column a source declares is referential unless you keep it plain with +`tag_only_columns`: + +```python +import orcapod as op + +rows = [{"unit": 1, "time": 10, "amp": 0.5}, {"unit": 2, "time": 20, "amp": 0.25}] + +spikes = op.sources.DictSource(rows, tag_columns=["unit", "time"], source_id="spikes") +print(spikes.keys()) +# (('unit', 'time'), ('unit', 'time', 'amp')) + +plain = op.sources.DictSource( + rows, tag_columns=["unit", "time"], tag_only_columns=True, source_id="spikes" +) +print(plain.keys()) +# (('unit', 'time'), ('amp',)) +``` + +`tag_only_columns` takes the names to keep plain, or `True` for all of them. A name that is +not a declared tag is an error. A source built with all its tags plain is exactly the +source v0.3 built: same columns, same hashes. + +The column is stored once: `as_table()` shows it once, and every row's tag and data hold +the same value. + +```python +print(spikes.as_table().column_names) +# ['unit', 'time', 'amp'] +``` + +## Functions read a referenced column by name + +```python +@op.function_pod(output_keys=["scaled"]) +def scale(time: int, amp: float) -> float: + return time * amp + + +scaled = scale.pod(spikes) +print(scaled.keys()) +# (('unit', 'time'), ('scaled',)) +``` + +A pod's output data is the function's outputs, so every tag of its output is plain, with +the value it had. A function output named like a tag is an error when the pipeline is +built. The automatic select of a pod call drops the data columns the function does not +take; a referential tag whose data column is dropped stays a plain tag. + +A function that does not read a referenced column keeps its result identity: retagging a +source does not make it compute again. + +## Operators + +| Operator | Referential tag `x` | +|---|---| +| `Join` | Plain with plain: as before. Plain with referential, or referential with referential: a referential tag and **one** data column `x`. Its source info lists both inputs' tokens. Every other data-name collision is still an error. | +| `GroupBy` | A group key stays scalar in both roles; a non-key becomes the same list in both roles, in the same member order. | +| `SelectDataColumns`, `DropDataColumns` | Removing the data column leaves `x` a plain tag with the same values, with a `ReferenceMaterializationWarning`. | +| `SelectTagColumns`, `DropTagColumns` | Removing the tag leaves `x` a data column. | +| `MapTags` / `map_tags` | Renames the column in both roles; `tag_only=True` renames the tag only and leaves a plain tag. | +| `MapData` / `map_data` | Always a data-only move: the data column is renamed and `x` stays a plain tag. | +| Filters, `DropNulls`, `SemiJoin`, `Pick`, `Index` | Keep the reference. | +| `Batch`, `MergeJoin` | Not supported with referential tags yet: they raise. | + +```python +units = op.sources.DictSource( + [{"unit": 1, "area": "V1"}, {"unit": 2, "area": "V2"}], + tag_columns=["unit"], + source_id="units", +) +joined = spikes.join(units) +print(joined.keys()) +# (('unit', 'time'), ('unit', 'area', 'time', 'amp')) +``` + +The materialization warning can be turned off; it never changes a result or a hash: + +```python +from orcapod.core import referential_tags + +referential_tags.reference_materialization = "silent" +# ... build the pipeline ... +referential_tags.reference_materialization = "warn" # the default +``` + +## Not supported yet + +These raise "not supported with referential tags yet" when a referential tag reaches them: +`Batch`, `MergeJoin`, storing an operator node's results (`LOG` and `REPLAY` cache modes), +`DerivedSource` over a stream with referential tags, and saving a pipeline or job. +`PollingSource` keeps its tags plain. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index dc7075718..2793856ac 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -353,7 +353,7 @@ experiments = DictSource( {"subject_id": "m1", "session_start": start_2, "experiment_index": 0, "experiment_start": datetime(2026, 1, 6, 9, 32, tzinfo=timezone.utc)}, ], - tag_columns=["subject_id", "session_start", "experiment_index"], + tag_columns=["subject_id", "session_start", "experiment_index"], tag_only_columns=True, source_id="experiments", ) @@ -452,7 +452,7 @@ from orcapod.sources import DictSource source = DictSource( data=[{"subject_id": "mouse_01", "age": 12, "weight": 25.3}], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, source_id="mice", ) @@ -490,7 +490,7 @@ source = DictSource( {"subject_id": "mouse_02", "age": 8, "weight": 22.1}, {"subject_id": "mouse_03", "age": 15, "weight": 27.8}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) # Inspect the schema diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index ebcee55ad..5d9bbec0c 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -52,7 +52,7 @@ stream = DictSource( {"subject_id": "mouse_01", "age": 12, "weight": 25.3}, {"subject_id": "mouse_02", "age": 8, "weight": 22.1}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) ``` @@ -117,7 +117,7 @@ subjects = DictSource( {"subject_id": f"mouse_{i:02d}", "age": 8 + i, "weight": 20.0 + i} for i in range(1, 26) ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, label="subjects", ) @@ -196,7 +196,7 @@ source = DictSource( {"subject_id": "mouse_01", "age": 12, "weight": 25.3}, {"subject_id": "mouse_02", "age": 8, "weight": 22.1}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) # Default: user-facing columns only @@ -228,7 +228,7 @@ source = DictSource( {"subject_id": "mouse_02", "age": 8, "weight": 22.1}, {"subject_id": "mouse_03", "age": 15, "weight": 27.8}, ], - tag_columns=["subject_id"], + tag_columns=["subject_id"], tag_only_columns=True, ) # Schema inspection diff --git a/docs/getting-started.md b/docs/getting-started.md index 289e1d1be..a437eea4a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,7 +18,7 @@ source = DictSource( {"experiment": "exp_002", "temperature": 22.3, "pressure": 0.98}, {"experiment": "exp_003", "temperature": 19.8, "pressure": 1.05}, ], - tag_columns=["experiment"], + tag_columns=["experiment"], tag_only_columns=True, source_id="lab_results", ) ``` @@ -187,7 +187,7 @@ source = DictSource( {"experiment": "exp_002", "temperature": 22.3, "pressure": 0.98}, {"experiment": "exp_003", "temperature": 19.8, "pressure": 1.05}, ], - tag_columns=["experiment"], + tag_columns=["experiment"], tag_only_columns=True, source_id="lab_results", ) diff --git a/docs/index.md b/docs/index.md index 45de2dc1a..78b92eb98 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,7 +38,7 @@ source = DictSource( {"name": "Bob", "age": 25}, {"name": "Charlie", "age": 35}, ], - tag_columns=["name"], + tag_columns=["name"], tag_only_columns=True, source_id="people", ) diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 00f170816..929a14e0d 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -89,7 +89,7 @@ rule_store = InMemoryArrowDatabase() rule_job = PipelineJob(name="my_pipeline", store=rule_store) with rule_job: numbers = DictSource( - data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="numbers" + data=[{"k": "a", "x": 1}], tag_columns=["k"], tag_only_columns=True, source_id="numbers" ) _ = double.pod(numbers, label="double") rule_job.run() @@ -144,12 +144,12 @@ from orcapod.side_effects import SideEffectPod, InvocationContext scores = DictSource( [{"student": "alice", "math": 90}, {"student": "bob", "math": 75}], - tag_columns=["student"], + tag_columns=["student"], tag_only_columns=True, source_id="scores_v1", ) attendance = DictSource( [{"student": "alice", "days": 180}, {"student": "bob", "days": 160}], - tag_columns=["student"], + tag_columns=["student"], tag_only_columns=True, source_id="attendance_v1", ) @@ -588,7 +588,7 @@ from orcapod.sources import DictSource def dict_row_hash(value): source = DictSource( data=[{"k": "r", "m": value}], - tag_columns=["k"], + tag_columns=["k"], tag_only_columns=True, data_schema={"k": str, "m": dict[str, int]}, ) _, data = next(iter(source.iter_data())) @@ -1252,7 +1252,7 @@ def double_for_contexts(x: int) -> int: context_store = InMemoryArrowDatabase() context_job = PipelineJob(name="contexts_demo", store=context_store) with context_job: - rows = DictSource(data=[{"k": "a", "x": 1}, {"k": "b", "x": 2}], tag_columns=["k"]) + rows = DictSource(data=[{"k": "a", "x": 1}, {"k": "b", "x": 2}], tag_columns=["k"], tag_only_columns=True) _ = double_for_contexts.pod(rows, label="dbl") context_job.run() diff --git a/examples/save_and_load_pipelines.py b/examples/save_and_load_pipelines.py index 69f67bab3..95d57efb7 100644 --- a/examples/save_and_load_pipelines.py +++ b/examples/save_and_load_pipelines.py @@ -13,11 +13,13 @@ source1 = DictSource( [{"id": 0, "x": 5}, {"id": 1, "x": 10}, {"id": 2, "x": 15}], tag_columns=["id"], + tag_only_columns=True, source_id="source1", ) source2 = DictSource( [{"id": 0, "y": 3}, {"id": 2, "y": 6}, {"id": 4, "y": 9}], tag_columns=["id"], + tag_only_columns=True, source_id="source2", ) diff --git a/mkdocs.yml b/mkdocs.yml index a152c656a..eb728f650 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,6 +38,7 @@ nav: - Streams: concepts/streams.md - Operators: concepts/operators.md - Function Pods: concepts/function-pods.md + - Referential Tags: concepts/referential-tags.md - Your Own Data Types: concepts/your-own-data-types.md - NumPy Arrays: concepts/arrays.md - Logical Type Identifiers: concepts/logical-type-identifiers.md diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index 8023198e7..ab3ae55e8 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -236,7 +236,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: f"Missing data columns: {missing_columns}. Make sure all specified columns to select are present or use strict=False to ignore missing columns" ) warn_materialized( - "SelectDataColumns", + self, [c for c in stream_references(stream) if c not in self.columns], ) @@ -512,7 +512,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: f"Missing data columns: {missing_columns}. Make sure all specified columns to drop are present or use strict=False to ignore missing columns" ) warn_materialized( - "DropDataColumns", + self, [c for c in stream_references(stream) if c in self.columns], ) diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index e6c9a046a..407471f99 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -937,18 +937,14 @@ def _merge_data( right_rest = right_table.drop_columns( [c for c in referential if c in right_table.column_names] ) - references = pa.table( - { - name: ( - left_table if name in left_table.column_names else right_table - ).column(name) - for name in referential - }, + owners = [ + left_table if name in left_table.column_names else right_table + for name in referential + ] + references = pa.Table.from_arrays( + [owner.column(name) for owner, name in zip(owners, referential)], schema=pa.schema( - ( - left_table if name in left_table.column_names else right_table - ).schema.field(name) - for name in referential + [owner.schema.field(name) for owner, name in zip(owners, referential)] ), ) return Data( diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py index ac9fe72f8..ba95c525a 100644 --- a/src/orcapod/core/referential_tags.py +++ b/src/orcapod/core/referential_tags.py @@ -20,12 +20,14 @@ from __future__ import annotations import warnings +import weakref from collections.abc import Iterable, Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar from typing import Any, Literal from orcapod.errors import InputValidationError +from orcapod.types import SourceInfoValue # Diagnostic emitted when a structural operator materializes a reference (section 7.4). # ``"warn"`` emits a ``ReferenceMaterializationWarning``; ``"silent"`` emits nothing. The @@ -44,6 +46,11 @@ class ReferenceMaterializationWarning(UserWarning): ) +# The operations that already reported, by id, holding the object so the id is not +# reused while the entry exists. +_reported: "weakref.WeakValueDictionary[int, object]" = weakref.WeakValueDictionary() + + @contextmanager def materialization_diagnostic_off() -> Iterator[None]: """Build the steps in this block without the materialization diagnostic. @@ -126,21 +133,28 @@ def check_namespaces( ) -def warn_materialized(operation: str, names: Iterable[str]) -> None: +def warn_materialized(operation: object, names: Iterable[str]) -> None: """Report that ``operation`` turned the referential tags ``names`` into plain tags. - Its job is the diagnostic of section 7.4, so it takes the operation's name. Emits - nothing when ``names`` is empty or ``reference_materialization`` is ``"silent"``. + Its job is the diagnostic of section 7.4, so it takes the operation. It reports + once per operation object: a job validates its operators again when it builds and + runs its nodes, and that is not a new structural step. An operation built while + ``materialization_diagnostic_off`` is active (Orcapod's own step) never reports. + Emits nothing when ``names`` is empty or ``reference_materialization`` is + ``"silent"``. Args: - operation: The structural operation, for the message. + operation: The structural operation (an operator instance). names: The tags that became plain. """ names = tuple(names) - if not names or reference_materialization == "silent" or _diagnostic_off.get(): + if not names or id(operation) in _reported: + return + _reported[id(operation)] = operation + if reference_materialization == "silent" or _diagnostic_off.get(): return warnings.warn( - f"{operation}: the data column of referential tag(s) {list(names)} was removed; " + f"{type(operation).__name__}: the data column of referential tag(s) {list(names)} was removed; " "they are now plain tags with the same values (set " "orcapod.core.referential_tags.reference_materialization = 'silent' to hide this)", ReferenceMaterializationWarning, @@ -205,7 +219,9 @@ def source_reference_tags( return tuple(name for name in tags if name not in plain) -def merged_source_tokens(left: object, right: object) -> list[str | None]: +def merged_source_tokens( + left: SourceInfoValue, right: SourceInfoValue +) -> list[SourceInfoValue]: """Return the source info of a data column merged by a join, keeping both inputs'. Section 9.4: a join that merges the data columns of two referential tags keeps the @@ -225,13 +241,13 @@ def merged_source_tokens(left: object, right: object) -> list[str | None]: NotImplementedError: If a value is a list nested more than one level (ITL-1016). """ - merged: list[str | None] = [] + merged: list[SourceInfoValue] = [] for value in (left, right): - items = value if isinstance(value, list) else [value] - for item in items: # pyright: ignore[reportUnknownVariableType] # a token or a list of tokens + items: list[SourceInfoValue] = value if isinstance(value, list) else [value] + for item in items: if isinstance(item, list): raise not_supported_yet( "Join of nested source info of a referential tag", "ITL-1016" ) - merged.append(item) # pyright: ignore[reportUnknownArgumentType] # a token + merged.append(item) return merged diff --git a/src/orcapod/core/sources/postgresql_table_source.py b/src/orcapod/core/sources/postgresql_table_source.py index 25f693039..58b1a805c 100644 --- a/src/orcapod/core/sources/postgresql_table_source.py +++ b/src/orcapod/core/sources/postgresql_table_source.py @@ -15,6 +15,7 @@ from collections.abc import Collection from typing import TYPE_CHECKING, Any +from orcapod.core.referential_tags import TagOnlyColumns from orcapod.core.sources.db_table_source import DBTableSource from orcapod.databases.postgresql_connector import PostgreSQLConnector @@ -54,6 +55,9 @@ class PostgreSQLTableSource(DBTableSource): label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: OrcaPod configuration. + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Every other tag column is referential: + also a data column of the same name (ITL-1023). Raises: ValueError: If the table is not found, is empty, or has no PK and @@ -72,6 +76,7 @@ def __init__( label: str | None = None, data_context: str | contexts.DataContext | None = None, config: OrcapodConfig | None = None, + tag_only_columns: TagOnlyColumns = (), ) -> None: self._dsn = dsn # store before try — needed by to_config even if super() raises connector = PostgreSQLConnector( @@ -88,6 +93,7 @@ def __init__( label=label, data_context=data_context, config=config, + tag_only_columns=tag_only_columns, ) finally: try: @@ -120,6 +126,7 @@ def from_config( dsn=config["dsn"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/sqlite_table_source.py b/src/orcapod/core/sources/sqlite_table_source.py index ba9837a51..ef0d40e9a 100644 --- a/src/orcapod/core/sources/sqlite_table_source.py +++ b/src/orcapod/core/sources/sqlite_table_source.py @@ -25,6 +25,7 @@ from collections.abc import Collection from typing import TYPE_CHECKING, Any +from orcapod.core.referential_tags import TagOnlyColumns from orcapod.core.sources.db_table_source import DBTableSource from orcapod.databases.sqlite_connector import SQLiteConnector from orcapod.utils.schema_utils import _normalize_column_list @@ -66,6 +67,9 @@ class SQLiteTableSource(DBTableSource): label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: OrcaPod configuration. + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Every other tag column is referential: + also a data column of the same name (ITL-1023). Raises: ValueError: If the table is not found or is empty. @@ -83,6 +87,7 @@ def __init__( label: str | None = None, data_context: str | contexts.DataContext | None = None, config: "OrcapodConfig | None" = None, + tag_only_columns: TagOnlyColumns = (), ) -> None: self._db_path = db_path connector = SQLiteConnector(db_path) @@ -115,6 +120,7 @@ def __init__( label=label, data_context=data_context, config=config, + tag_only_columns=tag_only_columns, _query=_query, ) finally: @@ -150,6 +156,7 @@ def from_config( db_path=config["db_path"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 32e909350..c9193bf41 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -50,7 +50,7 @@ def __init__( producer: PodProtocol | None = None, upstreams: tuple[StreamProtocol, ...] = (), reference_tags: Collection[str] = (), - **kwargs, + **kwargs: Any, ) -> None: """Wrap ``table`` as a stream. diff --git a/tests/test_arrays/test_array_operators.py b/tests/test_arrays/test_array_operators.py index f53962ddc..93c1a6e82 100644 --- a/tests/test_arrays/test_array_operators.py +++ b/tests/test_arrays/test_array_operators.py @@ -51,6 +51,7 @@ def arrays_source(source_id: str = "arrays", shift: int = 0) -> DictSource: for i, name in enumerate(NAMES) ], tag_columns=["k", "g"], + tag_only_columns=True, source_id=source_id, ) @@ -75,6 +76,7 @@ def other_source() -> DictSource: return DictSource( data=[{"k": name, "y": i} for i, name in enumerate(NAMES)], tag_columns=["k"], + tag_only_columns=True, source_id="other", ) @@ -91,7 +93,10 @@ def test_semijoin() -> None: """``SemiJoin`` keeps the matching rows with their arrays unchanged.""" keep = NAMES[::2] right = DictSource( - data=[{"k": name, "y": 0} for name in keep], tag_columns=["k"], source_id="r" + data=[{"k": name, "y": 0} for name in keep], + tag_columns=["k"], + tag_only_columns=True, + source_id="r", ) got = rows(SemiJoin()(arrays_source(), right)) assert sorted(got) == keep diff --git a/tests/test_arrays/test_array_pipeline_stores.py b/tests/test_arrays/test_array_pipeline_stores.py index 229870037..5824ad910 100644 --- a/tests/test_arrays/test_array_pipeline_stores.py +++ b/tests/test_arrays/test_array_pipeline_stores.py @@ -36,11 +36,13 @@ def _build(store: Any) -> PipelineJob: numeric = DictSource( data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], tag_columns=["k"], + tag_only_columns=True, source_id="numeric", ) every = DictSource( data=[{"k": k, "x": v} for k, v in ARRAYS.items()], tag_columns=["k"], + tag_only_columns=True, source_id="every", ) job = PipelineJob(name="arrays", store=store) @@ -116,6 +118,7 @@ def test_pod_output_list_of_arrays(tmp_path: Path, store: str, runner: str) -> N source = DictSource( data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], tag_columns=["k"], + tag_only_columns=True, source_id="numeric", ) job = PipelineJob(name="array_lists", store=STORES[store](tmp_path)) @@ -149,6 +152,7 @@ def test_batch_and_group_by_over_an_array_column( for i in range(4) ], tag_columns=["g", "k"], + tag_only_columns=True, source_id="grouped", ) job = PipelineJob(name="array_groups", store=STORES[store](tmp_path)) diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index a6f42a2fe..df8c45313 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -33,14 +33,18 @@ def stored(value: Any) -> bytes: """The stored cell of a one-row ``x`` column.""" - table = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]).as_table() + table = DictSource( + data=[{"k": "r", "x": value}], tag_columns=["k"], tag_only_columns=True + ).as_table() chunk = table.column("x").chunk(0) assert isinstance(chunk, pa.ExtensionArray) return chunk.storage[0].as_py() def read_back(value: Any) -> Any: - source = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": value}], tag_columns=["k"], tag_only_columns=True + ) return next(iter(source.iter_data()))[1].as_dict()["x"] @@ -49,7 +53,9 @@ def semantic_hash(value: Any) -> Any: def data_hash(value: Any) -> Any: - source = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": value}], tag_columns=["k"], tag_only_columns=True + ) return next(iter(source.iter_data()))[1].content_hash() @@ -61,7 +67,7 @@ def data_hash(value: Any) -> Any: def test_column_type_is_orcapod_numpy_ndarray_over_large_binary() -> None: """An array column is ``orcapod.numpy.ndarray`` over ``large_binary`` with the ignore list.""" table = DictSource( - data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"] + data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"], tag_only_columns=True ).as_table() ext = table.schema.field("x").type assert isinstance(ext, pa.ExtensionType) @@ -424,7 +430,9 @@ def test_table_token_carries_the_identifier_and_the_digest() -> None: hasher = get_default_context().arrow_hasher assert isinstance(hasher, StarfixArrowHasher) array = np.arange(3.0) - table = DictSource(data=[{"k": "r", "x": array}], tag_columns=["k"]).as_table() + table = DictSource( + data=[{"k": "r", "x": array}], tag_columns=["k"], tag_only_columns=True + ).as_table() # Non-public helper: this test is about the token the hasher builds for a cell. processed = hasher._process_table_columns(table.select(["x"])) # pyright: ignore[reportPrivateUsage] token = processed.column("x")[0].as_py() @@ -494,7 +502,11 @@ def test_unsupported_arrays_are_refused_naming_column_and_dtype( ) -> None: """Unsupported arrays are refused at write time, naming the column and the dtype.""" with pytest.raises(ValueError, match="'payload'") as info: - DictSource(data=[{"k": "r", "payload": value}], tag_columns=["k"]).as_table() + DictSource( + data=[{"k": "r", "payload": value}], + tag_columns=["k"], + tag_only_columns=True, + ).as_table() assert reason.lower() in str(info.value).lower() @@ -525,7 +537,7 @@ def test_a_cell_that_is_not_a_stored_array_raises_naming_the_column( ) # The first read of the value (hashing the source rows) must fail naming the column. with pytest.raises((ValueError, RuntimeError), match="'x'"): - source = ArrowTableSource(table, tag_columns=["k"]) + source = ArrowTableSource(table, tag_columns=["k"], tag_only_columns=True) for _, data in source.iter_data(): data.as_dict() @@ -541,7 +553,7 @@ def _source_of_cell(cell: bytes) -> ArrowTableSource: "x": pa.ExtensionArray.from_storage(ext, storage), } ) - return ArrowTableSource(table, tag_columns=["k"]) + return ArrowTableSource(table, tag_columns=["k"], tag_only_columns=True) def test_a_cell_that_declares_more_than_it_holds_is_refused_before_allocation() -> None: @@ -614,7 +626,7 @@ def test_a_stored_object_array_is_never_unpickled() -> None: } ) with pytest.raises((ValueError, RuntimeError), match="allow_pickle=False"): - source = ArrowTableSource(table, tag_columns=["k"]) + source = ArrowTableSource(table, tag_columns=["k"], tag_only_columns=True) for _, data in source.iter_data(): data.as_dict() @@ -632,7 +644,9 @@ def double(x: np.ndarray) -> np.ndarray: # pyright: ignore[reportMissingTypeArg return x * 2 # a 0-d input gives a NumPy scalar source = DictSource( - data=[{"k": "a", "x": np.array(7.5, np.float32)}], tag_columns=["k"] + data=[{"k": "a", "x": np.array(7.5, np.float32)}], + tag_columns=["k"], + tag_only_columns=True, ) with pytest.raises(TypeError, match="float32.*'y'.*ndarray"): list(double.pod(source).iter_data()) @@ -645,7 +659,9 @@ def test_none_returned_for_optional_ndarray_is_accepted() -> None: def nothing(x: np.ndarray) -> np.ndarray | None: # pyright: ignore[reportMissingTypeArgument] # the bare np.ndarray annotation is under test return None - source = DictSource(data=[{"k": "a", "x": np.arange(2.0)}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "a", "x": np.arange(2.0)}], tag_columns=["k"], tag_only_columns=True + ) assert [d.as_dict()["y"] for _, d in nothing.pod(source).iter_data()] == [None] @@ -666,11 +682,13 @@ def test_merge_join_orders_arrays_by_their_value_hash() -> None: left = DictSource( data=[{"k": "a", "x": large}, {"k": "b", "x": other}], tag_columns=["k"], + tag_only_columns=True, source_id="l", ) right = DictSource( data=[{"k": "a", "x": small}, {"k": "b", "x": small}], tag_columns=["k"], + tag_only_columns=True, source_id="r", ) for pair in ((left, right), (right, left)): @@ -688,8 +706,18 @@ def test_merge_join_orders_arrays_that_differ_only_in_field_order() -> None: """``MergeJoin`` orders arrays that differ only in field order the same way for either input order.""" ab = np.array([(1, 2.0)], dtype=[("a", " None: """Merged arrays read back as each side gave them.""" base = np.arange(6, dtype=np.float32).reshape(2, 3) fortran_big = np.asfortranarray(base.astype(">f4")) - left = DictSource(data=[{"k": "a", "x": base}], tag_columns=["k"], source_id="l") + left = DictSource( + data=[{"k": "a", "x": base}], + tag_columns=["k"], + tag_only_columns=True, + source_id="l", + ) right = DictSource( - data=[{"k": "a", "x": fortran_big}], tag_columns=["k"], source_id="r" + data=[{"k": "a", "x": fortran_big}], + tag_columns=["k"], + tag_only_columns=True, + source_id="r", ) got: Any = next(d.as_dict()["x"] for _, d in MergeJoin()(left, right).iter_data()) assert sorted(x.dtype.str for x in got) == ["f4"] @@ -724,7 +760,9 @@ def test_pick_on_an_array_column_raises_input_validation_error() -> None: from orcapod.errors import InputValidationError from orcapod.operators import Pick - source = DictSource(data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"], tag_only_columns=True + ) with pytest.raises(InputValidationError, match="'x'.*does not support pick"): Pick("x", "dtype")(source) @@ -740,7 +778,9 @@ def test_pick_on_an_array_column_raises_while_building_a_pipeline( from orcapod.errors import InputValidationError from orcapod.operators import Pick - source = DictSource(data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"], tag_only_columns=True + ) job = PipelineJob(name="pick", store=InMemoryArrowDatabase()) with pytest.raises(InputValidationError, match="'x'"), job: Pick("x", "dtype")(source) diff --git a/tests/test_core/function_pod/test_cached_function_pod_job.py b/tests/test_core/function_pod/test_cached_function_pod_job.py index eaf30a2c2..5c801b8be 100644 --- a/tests/test_core/function_pod/test_cached_function_pod_job.py +++ b/tests/test_core/function_pod/test_cached_function_pod_job.py @@ -35,7 +35,9 @@ def make_source(rows: list[dict[str, int]] | None = None) -> DictSource: - return DictSource(rows or ROWS, tag_columns=["k"], source_id="s") + return DictSource( + rows or ROWS, tag_columns=["k"], tag_only_columns=True, source_id="s" + ) class Counter: @@ -185,8 +187,12 @@ def test_inputs_are_joined_once() -> None: cached_pod = CachedFunctionPod( function_pod=counter.pod(), result_database=InMemoryArrowDatabase() ) - left = DictSource([{"k": 1, "x": 2}], tag_columns=["k"], source_id="l") - right = DictSource([{"k": 1, "y": 3}], tag_columns=["k"], source_id="r") + left = DictSource( + [{"k": 1, "x": 2}], tag_columns=["k"], tag_only_columns=True, source_id="l" + ) + right = DictSource( + [{"k": 1, "y": 3}], tag_columns=["k"], tag_only_columns=True, source_id="r" + ) joins: list[int] = [] original = Join.process diff --git a/tests/test_core/function_pod/test_function_node_caching.py b/tests/test_core/function_pod/test_function_node_caching.py index c58ba463b..a3c8467d1 100644 --- a/tests/test_core/function_pod/test_function_node_caching.py +++ b/tests/test_core/function_pod/test_function_node_caching.py @@ -61,7 +61,11 @@ def _make_source_stream( {k: pa.array([r[k] for r in rows], type=pa.int64()) for k in rows[0]} ) source = ArrowTableSource( - table, tag_columns=tag_columns, source_id=source_id, infer_nullable=True + table, + tag_columns=tag_columns, + tag_only_columns=True, + source_id=source_id, + infer_nullable=True, ) return source diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py index 10e4cf777..de754029b 100644 --- a/tests/test_core/function_pod/test_function_pod_auto_select.py +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -94,6 +94,7 @@ def _wide(extra: int = 9, source_id: str = "wide") -> DictSource: {"k": 2, "x": 4, "y": 5, "extra": extra}, ], tag_columns=["k"], + tag_only_columns=True, source_id=source_id, ) @@ -111,6 +112,7 @@ def _xs(source_id: str = "xs", **more: int) -> DictSource: return DictSource( [{"k": 1, "x": 2, **more}, {"k": 2, "x": 4, **more}], tag_columns=["k"], + tag_only_columns=True, source_id=source_id, ) @@ -128,6 +130,7 @@ def _ys(source_id: str = "ys", **more: int) -> DictSource: return DictSource( [{"k": 1, "y": 3, **more}, {"k": 2, "y": 5, **more}], tag_columns=["k"], + tag_only_columns=True, source_id=source_id, ) @@ -223,6 +226,7 @@ def test_tags_are_untouched() -> None: source = DictSource( [{"k": 1, "session": "a", "x": 2, "y": 3, "extra": 9}], tag_columns=["k", "session"], + tag_only_columns=True, source_id="two_tags", ) @@ -237,9 +241,15 @@ def test_stream_passed_for_its_tags_restricts_the_rows() -> None: full = DictSource( [{"k": 1, "x": 2, "y": 3}, {"k": 2, "x": 4, "y": 5}], tag_columns=["k"], + tag_only_columns=True, source_id="full", ) - chosen = DictSource([{"k": 2, "note": 0}], tag_columns=["k"], source_id="chosen") + chosen = DictSource( + [{"k": 2, "note": 0}], + tag_columns=["k"], + tag_only_columns=True, + source_id="chosen", + ) result = add.pod(full, chosen) @@ -295,7 +305,10 @@ def test_matching_input_adds_no_select_node_to_a_job() -> None: job = PipelineJob(name="narrow", store=InMemoryArrowDatabase()) with job: narrow = DictSource( - [{"k": 1, "x": 2, "y": 3}], tag_columns=["k"], source_id="narrow" + [{"k": 1, "x": 2, "y": 3}], + tag_columns=["k"], + tag_only_columns=True, + source_id="narrow", ) add.pod(narrow, label="add") @@ -358,7 +371,10 @@ def test_missing_required_column_adds_no_step_to_a_job() -> None: def test_input_with_no_parameter_column_is_rejected() -> None: """A stream cannot be left without data columns: the exact error is raised.""" only_extra = DictSource( - [{"k": 1, "extra": 9}], tag_columns=["k"], source_id="only_extra" + [{"k": 1, "extra": 9}], + tag_columns=["k"], + tag_only_columns=True, + source_id="only_extra", ) with pytest.raises(ValueError, match="column 'extra' is not a parameter"): @@ -373,10 +389,16 @@ def test_data_column_shared_by_two_inputs_fails_at_the_join(shared: str) -> None shared: The data column both inputs carry. """ left = DictSource( - [{"k": 1, "x": 2, "extra": 1}], tag_columns=["k"], source_id="left" + [{"k": 1, "x": 2, "extra": 1}], + tag_columns=["k"], + tag_only_columns=True, + source_id="left", ) right = DictSource( - [{"k": 1, "y": 3, shared: 7}], tag_columns=["k"], source_id="right" + [{"k": 1, "y": 3, shared: 7}], + tag_columns=["k"], + tag_only_columns=True, + source_id="right", ) with pytest.raises(InputValidationError, match="overlapping"): @@ -415,7 +437,10 @@ def test_misspelt_column_for_a_defaulted_parameter_is_dropped_and_logged( ) -> None: """A misspelt column is dropped and the default used; the INFO record is the trace.""" misspelt = DictSource( - [{"k": 1, "x": 2, "Y": 3}], tag_columns=["k"], source_id="misspelt" + [{"k": 1, "x": 2, "Y": 3}], + tag_columns=["k"], + tag_only_columns=True, + source_id="misspelt", ) with caplog.at_level(logging.DEBUG): diff --git a/tests/test_core/function_pod/test_function_pod_ctx_arg.py b/tests/test_core/function_pod/test_function_pod_ctx_arg.py index 0420d0d8e..9eb7178e9 100644 --- a/tests/test_core/function_pod/test_function_pod_ctx_arg.py +++ b/tests/test_core/function_pod/test_function_pod_ctx_arg.py @@ -191,6 +191,7 @@ def transform(value: int, ctx: InvocationContext) -> str: source = DictSource( [{"id": 0, "value": 10}, {"id": 1, "value": 20}], tag_columns=["id"], + tag_only_columns=True, ) pod.process(source) diff --git a/tests/test_core/function_pod/test_node_config_on_job_node.py b/tests/test_core/function_pod/test_node_config_on_job_node.py index 67389e5dd..be1c392a1 100644 --- a/tests/test_core/function_pod/test_node_config_on_job_node.py +++ b/tests/test_core/function_pod/test_node_config_on_job_node.py @@ -15,7 +15,9 @@ def double(x: int) -> int: pf = PythonDataFunction(double, output_keys="result") pod = FunctionPod(pf) table = pa.table({"id": [1], "x": [10]}) - source = ArrowTableSource(table, tag_columns=["id"], infer_nullable=True) + source = ArrowTableSource( + table, tag_columns=["id"], tag_only_columns=True, infer_nullable=True + ) db = InMemoryArrowDatabase() return FunctionJobNode(pod, source, pipeline_database=db) diff --git a/tests/test_core/function_pod/test_node_content_hash_redundancy.py b/tests/test_core/function_pod/test_node_content_hash_redundancy.py index 647c940f5..757292527 100644 --- a/tests/test_core/function_pod/test_node_content_hash_redundancy.py +++ b/tests/test_core/function_pod/test_node_content_hash_redundancy.py @@ -84,7 +84,9 @@ def _make_source_stream(values: list[int]) -> ArrowTableSource: "x": pa.array(values, type=pa.int64()), } ) - return ArrowTableSource(table=table, tag_columns=["id"], infer_nullable=True) + return ArrowTableSource( + table=table, tag_columns=["id"], tag_only_columns=True, infer_nullable=True + ) def _all_base_entry_ids(node: FunctionJobNode, stream) -> set[bytes]: @@ -349,7 +351,9 @@ def _make_random_source(rng: random.Random) -> ArrowTableSource: "x": pa.array(values, type=pa.int64()), } ) - return ArrowTableSource(table=table, tag_columns=["id"], infer_nullable=True) + return ArrowTableSource( + table=table, tag_columns=["id"], tag_only_columns=True, infer_nullable=True + ) @staticmethod def _system_tags_key(source: ArrowTableSource) -> tuple: @@ -551,7 +555,9 @@ def test_tdb_v1_path_used_after_attach(self): from orcapod.system_constants import TRACKING_DB_SCHEMA_VERSION table = pa.table({"x": pa.array([1], type=pa.int64())}) - source = ArrowTableSource(table=table, tag_columns=[], infer_nullable=True) + source = ArrowTableSource( + table=table, tag_columns=[], tag_only_columns=True, infer_nullable=True + ) def noop(x: int, ctx=None) -> None: pass @@ -579,7 +585,9 @@ def test_tdb_v0_entries_not_visible_to_new_code(self): from orcapod.side_effects import SideEffectJobNode, SideEffectPod table = pa.table({"x": pa.array([1], type=pa.int64())}) - source = ArrowTableSource(table=table, tag_columns=[], infer_nullable=True) + source = ArrowTableSource( + table=table, tag_columns=[], tag_only_columns=True, infer_nullable=True + ) def noop(x: int, ctx=None) -> None: pass diff --git a/tests/test_core/function_pod/test_output_conformance.py b/tests/test_core/function_pod/test_output_conformance.py index 78533335d..face69896 100644 --- a/tests/test_core/function_pod/test_output_conformance.py +++ b/tests/test_core/function_pod/test_output_conformance.py @@ -71,7 +71,10 @@ async def async_fn(x: int) -> Any: def make_source(xs: tuple[int, ...] = (1,)) -> DictSource: """A source with one row per ``x`` (tag ``k``, data ``x: int``).""" return DictSource( - data=[{"k": f"r{x}", "x": x} for x in xs], tag_columns=["k"], source_id="src" + data=[{"k": f"r{x}", "x": x} for x in xs], + tag_columns=["k"], + tag_only_columns=True, + source_id="src", ) @@ -871,6 +874,7 @@ def test_inferred_source_uses_value_class() -> None: source = DictSource( data=[{"y": cast(Any, ExtendedReading(3, 9))}], tag_columns=[], + tag_only_columns=True, source_id="subclass_source", ) table = source.as_table() diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py index 4d6eb6755..e529d7cc2 100644 --- a/tests/test_core/function_pod/test_polars_dataframe_values.py +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -92,6 +92,7 @@ def _int_source() -> DictSource: return DictSource( [{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"], + tag_only_columns=True, ) @@ -105,6 +106,7 @@ def _frame_source() -> DictSource: return DictSource( rows, tag_columns=["g", "k"], + tag_only_columns=True, data_schema={"g": str, "k": str, "df": pl.DataFrame}, ) @@ -259,6 +261,7 @@ def test_join_passes_frames_through(self) -> None: weights = DictSource( [{"g": "a", "k": "1", "w": 10}, {"g": "b", "k": "3", "w": 30}], tag_columns=["g", "k"], + tag_only_columns=True, ) stream = Join()(_frame_source(), weights) got = { @@ -274,6 +277,7 @@ def test_semi_join_keeps_frames_of_matching_rows(self) -> None: keys = DictSource( [{"g": "a", "k": "2", "w": 1}, {"g": "b", "k": "3", "w": 2}], tag_columns=["g", "k"], + tag_only_columns=True, ) stream = SemiJoin()(_frame_source(), keys) got = {str(tag["k"]): data["df"] for tag, data in stream.iter_data()} @@ -328,6 +332,7 @@ def _order_source() -> DictSource: for k, (columns, fields) in _ORDERS.items() ], tag_columns=["k"], + tag_only_columns=True, ) @@ -360,6 +365,7 @@ def _null_struct_source() -> DictSource: {"k": "struct_of_nulls", "whole_struct_null": False}, ], tag_columns=["k"], + tag_only_columns=True, ) @@ -528,7 +534,7 @@ def frame_height(df: pl.DataFrame) -> int: fh.write("height\\n") return df.height - source = DictSource([{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"]) + source = DictSource([{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"], tag_only_columns=True) job = PipelineJob(name="frames", store=DeltaTableDatabase(store_path)) with job: frames = make_frame.pod(source, label="make") diff --git a/tests/test_core/nodes/test_function_job_node_async_execute.py b/tests/test_core/nodes/test_function_job_node_async_execute.py index 72fdd4e13..e722d67c6 100644 --- a/tests/test_core/nodes/test_function_job_node_async_execute.py +++ b/tests/test_core/nodes/test_function_job_node_async_execute.py @@ -29,7 +29,9 @@ def _make_source(n: int = 3) -> ArrowTableSource: "value": pa.array(list(range(n)), type=pa.int64()), } ) - return ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + return ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) def double_no_db(value: int) -> int: diff --git a/tests/test_core/nodes/test_function_node_empty_data.py b/tests/test_core/nodes/test_function_node_empty_data.py index 3158dc148..7bb12fd36 100644 --- a/tests/test_core/nodes/test_function_node_empty_data.py +++ b/tests/test_core/nodes/test_function_node_empty_data.py @@ -31,7 +31,9 @@ def persistent_node(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) return FunctionJobNode( pod, @@ -79,7 +81,9 @@ def ephemeral_node(): "value": pa.array([1], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) node = FunctionJobNode( pod, @@ -189,7 +193,9 @@ def test_cache_hit_returns_cached_result(self, ephemeral_node): "value": pa.array([1], type=pa.int64()), } ) - ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) def triple_value(result: int) -> int: return result * 3 @@ -239,7 +245,9 @@ def test_cache_miss_raises_ephemeral_result_missing_error(self, ephemeral_node): "value": pa.array([1], type=pa.int64()), } ) - ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) def triple_value(result: int) -> int: return result * 3 diff --git a/tests/test_core/nodes/test_function_node_fetch_joined.py b/tests/test_core/nodes/test_function_node_fetch_joined.py index 52aa89d06..f7f5891e4 100644 --- a/tests/test_core/nodes/test_function_node_fetch_joined.py +++ b/tests/test_core/nodes/test_function_node_fetch_joined.py @@ -30,7 +30,9 @@ def node_without_db(): "value": pa.array([1], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) return FunctionJobNode(pod, src) @@ -44,7 +46,9 @@ def node_with_empty_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) return FunctionJobNode( pod, diff --git a/tests/test_core/nodes/test_function_node_get_cached.py b/tests/test_core/nodes/test_function_node_get_cached.py index 26b605bbf..d501938f6 100644 --- a/tests/test_core/nodes/test_function_node_get_cached.py +++ b/tests/test_core/nodes/test_function_node_get_cached.py @@ -24,7 +24,9 @@ def function_node_with_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) pipeline_db = InMemoryArrowDatabase() @@ -46,7 +48,9 @@ def test_returns_empty_dict_when_no_db(self): "value": pa.array([1], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) node = FunctionJobNode(pod, src) diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index 2e1f10c83..48b11c8cb 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -33,7 +33,7 @@ def _make_source(n: int = 3) -> ArrowTableSource: ] ), ) - return ArrowTableSource(table, tag_columns=["id"]) + return ArrowTableSource(table, tag_columns=["id"], tag_only_columns=True) def _make_node(n: int = 3, db: InMemoryArrowDatabase | None = None) -> FunctionJobNode: diff --git a/tests/test_core/nodes/test_node_execute.py b/tests/test_core/nodes/test_node_execute.py index 4ff84e053..447ad1c1e 100644 --- a/tests/test_core/nodes/test_node_execute.py +++ b/tests/test_core/nodes/test_node_execute.py @@ -31,7 +31,9 @@ def function_node_with_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) pipeline_db = InMemoryArrowDatabase() @@ -53,7 +55,9 @@ def function_node_no_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) return FunctionJobNode(pod, src) @@ -129,7 +133,7 @@ def operator_with_db(): "value": pa.array([10, 20], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"]) + src = ArrowTableSource(table, tag_columns=["key"], tag_only_columns=True) op = SelectDataColumns(columns=["value"]) db = InMemoryArrowDatabase() node = OperatorNode( @@ -149,7 +153,7 @@ def operator_no_db(): "value": pa.array([10, 20], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"]) + src = ArrowTableSource(table, tag_columns=["key"], tag_only_columns=True) op = SelectDataColumns(columns=["value"]) node = OperatorNode(op, input_streams=[src]) return node, src diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 50532f33b..ebb582581 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -149,7 +149,9 @@ def test_validate_compatible_source(self, tag_schema, data_schema): from orcapod.core.sources.dict_source import DictSource node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) node.validate(src) # must not raise def test_validate_incompatible_raises(self, tag_schema, data_schema): @@ -157,7 +159,9 @@ def test_validate_incompatible_raises(self, tag_schema, data_schema): from orcapod.core.sources.dict_source import DictSource node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(data=[{"id": 1, "wrong": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "wrong": 1.0}], tag_columns=["id"], tag_only_columns=True + ) with pytest.raises(SourceSpecMismatchError): node.validate(src) @@ -196,7 +200,9 @@ def test_bound_content_hash_is_concrete_hash(self, tag_schema, data_schema): from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -207,7 +213,9 @@ def test_bound_pipeline_hash_still_schema_based(self, tag_schema, data_schema): from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src @@ -233,7 +241,9 @@ def test_bound_source_property_updates_in_place(self, tag_schema, data_schema): name="x", tag_schema=tag_schema, data_schema=data_schema ) assert job_node.bound_source is None - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node.bound_source = src assert job_node.bound_source is src @@ -249,7 +259,9 @@ def test_bound_source_setter_clears_content_hash_cache( ) schema_hash = job_node.content_hash() # populates cache with schema-based hash - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node.bound_source = src # should clear the cache bound_hash = job_node.content_hash() @@ -275,7 +287,9 @@ def test_source_job_node_as_table_delegates_to_concrete( from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -371,7 +385,9 @@ def test_unbound_system_tag_names_match_bound(self, tag_schema, data_schema): from orcapod.core.sources.dict_source import DictSource from orcapod.types import ColumnConfig - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) unbound = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema ) @@ -405,7 +421,9 @@ def test_bound_keys_all_info_matches_source_directly(self, tag_schema, data_sche from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -436,7 +454,9 @@ def test_bound_output_schema_all_info_matches_source_directly( from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) diff --git a/tests/test_core/operators/test_aggregated_list_types.py b/tests/test_core/operators/test_aggregated_list_types.py index 3b115bca2..26211227b 100644 --- a/tests/test_core/operators/test_aggregated_list_types.py +++ b/tests/test_core/operators/test_aggregated_list_types.py @@ -60,7 +60,7 @@ def _source() -> ArrowTableSource: ] ), ) - return ArrowTableSource(table, tag_columns=["g", "i"]) + return ArrowTableSource(table, tag_columns=["g", "i"], tag_only_columns=True) OPERATORS = { diff --git a/tests/test_core/operators/test_chained_aggregation.py b/tests/test_core/operators/test_chained_aggregation.py index 544c511e4..6e9a37971 100644 --- a/tests/test_core/operators/test_chained_aggregation.py +++ b/tests/test_core/operators/test_chained_aggregation.py @@ -64,7 +64,9 @@ def _source( [pa.field(name, t, nullable=False) for name, (t, _) in columns.items()] ) return ArrowTableSource( - pa.Table.from_arrays(arrays, schema=schema), tag_columns=list(tags) + pa.Table.from_arrays(arrays, schema=schema), + tag_columns=list(tags), + tag_only_columns=True, ) diff --git a/tests/test_core/operators/test_drop_nulls.py b/tests/test_core/operators/test_drop_nulls.py index 654f4be06..79baf3f5c 100644 --- a/tests/test_core/operators/test_drop_nulls.py +++ b/tests/test_core/operators/test_drop_nulls.py @@ -41,7 +41,10 @@ def _source( table = pa.table(list(columns.values()), names=list(columns)) schema = pa.schema([f.with_nullable(f.name in nullable) for f in table.schema]) return ArrowTableSource( - table.cast(schema), tag_columns=list(tags), source_id=source_id + table.cast(schema), + tag_columns=list(tags), + tag_only_columns=True, + source_id=source_id, ) @@ -345,6 +348,7 @@ def logical() -> DictSource: {"k": "c", "p": Path("/c"), "arr": None}, ], tag_columns=["k"], + tag_only_columns=True, source_id="logical", ) diff --git a/tests/test_core/operators/test_empty_inputs.py b/tests/test_core/operators/test_empty_inputs.py index 0f69fd71e..b94bdbcbc 100644 --- a/tests/test_core/operators/test_empty_inputs.py +++ b/tests/test_core/operators/test_empty_inputs.py @@ -100,7 +100,7 @@ def _filtered_empty(tags: list[str], data: list[str]) -> StreamProtocol: source-info columns, which the join operators rename and reorder. """ table = pa.table({name: pa.array([1, 2], type=INT) for name in [*tags, *data]}) - source = ArrowTableSource(table, tag_columns=tags) + source = ArrowTableSource(table, tag_columns=tags, tag_only_columns=True) empty = PolarsFilter(predicates=[pl.lit(False)]).static_process(source) assert len(empty.as_table()) == 0 return empty diff --git a/tests/test_core/operators/test_group_by.py b/tests/test_core/operators/test_group_by.py index 8f2151957..01dc72ae1 100644 --- a/tests/test_core/operators/test_group_by.py +++ b/tests/test_core/operators/test_group_by.py @@ -36,6 +36,7 @@ def session_source(session_table) -> ArrowTableSource: return ArrowTableSource( session_table, tag_columns=["subject", "date", "probe"], + tag_only_columns=True, infer_nullable=True, ) @@ -122,7 +123,9 @@ def test_non_key_tag_keeps_its_arrow_type(self): "path": pa.array(["b", "a"], pa.large_string()), } ) - source = ArrowTableSource(table, tag_columns=["subject", "probe"]) + source = ArrowTableSource( + table, tag_columns=["subject", "probe"], tag_only_columns=True + ) out = GroupBy(by=["subject"])(source) assert out.as_table().schema.field("probe").type == pa.large_list(pa.int8()) @@ -176,7 +179,10 @@ def test_regroup_orders_list_tags_containing_nulls(self): } ) source = ArrowTableSource( - table, tag_columns=["a", "b", "c"], infer_nullable=True + table, + tag_columns=["a", "b", "c"], + tag_only_columns=True, + infer_nullable=True, ) first = GroupBy(by=["a", "b"])(source) # Regrouping on ``a`` alone leaves ``b`` scalar and ``c`` list-valued: @@ -198,7 +204,10 @@ def test_members_tied_on_scalar_tags_order_by_list_tag_with_null(self): } ) source = ArrowTableSource( - table, tag_columns=["a", "b", "c"], infer_nullable=True + table, + tag_columns=["a", "b", "c"], + tag_only_columns=True, + infer_nullable=True, ) # Drop ``b`` after the first grouping so the two members differ only in # the list-valued tag ``c``: [1, None] against [1, 2]. @@ -247,7 +256,10 @@ def test_scalar_non_key_tags_sort_ascending_with_nulls_last(self): } ) source = ArrowTableSource( - table, tag_columns=["g", "n", "s"], infer_nullable=True + table, + tag_columns=["g", "n", "s"], + tag_only_columns=True, + infer_nullable=True, ) row = GroupBy(by=["g"])(source).as_table().to_pylist()[0] assert row["n"] == [1, 1, 2, 3, None] @@ -260,7 +272,10 @@ def test_row_order_does_not_affect_output(self, session_table): def run(tbl): src = ArrowTableSource( - tbl, tag_columns=["subject", "date", "probe"], infer_nullable=True + tbl, + tag_columns=["subject", "date", "probe"], + tag_only_columns=True, + infer_nullable=True, ) return GroupBy(by=["subject", "date"]).process(src).as_table() @@ -277,7 +292,9 @@ def test_falls_back_to_record_id_when_key_covers_all_tags(self): when the default became ``arrow_table: