diff --git a/.zed/rules b/.zed/rules index a646e3419..22af2b746 100644 --- a/.zed/rules +++ b/.zed/rules @@ -456,6 +456,11 @@ Use a non-public entry point (`unary_static_process`, `static_process`, entry point. The standing exception is verifying a schema prediction, which must compare against `static_process(...).output_schema(...)`; see "Important implementation details". +Tests and golden fixtures that build a source with tag columns pass `tag_only_columns=True` +unless the test is about the default (developer, November 16, 2026, ITL-1159). This covers the +sources that take the option (the table, file and database sources); `ListSource`, +`DerivedSource`, `CachedSource` and `PollingSource` do not take it. + --- ## Project layout diff --git a/CLAUDE.md b/CLAUDE.md index a2b085b31..f7d9ad41c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -467,6 +467,11 @@ Use a non-public entry point (`unary_static_process`, `static_process`, entry point. The standing exception is verifying a schema prediction, which must compare against `static_process(...).output_schema(...)`; see "Important implementation details". +Tests and golden fixtures that build a source with tag columns pass `tag_only_columns=True` +unless the test is about the default (developer, November 16, 2026, ITL-1159). This covers the +sources that take the option (the table, file and database sources); `ListSource`, +`DerivedSource`, `CachedSource` and `PollingSource` do not take it. + --- ## Project layout diff --git a/changelog.d/ITL-1159.md b/changelog.d/ITL-1159.md new file mode 100644 index 000000000..bd421726e --- /dev/null +++ b/changelog.d/ITL-1159.md @@ -0,0 +1,21 @@ +### Added + +- Every source built from a table takes the keyword-only option `tag_only_columns`: + `ArrowTableSource`, `DictSource`, `DataFrameSource`, `CSVSource`, `DeltaTableSource`, + `DBTableSource`, `SQLiteTableSource`, `PostgreSQLTableSource` and `SpiralDBTableSource` + (ITL-1159). In v0.3 every tag is plain (a tag only, never also a data column), and the option + has one accepted value, `True` (all tags plain), which is also the default. A source built + with `tag_only_columns=True` is the source built without it: the same content and pipeline + hashes, source id, record ids, rows, stored paths and saved config. Any other value (`False`, + an empty collection, column names, even all of them) raises `ValueError`, because referential + tags arrive in v0.4. In v0.4 a declared tag becomes referential by default (a tag that is also + the data column of the same name) and `tag_only_columns` keeps a tag plain, so code that must + keep today's behaviour can pass `tag_only_columns=True` now. `ListSource`, `DerivedSource`, + `CachedSource` and `PollingSource` do not take the option. No hash or golden value changes; + the golden fixtures that build a source with tag columns now pass `tag_only_columns=True` + explicitly. +- A saved pipeline or job file does not record `tag_only_columns`: v0.3 writes no such key, and + both file format versions stay `0.2.0`. Plain tags are decided by + the format version: every source in a file at `0.1.0` or `0.2.0` has all its tags plain. The + format that brings referential tags (v0.4) writes `tag_only_columns` for every source, and its + reader refuses a source config without it, so a v0.3 file reads back plain in v0.4. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 3c6c28580..e86604edd 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -44,6 +44,25 @@ Two construction rules apply to every source and are the most common first error - At least one column must be left over as a data column, otherwise it raises `ValueError: No data columns found in the table`. +**Plain tags and `tag_only_columns`.** The sources built from a table (`ArrowTableSource`, +`DictSource`, `DataFrameSource`, `CSVSource`, `DeltaTableSource` and the database sources) also +take the keyword-only option `tag_only_columns`. In v0.3 every tag column is plain: it is a tag only, +never also a data column. `tag_only_columns=True` says so explicitly; it is the default and +the only value v0.3 accepts, and it changes no hash, source id, record id or saved config. Any +other value is refused. In v0.4 tags become referential by default (a tag that is also the +data column of the same name), and `tag_only_columns` keeps a tag plain, so code that must +keep today's behaviour passes `tag_only_columns=True` now. + +```python +from orcapod.sources import DictSource + +plain = DictSource( + data=[{"k": "a", "x": 1}], tag_columns=["k"], tag_only_columns=True +) +print(plain.keys()) +# (('k',), ('x',)) +``` + Data with no rows has no columns to read, so `DictSource(data=[], ...)` raises `ValueError: Table is empty`. To build a source with no rows, pass `data_schema=` so the columns are known. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 29fb94d3f..32934838e 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -689,6 +689,12 @@ sequence before the stream is returned: 3. **Site 5 — Per-row `record_id`:** a UUID v5 computed from `(source_id, row_provenance_token)` and stored per row. 4. Two system tag columns, `_tag_source_id::{schema_hash}` and `_tag_record_id::{schema_hash}`, are appended to every row. +`tag_only_columns=True` (every tag plain; the default and the only value v0.3 accepts) enters +none of these sites: a source built with it has the schema hash, `source_id`, record ids and +saved config of the same source built without it. Every golden fixture that builds a source +with tag columns passes `tag_only_columns=True`, so the golden values of data context `v1` pin +plain tags, and stay pinned when v0.4 makes tags referential by default. + At Join time (site 6), all existing system tag columns are **renamed** by appending a topology suffix, encoding the full join lineage into each column name. diff --git a/docs/tutorial/01-first-stream.md b/docs/tutorial/01-first-stream.md index 55225e2d1..aa3af5b7a 100644 --- a/docs/tutorial/01-first-stream.md +++ b/docs/tutorial/01-first-stream.md @@ -54,6 +54,11 @@ Every column of a source is either a tag column or a data column. uses tags to match rows when it combines tables. - **Data columns** are everything else: the values that were measured or recorded. +A tag column is a tag only, never also a data column. `CSVSource`, like every source built +from a table, accepts `tag_only_columns=True`, which says exactly that and is the default in +this version. From Orcapod v0.4 tags become referential by default (a tag that is also the +data column of the same name), and `tag_only_columns=True` keeps them plain as they are here. + You choose the tag columns; all remaining columns become data columns. `output_schema()` returns both groups, with their types: diff --git a/src/orcapod/core/referential_tags.py b/src/orcapod/core/referential_tags.py new file mode 100644 index 000000000..ff9a8b2f2 --- /dev/null +++ b/src/orcapod/core/referential_tags.py @@ -0,0 +1,39 @@ +"""The ``tag_only_columns`` option of a source (ITL-1159). + +Referential tags arrive in Orcapod v0.4: a declared source tag will then be referential by +default (a handle to the data column of the same name), and ``tag_only_columns`` names the +tags kept plain. In v0.3 every tag is plain. Every source that declares tag columns already +accepts the option, with exactly one value, ``True`` (all tags plain), which is also the +default, so code and tests written for v0.3 can state the plain behaviour explicitly and +keep it when the default changes. Any other value would make a tag referential and is +refused. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +TagOnlyColumns = str | Iterable[str] | bool +"""The ``tag_only_columns`` option of a source: names, or ``True`` for every tag.""" + + +def check_tag_only_columns(tag_only_columns: TagOnlyColumns) -> None: + """Refuse a ``tag_only_columns`` value that would make a tag referential. + + ``tag_only_columns=True`` keeps every declared tag plain (tag only, not data), which is + what every source does in v0.3; it is also the default. Any other value (``False``, + names, an empty collection, even a collection listing every tag) is refused: + referential tags arrive in v0.4. + + Args: + tag_only_columns: The tags kept plain; only ``True`` (all of them) is accepted. + + Raises: + ValueError: If ``tag_only_columns`` is anything but ``True``. + """ + if tag_only_columns is not True: + raise ValueError( + f"tag_only_columns={tag_only_columns!r}: every tag is plain in Orcapod v0.3, " + "so only True (the default) is accepted. Referential tags (tag_only_columns " + "naming some of the tags, an empty collection, or False) arrive in v0.4." + ) diff --git a/src/orcapod/core/sources/arrow_table_source.py b/src/orcapod/core/sources/arrow_table_source.py index b573d76f9..b4d78a254 100644 --- a/src/orcapod/core/sources/arrow_table_source.py +++ b/src/orcapod/core/sources/arrow_table_source.py @@ -4,6 +4,7 @@ from collections.abc import Collection from typing import TYPE_CHECKING, Any, ClassVar +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.utils import arrow_utils @@ -67,6 +68,11 @@ class ArrowTableSource(RootSource): table hash, which changes with row order. Whenever row order can vary between runs, pass an explicit ``source_id`` and set ``record_id_column`` to a column that is unique per row. + + Plain tags: ``tag_only_columns`` (keyword-only) is ``True``, the default, and keeps + every tag column a tag only, not data, as every source does in v0.3. Any other value + is refused: referential tags arrive in v0.4, where tags become referential by default + and ``tag_only_columns`` names the tags kept plain. """ # The config type key; also the kind prefix of the default source id. @@ -79,8 +85,11 @@ def __init__( system_tag_columns: Collection[str] = (), record_id_column: str | None = None, infer_nullable: bool = False, + *, + tag_only_columns: TagOnlyColumns = True, **kwargs: Any, ) -> None: + check_tag_only_columns(tag_only_columns) super().__init__(**kwargs) tag_columns = _normalize_column_list(tag_columns) diff --git a/src/orcapod/core/sources/csv_source.py b/src/orcapod/core/sources/csv_source.py index 05470c874..5d48d31b7 100644 --- a/src/orcapod/core/sources/csv_source.py +++ b/src/orcapod/core/sources/csv_source.py @@ -6,6 +6,7 @@ from typing_extensions import override +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.reload import keep_nullable_columns, same_rows from orcapod.core.sources.stream_builder import ( @@ -41,6 +42,11 @@ class CSVSource(RootSource): positions are record ids; in any order with one), holds the new rows and moves its modified time. Record ids keep their rule: the ``record_id_column`` value, or the row's position. + + Plain tags: ``tag_only_columns`` (keyword-only) is ``True``, the default, and keeps + every tag column a tag only, not data, as every source does in v0.3. Any other value + is refused: referential tags arrive in v0.4, where tags become referential by default + and ``tag_only_columns`` names the tags kept plain. """ def __init__( @@ -51,8 +57,11 @@ def __init__( record_id_column: str | None = None, source_id: str | None = None, schema: pa.Schema | None = None, + *, + tag_only_columns: TagOnlyColumns = True, **kwargs: Any, ) -> None: + check_tag_only_columns(tag_only_columns) if source_id is None: source_id = file_path super().__init__(source_id=source_id, **kwargs) @@ -159,6 +168,7 @@ def from_config(cls, config: dict[str, Any], db_registry=None) -> CSVSource: return cls( file_path=config["file_path"], tag_columns=config.get("tag_columns", ()), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/data_frame_source.py b/src/orcapod/core/sources/data_frame_source.py index 00268bf19..7f7cad859 100644 --- a/src/orcapod/core/sources/data_frame_source.py +++ b/src/orcapod/core/sources/data_frame_source.py @@ -4,6 +4,7 @@ from collections.abc import Collection from typing import TYPE_CHECKING, Any, ClassVar +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.utils import arrow_utils, polars_data_utils @@ -26,6 +27,11 @@ class DataFrameSource(RootSource): The DataFrame is converted to an Arrow table and enriched by ``SourceStreamBuilder`` (source-info, schema-hash, system tags). + + Plain tags: ``tag_only_columns`` (keyword-only) is ``True``, the default, and keeps + every tag column a tag only, not data, as every source does in v0.3. Any other value + is refused: referential tags arrive in v0.4, where tags become referential by default + and ``tag_only_columns`` names the tags kept plain. """ # The config type key; also the kind prefix of the default source id. @@ -38,8 +44,11 @@ def __init__( system_tag_columns: Collection[str] = (), source_id: str | None = None, schema: pa.Schema | None = None, + *, + tag_only_columns: TagOnlyColumns = True, **kwargs: Any, ) -> None: + check_tag_only_columns(tag_only_columns) super().__init__(source_id=source_id, **kwargs) df = pl.DataFrame(data) diff --git a/src/orcapod/core/sources/db_table_source.py b/src/orcapod/core/sources/db_table_source.py index 3dc4f2fd2..b73248d15 100644 --- a/src/orcapod/core/sources/db_table_source.py +++ b/src/orcapod/core/sources/db_table_source.py @@ -18,6 +18,7 @@ from typing_extensions import override +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.reload import keep_nullable_columns, same_rows from orcapod.core.sources.stream_builder import ( @@ -63,6 +64,11 @@ class DBTableSource(RootSource): label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: Orcapod configuration (controls hash character counts, etc.). + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Keyword-only. ``True`` is the default and + the only value v0.3 accepts: any other value is refused, because + referential tags arrive in v0.4 (tags then become referential by default + and ``tag_only_columns`` keeps a tag plain). Raises: ValueError: If the table is not found, has no PK columns and none are @@ -81,8 +87,10 @@ def __init__( data_context: str | contexts.DataContext | None = None, config: OrcapodConfig | None = None, *, + tag_only_columns: TagOnlyColumns = True, _query: str | None = None, ) -> None: + check_tag_only_columns(tag_only_columns) if source_id is None: source_id = table_name super().__init__( diff --git a/src/orcapod/core/sources/delta_table_source.py b/src/orcapod/core/sources/delta_table_source.py index f4762d497..1d82ad945 100644 --- a/src/orcapod/core/sources/delta_table_source.py +++ b/src/orcapod/core/sources/delta_table_source.py @@ -6,6 +6,7 @@ from typing_extensions import override +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.reload import same_rows from orcapod.core.sources.stream_builder import ( @@ -38,6 +39,11 @@ class DeltaTableSource(RootSource): version that only compacted files or changed metadata changes nothing), holds the new rows and moves its modified time. Record ids keep their rule: the ``record_id_column`` value, or the row's position. + + Plain tags: ``tag_only_columns`` (keyword-only) is ``True``, the default, and keeps + every tag column a tag only, not data, as every source does in v0.3. Any other value + is refused: referential tags arrive in v0.4, where tags become referential by default + and ``tag_only_columns`` names the tags kept plain. """ def __init__( @@ -47,8 +53,11 @@ def __init__( system_tag_columns: Collection[str] = (), record_id_column: str | None = None, source_id: str | None = None, + *, + tag_only_columns: TagOnlyColumns = True, **kwargs: Any, ) -> None: + check_tag_only_columns(tag_only_columns) resolved = Path(delta_table_path).resolve() if source_id is None: @@ -156,6 +165,7 @@ def from_config( return cls( delta_table_path=config["delta_table_path"], tag_columns=config.get("tag_columns", ()), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/dict_source.py b/src/orcapod/core/sources/dict_source.py index 8b6cecd82..584ee2a96 100644 --- a/src/orcapod/core/sources/dict_source.py +++ b/src/orcapod/core/sources/dict_source.py @@ -3,6 +3,7 @@ from collections.abc import Collection, Mapping from typing import TYPE_CHECKING, Any, ClassVar +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.stream_builder import SourceStreamBuilder from orcapod.types import DataValue, SchemaLike @@ -23,6 +24,11 @@ class DictSource(RootSource): Each dict becomes one (tag, data) pair in the stream. The dicts are converted to an Arrow table via the data-context type converter, then enriched by ``SourceStreamBuilder`` (source-info, schema-hash, system tags). + + Plain tags: ``tag_only_columns`` (keyword-only) is ``True``, the default, and keeps + every tag column a tag only, not data, as every source does in v0.3. Any other value + is refused: referential tags arrive in v0.4, where tags become referential by default + and ``tag_only_columns`` names the tags kept plain. """ # The config type key; also the kind prefix of the default source id. @@ -35,8 +41,11 @@ def __init__( system_tag_columns: Collection[str] = (), data_schema: SchemaLike | pa.Schema | None = None, source_id: str | None = None, + *, + tag_only_columns: TagOnlyColumns = True, **kwargs: Any, ) -> None: + check_tag_only_columns(tag_only_columns) super().__init__(source_id=source_id, **kwargs) tag_columns = _normalize_column_list(tag_columns) diff --git a/src/orcapod/core/sources/postgresql_table_source.py b/src/orcapod/core/sources/postgresql_table_source.py index 4492f410e..ad66a07c5 100644 --- a/src/orcapod/core/sources/postgresql_table_source.py +++ b/src/orcapod/core/sources/postgresql_table_source.py @@ -18,6 +18,7 @@ from typing_extensions import override +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.db_table_source import DBTableSource from orcapod.databases.postgresql_connector import PostgreSQLConnector @@ -57,6 +58,11 @@ class PostgreSQLTableSource(DBTableSource): label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: OrcaPod configuration. + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Keyword-only. ``True`` is the default and + the only value v0.3 accepts: any other value is refused, because + referential tags arrive in v0.4 (tags then become referential by default + and ``tag_only_columns`` keeps a tag plain). Raises: ValueError: If the table is not found, is empty, or has no PK and @@ -75,7 +81,10 @@ def __init__( label: str | None = None, data_context: str | contexts.DataContext | None = None, config: OrcapodConfig | None = None, + *, + tag_only_columns: TagOnlyColumns = True, ) -> None: + check_tag_only_columns(tag_only_columns) self._dsn = dsn # store before try — needed by to_config even if super() raises connector = PostgreSQLConnector( dsn @@ -91,6 +100,7 @@ def __init__( label=label, data_context=data_context, config=config, + tag_only_columns=tag_only_columns, ) finally: try: @@ -136,6 +146,7 @@ def from_config( dsn=config["dsn"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 81d8f0a22..e812555c6 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -42,6 +42,7 @@ from typing_extensions import override +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.base import RootSource from orcapod.core.sources.derived_source import validate_tags_as_data from orcapod.core.sources.stream_builder import SourceStreamBuilder @@ -221,6 +222,11 @@ class SpiralDBTableSource(RootSource): config: OrcaPod configuration. overrides: Optional pyspiral client config overrides passed through to ``SpiralDBConnector``, e.g. ``{"server.url": "https://api.spiraldb.dev"}``. + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Keyword-only. ``True`` is the default and + the only value v0.3 accepts: any other value is refused, because + referential tags arrive in v0.4 (tags then become referential by default + and ``tag_only_columns`` keeps a tag plain). tags_as_data: Maps a key column that is a tag column of this source to the name of a data column that receives a copy of it, as in ``DerivedSource``. ``None`` (default) or an empty mapping copies nothing. @@ -257,7 +263,10 @@ def __init__( config: "OrcapodConfig | None" = None, overrides: dict[str, str] | None = None, tags_as_data: Mapping[str, str] | None = None, + *, + tag_only_columns: TagOnlyColumns = True, ) -> None: + check_tag_only_columns(tag_only_columns) for name, value in (("limit", limit), ("max_rows", max_rows)): if value is not None and not _is_positive_int(value): raise ValueError( @@ -847,6 +856,7 @@ def from_config( table_name=config["table_name"], dataset=config.get("dataset", "default"), tag_columns=config.get("tag_columns"), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), where=_locationfrom_config(config.get("where") or {}), limit=config.get("limit"), diff --git a/src/orcapod/core/sources/sqlite_table_source.py b/src/orcapod/core/sources/sqlite_table_source.py index 76f207961..1393933e6 100644 --- a/src/orcapod/core/sources/sqlite_table_source.py +++ b/src/orcapod/core/sources/sqlite_table_source.py @@ -28,6 +28,7 @@ from typing_extensions import override +from orcapod.core.referential_tags import TagOnlyColumns, check_tag_only_columns from orcapod.core.sources.db_table_source import DBTableSource from orcapod.databases.sqlite_connector import SQLiteConnector from orcapod.utils.schema_utils import _normalize_column_list @@ -80,6 +81,11 @@ class SQLiteTableSource(DBTableSource): label: Human-readable label for this source node. data_context: Data context governing type conversion and hashing. config: OrcaPod configuration. + tag_only_columns: Tag columns kept as tags only, not data, exactly as in + v0.3; ``True`` for all of them. Keyword-only. ``True`` is the default and + the only value v0.3 accepts: any other value is refused, because + referential tags arrive in v0.4 (tags then become referential by default + and ``tag_only_columns`` keeps a tag plain). Raises: ValueError: If the table is not found or is empty. @@ -97,7 +103,10 @@ def __init__( label: str | None = None, data_context: str | contexts.DataContext | None = None, config: "OrcapodConfig | None" = None, + *, + tag_only_columns: TagOnlyColumns = True, ) -> None: + check_tag_only_columns(tag_only_columns) self._db_path = db_path # A check (ITL-1102) reopens the database found now, whatever the working # directory is then; an in-memory database or a URI is kept as given. @@ -132,6 +141,7 @@ def __init__( label=label, data_context=data_context, config=config, + tag_only_columns=tag_only_columns, _query=_query, ) finally: @@ -180,6 +190,7 @@ def from_config( db_path=config["db_path"], table_name=config["table_name"], tag_columns=config.get("tag_columns"), + tag_only_columns=config.get("tag_only_columns", True), system_tag_columns=config.get("system_tag_columns", ()), record_id_column=config.get("record_id_column"), source_id=config.get("source_id"), diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index 2ad9843d9..5785266b3 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -31,6 +31,11 @@ # only test membership (``Pipeline.load``). # 0.1.0: first format (v0.2.0). # 0.2.0: adds ``orcapod_version`` (v0.3.0); a 0.1.0 file loads unchanged. +# Plain tags are decided by this version, never by a missing key (ITL-1159): every +# source in a file at 0.1.0 or 0.2.0 has all its tags plain, and its config never +# writes ``tag_only_columns`` (v0.3 accepts only ``tag_only_columns=True``). The +# format that brings referential tags (v0.4) writes ``tag_only_columns`` for every +# source; a reader of that format refuses a source config without it. PIPELINE_FORMAT_VERSION = "0.2.0" SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0"}) @@ -1289,6 +1294,7 @@ def _get_arrow_primitive_types() -> dict[str, Any]: # --------------------------------------------------------------------------- # Same scheme as ``PIPELINE_FORMAT_VERSION``: the written version, and the set a -# load accepts (``PipelineJob.load``). +# load accepts (``PipelineJob.load``). The plain-tags rule stated there holds for a +# job file too: at 0.1.0 and 0.2.0 every source's tags are plain (ITL-1159). PIPELINE_JOB_FORMAT_VERSION = "0.2.0" SUPPORTED_JOB_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2.0"}) diff --git a/tests/test_arrays/test_array_cached_rerun.py b/tests/test_arrays/test_array_cached_rerun.py index ac6e75e70..fae770a11 100644 --- a/tests/test_arrays/test_array_cached_rerun.py +++ b/tests/test_arrays/test_array_cached_rerun.py @@ -44,11 +44,13 @@ def total(y: np.ndarray) -> float: src = DictSource( data=[{"k": k, "x": v} for k, v in ARRAYS.items()], tag_columns=["k"], + tag_only_columns=True, source_id="every", ) numeric = DictSource( data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], tag_columns=["k"], + tag_only_columns=True, source_id="numeric", ) job = PipelineJob(name="rerun", database=DeltaTableDatabase(base_path=store)) diff --git a/tests/test_arrays/test_array_hash_stability.py b/tests/test_arrays/test_array_hash_stability.py index 1344a1c71..c58227c88 100644 --- a/tests/test_arrays/test_array_hash_stability.py +++ b/tests/test_arrays/test_array_hash_stability.py @@ -21,6 +21,7 @@ src = DictSource( data=[{"k": k, "x": v} for k, v in ARRAYS.items()], tag_columns=["k"], + tag_only_columns=True, source_id="every", ) out = { diff --git a/tests/test_arrays/test_array_operators.py b/tests/test_arrays/test_array_operators.py index 83a3b7966..999c65890 100644 --- a/tests/test_arrays/test_array_operators.py +++ b/tests/test_arrays/test_array_operators.py @@ -51,6 +51,7 @@ def arrays_source(source_id: str = "arrays", shift: int = 0) -> DictSource: for i, name in enumerate(NAMES) ], tag_columns=["k", "g"], + tag_only_columns=True, source_id=source_id, ) @@ -75,6 +76,7 @@ def other_source() -> DictSource: return DictSource( data=[{"k": name, "y": i} for i, name in enumerate(NAMES)], tag_columns=["k"], + tag_only_columns=True, source_id="other", ) @@ -91,7 +93,10 @@ def test_semijoin() -> None: """``SemiJoin`` keeps the matching rows with their arrays unchanged.""" keep = NAMES[::2] right = DictSource( - data=[{"k": name, "y": 0} for name in keep], tag_columns=["k"], source_id="r" + data=[{"k": name, "y": 0} for name in keep], + tag_columns=["k"], + tag_only_columns=True, + source_id="r", ) got = rows(SemiJoin()(arrays_source(), right)) assert sorted(got) == keep diff --git a/tests/test_arrays/test_array_pipeline_stores.py b/tests/test_arrays/test_array_pipeline_stores.py index 1060fa197..a599abebf 100644 --- a/tests/test_arrays/test_array_pipeline_stores.py +++ b/tests/test_arrays/test_array_pipeline_stores.py @@ -36,11 +36,13 @@ def _build(store: Any) -> PipelineJob: numeric = DictSource( data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], tag_columns=["k"], + tag_only_columns=True, source_id="numeric", ) every = DictSource( data=[{"k": k, "x": v} for k, v in ARRAYS.items()], tag_columns=["k"], + tag_only_columns=True, source_id="every", ) job = PipelineJob(name="arrays", database=store) @@ -116,6 +118,7 @@ def test_pod_output_list_of_arrays(tmp_path: Path, store: str, runner: str) -> N source = DictSource( data=[{"k": k, "x": ARRAYS[k]} for k in NUMERIC], tag_columns=["k"], + tag_only_columns=True, source_id="numeric", ) job = PipelineJob(name="array_lists", database=STORES[store](tmp_path)) @@ -149,6 +152,7 @@ def test_batch_and_group_by_over_an_array_column( for i in range(4) ], tag_columns=["g", "k"], + tag_only_columns=True, source_id="grouped", ) job = PipelineJob(name="array_groups", database=STORES[store](tmp_path)) diff --git a/tests/test_arrays/test_array_round_trip.py b/tests/test_arrays/test_array_round_trip.py index 7f7b602c7..abc47acbc 100644 --- a/tests/test_arrays/test_array_round_trip.py +++ b/tests/test_arrays/test_array_round_trip.py @@ -18,6 +18,7 @@ def _source(arrays: dict[str, Any]) -> DictSource: data=[{"k": k, "x": v} for k, v in arrays.items()], tag_columns=["k"], source_id="arrays", + tag_only_columns=True, ) diff --git a/tests/test_arrays/test_array_strict_hash.py b/tests/test_arrays/test_array_strict_hash.py index 6827ac181..c4218def4 100644 --- a/tests/test_arrays/test_array_strict_hash.py +++ b/tests/test_arrays/test_array_strict_hash.py @@ -105,6 +105,7 @@ def _source(arrays: dict[str, Any], column: str = "x") -> DictSource: data=[{"k": k, column: v} for k, v in arrays.items()], tag_columns=["k"], source_id="arrays", + tag_only_columns=True, ) @@ -351,7 +352,9 @@ def stream(context: DataContext) -> ArrowTableSource: table = context.type_converter.python_dicts_to_arrow_table( [{"k": "r1", "x": BASE}], python_schema={"k": str, "x": np.ndarray} ) - return ArrowTableSource(table, tag_columns=["k"], source_id="s") + return ArrowTableSource( + table, tag_columns=["k"], source_id="s", tag_only_columns=True + ) with pytest.raises(LogicalTypeIdentifierError, match="'x'") as raised: MergeJoin()(stream(get_default_context()), stream(lenient)).as_table() diff --git a/tests/test_arrays/test_array_value_format.py b/tests/test_arrays/test_array_value_format.py index 2cc177a1c..416f5ad75 100644 --- a/tests/test_arrays/test_array_value_format.py +++ b/tests/test_arrays/test_array_value_format.py @@ -34,14 +34,18 @@ def stored(value: Any) -> bytes: """The stored cell of a one-row ``x`` column.""" - table = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]).as_table() + table = DictSource( + data=[{"k": "r", "x": value}], tag_columns=["k"], tag_only_columns=True + ).as_table() chunk = table.column("x").chunk(0) assert isinstance(chunk, pa.ExtensionArray) return chunk.storage[0].as_py() def read_back(value: Any) -> Any: - source = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": value}], tag_columns=["k"], tag_only_columns=True + ) return next(iter(source.iter_data()))[1].as_dict()["x"] @@ -50,7 +54,9 @@ def semantic_hash(value: Any) -> Any: def data_hash(value: Any) -> Any: - source = DictSource(data=[{"k": "r", "x": value}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": value}], tag_columns=["k"], tag_only_columns=True + ) return next(iter(source.iter_data()))[1].content_hash() @@ -62,7 +68,7 @@ def data_hash(value: Any) -> Any: def test_column_type_is_orcapod_numpy_ndarray_over_large_binary() -> None: """An array column is ``orcapod.numpy.ndarray`` over ``large_binary`` with the ignore list.""" table = DictSource( - data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"] + data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"], tag_only_columns=True ).as_table() ext = table.schema.field("x").type assert isinstance(ext, pa.ExtensionType) @@ -428,7 +434,9 @@ def test_table_token_carries_the_identifier_and_the_digest() -> None: hasher = get_default_context().arrow_hasher assert isinstance(hasher, StarfixArrowHasher) array = np.arange(3.0) - table = DictSource(data=[{"k": "r", "x": array}], tag_columns=["k"]).as_table() + table = DictSource( + data=[{"k": "r", "x": array}], tag_columns=["k"], tag_only_columns=True + ).as_table() # Non-public helper: this test is about the token the hasher builds for a cell. processed = hasher._process_table_columns(table.select(["x"])) # pyright: ignore[reportPrivateUsage] token = processed.column("x")[0].as_py() @@ -498,7 +506,11 @@ def test_unsupported_arrays_are_refused_naming_column_and_dtype( ) -> None: """Unsupported arrays are refused at write time, naming the column and the dtype.""" with pytest.raises(ValueError, match="'payload'") as info: - DictSource(data=[{"k": "r", "payload": value}], tag_columns=["k"]).as_table() + DictSource( + data=[{"k": "r", "payload": value}], + tag_columns=["k"], + tag_only_columns=True, + ).as_table() assert reason.lower() in str(info.value).lower() @@ -529,7 +541,7 @@ def test_a_cell_that_is_not_a_stored_array_raises_naming_the_column( ) # The first read of the value (hashing the source rows) must fail naming the column. with pytest.raises((ValueError, RuntimeError), match="'x'"): - source = ArrowTableSource(table, tag_columns=["k"]) + source = ArrowTableSource(table, tag_columns=["k"], tag_only_columns=True) for _, data in source.iter_data(): data.as_dict() @@ -545,7 +557,7 @@ def _source_of_cell(cell: bytes) -> ArrowTableSource: "x": pa.ExtensionArray.from_storage(ext, storage), } ) - return ArrowTableSource(table, tag_columns=["k"]) + return ArrowTableSource(table, tag_columns=["k"], tag_only_columns=True) def test_a_cell_that_declares_more_than_it_holds_is_refused_before_allocation() -> None: @@ -618,7 +630,7 @@ def test_a_stored_object_array_is_never_unpickled() -> None: } ) with pytest.raises((ValueError, RuntimeError), match="allow_pickle=False"): - source = ArrowTableSource(table, tag_columns=["k"]) + source = ArrowTableSource(table, tag_columns=["k"], tag_only_columns=True) for _, data in source.iter_data(): data.as_dict() @@ -636,7 +648,9 @@ def double(x: np.ndarray) -> np.ndarray: # pyright: ignore[reportMissingTypeArg return x * 2 # a 0-d input gives a NumPy scalar source = DictSource( - data=[{"k": "a", "x": np.array(7.5, np.float32)}], tag_columns=["k"] + data=[{"k": "a", "x": np.array(7.5, np.float32)}], + tag_columns=["k"], + tag_only_columns=True, ) with pytest.raises(TypeError, match="float32.*'y'.*ndarray"): list(double.pod(source).iter_data()) @@ -649,7 +663,9 @@ def test_none_returned_for_optional_ndarray_is_accepted() -> None: def nothing(x: np.ndarray) -> np.ndarray | None: # pyright: ignore[reportMissingTypeArgument] # the bare np.ndarray annotation is under test return None - source = DictSource(data=[{"k": "a", "x": np.arange(2.0)}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "a", "x": np.arange(2.0)}], tag_columns=["k"], tag_only_columns=True + ) assert [d.as_dict()["y"] for _, d in nothing.pod(source).iter_data()] == [None] @@ -670,11 +686,13 @@ def test_merge_join_orders_arrays_by_their_value_hash() -> None: left = DictSource( data=[{"k": "a", "x": large}, {"k": "b", "x": other}], tag_columns=["k"], + tag_only_columns=True, source_id="l", ) right = DictSource( data=[{"k": "a", "x": small}, {"k": "b", "x": small}], tag_columns=["k"], + tag_only_columns=True, source_id="r", ) for pair in ((left, right), (right, left)): @@ -692,8 +710,18 @@ def test_merge_join_orders_arrays_that_differ_only_in_field_order() -> None: """``MergeJoin`` orders arrays that differ only in field order the same way for either input order.""" ab = np.array([(1, 2.0)], dtype=[("a", " None: """Merged arrays read back as each side gave them.""" base = np.arange(6, dtype=np.float32).reshape(2, 3) fortran_big = np.asfortranarray(base.astype(">f4")) - left = DictSource(data=[{"k": "a", "x": base}], tag_columns=["k"], source_id="l") + left = DictSource( + data=[{"k": "a", "x": base}], + tag_columns=["k"], + tag_only_columns=True, + source_id="l", + ) right = DictSource( - data=[{"k": "a", "x": fortran_big}], tag_columns=["k"], source_id="r" + data=[{"k": "a", "x": fortran_big}], + tag_columns=["k"], + tag_only_columns=True, + source_id="r", ) got: Any = next(d.as_dict()["x"] for _, d in MergeJoin()(left, right).iter_data()) assert sorted(x.dtype.str for x in got) == ["f4"] @@ -728,7 +764,9 @@ def test_pick_on_an_array_column_raises_input_validation_error() -> None: from orcapod.errors import InputValidationError from orcapod.operators import Pick - source = DictSource(data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"], tag_only_columns=True + ) with pytest.raises(InputValidationError, match="'x'.*does not support pick"): Pick("x", "dtype")(source) @@ -744,7 +782,9 @@ def test_pick_on_an_array_column_raises_while_building_a_pipeline( from orcapod.errors import InputValidationError from orcapod.operators import Pick - source = DictSource(data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"]) + source = DictSource( + data=[{"k": "r", "x": np.arange(3.0)}], tag_columns=["k"], tag_only_columns=True + ) job = PipelineJob(name="pick", database=InMemoryArrowDatabase()) with pytest.raises(InputValidationError, match="'x'"), job: Pick("x", "dtype")(source) diff --git a/tests/test_arrays/v02_full_store_pods.py b/tests/test_arrays/v02_full_store_pods.py index 43f1c8411..fb3184ed7 100644 --- a/tests/test_arrays/v02_full_store_pods.py +++ b/tests/test_arrays/v02_full_store_pods.py @@ -81,7 +81,9 @@ def notify(m: int, ctx: Any) -> int: job = PipelineJob(name=JOB_NAME, database=store) with job: - ints = DictSource(data=ROWS, tag_columns=["k"], source_id="ints") + ints = DictSource( + data=ROWS, tag_columns=["k"], source_id="ints", tag_only_columns=True + ) ones = plus_one.pod(ints, label="plus_one") plus_two.pod(ones, label="plus_two") if with_notify: @@ -111,7 +113,9 @@ def build_cached_source(store: Any) -> Any: } ) return CachedSource( - ArrowTableSource(table, tag_columns=["k"], source_id="fixed"), + ArrowTableSource( + table, tag_columns=["k"], source_id="fixed", tag_only_columns=True + ), cache_database=store, cache_path_prefix=(JOB_NAME,), ) diff --git a/tests/test_arrays/v02_store_pods.py b/tests/test_arrays/v02_store_pods.py index 560b2eae9..663e83df0 100644 --- a/tests/test_arrays/v02_store_pods.py +++ b/tests/test_arrays/v02_store_pods.py @@ -55,16 +55,19 @@ def plus_two(n: int) -> int: data=[{"k": k, "x": v} for k, v in ARRAYS.items()], tag_columns=["k"], source_id="arrays", + tag_only_columns=True, ) ints = DictSource( data=[{"k": k, "n": i} for i, k in enumerate(ARRAYS)], tag_columns=["k"], source_id="ints", + tag_only_columns=True, ) mixed = DictSource( data=[{"k": k, "x": v, "n": i} for i, (k, v) in enumerate(ARRAYS.items())], tag_columns=["k"], source_id="mixed", + tag_only_columns=True, ) job = PipelineJob(name="v02", database=DeltaTableDatabase(base_path=store_root)) with job: diff --git a/tests/test_channels/test_native_async_operators.py b/tests/test_channels/test_native_async_operators.py index cd2e17c68..8f47f3d12 100644 --- a/tests/test_channels/test_native_async_operators.py +++ b/tests/test_channels/test_native_async_operators.py @@ -296,6 +296,7 @@ async def test_system_tags_preserved(self): ), tag_columns=["region", "animal"], infer_nullable=True, + tag_only_columns=True, ) op = SelectTagColumns(columns=["region"]) results = await run_unary(op, src) @@ -389,6 +390,7 @@ async def test_source_info_for_dropped_columns_not_surfaced(self): ), tag_columns=["animal"], infer_nullable=True, + tag_only_columns=True, ) op = SelectDataColumns(columns=["weight"]) results = await run_unary(op, src) @@ -543,6 +545,7 @@ async def test_source_info_for_dropped_columns_not_surfaced(self): ), tag_columns=["animal"], infer_nullable=True, + tag_only_columns=True, ) op = DropDataColumns(columns=["legs"]) results = await run_unary(op, src) @@ -700,6 +703,7 @@ async def test_source_info_renamed(self): ), tag_columns=["animal"], infer_nullable=True, + tag_only_columns=True, ) op = MapData(name_map={"weight": "mass"}) results = await run_unary(op, src) @@ -2054,7 +2058,9 @@ def _make_source(tag_col: str, data_col: str, data: dict) -> ArrowTableStream: data_col: pa.array(data[data_col], type=pa.int64()), } ) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) + return ArrowTableSource( # pyright: ignore[reportReturnType] # the helpers below take this source as their stream + table, tag_columns=[tag_col], infer_nullable=True, tag_only_columns=True + ) async def run_binary_validated( diff --git a/tests/test_channels/test_pipeline_async_integration.py b/tests/test_channels/test_pipeline_async_integration.py index 661487d63..3f91fb5a7 100644 --- a/tests/test_channels/test_pipeline_async_integration.py +++ b/tests/test_channels/test_pipeline_async_integration.py @@ -92,10 +92,16 @@ def _build_job() -> PipelineJob: with job: students = ArrowTableSource( - STUDENTS, tag_columns=["student_id"], infer_nullable=True + STUDENTS, + tag_columns=["student_id"], + infer_nullable=True, + tag_only_columns=True, ) grades = ArrowTableSource( - GRADES, tag_columns=["student_id"], infer_nullable=True + GRADES, + tag_columns=["student_id"], + infer_nullable=True, + tag_only_columns=True, ) joined = Join()(students, grades, label="join") diff --git a/tests/test_channels/test_polling_source.py b/tests/test_channels/test_polling_source.py index 5805d23e0..1b6a9a887 100644 --- a/tests/test_channels/test_polling_source.py +++ b/tests/test_channels/test_polling_source.py @@ -246,7 +246,9 @@ async def test_source_node_async_execute_uses_async_iter(self): from orcapod.types import Schema # A bound source needs one system-tag pair (ITL-1126); a raw stream has none. - stream = ArrowTableSource(_make_arrow_stream(2).as_table(), tag_columns=["id"]) + stream = ArrowTableSource( + _make_arrow_stream(2).as_table(), tag_columns=["id"], tag_only_columns=True + ) node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), @@ -265,7 +267,9 @@ async def test_source_node_closes_channel_after_exhaustion(self): from orcapod.types import Schema # A bound source needs one system-tag pair (ITL-1126); a raw stream has none. - stream = ArrowTableSource(_make_arrow_stream(1).as_table(), tag_columns=["id"]) + stream = ArrowTableSource( + _make_arrow_stream(1).as_table(), tag_columns=["id"], tag_only_columns=True + ) node = SourceJobNode( name="test", tag_schema=Schema({"id": int}), diff --git a/tests/test_core/alignment/fixtures.py b/tests/test_core/alignment/fixtures.py index b51a325df..611e14f87 100644 --- a/tests/test_core/alignment/fixtures.py +++ b/tests/test_core/alignment/fixtures.py @@ -217,6 +217,7 @@ def make_sources(scenario: str, input_order: str = "shuffled") -> Sources: tag_columns=list(TAG_COLUMNS), source_id=name, record_id_column=record_id_column, + tag_only_columns=True, ) oracle.add_source( name, diff --git a/tests/test_core/alignment/test_spiraldb.py b/tests/test_core/alignment/test_spiraldb.py index b7b183140..d9aa02c93 100644 --- a/tests/test_core/alignment/test_spiraldb.py +++ b/tests/test_core/alignment/test_spiraldb.py @@ -75,7 +75,12 @@ def source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Any]: project = fake_pyspiral.FakeSpiral().project(PROJECT) project.create_table("default.event", key_schema=KEYS).write(written_table()) yield SpiralDBTableSource( - PROJECT, "event", tag_columns=["grp"], source_id=SOURCE_ID, batch_rows=2 + PROJECT, + "event", + tag_columns=["grp"], + source_id=SOURCE_ID, + batch_rows=2, + tag_only_columns=True, ) diff --git a/tests/test_core/function_pod/test_as_table_stale_between_passes.py b/tests/test_core/function_pod/test_as_table_stale_between_passes.py index e3d0c1c6a..cdfdff737 100644 --- a/tests/test_core/function_pod/test_as_table_stale_between_passes.py +++ b/tests/test_core/function_pod/test_as_table_stale_between_passes.py @@ -43,7 +43,9 @@ def source() -> ArrowTableSource: ] ), ) - return ArrowTableSource(table, tag_columns=["k"], source_id="src") + return ArrowTableSource( + table, tag_columns=["k"], source_id="src", tag_only_columns=True + ) def chain(cached: bool = False) -> tuple[Any, Any]: diff --git a/tests/test_core/function_pod/test_cached_function_pod_job.py b/tests/test_core/function_pod/test_cached_function_pod_job.py index 533f33fab..18cda53ab 100644 --- a/tests/test_core/function_pod/test_cached_function_pod_job.py +++ b/tests/test_core/function_pod/test_cached_function_pod_job.py @@ -35,7 +35,9 @@ def make_source(rows: list[dict[str, int]] | None = None) -> DictSource: - return DictSource(rows or ROWS, tag_columns=["k"], source_id="s") + return DictSource( + rows or ROWS, tag_columns=["k"], tag_only_columns=True, source_id="s" + ) class Counter: @@ -185,8 +187,12 @@ def test_inputs_are_joined_once() -> None: cached_pod = CachedFunctionPod( function_pod=counter.pod(), result_database=InMemoryArrowDatabase() ) - left = DictSource([{"k": 1, "x": 2}], tag_columns=["k"], source_id="l") - right = DictSource([{"k": 1, "y": 3}], tag_columns=["k"], source_id="r") + left = DictSource( + [{"k": 1, "x": 2}], tag_columns=["k"], tag_only_columns=True, source_id="l" + ) + right = DictSource( + [{"k": 1, "y": 3}], tag_columns=["k"], tag_only_columns=True, source_id="r" + ) joins: list[int] = [] original = Join.process diff --git a/tests/test_core/function_pod/test_ephemeral_result.py b/tests/test_core/function_pod/test_ephemeral_result.py index a5135b852..799c83bc1 100644 --- a/tests/test_core/function_pod/test_ephemeral_result.py +++ b/tests/test_core/function_pod/test_ephemeral_result.py @@ -57,7 +57,11 @@ def _make_source_stream( {k: pa.array([r[k] for r in rows], type=pa.int64()) for k in rows[0]} ) source = ArrowTableSource( - table, tag_columns=tag_columns, source_id="test_src", infer_nullable=True + table, + tag_columns=tag_columns, + source_id="test_src", + infer_nullable=True, + tag_only_columns=True, ) return source @@ -655,6 +659,7 @@ def test_pipeline_job_set_ephemeral_database_propagates(self): stream = ArrowTableSource( _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]).as_table(), tag_columns=["id"], + tag_only_columns=True, ) pod = _make_pod() @@ -677,7 +682,9 @@ def test_pipeline_job_set_ephemeral_database_none_detaches(self): # A job's input must be a source (ITL-1126: a raw stream has no system tags). stream = ArrowTableSource( - _make_stream([{"id": 0, "x": 10}]).as_table(), tag_columns=["id"] + _make_stream([{"id": 0, "x": 10}]).as_table(), + tag_columns=["id"], + tag_only_columns=True, ) pod = _make_pod() diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py index 1cd7bf732..6886d8715 100644 --- a/tests/test_core/function_pod/test_function_node_arrow_types.py +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -132,7 +132,9 @@ def _run(job: PipelineJob, mode: str) -> None: def _build(store, table: pa.Table, *, chained: bool = False) -> PipelineJob: """A job: source -> ``double`` (-> ``increment`` when chained).""" with PipelineJob(name="types", database=store) as job: - src = ArrowTableSource(table, tag_columns=["k"], source_id="src") + src = ArrowTableSource( + table, tag_columns=["k"], source_id="src", tag_only_columns=True + ) out = double.pod(src, label="double") if chained: increment.pod(out, label="increment") @@ -164,7 +166,12 @@ def test_as_table_after_run_keeps_tag_type( def test_pod_stream_outside_a_job_keeps_tag_type(name: str) -> None: """The same pod outside a job: its stream's table keeps the tag type too.""" stream = double.pod( - ArrowTableSource(_source_table(name), tag_columns=["k"], source_id="src") + ArrowTableSource( + _source_table(name), + tag_columns=["k"], + source_id="src", + tag_only_columns=True, + ) ) table = stream.as_table() @@ -178,7 +185,9 @@ def test_as_table_all_info_matches_the_pod_outside_a_job(name: str) -> None: """With every column group, the node's table equals the pod's stream outside a job.""" table = _source_table(name) outside = double.pod( - ArrowTableSource(table, tag_columns=["k"], source_id="src") + ArrowTableSource( + table, tag_columns=["k"], source_id="src", tag_only_columns=True + ) ).as_table(all_info=True) job = _build(InMemoryArrowDatabase(), table) job.run() @@ -197,7 +206,12 @@ def test_join_after_a_function_node_on_a_typed_tag(name: str) -> None: """The ITL-990 case: join a function node's output with another source on the tag.""" job = _build(InMemoryArrowDatabase(), _source_table(name)) job.run() - other = ArrowTableSource(_source_table(name, "w"), tag_columns=["k"], source_id="o") + other = ArrowTableSource( + _source_table(name, "w"), + tag_columns=["k"], + source_id="o", + tag_only_columns=True, + ) joined = job.nodes["double"].join(other).as_table() diff --git a/tests/test_core/function_pod/test_function_node_caching.py b/tests/test_core/function_pod/test_function_node_caching.py index 2e194db9b..e8352be97 100644 --- a/tests/test_core/function_pod/test_function_node_caching.py +++ b/tests/test_core/function_pod/test_function_node_caching.py @@ -61,7 +61,11 @@ def _make_source_stream( {k: pa.array([r[k] for r in rows], type=pa.int64()) for k in rows[0]} ) source = ArrowTableSource( - table, tag_columns=tag_columns, source_id=source_id, infer_nullable=True + table, + tag_columns=tag_columns, + tag_only_columns=True, + source_id=source_id, + infer_nullable=True, ) return source diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py index 94e0ca59f..8aa1c553b 100644 --- a/tests/test_core/function_pod/test_function_pod_auto_select.py +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -94,6 +94,7 @@ def _wide(extra: int = 9, source_id: str = "wide") -> DictSource: {"k": 2, "x": 4, "y": 5, "extra": extra}, ], tag_columns=["k"], + tag_only_columns=True, source_id=source_id, ) @@ -111,6 +112,7 @@ def _xs(source_id: str = "xs", **more: int) -> DictSource: return DictSource( [{"k": 1, "x": 2, **more}, {"k": 2, "x": 4, **more}], tag_columns=["k"], + tag_only_columns=True, source_id=source_id, ) @@ -128,6 +130,7 @@ def _ys(source_id: str = "ys", **more: int) -> DictSource: return DictSource( [{"k": 1, "y": 3, **more}, {"k": 2, "y": 5, **more}], tag_columns=["k"], + tag_only_columns=True, source_id=source_id, ) @@ -223,6 +226,7 @@ def test_tags_are_untouched() -> None: source = DictSource( [{"k": 1, "session": "a", "x": 2, "y": 3, "extra": 9}], tag_columns=["k", "session"], + tag_only_columns=True, source_id="two_tags", ) @@ -237,9 +241,15 @@ def test_stream_passed_for_its_tags_restricts_the_rows() -> None: full = DictSource( [{"k": 1, "x": 2, "y": 3}, {"k": 2, "x": 4, "y": 5}], tag_columns=["k"], + tag_only_columns=True, source_id="full", ) - chosen = DictSource([{"k": 2, "note": 0}], tag_columns=["k"], source_id="chosen") + chosen = DictSource( + [{"k": 2, "note": 0}], + tag_columns=["k"], + tag_only_columns=True, + source_id="chosen", + ) result = add.pod(full, chosen) @@ -295,7 +305,10 @@ def test_matching_input_adds_no_select_node_to_a_job() -> None: job = PipelineJob(name="narrow", database=InMemoryArrowDatabase()) with job: narrow = DictSource( - [{"k": 1, "x": 2, "y": 3}], tag_columns=["k"], source_id="narrow" + [{"k": 1, "x": 2, "y": 3}], + tag_columns=["k"], + tag_only_columns=True, + source_id="narrow", ) add.pod(narrow, label="add") @@ -358,7 +371,10 @@ def test_missing_required_column_adds_no_step_to_a_job() -> None: def test_input_with_no_parameter_column_is_rejected() -> None: """A stream cannot be left without data columns: the exact error is raised.""" only_extra = DictSource( - [{"k": 1, "extra": 9}], tag_columns=["k"], source_id="only_extra" + [{"k": 1, "extra": 9}], + tag_columns=["k"], + tag_only_columns=True, + source_id="only_extra", ) with pytest.raises(ValueError, match="column 'extra' is not a parameter"): @@ -373,10 +389,16 @@ def test_data_column_shared_by_two_inputs_fails_at_the_join(shared: str) -> None shared: The data column both inputs carry. """ left = DictSource( - [{"k": 1, "x": 2, "extra": 1}], tag_columns=["k"], source_id="left" + [{"k": 1, "x": 2, "extra": 1}], + tag_columns=["k"], + tag_only_columns=True, + source_id="left", ) right = DictSource( - [{"k": 1, "y": 3, shared: 7}], tag_columns=["k"], source_id="right" + [{"k": 1, "y": 3, shared: 7}], + tag_columns=["k"], + tag_only_columns=True, + source_id="right", ) with pytest.raises(InputValidationError, match="overlapping"): @@ -415,7 +437,10 @@ def test_misspelt_column_for_a_defaulted_parameter_is_dropped_and_logged( ) -> None: """A misspelt column is dropped and the default used; the INFO record is the trace.""" misspelt = DictSource( - [{"k": 1, "x": 2, "Y": 3}], tag_columns=["k"], source_id="misspelt" + [{"k": 1, "x": 2, "Y": 3}], + tag_columns=["k"], + tag_only_columns=True, + source_id="misspelt", ) with caplog.at_level(logging.DEBUG): diff --git a/tests/test_core/function_pod/test_function_pod_ctx_arg.py b/tests/test_core/function_pod/test_function_pod_ctx_arg.py index f8799716e..9a4460448 100644 --- a/tests/test_core/function_pod/test_function_pod_ctx_arg.py +++ b/tests/test_core/function_pod/test_function_pod_ctx_arg.py @@ -191,6 +191,7 @@ def transform(value: int, ctx: InvocationContext) -> str: source = DictSource( [{"id": 0, "value": 10}, {"id": 1, "value": 20}], tag_columns=["id"], + tag_only_columns=True, ) pod.process(source) diff --git a/tests/test_core/function_pod/test_node_config_on_job_node.py b/tests/test_core/function_pod/test_node_config_on_job_node.py index 67389e5dd..be1c392a1 100644 --- a/tests/test_core/function_pod/test_node_config_on_job_node.py +++ b/tests/test_core/function_pod/test_node_config_on_job_node.py @@ -15,7 +15,9 @@ def double(x: int) -> int: pf = PythonDataFunction(double, output_keys="result") pod = FunctionPod(pf) table = pa.table({"id": [1], "x": [10]}) - source = ArrowTableSource(table, tag_columns=["id"], infer_nullable=True) + source = ArrowTableSource( + table, tag_columns=["id"], tag_only_columns=True, infer_nullable=True + ) db = InMemoryArrowDatabase() return FunctionJobNode(pod, source, pipeline_database=db) diff --git a/tests/test_core/function_pod/test_node_content_hash_redundancy.py b/tests/test_core/function_pod/test_node_content_hash_redundancy.py index 647c940f5..757292527 100644 --- a/tests/test_core/function_pod/test_node_content_hash_redundancy.py +++ b/tests/test_core/function_pod/test_node_content_hash_redundancy.py @@ -84,7 +84,9 @@ def _make_source_stream(values: list[int]) -> ArrowTableSource: "x": pa.array(values, type=pa.int64()), } ) - return ArrowTableSource(table=table, tag_columns=["id"], infer_nullable=True) + return ArrowTableSource( + table=table, tag_columns=["id"], tag_only_columns=True, infer_nullable=True + ) def _all_base_entry_ids(node: FunctionJobNode, stream) -> set[bytes]: @@ -349,7 +351,9 @@ def _make_random_source(rng: random.Random) -> ArrowTableSource: "x": pa.array(values, type=pa.int64()), } ) - return ArrowTableSource(table=table, tag_columns=["id"], infer_nullable=True) + return ArrowTableSource( + table=table, tag_columns=["id"], tag_only_columns=True, infer_nullable=True + ) @staticmethod def _system_tags_key(source: ArrowTableSource) -> tuple: @@ -551,7 +555,9 @@ def test_tdb_v1_path_used_after_attach(self): from orcapod.system_constants import TRACKING_DB_SCHEMA_VERSION table = pa.table({"x": pa.array([1], type=pa.int64())}) - source = ArrowTableSource(table=table, tag_columns=[], infer_nullable=True) + source = ArrowTableSource( + table=table, tag_columns=[], tag_only_columns=True, infer_nullable=True + ) def noop(x: int, ctx=None) -> None: pass @@ -579,7 +585,9 @@ def test_tdb_v0_entries_not_visible_to_new_code(self): from orcapod.side_effects import SideEffectJobNode, SideEffectPod table = pa.table({"x": pa.array([1], type=pa.int64())}) - source = ArrowTableSource(table=table, tag_columns=[], infer_nullable=True) + source = ArrowTableSource( + table=table, tag_columns=[], tag_only_columns=True, infer_nullable=True + ) def noop(x: int, ctx=None) -> None: pass diff --git a/tests/test_core/function_pod/test_output_conformance.py b/tests/test_core/function_pod/test_output_conformance.py index 4c0fcc168..6a2f0b549 100644 --- a/tests/test_core/function_pod/test_output_conformance.py +++ b/tests/test_core/function_pod/test_output_conformance.py @@ -71,7 +71,10 @@ async def async_fn(x: int) -> Any: def make_source(xs: tuple[int, ...] = (1,)) -> DictSource: """A source with one row per ``x`` (tag ``k``, data ``x: int``).""" return DictSource( - data=[{"k": f"r{x}", "x": x} for x in xs], tag_columns=["k"], source_id="src" + data=[{"k": f"r{x}", "x": x} for x in xs], + tag_columns=["k"], + tag_only_columns=True, + source_id="src", ) @@ -873,6 +876,7 @@ def test_inferred_source_uses_value_class() -> None: source = DictSource( data=[{"y": cast(Any, ExtendedReading(3, 9))}], tag_columns=[], + tag_only_columns=True, source_id="subclass_source", ) table = source.as_table() diff --git a/tests/test_core/function_pod/test_pandas_dataframe_order.py b/tests/test_core/function_pod/test_pandas_dataframe_order.py index c642ae1b8..35545a0b7 100644 --- a/tests/test_core/function_pod/test_pandas_dataframe_order.py +++ b/tests/test_core/function_pod/test_pandas_dataframe_order.py @@ -121,7 +121,9 @@ def _run( ) -> PipelineJob: """Run ``make_frame -> leading_names`` over ``rows`` on ``store`` with the named orchestrator.""" source = DictSource( - [{"k": key, "reverse": reverse} for key, reverse in rows], tag_columns=["k"] + [{"k": key, "reverse": reverse} for key, reverse in rows], + tag_columns=["k"], + tag_only_columns=True, ) job = PipelineJob(name="pandas_order", database=store) with job: @@ -187,7 +189,11 @@ def test_equal_frame_is_served_from_the_cache( _COUNTED_CALLS.clear() got: dict[str, str] = {} for key, reverse in (("first", False), ("equal", False), ("reordered", True)): - source = DictSource([{"k": key, "reverse": reverse}], tag_columns=["k"]) + source = DictSource( + [{"k": key, "reverse": reverse}], + tag_columns=["k"], + tag_only_columns=True, + ) job = PipelineJob(name="pandas_order_counted", database=store) with job: frames = make_frame.pod(source, label="make") @@ -215,7 +221,11 @@ def test_each_order_alone_is_a_cache_miss( ("columns", "columns"), ("levels", "levels"), ): - source = DictSource([{"k": key, "variant": variant}], tag_columns=["k"]) + source = DictSource( + [{"k": key, "variant": variant}], + tag_columns=["k"], + tag_only_columns=True, + ) job = PipelineJob(name="pandas_order_variants", database=store) with job: frames = make_variant_frame.pod(source, label="make") @@ -250,7 +260,9 @@ def test_frame_read_back_from_the_store_keeps_its_cache_key( _CONSUMER_CALLS.clear() totals: list[int] = [] for _ in range(2): - source = DictSource([{"k": "a", "n": 3}], tag_columns=["k"]) + source = DictSource( + [{"k": "a", "n": 3}], tag_columns=["k"], tag_only_columns=True + ) job = PipelineJob(name="pandas_storage_form", database=store) with job: frames = make_frame_storage_changes.pod(source, label="make") diff --git a/tests/test_core/function_pod/test_pipeline_hash_integration.py b/tests/test_core/function_pod/test_pipeline_hash_integration.py index c67cdbc4c..53fd08a32 100644 --- a/tests/test_core/function_pod/test_pipeline_hash_integration.py +++ b/tests/test_core/function_pod/test_pipeline_hash_integration.py @@ -234,7 +234,7 @@ def test_different_tag_column_different_pipeline_hash(self): } ) src_with_tag = ArrowTableSource( - table=table, tag_columns=["id"], infer_nullable=True + table=table, tag_columns=["id"], infer_nullable=True, tag_only_columns=True ) src_no_tag = ArrowTableSource(table=table, infer_nullable=True) assert src_with_tag.pipeline_hash() != src_no_tag.pipeline_hash() diff --git a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py index 98d633ea2..89e137c58 100644 --- a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py +++ b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py @@ -81,7 +81,9 @@ def _source(extra: bool) -> ArrowTableSource: fields.append(pa.field("e", pa.int64(), nullable=False)) arrays.append(pa.array([-i for i in IDS], pa.int64())) table = pa.Table.from_arrays(arrays, schema=pa.schema(fields)) - return ArrowTableSource(table, tag_columns=["id"], source_id="src") + return ArrowTableSource( + table, tag_columns=["id"], source_id="src", tag_only_columns=True + ) def _other() -> ArrowTableSource: @@ -97,7 +99,9 @@ def _other() -> ArrowTableSource: [pa.array(ids, pa.int64()), pa.array([7 * i for i in ids], pa.int64())], schema=schema, ) - return ArrowTableSource(table, tag_columns=["id"], source_id="other") + return ArrowTableSource( + table, tag_columns=["id"], source_id="other", tag_only_columns=True + ) def _pod( diff --git a/tests/test_core/function_pod/test_pod_stream_staleness.py b/tests/test_core/function_pod/test_pod_stream_staleness.py index 478183eda..d2a495510 100644 --- a/tests/test_core/function_pod/test_pod_stream_staleness.py +++ b/tests/test_core/function_pod/test_pod_stream_staleness.py @@ -134,7 +134,9 @@ def _source() -> ArrowTableSource: [pa.array(IDS, type=pa.int64()), pa.array([10 * i for i in IDS], pa.int64())], schema=schema, ) - return ArrowTableSource(table, tag_columns=["id"], source_id="src") + return ArrowTableSource( + table, tag_columns=["id"], source_id="src", tag_only_columns=True + ) def _other() -> ArrowTableSource: @@ -150,7 +152,9 @@ def _other() -> ArrowTableSource: [pa.array(ids, type=pa.int64()), pa.array([7 * i for i in ids], pa.int64())], schema=schema, ) - return ArrowTableSource(table, tag_columns=["id"], source_id="other") + return ArrowTableSource( + table, tag_columns=["id"], source_id="other", tag_only_columns=True + ) def _pod( diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py index 17e5da0ac..ae67cb22a 100644 --- a/tests/test_core/function_pod/test_polars_dataframe_values.py +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -92,6 +92,7 @@ def _int_source() -> DictSource: return DictSource( [{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"], + tag_only_columns=True, ) @@ -105,6 +106,7 @@ def _frame_source() -> DictSource: return DictSource( rows, tag_columns=["g", "k"], + tag_only_columns=True, data_schema={"g": str, "k": str, "df": pl.DataFrame}, ) @@ -259,6 +261,7 @@ def test_join_passes_frames_through(self) -> None: weights = DictSource( [{"g": "a", "k": "1", "w": 10}, {"g": "b", "k": "3", "w": 30}], tag_columns=["g", "k"], + tag_only_columns=True, ) stream = Join()(_frame_source(), weights) got = { @@ -274,6 +277,7 @@ def test_semi_join_keeps_frames_of_matching_rows(self) -> None: keys = DictSource( [{"g": "a", "k": "2", "w": 1}, {"g": "b", "k": "3", "w": 2}], tag_columns=["g", "k"], + tag_only_columns=True, ) stream = SemiJoin()(_frame_source(), keys) got = {str(tag["k"]): data["df"] for tag, data in stream.iter_data()} @@ -328,6 +332,7 @@ def _order_source() -> DictSource: for k, (columns, fields) in _ORDERS.items() ], tag_columns=["k"], + tag_only_columns=True, ) @@ -360,6 +365,7 @@ def _null_struct_source() -> DictSource: {"k": "struct_of_nulls", "whole_struct_null": False}, ], tag_columns=["k"], + tag_only_columns=True, ) @@ -534,7 +540,9 @@ def frame_height(df: pl.DataFrame) -> int: fh.write("height\\n") return df.height - source = DictSource([{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"]) + source = DictSource( + [{"k": "a", "n": 1}, {"k": "b", "n": 3}], tag_columns=["k"], tag_only_columns=True + ) job = PipelineJob(name="frames", database=DeltaTableDatabase(database_path)) with job: frames = make_frame.pod(source, label="make") diff --git a/tests/test_core/function_pod/test_readback_datagram_id.py b/tests/test_core/function_pod/test_readback_datagram_id.py index 46ae521a3..395809034 100644 --- a/tests/test_core/function_pod/test_readback_datagram_id.py +++ b/tests/test_core/function_pod/test_readback_datagram_id.py @@ -29,7 +29,9 @@ def source() -> ArrowTableSource: ] ), ) - return ArrowTableSource(table, tag_columns=["k"], source_id="src") + return ArrowTableSource( + table, tag_columns=["k"], source_id="src", tag_only_columns=True + ) def build(store: InMemoryArrowDatabase) -> PipelineJob: diff --git a/tests/test_core/function_pod/test_resumed_iteration.py b/tests/test_core/function_pod/test_resumed_iteration.py index e272a262f..a09974573 100644 --- a/tests/test_core/function_pod/test_resumed_iteration.py +++ b/tests/test_core/function_pod/test_resumed_iteration.py @@ -57,7 +57,11 @@ def supports_concurrent_execution(self) -> bool: def source() -> DictSource: """Six rows: tag ``k`` and data ``x``, both ``0..5``.""" - return DictSource([{"k": i, "x": i} for i in range(ROWS)], tag_columns=["k"]) + return DictSource( + [{"k": i, "x": i} for i in range(ROWS)], + tag_columns=["k"], + tag_only_columns=True, + ) def stream(kind: str) -> Any: diff --git a/tests/test_core/large_types_cases.py b/tests/test_core/large_types_cases.py index 0d5ae54b8..469a38429 100644 --- a/tests/test_core/large_types_cases.py +++ b/tests/test_core/large_types_cases.py @@ -128,7 +128,9 @@ def _table(n_rows: int, columns: list[str] | None = None) -> pa.Table: def _arrow_table_source(n_rows: int, tmp: Path) -> StreamProtocol: - return ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="arrow") + return ArrowTableSource( + _table(n_rows), tag_columns=TAGS, tag_only_columns=True, source_id="arrow" + ) def _arrow_table_stream(n_rows: int, tmp: Path) -> StreamProtocol: @@ -141,6 +143,7 @@ def _dict_source_python(n_rows: int, tmp: Path) -> StreamProtocol: return DictSource( [], tag_columns=TAGS, + tag_only_columns=True, data_schema={ "t": str, "k": int, @@ -151,13 +154,19 @@ def _dict_source_python(n_rows: int, tmp: Path) -> StreamProtocol: }, source_id="dict", ) - return DictSource(_rows(n_rows, columns), tag_columns=TAGS, source_id="dict") + return DictSource( + _rows(n_rows, columns), + tag_columns=TAGS, + tag_only_columns=True, + source_id="dict", + ) def _dict_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: return DictSource( _rows(n_rows, FULL_SCHEMA.names), tag_columns=TAGS, + tag_only_columns=True, data_schema=FULL_SCHEMA, source_id="dict_schema", ) @@ -170,13 +179,19 @@ def _csv_path(n_rows: int, tmp: Path) -> Path: def _csv_source(n_rows: int, tmp: Path) -> StreamProtocol: - return CSVSource(str(_csv_path(n_rows, tmp)), tag_columns=TAGS, source_id="csv") + return CSVSource( + str(_csv_path(n_rows, tmp)), + tag_columns=TAGS, + tag_only_columns=True, + source_id="csv", + ) def _csv_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: return CSVSource( str(_csv_path(n_rows, tmp)), tag_columns=TAGS, + tag_only_columns=True, source_id="csv_schema", schema=pa.schema([FULL_SCHEMA.field(name) for name in ["t", "k", "s", "u"]]), ) @@ -184,7 +199,10 @@ def _csv_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: def _data_frame_source_polars(n_rows: int, tmp: Path) -> StreamProtocol: return DataFrameSource( - pl.from_arrow(_table(n_rows)), tag_columns=TAGS, source_id="polars" + pl.from_arrow(_table(n_rows)), + tag_columns=TAGS, + tag_only_columns=True, + source_id="polars", ) @@ -194,18 +212,22 @@ def _data_frame_source_pandas(n_rows: int, tmp: Path) -> StreamProtocol: return DataFrameSource( _table(n_rows, ["t", "k", "s", "u"]).to_pandas(), tag_columns=TAGS, + tag_only_columns=True, source_id="pandas", ) def _data_frame_source_arrow_table(n_rows: int, tmp: Path) -> StreamProtocol: - return DataFrameSource(_table(n_rows), tag_columns=TAGS, source_id="df_arrow") + return DataFrameSource( + _table(n_rows), tag_columns=TAGS, tag_only_columns=True, source_id="df_arrow" + ) def _data_frame_source_arrow_schema(n_rows: int, tmp: Path) -> StreamProtocol: return DataFrameSource( pl.from_arrow(_table(n_rows)), tag_columns=TAGS, + tag_only_columns=True, source_id="df_schema", schema=FULL_SCHEMA, ) @@ -215,7 +237,9 @@ def _delta_table_source(n_rows: int, tmp: Path) -> StreamProtocol: path = tmp / f"delta{n_rows}" if not path.exists(): deltalake.write_deltalake(str(path), _table(n_rows)) - return DeltaTableSource(path, tag_columns=TAGS, source_id="delta") + return DeltaTableSource( + path, tag_columns=TAGS, tag_only_columns=True, source_id="delta" + ) def _sqlite_table_source(n_rows: int, tmp: Path) -> StreamProtocol: @@ -233,19 +257,31 @@ def _sqlite_table_source(n_rows: int, tmp: Path) -> StreamProtocol: ) connection.commit() connection.close() - return SQLiteTableSource(path, "r", tag_columns=TAGS, source_id="sqlite") + return SQLiteTableSource( + path, "r", tag_columns=TAGS, tag_only_columns=True, source_id="sqlite" + ) def _cached_source_in_memory(n_rows: int, tmp: Path) -> StreamProtocol: return CachedSource( - ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="cached_mem"), + ArrowTableSource( + _table(n_rows), + tag_columns=TAGS, + tag_only_columns=True, + source_id="cached_mem", + ), cache_database=InMemoryArrowDatabase(), ) def _cached_source_delta(n_rows: int, tmp: Path) -> StreamProtocol: return CachedSource( - ArrowTableSource(_table(n_rows), tag_columns=TAGS, source_id="cached_delta"), + ArrowTableSource( + _table(n_rows), + tag_columns=TAGS, + tag_only_columns=True, + source_id="cached_delta", + ), cache_database=DeltaTableDatabase(base_path=tmp / f"cache{n_rows}"), ) diff --git a/tests/test_core/nodes/test_function_job_node_async_execute.py b/tests/test_core/nodes/test_function_job_node_async_execute.py index 23ea63be8..1e272ef25 100644 --- a/tests/test_core/nodes/test_function_job_node_async_execute.py +++ b/tests/test_core/nodes/test_function_job_node_async_execute.py @@ -29,7 +29,9 @@ def _make_source(n: int = 3) -> ArrowTableSource: "value": pa.array(list(range(n)), type=pa.int64()), } ) - return ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + return ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) def double_no_db(value: int) -> int: diff --git a/tests/test_core/nodes/test_function_node_dataclass.py b/tests/test_core/nodes/test_function_node_dataclass.py index b2b6e470e..04ffec518 100644 --- a/tests/test_core/nodes/test_function_node_dataclass.py +++ b/tests/test_core/nodes/test_function_node_dataclass.py @@ -42,7 +42,9 @@ def test_as_df_dataclass_column_populated(): """FunctionNode with a dataclass return type produces a correct Polars DataFrame.""" store = InMemoryArrowDatabase() job = PipelineJob(database=store) - source = DictSource([{"id": 0, "a": 5, "b": 3}], tag_columns=["id"]) + source = DictSource( + [{"id": 0, "a": 5, "b": 3}], tag_columns=["id"], tag_only_columns=True + ) with job: take_sum.pod(source) @@ -67,7 +69,9 @@ def test_as_df_dataclass_column_populated(): def test_as_df_dataclass_column_empty_node(): """Unrun FunctionNode returns a zero-row DataFrame with the dataclass column present.""" job = PipelineJob(database=InMemoryArrowDatabase()) - source = DictSource([{"id": 0, "a": 5, "b": 3}], tag_columns=["id"]) + source = DictSource( + [{"id": 0, "a": 5, "b": 3}], tag_columns=["id"], tag_only_columns=True + ) with job: take_sum.pod(source) @@ -84,7 +88,9 @@ def test_as_df_dataclass_column_empty_node(): def test_as_df_empty_schema_matches_nonempty_schema(): """Empty and populated as_df() for the same node type share identical schemas.""" # Populated node - source_full = DictSource([{"id": 0, "a": 5, "b": 3}], tag_columns=["id"]) + source_full = DictSource( + [{"id": 0, "a": 5, "b": 3}], tag_columns=["id"], tag_only_columns=True + ) job_full = PipelineJob(database=InMemoryArrowDatabase()) with job_full: take_sum.pod(source_full) @@ -92,7 +98,9 @@ def test_as_df_empty_schema_matches_nonempty_schema(): full_df = job_full.nodes["take_sum"].as_df() # Unrun node — same pod, fresh job - source_empty = DictSource([{"id": 0, "a": 5, "b": 3}], tag_columns=["id"]) + source_empty = DictSource( + [{"id": 0, "a": 5, "b": 3}], tag_columns=["id"], tag_only_columns=True + ) job_empty = PipelineJob(database=InMemoryArrowDatabase()) with job_empty: take_sum.pod(source_empty) diff --git a/tests/test_core/nodes/test_function_node_empty_data.py b/tests/test_core/nodes/test_function_node_empty_data.py index a6d30559a..ab87453a3 100644 --- a/tests/test_core/nodes/test_function_node_empty_data.py +++ b/tests/test_core/nodes/test_function_node_empty_data.py @@ -32,7 +32,9 @@ def persistent_node(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) return FunctionJobNode( pod, @@ -80,7 +82,9 @@ def ephemeral_node(): "value": pa.array([1], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) node = FunctionJobNode( pod, @@ -204,7 +208,9 @@ def test_cache_hit_returns_cached_result(self, ephemeral_node): "value": pa.array([1], type=pa.int64()), } ) - ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) def triple_value(result: int) -> int: return result * 3 @@ -254,7 +260,9 @@ def test_cache_miss_raises_ephemeral_result_missing_error(self, ephemeral_node): "value": pa.array([1], type=pa.int64()), } ) - ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) def triple_value(result: int) -> int: return result * 3 diff --git a/tests/test_core/nodes/test_function_node_fetch_joined.py b/tests/test_core/nodes/test_function_node_fetch_joined.py index 52aa89d06..f7f5891e4 100644 --- a/tests/test_core/nodes/test_function_node_fetch_joined.py +++ b/tests/test_core/nodes/test_function_node_fetch_joined.py @@ -30,7 +30,9 @@ def node_without_db(): "value": pa.array([1], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) return FunctionJobNode(pod, src) @@ -44,7 +46,9 @@ def node_with_empty_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pod = FunctionPod(PythonDataFunction(double_value, output_keys="result")) return FunctionJobNode( pod, diff --git a/tests/test_core/nodes/test_function_node_get_cached.py b/tests/test_core/nodes/test_function_node_get_cached.py index 26b605bbf..d501938f6 100644 --- a/tests/test_core/nodes/test_function_node_get_cached.py +++ b/tests/test_core/nodes/test_function_node_get_cached.py @@ -24,7 +24,9 @@ def function_node_with_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) pipeline_db = InMemoryArrowDatabase() @@ -46,7 +48,9 @@ def test_returns_empty_dict_when_no_db(self): "value": pa.array([1], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) node = FunctionJobNode(pod, src) diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index 10a5266a9..29b5123eb 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -33,7 +33,7 @@ def _make_source(n: int = 3) -> ArrowTableSource: ] ), ) - return ArrowTableSource(table, tag_columns=["id"]) + return ArrowTableSource(table, tag_columns=["id"], tag_only_columns=True) def _make_node(n: int = 3, db: InMemoryArrowDatabase | None = None) -> FunctionJobNode: diff --git a/tests/test_core/nodes/test_node_execute.py b/tests/test_core/nodes/test_node_execute.py index 4ff84e053..447ad1c1e 100644 --- a/tests/test_core/nodes/test_node_execute.py +++ b/tests/test_core/nodes/test_node_execute.py @@ -31,7 +31,9 @@ def function_node_with_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) pipeline_db = InMemoryArrowDatabase() @@ -53,7 +55,9 @@ def function_node_no_db(): "value": pa.array([1, 2], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) + src = ArrowTableSource( + table, tag_columns=["key"], tag_only_columns=True, infer_nullable=True + ) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) return FunctionJobNode(pod, src) @@ -129,7 +133,7 @@ def operator_with_db(): "value": pa.array([10, 20], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"]) + src = ArrowTableSource(table, tag_columns=["key"], tag_only_columns=True) op = SelectDataColumns(columns=["value"]) db = InMemoryArrowDatabase() node = OperatorNode( @@ -149,7 +153,7 @@ def operator_no_db(): "value": pa.array([10, 20], type=pa.int64()), } ) - src = ArrowTableSource(table, tag_columns=["key"]) + src = ArrowTableSource(table, tag_columns=["key"], tag_only_columns=True) op = SelectDataColumns(columns=["value"]) node = OperatorNode(op, input_streams=[src]) return node, src diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 7c1cedbc8..82ce85fee 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -152,7 +152,9 @@ def test_validate_compatible_source(self, tag_schema, data_schema): from orcapod.core.sources.dict_source import DictSource node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) node.validate(src) # must not raise def test_validate_incompatible_raises(self, tag_schema, data_schema): @@ -160,7 +162,9 @@ def test_validate_incompatible_raises(self, tag_schema, data_schema): from orcapod.core.sources.dict_source import DictSource node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - src = DictSource(data=[{"id": 1, "wrong": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "wrong": 1.0}], tag_columns=["id"], tag_only_columns=True + ) with pytest.raises(SourceSpecMismatchError): node.validate(src) @@ -199,7 +203,9 @@ def test_bound_content_hash_is_concrete_hash(self, tag_schema, data_schema): from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -210,7 +216,9 @@ def test_bound_pipeline_hash_still_schema_based(self, tag_schema, data_schema): from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src @@ -236,7 +244,9 @@ def test_bound_source_property_updates_in_place(self, tag_schema, data_schema): name="x", tag_schema=tag_schema, data_schema=data_schema ) assert job_node.bound_source is None - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node.bound_source = src assert job_node.bound_source is src @@ -252,7 +262,9 @@ def test_bound_source_setter_clears_content_hash_cache( ) schema_hash = job_node.content_hash() # populates cache with schema-based hash - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node.bound_source = src # should clear the cache bound_hash = job_node.content_hash() @@ -278,7 +290,9 @@ def test_source_job_node_as_table_delegates_to_concrete( from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -409,7 +423,9 @@ def test_unbound_system_tag_names_match_bound(self, tag_schema, data_schema): from orcapod.core.sources.dict_source import DictSource from orcapod.types import ColumnConfig - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) unbound = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema ) @@ -445,7 +461,9 @@ def test_bound_keys_all_info_matches_source_directly(self, tag_schema, data_sche from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -478,7 +496,9 @@ def test_bound_output_schema_all_info_matches_source_directly( from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource - src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + src = DictSource( + data=[{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) job_node = SourceJobNode( name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) @@ -526,7 +546,9 @@ def test_source_job_node_from_stream_slot_name_uses_full_hash(self): from orcapod.core.operators import MapData from orcapod.core.sources import DictSource - source = DictSource([{"id": 1, "value": 1.0}], tag_columns=["id"]) + source = DictSource( + [{"id": 1, "value": 1.0}], tag_columns=["id"], tag_only_columns=True + ) stream = MapData({"value": "v"})(source) job_node = SourceJobNode.from_stream(stream) diff --git a/tests/test_core/operators/test_aggregated_list_types.py b/tests/test_core/operators/test_aggregated_list_types.py index 00de01aee..22add24f1 100644 --- a/tests/test_core/operators/test_aggregated_list_types.py +++ b/tests/test_core/operators/test_aggregated_list_types.py @@ -60,7 +60,7 @@ def _source() -> ArrowTableSource: ] ), ) - return ArrowTableSource(table, tag_columns=["g", "i"]) + return ArrowTableSource(table, tag_columns=["g", "i"], tag_only_columns=True) OPERATORS = { diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index d94b76545..46679eb08 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -105,7 +105,10 @@ def _source( table = pa.table(list(columns.values()), names=list(columns)) schema = pa.schema([f.with_nullable(f.name in nullable) for f in table.schema]) return ArrowTableSource( - table.cast(schema), tag_columns=list(tags), source_id=source_id + table.cast(schema), + tag_columns=list(tags), + source_id=source_id, + tag_only_columns=True, ) @@ -917,7 +920,9 @@ def _narrow_source(column: str, source_id: str) -> ArrowTableSource: table = pa.table( {"k": pa.array([1, 2], pa.int16()), column: pa.array([3, 4], pa.int8())} ) - return ArrowTableSource(table, tag_columns=["k"], source_id=source_id) + return ArrowTableSource( + table, tag_columns=["k"], source_id=source_id, tag_only_columns=True + ) # Recorded from the sync path on main (ab544982) before the ITL-837 fix. The sync @@ -965,7 +970,9 @@ def _key_source(key: Column, column: str, source_id: str) -> ArrowTableSource: table = pa.Table.from_arrays( [key, pa.array([1, 2], pa.int8())], names=["k", column] ) - return ArrowTableSource(table, tag_columns=["k"], source_id=source_id) + return ArrowTableSource( + table, tag_columns=["k"], source_id=source_id, tag_only_columns=True + ) DISAGREEING_KEYS = [ @@ -1080,7 +1087,12 @@ def _tag_source( ], names=[id_name, "k", data_name], ) - return ArrowTableSource(table, tag_columns=[id_name, "k"], source_id=source_id) + return ArrowTableSource( + table, + tag_columns=[id_name, "k"], + source_id=source_id, + tag_only_columns=True, + ) _LIST_INT = pa.large_list(pa.int64()) @@ -1264,6 +1276,7 @@ def _merge_null_key_source( ), tag_columns=[f"z{source_id}", "g"], source_id=source_id, + tag_only_columns=True, ) @@ -1372,7 +1385,12 @@ def test_async_null_logical_key_matches_nothing_as_the_sync_path_does() -> None: names=["z" + name, "k", name], ) sources.append( - ArrowTableSource(table, tag_columns=["z" + name, "k"], source_id=source_id) + ArrowTableSource( + table, + tag_columns=["z" + name, "k"], + source_id=source_id, + tag_only_columns=True, + ) ) left, right = sources for operator in (Join, SemiJoin): @@ -1586,7 +1604,10 @@ def test_a_source_refuses_a_tag_named_with_the_meta_prefix() -> None: """Not "tag column not found": the source would drop the meta column first.""" with pytest.raises(InputValidationError, match="'__g'.*reserved for meta"): _ = ArrowTableSource( - _table_with_meta_named_column(), tag_columns=["k", "__g"], source_id="s" + _table_with_meta_named_column(), + tag_columns=["k", "__g"], + source_id="s", + tag_only_columns=True, ) diff --git a/tests/test_core/operators/test_chained_aggregation.py b/tests/test_core/operators/test_chained_aggregation.py index 006d4f713..e6061b848 100644 --- a/tests/test_core/operators/test_chained_aggregation.py +++ b/tests/test_core/operators/test_chained_aggregation.py @@ -64,7 +64,9 @@ def _source( [pa.field(name, t, nullable=False) for name, (t, _) in columns.items()] ) return ArrowTableSource( - pa.Table.from_arrays(arrays, schema=schema), tag_columns=list(tags) + pa.Table.from_arrays(arrays, schema=schema), + tag_columns=list(tags), + tag_only_columns=True, ) diff --git a/tests/test_core/operators/test_drop_nulls.py b/tests/test_core/operators/test_drop_nulls.py index 4af8ee429..72fe31e41 100644 --- a/tests/test_core/operators/test_drop_nulls.py +++ b/tests/test_core/operators/test_drop_nulls.py @@ -41,7 +41,10 @@ def _source( table = pa.table(list(columns.values()), names=list(columns)) schema = pa.schema([f.with_nullable(f.name in nullable) for f in table.schema]) return ArrowTableSource( - table.cast(schema), tag_columns=list(tags), source_id=source_id + table.cast(schema), + tag_columns=list(tags), + tag_only_columns=True, + source_id=source_id, ) @@ -347,6 +350,7 @@ def logical() -> DictSource: {"k": "c", "p": Path("/c"), "arr": None}, ], tag_columns=["k"], + tag_only_columns=True, source_id="logical", ) diff --git a/tests/test_core/operators/test_empty_inputs.py b/tests/test_core/operators/test_empty_inputs.py index 22e9ee1c8..22970a9f2 100644 --- a/tests/test_core/operators/test_empty_inputs.py +++ b/tests/test_core/operators/test_empty_inputs.py @@ -102,7 +102,7 @@ def _filtered_empty(tags: list[str], data: list[str]) -> StreamProtocol: source-info columns, which the join operators rename and reorder. """ table = pa.table({name: pa.array([1, 2], type=INT) for name in [*tags, *data]}) - source = ArrowTableSource(table, tag_columns=tags) + source = ArrowTableSource(table, tag_columns=tags, tag_only_columns=True) empty = PolarsFilter(predicates=[pl.lit(False)]).static_process(source) assert len(empty.as_table()) == 0 return empty diff --git a/tests/test_core/operators/test_group_by.py b/tests/test_core/operators/test_group_by.py index 6e8c24f88..d4c138c29 100644 --- a/tests/test_core/operators/test_group_by.py +++ b/tests/test_core/operators/test_group_by.py @@ -37,6 +37,7 @@ def session_source(session_table) -> ArrowTableSource: session_table, tag_columns=["subject", "date", "probe"], infer_nullable=True, + tag_only_columns=True, ) @@ -184,7 +185,9 @@ def test_list_valued_non_key_tag_orders_members(self): "v": pa.array([1, 2, 3], pa.int64()), } ) - source = ArrowTableSource(table, tag_columns=["a", "c"], infer_nullable=True) + source = ArrowTableSource( + table, tag_columns=["a", "c"], infer_nullable=True, tag_only_columns=True + ) out = GroupBy(by=["a"])(source) assert out.keys() == (("a",), ("v",)) assert out.as_table().to_pylist() == [{"a": "x", "v": [3, 2, 1]}] @@ -232,7 +235,10 @@ def test_scalar_non_key_tags_sort_ascending_with_nulls_last(self): } ) source = ArrowTableSource( - table, tag_columns=["g", "n", "s"], infer_nullable=True + table, + tag_columns=["g", "n", "s"], + infer_nullable=True, + tag_only_columns=True, ) # Members by (n, s): (1, "a"), (1, None), (2, "c"), (3, "b"), (None, "a"). assert GroupBy(by=["g"])(source).as_table().to_pylist() == [ @@ -245,7 +251,10 @@ def test_row_order_does_not_affect_output(self, session_table): def run(tbl): src = ArrowTableSource( - tbl, tag_columns=["subject", "date", "probe"], infer_nullable=True + tbl, + tag_columns=["subject", "date", "probe"], + infer_nullable=True, + tag_only_columns=True, ) return GroupBy(by=["subject", "date"]).process(src).as_table() @@ -262,7 +271,9 @@ def test_falls_back_to_record_id_when_key_covers_all_tags(self): when the default became ``arrow_table: