diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index 6b38e5d80..d7ddbbfc9 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -138,6 +138,24 @@ Order- or role-dependent behaviour that reaches a hash or a stored table first. - `bind(sources=)` with two sources: both bound, nothing moves; one refused, neither bound. A plain or data-tag source in any arrangement binds and reads; a mixed one is ITL-1245. +## Checked and found as documented (part 3) + +- `DropNulls` naming plain tags, data-tags holding nulls and data, in every order: one row + set and schema, each named column non-nullable (a data-tag in both roles), sync equal to + async. +- `Index` with `out=` over a list-valued data-tag (after `Batch`): reads it as a data + column; the data-tag stays; sync equal to async. +- A `DictSource` `data_schema` in another field order: the same source (rows, hashes). +- Every stream method given several items equals its operator (rows, hashes, column + order); `polars_filter` keywords win over `constraint_map`. +- Every combination of `ColumnConfig` groups gives the `all_info` columns of those groups + in the same order, for a source, a join and a batch, in every role and layout. + +## Remaining + +- `MapTags` has no golden entry (adding golden entries is out of scope for this issue). +- The defects and the two undecided rules above: ITL-1241 to ITL-1247. + ## Tests added `tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`, @@ -146,4 +164,7 @@ three layouts and three role settings. `ops/test_several_items.py` uses it for e operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). Part 2: `tests/test_core/sources/test_several_items.py` (gaps 4, 13, 14, 15: CachedSource), `tests/test_core/function_pod/test_several_items.py` (gaps 1, 10; ITL-1244, ITL-1246) and -`tests/test_pipeline/test_bind_several_items.py` (gap 8; ITL-1245). +`tests/test_pipeline/test_bind_several_items.py` (gap 8; ITL-1245). Part 3: +`tests/test_core/data_tags/test_several_items.py` (gaps 12, 15: `Index`), +`tests/test_core/streams/test_several_items_methods.py` (gap 15: stream methods; +`ColumnConfig` groups) and a schema-order test in the sources file (gap 15). diff --git a/tests/test_core/data_tags/test_several_items.py b/tests/test_core/data_tags/test_several_items.py new file mode 100644 index 000000000..f59035b20 --- /dev/null +++ b/tests/test_core/data_tags/test_several_items.py @@ -0,0 +1,121 @@ +"""Data-tags among several named columns: DropNulls and Index (ITL-1236). + +* ``DropNulls`` naming several columns, plain tags, data-tags holding nulls and data, + in several orders: one row set, one schema, the same on the async path; each named + column, in both roles for a data-tag, becomes non-nullable. +* ``Index`` with ``out=`` reads a list-valued data-tag (after ``Batch``) as it reads a + data column, on both paths. +""" + +from __future__ import annotations + +import itertools + +import pyarrow as pa +import pytest + +from orcapod.core.operators import Batch, DropNulls, Index +from orcapod.core.sources import ArrowTableSource + +from ..operators.several_items import ( + DATA_TAGS, + MAIN, + MIXED, + TAGS_FIRST, + Arrangement, + async_rows, + fingerprint, + make_source, + presentation, + rows_fingerprint, +) + +# Tags t1 and t2 and data d1, each null in one row: only row 0 has no null. +NULLS = pa.table( + { + "t1": pa.array(["a", None, "b", "c"], pa.large_string()), + "t2": pa.array([1, 2, None, 4], pa.int64()), + "d1": pa.array([1, 2, 3, None], pa.int64()), + "d2": pa.array([5, 6, 7, 8], pa.int64()), + } +) + + +def _nullable_source(tag_only: list[str]) -> ArrowTableSource: + return ArrowTableSource( + NULLS, tag_columns=["t1", "t2"], source_id="n", tag_only_columns=tag_only + ) + + +@pytest.mark.parametrize("tag_only", [[], ["t2"]], ids=["data_tags", "mixed"]) +def test_drop_nulls_of_several_columns_with_nullable_data_tags( + tag_only: list[str], +) -> None: + source = _nullable_source(tag_only) + results = {} + for names in itertools.permutations(["t1", "t2", "d1"]): + op = DropNulls(list(names)) + kept = op(source) + tag_schema, data_schema = kept.output_schema() + assert tag_schema["t1"] is str and tag_schema["t2"] is int + assert data_schema["d1"] is int + # A data-tag is one column: narrowed in both roles, each data-tag. + for name, python_type in (("t1", str), ("t2", int)): + if name not in tag_only: + assert data_schema[name] is python_type + else: + assert name not in data_schema + # The prediction equals the computed stream's schema (a prediction is + # verified against the static result, CLAUDE.md), with system tags and + # source info too. + for config in ( + {}, + {"system_tags": True}, + {"source": True, "system_tags": True}, + ): + computed = op.static_process(source).output_schema(columns=config) + assert kept.output_schema(columns=config) == computed + # The computed rows hold no null in a named column, in either role. + table = kept.as_table() + assert all(table.column(name).null_count == 0 for name in names) + assert not any(table.schema.field(name).nullable for name in names) + rows = rows_fingerprint(list(kept.iter_data())) + assert rows_fingerprint(async_rows(DropNulls(list(names)), source)) == rows + results[names] = {**fingerprint(kept), "presentation": presentation(kept)} + assert [data["d2"] for _, data in kept.iter_data()] == [5] + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +@pytest.mark.parametrize("tag_only", [[], ["t2"]], ids=["data_tags", "mixed"]) +def test_drop_nulls_of_one_data_tag_keeps_the_rows_where_it_has_a_value( + tag_only: list[str], +) -> None: + kept = DropNulls(["t1"])(_nullable_source(tag_only)) + assert sorted(data["d2"] for _, data in kept.iter_data()) == [5, 7, 8] # pyright: ignore[reportArgumentType] + + +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +@pytest.mark.parametrize("column", ["t1", "t3", "d1"]) +def test_index_reads_a_list_valued_data_tag_as_a_data_column( + role: str, column: str +) -> None: + """After ``Batch``, t1 and t3 are list-valued data-tags (t2 is plain in MIXED).""" + batched = Batch(batch_size=3)( + make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + ) + op = Index(column, 0, out=f"first_{column}") + indexed = op(batched) + first = [data[f"first_{column}"] for _, data in indexed.iter_data()] + assert first == {"t1": ["b", "b"], "t3": [1, 2], "d1": [10, 40]}[column] + # The data-tag itself is unchanged: still a tag and a data column, with its lists. + assert presentation(indexed)[0] == ("t1", "t2", "t3") + assert column in presentation(indexed)[1] + lists = {"t1": [["b", "a", "a"], ["b", "a", "b"]], "t3": [[1, 2, 1], [2, 1, 2]]} + lists["d1"] = [[10, 20, 30], [40, 50, 60]] + assert [data[column] for _, data in indexed.iter_data()] == lists[column] + if column != "d1": + assert [tag[column] for tag, _ in indexed.iter_data()] == lists[column] + assert rows_fingerprint( + async_rows(Index(column, 0, out=f"first_{column}"), batched) + ) == rows_fingerprint(list(indexed.iter_data())) diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index ff7ee7c8a..3e052c1b0 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -22,7 +22,12 @@ import pyarrow as pa import pytest -from orcapod.core.sources import ArrowTableSource, CachedSource, DerivedSource +from orcapod.core.sources import ( + ArrowTableSource, + CachedSource, + DerivedSource, + DictSource, +) from orcapod.databases import InMemoryArrowDatabase from ..operators.several_items import ( @@ -202,3 +207,39 @@ def no_rows(*args: object, **kwargs: object) -> pa.Table: again = CachedSource(unavailable, cache_database=database) assert presentation(again) == presentation(make_source(MAIN, arrangement)) assert fingerprint(again) == fingerprint(cached) + + +@pytest.mark.parametrize( + "data_schema", + [ + {"t1": str, "t2": int, "d1": int, "d3": str}, + {"d3": str, "t2": int, "d1": int, "t1": str}, + ], + ids=["table_order", "another_order"], +) +@pytest.mark.parametrize("role", ROLES) +def test_a_schema_in_another_field_order_gives_the_same_source( + data_schema: dict[str, type], role: str +) -> None: + """A ``DictSource`` schema with several fields, in the rows' order or another.""" + rows = [ + {"t1": "a", "t2": 1, "d1": 10, "d3": "x"}, + {"t1": "b", "t2": 2, "d1": 20, "d3": "y"}, + ] + tag_only = {PLAIN: True, DATA_TAGS: [], MIXED: ["t2"]}[role] + source = DictSource( + rows, + tag_columns=["t1", "t2"], + source_id="d", + data_schema=data_schema, + tag_only_columns=tag_only, + ) + reference = DictSource( + rows, + tag_columns=["t1", "t2"], + source_id="d", + data_schema={"t1": str, "t2": int, "d1": int, "d3": str}, + tag_only_columns=tag_only, + ) + assert fingerprint(source) == fingerprint(reference) + assert presentation(source)[0] == ("t1", "t2") diff --git a/tests/test_core/streams/test_several_items_methods.py b/tests/test_core/streams/test_several_items_methods.py new file mode 100644 index 000000000..fa31c955f --- /dev/null +++ b/tests/test_core/streams/test_several_items_methods.py @@ -0,0 +1,224 @@ +"""Stream methods given several items equal their operators; several column groups (ITL-1236). + +Every stream method that builds an operator from several items (names, a mapping, +keywords, predicates) gives the stream that operator gives: same rows, hashes and +column order, for plain tags, data-tags and a mix, on a stream with three tags and three +data columns. ``polars_filter`` merges ``constraint_map`` and keyword constraints, the +keywords winning. Several ``ColumnConfig`` groups together give the ``all_info`` +columns of those groups, in the same order. +""" + +from __future__ import annotations + +import itertools +from collections.abc import Callable +from dataclasses import dataclass + +import polars as pl +import pytest + +from orcapod.core.operators import ( + AddTags, + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + Join, + MapData, + MapTags, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, + TagBy, +) +from orcapod.core.sources import ArrowTableSource +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.system_constants import constants + +from ..operators.several_items import ( + LAYOUTS, + MAIN, + P, + ROLES, + TAGS_FIRST, + Arrangement, + fingerprint, + make_source, + presentation, +) + +pytestmark = pytest.mark.filterwarnings( + "ignore::orcapod.core.data_tags.DataTagMaterializationWarning" +) + +# The stream methods are on the stream classes, not on ``StreamProtocol``. +Build = Callable[[ArrowTableSource], StreamProtocol] + + +@dataclass(frozen=True) +class MethodCase: + """A stream method call and the operator call it must equal.""" + + label: str + method: Build + operator: Build + + +CASES = [ + MethodCase( + "select_tag_columns", + lambda s: s.select_tag_columns(["t3", "t1"]), + lambda s: SelectTagColumns(["t3", "t1"])(s), + ), + MethodCase( + "select_data_columns", + lambda s: s.select_data_columns(["d3", "d1"]), + lambda s: SelectDataColumns(["d3", "d1"])(s), + ), + MethodCase( + "drop_tag_columns", + lambda s: s.drop_tag_columns(["t3", "t1"]), + lambda s: DropTagColumns(["t3", "t1"])(s), + ), + MethodCase( + "drop_data_columns", + lambda s: s.drop_data_columns(["d3", "d1"]), + lambda s: DropDataColumns(["d3", "d1"])(s), + ), + MethodCase( + "select_columns_positional", + lambda s: s.select_columns("d3", "t2", "d1"), + lambda s: SelectColumns(["d3", "t2", "d1"])(s), + ), + MethodCase( + "drop_columns_positional", + lambda s: s.drop_columns("d3", "t2"), + lambda s: DropColumns(["d3", "t2"])(s), + ), + MethodCase( + "map_tags", + lambda s: s.map_tags({"t3": "u3", "t1": "u1"}), + lambda s: MapTags({"t3": "u3", "t1": "u1"})(s), + ), + MethodCase( + "map_data", + lambda s: s.map_data({"d3": "e3", "d1": "e1"}), + lambda s: MapData({"d3": "e3", "d1": "e1"})(s), + ), + MethodCase( + "drop_nulls", + lambda s: s.drop_nulls(["d2", "t2"]), + lambda s: DropNulls(["d2", "t2"])(s), + ), + MethodCase( + "polars_filter_predicates", + lambda s: s.polars_filter(pl.col("d1") > 10, pl.col("t3") == 1), + lambda s: PolarsFilter(predicates=[pl.col("d1") > 10, pl.col("t3") == 1])(s), + ), + MethodCase( + "polars_filter_constraints", + lambda s: s.polars_filter(t1="a", t2=1), + lambda s: PolarsFilter(constraints={"t1": "a", "t2": 1})(s), + ), + # Keyword constraints win over the map's value for the same name. + MethodCase( + "polars_filter_constraint_map_and_keywords", + lambda s: s.polars_filter(constraint_map={"t1": "a", "t2": 2}, t2=1), + lambda s: PolarsFilter(constraints={"t1": "a", "t2": 1})(s), + ), + MethodCase( + "add_tags", + lambda s: s.add_tags(x=1, y="z"), + lambda s: AddTags({"x": 1, "y": "z"})(s), + ), + MethodCase( + "tag_by", + lambda s: s.tag_by(["d3", "d1"]), + lambda s: TagBy(["d3", "d1"])(s), + ), +] + + +@pytest.mark.parametrize("role", ROLES) +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.label) +def test_a_stream_method_with_several_items_equals_its_operator( + case: MethodCase, role: str +) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + by_method, by_operator = case.method(source), case.operator(source) + assert presentation(by_method) == presentation(by_operator) + assert fingerprint(by_method) == fingerprint(by_operator) + + +@pytest.mark.parametrize("role", ROLES) +def test_join_and_semi_join_methods_equal_their_operators(role: str) -> None: + def build(spec): + return make_source(spec, Arrangement(spec.tags, TAGS_FIRST, role)) + + main, partner = build(MAIN), build(P) + assert fingerprint(main.join(partner)) == fingerprint(Join()(main, partner)) + assert fingerprint(partner.join(main)) == fingerprint(Join()(main, partner)) + assert fingerprint(main.semi_join(partner)) == fingerprint( + SemiJoin()(main, partner) + ) + + +def test_polars_filter_keywords_win_over_the_constraint_map() -> None: + kept = make_source(MAIN).polars_filter(constraint_map={"t1": "a", "t2": 2}, t2=1) + assert sorted(data["d1"] for _, data in kept.iter_data()) == [20, 50] # pyright: ignore[reportArgumentType] + + +COLUMN_GROUPS = ("meta", "context", "source", "system_tags", "content_hash") +CONTENT_HASH_COLUMN = "_content_hash" + + +def _group_of(name: str) -> str | None: + """The ``ColumnConfig`` group of a column name, or ``None`` for a user column. + + Derived from the column naming convention (CLAUDE.md, "Column naming + conventions"), not from the stream under test. + """ + if name.startswith(constants.META_PREFIX): + return "meta" + if name == constants.CONTEXT_KEY: + return "context" + if name.startswith(constants.SOURCE_PREFIX): + return "source" + if name.startswith(constants.SYSTEM_TAG_PREFIX): + return "system_tags" + if name == CONTENT_HASH_COLUMN: + return "content_hash" + return None + + +@pytest.mark.parametrize("layout", LAYOUTS) +@pytest.mark.parametrize("role", ROLES) +def test_several_column_groups_give_exactly_their_columns_in_the_all_info_order( + role: str, layout: str +) -> None: + """Every combination of groups: the user columns and those groups' columns, in order. + + The expected list is the ``all_info`` table's columns kept by their group, so a + missing requested column and an unrequested group both fail. + """ + source = make_source(MAIN, Arrangement(MAIN.tags, layout, role)) + partner = make_source(P, Arrangement(P.tags, layout, role)) + for stream in (source, Join()(source, partner), Batch(batch_size=2)(source)): + full = stream.as_table(all_info=True).column_names + present = {_group_of(name) for name in full} - {None} + # Every group but meta shows up in all_info for these streams. + assert present == {"context", "source", "system_tags", "content_hash"} + for n in range(1, len(COLUMN_GROUPS) + 1): + for groups in itertools.combinations(COLUMN_GROUPS, n): + expected = [ + name + for name in full + if _group_of(name) is None or _group_of(name) in groups + ] + columns = stream.as_table( + columns=dict.fromkeys(groups, True) + ).column_names + assert columns == expected, groups