From 0bf5e72665a4c18d5dacf028e2e997db0cb930bc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:12:29 -0800 Subject: [PATCH 01/10] test(operators): several columns, names, mappings and inputs, in every order, role and arrangement (ITL-1236) A shared family of inputs with three tags and three data columns, in several tag orders, table layouts and role settings (plain, data-tags, mixed), and tests of every operator over it, sync and async: the result does not depend on the arrangement of the input; the order of a set argument (names, rename mappings, constraints) changes nothing; Join over 2, 3 and 4 inputs and MergeJoin are commutative; SemiJoin matches on every shared tag. Defects found are xfail(strict=True): ITL-1241 (MergeJoin ties), ITL-1242 (MapTags swap of data-tags), MergeJoin null crash (issue pending). The inventory of every several-item argument, with the evidence and the ranked gaps, is docs/dev/specs/itl-1236-several-item-test-audit.md. CLAUDE.md and .zed/rules state the rule in "Testing conventions". Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 12 + CLAUDE.md | 12 + .../specs/itl-1236-several-item-test-audit.md | 129 ++++ tests/test_core/operators/several_items.py | 283 ++++++++ .../test_core/operators/test_several_items.py | 610 ++++++++++++++++++ 5 files changed, 1046 insertions(+) create mode 100644 docs/dev/specs/itl-1236-several-item-test-audit.md create mode 100644 tests/test_core/operators/several_items.py create mode 100644 tests/test_core/operators/test_several_items.py diff --git a/.zed/rules b/.zed/rules index 915a08c69..7d1dd6ed3 100644 --- a/.zed/rules +++ b/.zed/rules @@ -582,6 +582,18 @@ unless the test is about the default (developer, November 16, 2026, ITL-1159). T sources that take the option (the table, file and database sources); `ListSource`, `DerivedSource`, `CachedSource` and `PollingSource` do not take it. +Every argument or input that can take several items (column names, a mapping, keywords, +input streams, sources to bind, output keys) is tested (developer, November 17, 2026, +ITL-1236): with two or more items; in more than one order, asserting what the order does or +must not change (rows, column order, content hash, pipeline hash, system-tag column names); +in each role it accepts (plain tag, data, data-tag: build the role you test, whatever the +`tag_only_columns` default above says); on inputs with two or more tag columns AND two or +more data columns, in more than one arrangement (tag order, table layout); and on the sync +and the async path. A test whose streams have one tag and one data column does not test an +operator. The shared family of such inputs is `tests/test_core/operators/several_items.py`; +the inventory is `docs/dev/specs/itl-1236-several-item-test-audit.md`. The docs show the +several-item form of such an argument, not only the one-item form. + --- ## Project layout diff --git a/CLAUDE.md b/CLAUDE.md index a06981d07..e4b82cbc3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -593,6 +593,18 @@ unless the test is about the default (developer, November 16, 2026, ITL-1159). T sources that take the option (the table, file and database sources); `ListSource`, `DerivedSource`, `CachedSource` and `PollingSource` do not take it. +Every argument or input that can take several items (column names, a mapping, keywords, +input streams, sources to bind, output keys) is tested (developer, November 17, 2026, +ITL-1236): with two or more items; in more than one order, asserting what the order does or +must not change (rows, column order, content hash, pipeline hash, system-tag column names); +in each role it accepts (plain tag, data, data-tag: build the role you test, whatever the +`tag_only_columns` default above says); on inputs with two or more tag columns AND two or +more data columns, in more than one arrangement (tag order, table layout); and on the sync +and the async path. A test whose streams have one tag and one data column does not test an +operator. The shared family of such inputs is `tests/test_core/operators/several_items.py`; +the inventory is `docs/dev/specs/itl-1236-several-item-test-audit.md`. The docs show the +several-item form of such an argument, not only the one-item form. + --- ## Project layout diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md new file mode 100644 index 000000000..ffd91122e --- /dev/null +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -0,0 +1,129 @@ +# ITL-1236: tests of every argument that takes several items + +Status: inventory of 2026-11-17 (phase 1), on `main` at 4da42681. Kept up to date as the +gaps are closed. + +## Why + +`GroupBy` was tested with one tag and one data column. That hid a result that depended on +the order of the input's tag columns (ITL-1226). The same pattern can hide a defect in any +argument that takes several items: column names, a mapping, several input streams. + +## The rule + +Every argument or input that can take several items is tested: + +- **(a)** with two or more items; +- **(b)** with the items in more than one order, with an assertion of what the order does or + must not do (rows, output schema and column order, content hash, pipeline hash, + system-tag column names); +- **(c)** in each role the argument accepts: plain tag, data, data-tag; +- **(d)** on inputs with two or more tag columns AND two or more data columns, in more than + one arrangement (tag order and table layout); +- **(e)** on the sync and the async path. + +A test whose streams have one tag and one data column does not test an operator. + +## How the inventory was made + +1. Every test that uses the argument was found by searching for the class and the stream + method (`grep -rn` over `tests/`), and read. +2. A probe ran every operator on one stream, three tags and three data columns, in 18 + arrangements (6 tag orders x 3 layouts), for plain tags, data-tags and a mix. It compared + rows, row hashes, system tags, source info, content and pipeline hashes. A second probe + compared each argument given in two orders. On `main` only `GroupBy` depended on the + arrangement (ITL-1226, PR #434). The other differences found are the defects below. + +Legend: **yes** (with the best test as evidence), **no** (with what the closest test does), +**n/a**. Paths: `ops/` = `tests/test_core/operators/`, `dt/` = `tests/test_core/data_tags/`, +`ASE` = `ops/test_async_sync_equivalence.py`, `WHF` = `tests/test_hashing/test_wrong_hash_findings.py`, +`NEW` = `ops/test_several_items.py` (this issue). + +## Inventory + +### Operators + +| Argument | (a) several | (b) order | (c) roles | (d) 2+ tags × 2+ data, arrangements | (e) sync / async | +|---|---|---|---|---|---| +| `Join` inputs, 2 | yes: `ops/test_operators.py::TestJoinBehavior::test_join_combines_streams_on_shared_tags` | yes: `ops/test_join_tag_order.py::test_operand_order_does_not_change_columns`; hashes `TestArgumentSymmetryIdentity::test_join_swapped_inputs_same_*` | yes: ASE `join_2`, `ref_join_2`, `ref_join_2_plain_left/right` | no (one arrangement) → NEW `test_join_result_does_not_depend_on_the_arrangement_of_its_inputs` | yes: ASE `join_2`, `join_2_reversed` | +| `Join` inputs, 3 | yes: `TestJoinSystemTagCanonicalOrdering::test_three_way_join_produces_six_system_tag_columns` | partial: system-tag names for 3 of 6 permutations; `dt/test_merged_source_info.py::test_join_token_does_not_depend_on_input_order`. No content/pipeline hash → NEW `test_join_result_does_not_depend_on_the_order_of_its_inputs[join_3/*]` (all 6) | yes: ASE `join_3`, `ref_join_3` | no → NEW | yes: ASE `join_3`, `join_3_rotated` | +| `Join` inputs, 4+ | async only: `tests/test_channels/test_native_async_operators.py::TestJoinNativeAsync::test_four_way_join` | no → NEW (all 24 permutations) | no → NEW (plain, data-tags, mixed) | no → NEW | no sync comparison → NEW | +| `Join` shared keys, several | schema only: `tests/test_pipeline/test_zero_row_column_order.py` step `join` | no (keys never in another order in one input) → NEW (tag orders per input) | plain, data-tag | no → NEW | schema only → NEW | +| `MergeJoin` colliding columns | yes: `ops/test_merge_join.py::TestMergeJoinBasic::test_values_sorted_independently_per_column`; ASE `merge_source` | left/right only, distinct values → NEW `test_merge_join_result_does_not_depend_on_the_order_of_its_inputs`; ties: **defect ITL-1241** | yes: ASE `merge_join`, `ref_merge_join_*` | no → NEW | yes: ASE | +| `MergeJoin` nullable colliding column with a null | no | n/a | n/a | n/a | n/a: **defect 3** (raises `TypeError`) → NEW xfail | +| `SemiJoin` match keys | rows with 1 key only (`ops/test_semijoin_tag_keys.py::test_semijoin_keys_on_shared_tags_only`) | no → NEW `test_semijoin_matches_on_every_shared_tag_in_any_arrangement` | yes: plain, `ref_semijoin`, mixed | no → NEW | 1 key: `test_semijoin_tag_keys.py::test_async_rows_equal_sync_rows` → NEW (2 keys) | +| `Batch` (tags and data become lists) | yes: ASE `batch_2` (2 tags, 9 data) | n/a (no column argument); member order is identity: `WHF::TestBatchGroupByFold::test_member_order_is_identity` | yes: `dt/test_batch.py` | no → NEW `test_unary_result_does_not_depend_on_the_input_arrangement[batch*]` | yes: ASE | +| `GroupBy` `by`, member order | yes: `ops/test_group_by.py::TestGroupByShape::test_one_row_per_distinct_key` | keys only on `main` | yes | no on `main` | 1 key on `main` | +| ↳ covered by PR #434 (ITL-1226) | | `test_group_by_member_order.py::test_the_order_of_by_is_not_identity` | plain, data-tag, mixed | `test_no_arrangement_of_the_input_changes_the_result` (24 × 4, source and raw stream) | `test_a_job_gives_the_same_rows_sync_and_async_for_any_arrangement` | +| `TagBy` columns | yes: `dt/test_tag_by.py::test_named_data_columns_become_data_tags_in_the_order_given` | yes: `test_the_order_of_the_names_is_part_of_the_identity` (PR #434 reverses this decision) | n/a / yes / refused: `test_an_existing_data_tag_is_refused` | no (1 tag) → NEW `[tag_by/*]` | yes: ASE `tag_by`, `tag_by_reversed` | +| `AddTags` tags | yes: `ops/test_add_tags.py::TestValues::test_arrow_type_of_each_accepted_value_type` | operator hash and keys: `TestIdentity::test_order_is_part_of_the_identity`; stream hashes → NEW `test_add_tags_order_is_the_order_of_the_new_tags_and_its_identity` | yes: `dt/test_add_tags.py` | no → NEW | yes: ASE `add_tags`, `ref_add_tags` | +| `SelectTagColumns` columns | **no** (only one name, or all) → NEW | **no** (`WHF::test_tag_column_spellings_share_identity` uses one name) → NEW | yes: `dt/test_roles.py::test_each_operator_acts_by_role` | no → NEW | yes | +| `DropTagColumns` columns | **no** → NEW | **no** → NEW | yes | no → NEW | yes | +| `SelectDataColumns` columns | yes: ASE `select_data_columns` | hash: `WHF::TestColumnSelectionIdentity`; keys, rows → NEW | yes | no → NEW | yes | +| `DropDataColumns` columns | yes: ASE `drop_data_columns` | hash only → NEW | yes | no → NEW | yes | +| `SelectColumns` columns | yes: `ops/test_tag_and_data_columns.py::test_select_columns_keeps_only_the_named_columns` | yes (keys and pipeline hash) | yes: `dt/test_operators.py::test_select_and_drop_columns_act_on_the_one_column_in_both_roles` | no → NEW | yes | +| `DropColumns` columns | yes: `test_drop_columns_drops_tags_and_data` | hash only → NEW | yes | no → NEW | yes | +| `MapTags` name_map | **almost none**: one chained rename with `tag_only` (`dt/test_operators.py::test_map_tags_tag_only_keeps_a_chained_rename_of_another_tag`) → NEW | **no** → NEW | yes; a swap of data-tags is refused: **defect ITL-1242** → NEW xfail | no → NEW | single rename only → NEW | +| `MapData` name_map | yes: `dt/test_operators.py::test_map_data_chained_rename_does_not_reach_the_data_tag_copy` | **no** → NEW (incl. a swap) | yes | no → NEW | single rename → NEW | +| `MapColumns` name_map | yes: `ops/test_mappers_strict.py::test_map_columns_renames_tags_and_data`; ASE `map_columns` | **no** → NEW | plain; refused with data-tags (ITL-1022): `dt/test_operators.py::test_uncovered_operators_raise` | no → NEW | yes | +| `PolarsFilter` predicates | **no** (always one) → NEW | **no**; the order IS in the identity (`filters.py` hashes an ordered tuple): **question Q1** | data | no → NEW | yes | +| `PolarsFilter` constraints | **no** (never two keys) → NEW | **no** → NEW (not in identity) | tag, data, data-tag | no → NEW | yes | +| `stream.polars_filter(*predicates, constraint_map, **constraints)` | **no** | no | — | no | sync only | +| `DropNulls` columns | yes: `ops/test_drop_nulls.py::test_tag_and_data_columns_in_one_call` | hashes: `test_identity_is_the_set_of_names`; rows → NEW | tag, data; a nullable data-tag holding a null: **no** | one layout → NEW | yes: ASE `drop_nulls_many` | +| `Pick` / `Index` | one item each (n/a); inputs with 2+ tags and 2+ data: `ops/test_index_pick_arrow_types.py::test_materialized_schema_matches_prediction` | n/a | data-tag with `out=`: **no**; ASE has no `ref_pick` | no | yes | +| Stream methods (`select_*`, `drop_*`, `map_*`, `join`, `semi_join`, `drop_nulls`) | partly (`test_stream_convenience_methods.py`, plain only) | no | plain only | no | sync only | + +### Sources, function pods, binding + +| Argument | (a) | (b) | (c) | (d) | (e) | +|---|---|---|---|---|---| +| `tag_columns` | yes: `sources/test_source_protocol_conformance.py::test_dict_source_multiple_tag_columns` (membership) | **no** source test (raw stream keys only: `dt/test_data_tag_order.py::test_data_tags_follow_the_tag_order`) | plain, data-tag | no | sync | +| `tag_only_columns` | True: `sources/test_tag_only_columns.py` (9 kinds, 1 tag); a list: `dt/test_error_messages.py::test_the_named_remedy_binds` | no (list order) | mixed: `dt/test_add_tags.py::test_a_mixed_stream_keeps_its_plain_tags_plain` | no | `dt/test_persistence.py` | +| `record_id_columns` | one composite key: `sources/test_stream_builder.py::TestKeyRowIdentifiers::test_record_id_and_token[composite]` | **no**: two key orders never compared (the order is in the identity, `sources/base.py`) | tag key only | no | no | +| schema arguments (`data_schema`, `schema`) | yes: `sources/test_large_normalization_sources.py` | **no**: schema always in the table's order | plain | no | no | +| `CachedSource` | 1 tag | no | **plain only** (data-tag read-back untested) | no | no | +| `DerivedSource(tags_as_data=)` | yes: `sources/test_derived_source_tags_as_data.py::TestColumns::test_several_copies_follow_map_order` | content hash only: `TestIdentity::test_mapping_order_does_not_change_content_hash` | plain; data-tag origin `dt/test_derived_source.py::test_tags_as_data_over_a_data_tag_node` | one arrangement | yes | +| function pod, several inputs | 2: `function_pod/test_function_pod_auto_select.py::test_call_selects_once_after_the_join`; 3: `dt/test_edi_end_to_end.py` | **no**: `pod(b, a)` never compared with `pod(a, b)` | data-tag parameter (EDI) | no | yes | +| `output_keys`, several | yes: `data_function/test_data_function.py::test_multiple_output_keys` | **no**; order is not in the identity: **defect or decision 4** | — | — | value only | +| automatic select of several extra columns | yes: `test_dropping_logs_one_info_message_naming_the_columns` | no (kept list sorted) | one data-tag falls through | no | yes | +| `PipelineJob.bind(sources=)`, several | yes: `test_pipeline/test_pipeline_job.py::TestPipelineJobBind::test_bind_sources_mutates_job` (membership) | n/a (dict order unused) | plain only | — | — | +| bind a source in another column order | yes: `test_bind_type_check.py::test_columns_match_by_name_within_each_group_whatever_their_order` (plain, run ok only) | yes (plain) | **mixed roles: defect 5** (accepted, then the node's table breaks) | partly | async read in `test_bind_narrower_source.py::test_columns_follow_the_declared_order` | +| `ColumnConfig` groups | all_info: `tests/test_types_column_config_all_info.py`; each group alone: `test_zero_row_column_order.py` | n/a | data-tag variants | one arrangement | yes | + +## Defects and questions found + +| # | What | Reaches | Issue | +|---|---|---|---| +| 1 | `MergeJoin(A, B)` and `MergeJoin(B, A)` give the source-info lists of tied values in argument order | stored source info (operator cache, downstream tables); not the hashes | ITL-1241 | +| 2 | `MapTags` refuses a swap or cycle of data-tags that it accepts for plain tags (`MapData` is right: a data-only move) | error only | ITL-1242 | +| 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | pending | +| 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | pending (decision) | +| 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | pending | +| Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | question | + +## Gaps, ranked + +Order- or role-dependent behaviour that reaches a hash or a stored table first. + +1. `output_keys` order: not in identity, stored values served under swapped names (defect 4). +2. `MergeJoin` tied values: source-info order follows the inputs (ITL-1241). +3. `PolarsFilter` predicate order in identity, constraint order not; never tested with two (Q1). +4. `record_id_columns` key order: moves record ids, tokens and the content hash; never compared. +5. `Join` with 3 and 4 inputs: content and pipeline hash under every permutation; 4 inputs never run sync. Closed by NEW. +6. `MapTags` / `MapData` / `MapColumns`: several renames, mapping order, swaps, sync and async; `MapTags` has no golden entry. Closed by NEW (except golden: out of scope). +7. Column arrangement (d) never varied for any operator but `GroupBy` (PR #434). Closed by NEW for every operator. +8. Bind of a mixed-role source in another column order (defect 5); several sources with one refused. +9. `SelectTagColumns` / `DropTagColumns` with two or more names, in two orders. Closed by NEW. +10. `function_pod` over several inputs: `pod(b, a)` against `pod(a, b)` (hashes, system-tag names, stored rows). +11. `SemiJoin` with two or more keys, rows sync and async. Closed by NEW. +12. `DropNulls` on a nullable data-tag holding a null, sync and async. +13. `tag_columns` order on a source: keys follow, hashes and system tags must not. +14. `DerivedSource(tags_as_data=)` reversed: default `source_id` and pipeline hash. +15. `CachedSource` over a data-tag source; `Pick` / `Index` from a data-tag with `out=`; schema arguments in another field order; the stream methods' several-argument forms (`polars_filter` with `constraint_map` and keywords). + +## Tests added + +`tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`, +`t2`, `t3`; data `d1`, `d2`, `d3`) and join partners `P`, `Q`, `R`, each in any tag order, +three layouts and three role settings. `ops/test_several_items.py` uses it for every +operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). diff --git a/tests/test_core/operators/several_items.py b/tests/test_core/operators/several_items.py new file mode 100644 index 000000000..49642b438 --- /dev/null +++ b/tests/test_core/operators/several_items.py @@ -0,0 +1,283 @@ +"""Streams with several tag and several data columns, in several arrangements (ITL-1236). + +An argument that takes several items (column names, a mapping, input streams) must be +tested with two or more items, in more than one order, in each role a column can have +(plain tag, data, data-tag), on inputs with two or more tags AND two or more data columns, +on the sync and the async path. A stream with one tag and one data column cannot show a +result that depends on column order or role: ``GroupBy``'s did (ITL-1226). + +This module is the shared family of such inputs. ``main_source`` has three tags +(``t1``, ``t2``, ``t3``) and three data columns (``d1``, ``d2``, ``d3``); the join +partners ``P``, ``Q`` and ``R`` share some of those tags and add data columns of their +own. Each source can be built in any ``Arrangement``: + +* the order of its tag columns (``tag_order``); +* the physical order of its table's columns (``layout``: tags first, data first and + reversed, or interleaved); +* the role of its tags (``roles``): all plain (``tag_only_columns=True``), all data-tags + (the default of a source), or mixed (one plain tag, the others data-tags). + +``fingerprint`` is what must not depend on an arrangement or on the order of an argument +that names a set: the rows (values, system tags, row hashes, source info), the stream's +content and pipeline hashes, and the column names of each group. ``presentation`` is the +column order, which follows the input as for every operator and enters no hash. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any, TypeAlias + +import pyarrow as pa + +from orcapod.channels import Channel +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.core.sources import ArrowTableSource +from orcapod.protocols.core_protocols import ( + DataProtocol, + StreamProtocol, + TagProtocol, +) + +Row: TypeAlias = tuple[TagProtocol, DataProtocol] + +# Roles of a source's tags. +PLAIN = "plain" +DATA_TAGS = "data_tags" +MIXED = "mixed" +ROLES = (PLAIN, DATA_TAGS, MIXED) + +# Layouts of a table: where the tag columns sit among the data columns. +TAGS_FIRST = "tags_first" +DATA_FIRST_REVERSED = "data_first_reversed" +INTERLEAVED = "interleaved" +LAYOUTS = (TAGS_FIRST, DATA_FIRST_REVERSED, INTERLEAVED) + + +@dataclass(frozen=True) +class Spec: + """The columns of one source: tags, data, values and the plain tag of ``MIXED``.""" + + source_id: str + tags: tuple[str, ...] + data: tuple[str, ...] + columns: dict[str, "pa.Array[Any]"] + mixed_plain: str + + def tag_orders(self) -> tuple[tuple[str, ...], ...]: + """The declared order, its reverse, and (with three tags) a rotation.""" + orders = [self.tags, tuple(reversed(self.tags))] + if len(self.tags) > 2: + orders.append(self.tags[1:] + self.tags[:1]) + return tuple(orders) + + +def _array(values: list[Any], arrow_type: pa.DataType) -> "pa.Array[Any]": + return pa.array(values, arrow_type) + + +# Six rows, each tag tuple once. ``t1`` and ``t2`` repeat on their own; ``d2`` holds a +# null (nullable); the other data columns hold none and are non-nullable. +MAIN = Spec( + source_id="s", + tags=("t1", "t2", "t3"), + data=("d1", "d2", "d3"), + columns={ + "t1": _array(["b", "a", "a", "b", "a", "b"], pa.large_string()), + "t2": _array([2, 1, 2, 1, 1, 2], pa.int64()), + "t3": _array([1, 2, 1, 2, 1, 2], pa.int64()), + "d1": _array([10, 20, 30, 40, 50, 60], pa.int64()), + "d2": _array([0.5, None, 2.5, 3.5, 4.5, None], pa.float64()), + "d3": _array(["w", "x", "y", "z", "v", "u"], pa.large_string()), + }, + mixed_plain="t2", +) + +# Join partners. Each shares tags with ``MAIN`` and with each other, never a data name. +P = Spec( + source_id="p", + tags=("t1", "t2"), + data=("p1", "p2"), + columns={ + "t1": _array(["a", "b", "a", "b"], pa.large_string()), + "t2": _array([2, 1, 1, 2], pa.int64()), + "p1": _array([7, 8, 9, 6], pa.int64()), + "p2": _array(["m", "n", "o", "q"], pa.large_string()), + }, + mixed_plain="t1", +) +Q = Spec( + source_id="q", + tags=("t2", "t3"), + data=("q1", "q2"), + columns={ + "t2": _array([1, 2, 2, 1], pa.int64()), + "t3": _array([2, 1, 2, 1], pa.int64()), + "q1": _array([0.25, 0.75, 1.25, 1.75], pa.float64()), + "q2": _array([100, 200, 300, 400], pa.int64()), + }, + mixed_plain="t3", +) +R = Spec( + source_id="r", + tags=("t1",), + data=("r1", "r2"), + columns={ + "t1": _array(["b", "a"], pa.large_string()), + "r1": _array([-1, -2], pa.int64()), + "r2": _array(["rb", "ra"], pa.large_string()), + }, + mixed_plain="t1", +) + + +@dataclass(frozen=True) +class Arrangement: + """How a source presents its columns: tag order, table layout and tag roles.""" + + tag_order: tuple[str, ...] + layout: str + roles: str + + @property + def id(self) -> str: + return f"{'-'.join(self.tag_order)}/{self.layout}/{self.roles}" + + +def arrangements(spec: Spec, roles: Sequence[str] = ROLES) -> list[Arrangement]: + """Every tag order of ``spec.tag_orders()`` in every layout, for each role.""" + return [ + Arrangement(order, layout, role) + for role in roles + for order in spec.tag_orders() + for layout in LAYOUTS + ] + + +def _layout(spec: Spec, tag_order: tuple[str, ...], layout: str) -> list[str]: + """The physical column order of the table.""" + if layout == TAGS_FIRST: + return [*tag_order, *spec.data] + if layout == DATA_FIRST_REVERSED: + return [*reversed(spec.data), *reversed(tag_order)] + if layout == INTERLEAVED: + names: list[str] = [] + for i in range(max(len(tag_order), len(spec.data))): + if i < len(spec.data): + names.append(spec.data[i]) + if i < len(tag_order): + names.append(tag_order[i]) + return names + raise ValueError(f"unknown layout {layout!r}") + + +def make_source( + spec: Spec, + arrangement: Arrangement | None = None, + *, + source_id: str | None = None, + rows: Sequence[int] | None = None, +) -> ArrowTableSource: + """Build ``spec`` as an ``ArrowTableSource`` in ``arrangement``. + + Args: + spec: The columns and values. + arrangement: Tag order, layout and roles; default: declared order, tags + first, plain tags. + source_id: The source id; default ``spec.source_id``. + rows: The row positions to keep, in this order; default every row. + + Returns: + The source. A column is nullable exactly when it holds a null. + """ + arrangement = arrangement or Arrangement(spec.tags, TAGS_FIRST, PLAIN) + names = _layout(spec, arrangement.tag_order, arrangement.layout) + columns = {name: spec.columns[name] for name in names} + if rows is not None: + columns = {name: column.take(list(rows)) for name, column in columns.items()} + schema = pa.schema( + pa.field(name, column.type, nullable=column.null_count > 0) + for name, column in columns.items() + ) + table = pa.table(list(columns.values()), schema=schema) + if arrangement.roles == PLAIN: + tag_only: bool | list[str] = True + elif arrangement.roles == DATA_TAGS: + tag_only = [] + elif arrangement.roles == MIXED: + tag_only = [spec.mixed_plain] + else: + raise ValueError(f"unknown roles {arrangement.roles!r}") + return ArrowTableSource( + table, + tag_columns=list(arrangement.tag_order), + source_id=source_id or spec.source_id, + tag_only_columns=tag_only, + ) + + +def row_fingerprint(tag: TagProtocol, data: DataProtocol) -> tuple[str, ...]: + """What identifies one row regardless of column order: values, system tags, hashes.""" + return ( + repr(sorted(tag.as_dict().items())), + repr(sorted(data.as_dict().items())), + repr(sorted(tag.system_tags().items())), + tag.content_hash().to_string(), + data.content_hash().to_string(), + repr(sorted(data.source_info().items())), + ) + + +def rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: + """The rows' fingerprints, sorted: row order is not part of it.""" + return sorted(row_fingerprint(tag, data) for tag, data in rows) + + +def fingerprint(stream: StreamProtocol) -> dict[str, Any]: + """What must not depend on the input's arrangement or a set argument's order. + + Args: + stream: An operator's result. + + Returns: + The rows (``rows_fingerprint``), the content and pipeline hashes, and the + sorted column names of the tag group (system tags included) and the data group. + """ + tag_keys, data_keys = stream.keys(columns={"system_tags": True}) + return { + "rows": rows_fingerprint(list(stream.iter_data())), + "content_hash": stream.content_hash().to_string(), + "pipeline_hash": stream.pipeline_hash().to_string(), + "tag_names": sorted(tag_keys), + "data_names": sorted(data_keys), + } + + +def presentation(stream: StreamProtocol) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The column order of each group: follows the input, enters no hash.""" + tag_keys, data_keys = stream.keys() + return tuple(tag_keys), tuple(data_keys) + + +async def _run_async( + op: StaticOutputOperatorPod, sources: Sequence[StreamProtocol] +) -> list[Row]: + channels: list[Channel[Row]] = [Channel(buffer_size=64) for _ in sources] + output: Channel[Row] = Channel(buffer_size=64) + for stream, channel in zip(sources, channels): + for row in stream.iter_data(): + await channel.writer.send(row) + await channel.writer.close() + await op.async_execute( + [channel.reader for channel in channels], + output.writer, + input_pipeline_hashes=[s.pipeline_hash() for s in sources], + ) + return await output.reader.collect() + + +def async_rows(op: StaticOutputOperatorPod, *sources: StreamProtocol) -> list[Row]: + """The rows of the async path, ``op.async_execute`` over channels.""" + return asyncio.run(_run_async(op, sources)) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py new file mode 100644 index 000000000..48432323b --- /dev/null +++ b/tests/test_core/operators/test_several_items.py @@ -0,0 +1,610 @@ +"""Operators with several items: several columns, names, mappings and inputs (ITL-1236). + +Every argument that takes several items is tested here with two or more, in more than +one order, in each role (plain tag, data, data-tag), on inputs with three tags and three +data columns in several arrangements (``several_items``), on the sync and the async path. + +What is asserted: + +* **Arrangement.** An operator's result does not depend on the arrangement of its input: + the order of the tag columns, the physical layout of the table, for each role. The + rows (values, system tags, row hashes, source info), the stream's content and pipeline + hashes and the column names are the same (``fingerprint``); only the column order + follows the input. +* **Order of a set argument.** Column names to select, drop or narrow, and the entries + of a rename mapping, name a set: their order changes nothing, not even the column + order of the result, nor the operator's identity. +* **Order of the inputs of a commutative operator** (``Join``, ``MergeJoin``) changes + nothing, with two, three and four inputs. + +``GroupBy`` is not here: its arrangement and ``by`` order are tested by ITL-1226 +(``test_group_by_member_order.py``). The order of ``AddTags``'s names is the order of +the new tags and part of its identity, and is tested as such below. +""" + +from __future__ import annotations + +import itertools +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import polars as pl +import pyarrow as pa +import pytest + +from orcapod.core.operators import ( + AddTags, + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + Join, + MapColumns, + MapData, + MapTags, + MergeJoin, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, + TagBy, +) +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.core.sources import ArrowTableSource +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import StreamProtocol + +from .several_items import ( + DATA_TAGS, + MAIN, + MIXED, + P, + PLAIN, + Q, + R, + ROLES, + TAGS_FIRST, + Arrangement, + Spec, + arrangements, + async_rows, + fingerprint, + make_source, + presentation, + rows_fingerprint, +) + +# Selecting or dropping the data role of a data-tag warns that the tag is materialized; +# that warning is tested in the data-tag tests, not here. +pytestmark = pytest.mark.filterwarnings( + "ignore::orcapod.core.data_tags.DataTagMaterializationWarning" +) + + +@dataclass(frozen=True) +class UnaryCase: + """An operator applied to ``MAIN``, and the roles it supports.""" + + label: str + make_op: Callable[[], StaticOutputOperatorPod] + roles: tuple[str, ...] = ROLES + + +UNARY_CASES = [ + UnaryCase("select_tag_columns", lambda: SelectTagColumns(["t3", "t1"])), + UnaryCase("select_data_columns", lambda: SelectDataColumns(["d3", "d1"])), + UnaryCase("drop_tag_columns", lambda: DropTagColumns(["t3", "t1"])), + UnaryCase("drop_data_columns", lambda: DropDataColumns(["d3", "d1"])), + UnaryCase("select_columns", lambda: SelectColumns(["d3", "t2", "d1", "t1"])), + UnaryCase("drop_columns", lambda: DropColumns(["d3", "t2"])), + UnaryCase("map_tags", lambda: MapTags({"t3": "u3", "t1": "u1"})), + UnaryCase( + "map_tags_drop_unmapped", + lambda: MapTags({"t3": "u3", "t1": "u1"}, drop_unmapped=True), + ), + UnaryCase("map_tags_tag_only", lambda: MapTags({"t3": "u3"}, tag_only=True)), + UnaryCase("map_data", lambda: MapData({"d3": "e3", "d1": "e1"})), + UnaryCase( + "map_data_drop_unmapped", + lambda: MapData({"d3": "e3", "d1": "e1"}, drop_unmapped=True), + ), + # MapColumns is refused with data-tags (ITL-1022). + UnaryCase( + "map_columns", lambda: MapColumns({"d3": "e3", "t1": "u1"}), roles=(PLAIN,) + ), + UnaryCase( + "polars_filter_constraints", + lambda: PolarsFilter(constraints={"t1": "a", "t2": 1}), + ), + UnaryCase( + "polars_filter_predicates", + lambda: PolarsFilter(predicates=[pl.col("d1") > 10, pl.col("t3") == 1]), + ), + UnaryCase("drop_nulls_named", lambda: DropNulls(["d2", "t2"])), + UnaryCase("drop_nulls_all", DropNulls), + UnaryCase("batch", lambda: Batch(batch_size=4)), + UnaryCase("batch_all", Batch), + UnaryCase("add_tags", lambda: AddTags({"x": 1, "y": "z"})), + UnaryCase("tag_by", lambda: TagBy(["d3", "d1"])), +] + + +def _unary_params() -> list[object]: + return [ + pytest.param(case, role, id=f"{case.label}/{role}") + for case in UNARY_CASES + for role in case.roles + ] + + +def _assert_all_equal(results: dict[str, dict[str, object]]) -> None: + """Assert every arrangement's fingerprint equals the first one's, naming the field.""" + (first_id, first), *others = results.items() + for arrangement_id, result in others: + for field, value in first.items(): + assert result[field] == value, ( + f"{field} differs between {first_id} and {arrangement_id}" + ) + + +# ── arrangement: one input ────────────────────────────────────────────────── + + +@pytest.mark.parametrize(("case", "role"), _unary_params()) +def test_unary_result_does_not_depend_on_the_input_arrangement( + case: UnaryCase, role: str +) -> None: + results = { + arrangement.id: fingerprint(case.make_op()(make_source(MAIN, arrangement))) + for arrangement in arrangements(MAIN, roles=(role,)) + } + _assert_all_equal(results) + + +@pytest.mark.parametrize(("case", "role"), _unary_params()) +def test_unary_async_rows_equal_sync_rows_in_every_arrangement( + case: UnaryCase, role: str +) -> None: + for arrangement in arrangements(MAIN, roles=(role,)): + source = make_source(MAIN, arrangement) + sync = rows_fingerprint(list(case.make_op()(source).iter_data())) + assert rows_fingerprint(async_rows(case.make_op(), source)) == sync, ( + arrangement.id + ) + + +# ── order of a set argument ───────────────────────────────────────────────── + + +@dataclass(frozen=True) +class OrderCase: + """An operator built from each permutation of its names or mapping entries.""" + + label: str + items: tuple[Any, ...] + make_op: Callable[[Any], StaticOutputOperatorPod] + roles: tuple[str, ...] = ROLES + + +ORDER_CASES = [ + OrderCase("select_tag_columns", ("t1", "t2", "t3"), SelectTagColumns), + OrderCase("select_data_columns", ("d1", "d2", "d3"), SelectDataColumns), + OrderCase("drop_tag_columns", ("t1", "t3"), DropTagColumns), + OrderCase("drop_data_columns", ("d1", "d3"), DropDataColumns), + # Tag and data names in one call. + OrderCase("select_columns", ("t1", "d3", "t3", "d1"), SelectColumns), + OrderCase("drop_columns", ("t1", "d3", "d1"), DropColumns), + OrderCase("drop_nulls", ("d2", "t2", "d1"), DropNulls), + OrderCase("map_tags", (("t1", "u1"), ("t2", "u2"), ("t3", "u3")), MapTags), + # A swap: each target is another entry's source. + OrderCase("map_tags_swap", (("t1", "t3"), ("t3", "t1")), MapTags, roles=(PLAIN,)), + OrderCase("map_data", (("d1", "e1"), ("d2", "e2"), ("d3", "e3")), MapData), + OrderCase("map_data_swap", (("d1", "d3"), ("d3", "d1")), MapData), + OrderCase( + "map_data_drop_unmapped", + (("d1", "e1"), ("d3", "e3")), + lambda items: MapData(items, drop_unmapped=True), + ), + OrderCase( + "map_columns", + (("t1", "u1"), ("d2", "e2"), ("t3", "u3")), + MapColumns, + roles=(PLAIN,), + ), + OrderCase( + "polars_filter_constraints", + (("t1", "a"), ("t2", 1), ("t3", 1)), + lambda items: PolarsFilter(constraints=dict(items)), + ), +] + + +def _make(case: OrderCase, items: tuple[Any, ...]) -> StaticOutputOperatorPod: + """Build the operator from ``items``: a dict of pairs, or a list of names.""" + if isinstance(case.items[0], tuple): + return case.make_op(dict(items)) + return case.make_op(list(items)) + + +def _order_params() -> list[object]: + return [ + pytest.param(case, role, id=f"{case.label}/{role}") + for case in ORDER_CASES + for role in case.roles + ] + + +@pytest.mark.parametrize(("case", "role"), _order_params()) +def test_the_order_of_a_set_argument_changes_nothing( + case: OrderCase, role: str +) -> None: + """Every permutation: same rows, hashes, column names AND column order.""" + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + results: dict[str, dict[str, object]] = {} + for items in itertools.permutations(case.items): + op = _make(case, items) + stream = op(make_source(MAIN, arrangement)) + results[repr(items)] = { + **fingerprint(stream), + "presentation": presentation(stream), + "operator_identity": op.content_hash().to_string(), + "async_rows": rows_fingerprint( + async_rows(op, make_source(MAIN, arrangement)) + ), + } + _assert_all_equal(results) + + +def test_polars_filter_predicates_in_any_order_choose_the_same_rows() -> None: + """Two predicates in either order keep the same rows (identity: see ITL-1236 doc).""" + predicates = (pl.col("d1") > 10, pl.col("t3") == 1, pl.col("d3") != "y") + rows = { + repr(order): rows_fingerprint( + list(PolarsFilter(predicates=list(order))(make_source(MAIN)).iter_data()) + ) + for order in itertools.permutations(predicates) + } + assert len({repr(v) for v in rows.values()}) == 1 + + +@pytest.mark.parametrize("role", ROLES) +def test_add_tags_order_is_the_order_of_the_new_tags_and_its_identity( + role: str, +) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + xy = AddTags({"x": 1, "y": "z"})(source) + yx = AddTags({"y": "z", "x": 1})(source) + assert xy.keys()[0] == ("t1", "t2", "t3", "x", "y") + assert yx.keys()[0] == ("t1", "t2", "t3", "y", "x") + assert xy.content_hash() != yx.content_hash() + assert xy.pipeline_hash() != yx.pipeline_hash() + # Same rows: the values of each row do not depend on the order. + assert rows_fingerprint(list(xy.iter_data())) == rows_fingerprint( + list(yx.iter_data()) + ) + + +# ── roles: names of several roles in one call ─────────────────────────────── + + +@dataclass(frozen=True) +class RoleCase: + """An operator naming columns of several roles, on ``MIXED``, and its result keys. + + In ``MIXED`` ``t2`` is a plain tag and ``t1``, ``t3`` are data-tags. + """ + + label: str + make_op: Callable[[], StaticOutputOperatorPod] + keys: tuple[tuple[str, ...], tuple[str, ...]] + + +ROLE_CASES = [ + # Dropping a data-tag's tag role keeps it as data; a plain tag is removed. + RoleCase( + "drop_tag_columns", + lambda: DropTagColumns(["t3", "t2"]), + (("t1",), ("t1", "t3", "d1", "d2", "d3")), + ), + # Dropping a data-tag's data role leaves a plain tag. + RoleCase( + "drop_data_columns", + lambda: DropDataColumns(["t3", "d1"]), + (("t1", "t2", "t3"), ("t1", "d2", "d3")), + ), + RoleCase( + "select_tag_columns", + lambda: SelectTagColumns(["t3", "t2"]), + # t1 lost its tag role: plain data, after the data-tag t3 (ITL-1205). + (("t2", "t3"), ("t3", "t1", "d1", "d2", "d3")), + ), + RoleCase( + "select_data_columns", + lambda: SelectDataColumns(["d2", "t3"]), + (("t1", "t2", "t3"), ("t3", "d2")), + ), + # Both roles at once: a data-tag kept or removed whole, a plain tag, data. + RoleCase( + "select_columns", + lambda: SelectColumns(["d2", "t3", "t2"]), + (("t2", "t3"), ("t3", "d2")), + ), + RoleCase( + "drop_columns", + lambda: DropColumns(["t3", "t2", "d1"]), + (("t1",), ("t1", "d2", "d3")), + ), + # A data-tag renamed in both roles; a plain tag renamed. + RoleCase( + "map_tags", + lambda: MapTags({"t3": "u3", "t2": "u2"}), + (("t1", "u2", "u3"), ("t1", "u3", "d1", "d2", "d3")), + ), + # A data-only move of a data-tag: the tag keeps its name, plain. + RoleCase( + "map_data", + lambda: MapData({"t3": "e3", "d1": "e1"}), + (("t1", "t2", "t3"), ("t1", "e3", "e1", "d2", "d3")), + ), + RoleCase( + "drop_nulls", + lambda: DropNulls(["t3", "t2", "d2"]), + (("t1", "t2", "t3"), ("t1", "t3", "d1", "d2", "d3")), + ), + RoleCase( + "polars_filter", + lambda: PolarsFilter(constraints={"t3": 1, "t2": 2, "d3": "w"}), + (("t1", "t2", "t3"), ("t1", "t3", "d1", "d2", "d3")), + ), +] + + +@pytest.mark.parametrize("case", ROLE_CASES, ids=lambda case: case.label) +def test_names_of_several_roles_in_one_call(case: RoleCase) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) + result = case.make_op()(source) + assert result.keys() == case.keys + assert rows_fingerprint(async_rows(case.make_op(), source)) == rows_fingerprint( + list(result.iter_data()) + ) + + +def _sorted_values(stream: StreamProtocol, name: str, role: str = "data") -> list[Any]: + """The values of column ``name`` of every row, sorted.""" + index = 0 if role == "tag" else 1 + values: list[Any] = [row[index][name] for row in stream.iter_data()] + return sorted(values) + + +def test_drop_nulls_of_several_roles_drops_a_row_with_a_null_in_any() -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) + kept = DropNulls(["t3", "t2", "d2"])(source) + assert _sorted_values(kept, "d1") == [10, 30, 40, 50] + + +def test_polars_filter_constraints_of_several_roles_are_all_applied() -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) + kept = PolarsFilter(constraints={"t1": "a", "t2": 1, "t3": 1})(source) + assert [data["d1"] for _, data in kept.iter_data()] == [50] + + +@pytest.mark.xfail( + strict=True, + reason="ITL-1242: MapTags refuses a swap of data-tags that it accepts for plain tags", + raises=InputValidationError, +) +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_map_tags_swaps_data_tags_as_it_swaps_plain_tags(role: str) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + swapped = MapTags({"t1": "t3", "t3": "t1"})(source) + plain = MapTags({"t1": "t3", "t3": "t1"})( + make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, PLAIN)) + ) + assert swapped.keys()[0] == plain.keys()[0] + assert _sorted_values(swapped, "t1", "tag") == _sorted_values(plain, "t1", "tag") + + +# ── several inputs: Join, MergeJoin, SemiJoin ─────────────────────────────── + + +JOIN_INPUTS: dict[str, tuple[Spec, ...]] = { + "2": (MAIN, P), + "3": (MAIN, P, Q), + "4": (MAIN, P, Q, R), +} + + +def _join_params() -> list[object]: + return [ + pytest.param(n, role, id=f"join_{n}/{role}") + for n in JOIN_INPUTS + for role in ROLES + ] + + +def _sources( + specs: tuple[Spec, ...], + order_index: int, + layout: str, + role: str, + rows: list[int] | None = None, +) -> list[ArrowTableSource]: + """Each spec in its ``order_index``-th tag order, in ``layout`` and ``role``.""" + return [ + make_source( + spec, + Arrangement( + spec.tag_orders()[order_index % len(spec.tag_orders())], layout, role + ), + rows=rows, + ) + for spec in specs + ] + + +@pytest.mark.parametrize(("n", "role"), _join_params()) +def test_join_result_does_not_depend_on_the_order_of_its_inputs( + n: str, role: str +) -> None: + """Every permutation of the inputs: same rows, hashes, system-tag names, order.""" + specs = JOIN_INPUTS[n] + results: dict[str, dict[str, object]] = {} + for permutation in itertools.permutations(range(len(specs))): + sources = _sources(specs, 0, TAGS_FIRST, role) + ordered = [sources[i] for i in permutation] + stream = Join()(*ordered) + results[repr(permutation)] = { + **fingerprint(stream), + "presentation": presentation(stream), + "system_tag_order": stream.keys(columns={"system_tags": True})[0], + "async_rows": rows_fingerprint(async_rows(Join(), *ordered)), + } + _assert_all_equal(results) + + +@pytest.mark.parametrize(("n", "role"), _join_params()) +def test_join_result_does_not_depend_on_the_arrangement_of_its_inputs( + n: str, role: str +) -> None: + specs = JOIN_INPUTS[n] + results: dict[str, dict[str, object]] = {} + for order_index in range(3): + for layout in ("tags_first", "data_first_reversed", "interleaved"): + sources = _sources(specs, order_index, layout, role) + stream = Join()(*sources) + results[f"{order_index}/{layout}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(Join(), *sources)), + } + _assert_all_equal(results) + + +def test_join_of_four_inputs_matches_on_every_shared_tag() -> None: + sources = _sources(JOIN_INPUTS["4"], 0, TAGS_FIRST, PLAIN) + joined = Join()(*sources) + rows = sorted( + ( + tag["t1"], + tag["t2"], + tag["t3"], + data["d1"], + data["p1"], + data["q2"], + data["r1"], + ) + for tag, data in joined.iter_data() + ) + # MAIN x P on (t1, t2), x Q on (t2, t3), x R on t1: every MAIN row has one match. + assert rows == [ + ("a", 1, 1, 50, 9, 400, -2), + ("a", 1, 2, 20, 9, 100, -2), + ("a", 2, 1, 30, 7, 200, -2), + ("b", 1, 2, 40, 8, 100, -1), + ("b", 2, 1, 10, 6, 200, -1), + ("b", 2, 2, 60, 6, 300, -1), + ] + + +# MergeJoin: two inputs with the same columns; every data column collides. The rows +# of ``MAIN`` without a null (``NON_NULL_ROWS``), and other values in every data column, +# so no merged pair ties: a tie is ITL-1241, a null ``MERGE_NULL_ISSUE``. +NON_NULL_ROWS = [0, 2, 3, 4] +MAIN_OTHER = Spec( + source_id="s2", + tags=MAIN.tags, + data=MAIN.data, + columns={ + **{tag: MAIN.columns[tag] for tag in MAIN.tags}, + "d1": pa.array([60, 55, 45, 35, 25, 15], pa.int64()), + "d2": pa.array([9.5, None, 7.5, 6.5, 5.5, 1.5], pa.float64()), + "d3": pa.array(["a", "b", "c", "d", "e", "f"], pa.large_string()), + }, + mixed_plain=MAIN.mixed_plain, +) + + +@pytest.mark.parametrize("role", ROLES) +def test_merge_join_result_does_not_depend_on_the_order_of_its_inputs( + role: str, +) -> None: + """Three colliding data columns: either input order, any tag order. + + Swapping the inputs changes nothing, not even the column order. Another tag order + of the inputs changes only the column order. + """ + results: dict[str, dict[str, object]] = {} + for order_index in range(3): + left, right = _sources( + (MAIN, MAIN_OTHER), order_index, TAGS_FIRST, role, rows=NON_NULL_ROWS + ) + ab, ba = MergeJoin()(left, right), MergeJoin()(right, left) + assert presentation(ab) == presentation(ba) + for label, inputs, stream in ( + ("ab", (left, right), ab), + ("ba", (right, left), ba), + ): + results[f"{order_index}/{label}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), + } + _assert_all_equal(results) + + +@pytest.mark.xfail( + strict=True, + reason="ITL-1241: MergeJoin orders tied values' source info by input position", +) +@pytest.mark.parametrize("role", ROLES) +def test_merge_join_of_tied_values_does_not_depend_on_the_order_of_its_inputs( + role: str, +) -> None: + """Two inputs with equal values in three colliding columns (ITL-1241).""" + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + left = make_source(MAIN, arrangement, source_id="s", rows=NON_NULL_ROWS) + right = make_source(MAIN, arrangement, source_id="s2", rows=NON_NULL_ROWS) + assert fingerprint(MergeJoin()(left, right)) == fingerprint( + MergeJoin()(right, left) + ) + + +MERGE_NULL_ISSUE = "ITL-1236 defect 3, issue pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{MERGE_NULL_ISSUE}: MergeJoin raises TypeError on a null in a merged column", + raises=TypeError, +) +@pytest.mark.parametrize("role", ROLES) +def test_merge_join_of_a_nullable_column_holding_nulls(role: str) -> None: + """``d2`` holds a null in rows 1 and 5 of ``MAIN``; the merge must not crash.""" + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + left = make_source(MAIN, arrangement) + right = make_source(MAIN_OTHER, arrangement) + assert fingerprint(MergeJoin()(left, right)) == fingerprint( + MergeJoin()(right, left) + ) + + +@pytest.mark.parametrize("role", ROLES) +def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> None: + """The right input's tag order and layout never change which rows are kept.""" + results: dict[str, dict[str, object]] = {} + for order_index in range(3): + for layout in ("tags_first", "data_first_reversed", "interleaved"): + left, right = _sources((MAIN, P), order_index, layout, role) + stream = SemiJoin()(left, right) + results[f"{order_index}/{layout}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(SemiJoin(), left, right)), + } + _assert_all_equal(results) + left, right = _sources((MAIN, P), 0, TAGS_FIRST, role) + kept = SemiJoin()( + left, make_source(P, Arrangement(P.tags, TAGS_FIRST, role), rows=[0, 1]) + ) + # P's first two rows are (a, 2) and (b, 1): matched on t1 AND t2. + assert _sorted_values(kept, "d1") == [30, 40] From 03f4a26ac4c14afd3aec34d0557ea05852223bd4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:17:56 -0800 Subject: [PATCH 02/10] test(sources, function_pod, pipeline): several tags, keys, inputs and bound sources, in every order and role (ITL-1236) Sources: tag order and layout enter no hash; tag_only_columns order changes nothing; record ids follow the key order (content hash moves, pipeline hash and source id do not); tags_as_data order is the new columns' order only; CachedSource reads back every role and arrangement. Function pods: every order of three inputs gives the same rows, hashes and columns, and a job over another order finds every result stored, sync and async; parameters match by name; a function reads several data-tags in any arrangement. Bind: two sources in one call, one refused leaves both unbound; a source in another arrangement binds without moving anything. Defects found are xfail(strict=True) with the issue pending: the fall-through tags' order in a function pod's identity (defect 6), swapped output_keys keep the uri (defect 4), a mixed-role source in another order breaks the bound node's table (defect 5). Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 21 +- .../function_pod/test_several_items.py | 215 ++++++++++++++++++ tests/test_core/operators/several_items.py | 29 ++- tests/test_core/sources/test_several_items.py | 169 ++++++++++++++ .../test_pipeline/test_bind_several_items.py | 152 +++++++++++++ 5 files changed, 578 insertions(+), 8 deletions(-) create mode 100644 tests/test_core/function_pod/test_several_items.py create mode 100644 tests/test_core/sources/test_several_items.py create mode 100644 tests/test_pipeline/test_bind_several_items.py diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index ffd91122e..ae870753b 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -99,6 +99,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | pending | | 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | pending (decision) | | 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | pending | +| 6 | A function pod over data-tags (`FallThroughPod`) hashes its fall-through tags in the input's tag order: two sources that differ only in tag order give the function node another content and pipeline hash | pipeline hash, pipeline table path | pending | | Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | question | ## Gaps, ranked @@ -121,9 +122,27 @@ Order- or role-dependent behaviour that reaches a hash or a stored table first. 14. `DerivedSource(tags_as_data=)` reversed: default `source_id` and pipeline hash. 15. `CachedSource` over a data-tag source; `Pick` / `Index` from a data-tag with `out=`; schema arguments in another field order; the stream methods' several-argument forms (`polars_filter` with `constraint_map` and keywords). +## Checked and found as documented (part 2) + +- A source's `tag_columns` order and table layout: keys follow the tag order; rows, record + ids, system tags, content and pipeline hashes do not change (plain, data-tags, mixed). +- `tag_only_columns` given in two orders: no change. +- `record_id_columns` in another key order: record ids follow the key order; the content + hash changes, the pipeline hash and the source id do not; a set is refused. +- `DerivedSource(tags_as_data=)` reversed: only the new columns' order changes. +- `CachedSource` over a source of each role and arrangement: reads back as the source. +- `pod(a, b, c)` in every input order: same rows, hashes and columns; a job over the inputs + in another order finds every result stored (sync and async). The in-memory output's + source info names a fresh random record per computation, so comparisons leave it out. +- `bind(sources=)` with two sources: both bound, nothing moves; one refused, neither bound. + A plain or data-tag source in any arrangement binds and reads; a mixed one is defect 5. + ## Tests added `tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`, `t2`, `t3`; data `d1`, `d2`, `d3`) and join partners `P`, `Q`, `R`, each in any tag order, three layouts and three role settings. `ops/test_several_items.py` uses it for every -operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). +operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). Part 2: +`tests/test_core/sources/test_several_items.py` (gaps 4, 13, 14, 15: CachedSource), +`tests/test_core/function_pod/test_several_items.py` (gaps 1, 10; defect 6) and +`tests/test_pipeline/test_bind_several_items.py` (gap 8; defect 5). diff --git a/tests/test_core/function_pod/test_several_items.py b/tests/test_core/function_pod/test_several_items.py new file mode 100644 index 000000000..24ed87f0b --- /dev/null +++ b/tests/test_core/function_pod/test_several_items.py @@ -0,0 +1,215 @@ +"""Function pods over several inputs, reading several parameters, with several outputs (ITL-1236). + +The rules tested: + +* ``pod(a, b, c)`` joins its inputs as ``Join`` does, so the order of the inputs changes + nothing: rows, hashes, system-tag names and column order, in memory and in a job (a + second job over the inputs in another order finds every result stored). +* Data columns that the function does not read are dropped by the automatic select + (ITL-807), whatever their number and position. +* The function's parameters are matched by name, never by position. +* A data-tag is a data column a function may read (ITL-1023). +""" + +from __future__ import annotations + +import itertools + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +from ..operators.several_items import ( + DATA_TAGS, + LAYOUTS, + MAIN, + MIXED, + ROLES, + TAGS_FIRST, + Arrangement, + P, + Q, + Spec, + fingerprint, + make_source, + presentation, + rows_fingerprint, +) + +INPUTS: tuple[Spec, ...] = (MAIN, P, Q) + + +# Parameters in another order than the columns: q2 is in Q, d1 in MAIN, p1 in P. +@function_pod(output_keys=["total", "product"]) +def combine(q2: int, d1: int, p1: int) -> tuple[int, int]: + return q2 + d1 + p1, d1 * p1 + + +def _inputs(role: str) -> list[ArrowTableSource]: + return [ + make_source(spec, Arrangement(spec.tags, TAGS_FIRST, role)) for spec in INPUTS + ] + + +@pytest.mark.parametrize("role", ROLES) +def test_the_order_of_the_inputs_changes_nothing(role: str) -> None: + """All six orders of three inputs: same rows, hashes and columns. + + Each in-memory computation names a fresh record in its outputs' source info, so the + rows are compared without it. + """ + results = {} + for permutation in itertools.permutations(range(len(INPUTS))): + inputs = _inputs(role) + stream = combine.pod(*(inputs[i] for i in permutation)) + results[permutation] = { + **fingerprint(stream, source_info=False), + "presentation": presentation(stream), + } + first = results[(0, 1, 2)] + for permutation, result in results.items(): + assert result == first, permutation + # The extra data columns (d2, d3, p2, q1) were dropped; every tag stays. + tag_keys, data_keys = first["presentation"] + assert sorted(tag_keys) == ["t1", "t2", "t3"] + assert {"d2", "d3", "p2", "q1"}.isdisjoint(data_keys) + assert {"total", "product"} <= set(data_keys) + + +@pytest.mark.parametrize("role", ROLES) +def test_values_are_matched_by_parameter_name(role: str) -> None: + stream = combine.pod(*_inputs(role)) + rows = sorted( + (tag["t1"], tag["t2"], tag["t3"], data["total"], data["product"]) + for tag, data in stream.iter_data() + ) + # MAIN x P on (t1, t2) x Q on (t2, t3); total = q2 + d1 + p1, product = d1 * p1. + assert rows == [ + ("a", 1, 1, 400 + 50 + 9, 50 * 9), + ("a", 1, 2, 100 + 20 + 9, 20 * 9), + ("a", 2, 1, 200 + 30 + 7, 30 * 7), + ("b", 1, 2, 100 + 40 + 8, 40 * 8), + ("b", 2, 1, 200 + 10 + 6, 10 * 6), + ("b", 2, 2, 300 + 60 + 6, 60 * 6), + ] + + +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +@pytest.mark.parametrize("role", ROLES) +def test_a_job_over_the_inputs_in_another_order_finds_every_result_stored( + role: str, orchestrator: type +) -> None: + database = InMemoryArrowDatabase() + stored = {} + for permutation in ((0, 1, 2), (2, 0, 1), (1, 2, 0)): + inputs = _inputs(role) + job = PipelineJob(name="several", database=database) + with job: + combine.pod(*(inputs[i] for i in permutation), label="combine") + summary = job.run(orchestrator=orchestrator()) + node = job.nodes["combine"] + stored[permutation] = ( + rows_fingerprint(list(node.iter_data()), source_info=False), + node.pipeline_hash(), + node.content_hash(), + presentation(node), + ) + if permutation != (0, 1, 2): + assert summary.nodes["combine"].computed == 0, permutation + assert summary.nodes["combine"].reused == 6, permutation + first = stored[(0, 1, 2)] + assert all(result == first for result in stored.values()) + + +@function_pod(output_keys="labelled") +def label_rows(t1: str, d3: str, t3: int) -> str: + return f"{t1}{t3}{d3}" + + +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_a_function_reads_several_data_tags_and_data_in_any_arrangement( + role: str, +) -> None: + """t1 and t3 are data-tags; the tag order and layout change no row.""" + results = {} + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + stream = label_rows.pod(make_source(MAIN, Arrangement(order, layout, role))) + results[(order, layout)] = rows_fingerprint( + list(stream.iter_data()), source_info=False + ) + values = sorted(str(data["labelled"]) for _, data in stream.iter_data()) + assert values == ["a1v", "a1y", "a2x", "b1w", "b2u", "b2z"] + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_a_layout_of_the_input_does_not_change_a_fall_through_pods_hashes( + role: str, +) -> None: + hashes = { + layout: fingerprint( + label_rows.pod(make_source(MAIN, Arrangement(MAIN.tags, layout, role))), + source_info=False, + ) + for layout in LAYOUTS + } + first = next(iter(hashes.values())) + assert all(result == first for result in hashes.values()) + + +FALL_THROUGH_ORDER_ISSUE = "ITL-1236 defect 6, issue pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{FALL_THROUGH_ORDER_ISSUE}: the fall-through tags enter the pod's " + + "identity in the input's tag order", +) +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_the_tag_order_of_the_input_does_not_change_a_fall_through_pods_hashes( + role: str, +) -> None: + results = { + order: fingerprint( + label_rows.pod(make_source(MAIN, Arrangement(order, TAGS_FIRST, role))), + source_info=False, + ) + for order in MAIN.tag_orders() + } + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +OUTPUT_KEYS_ISSUE = "ITL-1236 defect 4, decision pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{OUTPUT_KEYS_ISSUE}: the order of output_keys is not in the identity", +) +def test_swapped_output_keys_are_another_function() -> None: + """Swapped output keys map the results to other names: not the same function. + + Today both have one uri, so a job serves one's stored values under the other's + names. + """ + + def make(keys: list[str]): + @function_pod(output_keys=keys) + def stats(d1: int, t2: int) -> tuple[int, int]: + return d1 + t2, d1 * t2 + + return stats + + forward, backward = make(["sum", "product"]), make(["product", "sum"]) + assert forward.pod.uri != backward.pod.uri diff --git a/tests/test_core/operators/several_items.py b/tests/test_core/operators/several_items.py index 49642b438..c405ab0c8 100644 --- a/tests/test_core/operators/several_items.py +++ b/tests/test_core/operators/several_items.py @@ -218,28 +218,43 @@ def make_source( ) -def row_fingerprint(tag: TagProtocol, data: DataProtocol) -> tuple[str, ...]: - """What identifies one row regardless of column order: values, system tags, hashes.""" +def row_fingerprint( + tag: TagProtocol, data: DataProtocol, *, source_info: bool = True +) -> tuple[str, ...]: + """What identifies one row regardless of column order: values, system tags, hashes. + + Args: + tag: The row's tag. + data: The row's data. + source_info: Include the data's source info. A function's in-memory output + names a fresh random record per computation, so a comparison of two + computations leaves it out. + """ return ( repr(sorted(tag.as_dict().items())), repr(sorted(data.as_dict().items())), repr(sorted(tag.system_tags().items())), tag.content_hash().to_string(), data.content_hash().to_string(), - repr(sorted(data.source_info().items())), + repr(sorted(data.source_info().items())) if source_info else "", ) -def rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: +def rows_fingerprint( + rows: Sequence[Row], *, source_info: bool = True +) -> list[tuple[str, ...]]: """The rows' fingerprints, sorted: row order is not part of it.""" - return sorted(row_fingerprint(tag, data) for tag, data in rows) + return sorted( + row_fingerprint(tag, data, source_info=source_info) for tag, data in rows + ) -def fingerprint(stream: StreamProtocol) -> dict[str, Any]: +def fingerprint(stream: StreamProtocol, *, source_info: bool = True) -> dict[str, Any]: """What must not depend on the input's arrangement or a set argument's order. Args: stream: An operator's result. + source_info: Include each row's source info (see ``row_fingerprint``). Returns: The rows (``rows_fingerprint``), the content and pipeline hashes, and the @@ -247,7 +262,7 @@ def fingerprint(stream: StreamProtocol) -> dict[str, Any]: """ tag_keys, data_keys = stream.keys(columns={"system_tags": True}) return { - "rows": rows_fingerprint(list(stream.iter_data())), + "rows": rows_fingerprint(list(stream.iter_data()), source_info=source_info), "content_hash": stream.content_hash().to_string(), "pipeline_hash": stream.pipeline_hash().to_string(), "tag_names": sorted(tag_keys), diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py new file mode 100644 index 000000000..ecfc017f7 --- /dev/null +++ b/tests/test_core/sources/test_several_items.py @@ -0,0 +1,169 @@ +"""Sources with several tag, key and data columns, in several orders and roles (ITL-1236). + +The rules tested (CLAUDE.md, "Important implementation details" and "Two identity +chains"): + +* The order of ``tag_columns`` is the order of the tags (and of the data-tags among the + data columns); it enters no hash, no record id and no system-tag column name. +* ``tag_only_columns`` names a set: its order changes nothing. +* ``record_id_columns`` is in key order: a record id is the canonical JSON of the key + columns in key order, so another key order is another source with the same pipeline + hash and source id, another content hash. An unordered collection is refused. +* ``DerivedSource(tags_as_data=...)``: the entries' order is the order of the new data + columns and nothing else. +* ``CachedSource`` reads back a source of each role as the source itself. +""" + +from __future__ import annotations + +import itertools +import json + +import pyarrow as pa +import pytest + +from orcapod.core.sources import ArrowTableSource, CachedSource, DerivedSource +from orcapod.databases import InMemoryArrowDatabase + +from ..operators.several_items import ( + DATA_TAGS, + LAYOUTS, + MAIN, + MIXED, + PLAIN, + ROLES, + TAGS_FIRST, + Arrangement, + fingerprint, + make_source, + presentation, +) + + +def _tag_only(role: str) -> bool | list[str]: + return {PLAIN: True, DATA_TAGS: [], MIXED: [MAIN.mixed_plain]}[role] + + +def _table() -> pa.Table: + return pa.table({name: MAIN.columns[name] for name in (*MAIN.tags, *MAIN.data)}) + + +def _record_ids(source: ArrowTableSource) -> list[str]: + ids: list[str] = [] + for tag, _ in source.iter_data(): + [record_id] = [ + value for name, value in tag.system_tags().items() if "record_id" in name + ] + ids.append(str(record_id)) + return sorted(ids) + + +@pytest.mark.parametrize("role", ROLES) +def test_tag_order_is_the_order_of_the_tags_and_enters_no_hash(role: str) -> None: + """Every tag order and layout: same rows, hashes and system tags; keys follow.""" + results = {} + for order in itertools.permutations(MAIN.tags): + for layout in LAYOUTS: + source = make_source(MAIN, Arrangement(order, layout, role)) + tag_keys, data_keys = presentation(source) + assert tag_keys == order + # Data-tags come first among the data columns, in tag order (ITL-1205). + data_tags = [t for t in order if role != PLAIN and t != _plain_tag(role)] + assert list(data_keys[: len(data_tags)]) == data_tags + results[(order, layout)] = fingerprint(source) + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +def _plain_tag(role: str) -> str | None: + return MAIN.mixed_plain if role == MIXED else None + + +def test_the_order_of_tag_only_columns_changes_nothing() -> None: + sources = [ + ArrowTableSource( + _table(), tag_columns=list(MAIN.tags), source_id="s", tag_only_columns=names + ) + for names in (["t1", "t3"], ["t3", "t1"]) + ] + assert presentation(sources[0]) == presentation(sources[1]) + assert presentation(sources[0]) == (("t1", "t2", "t3"), ("t2", "d1", "d2", "d3")) + assert fingerprint(sources[0]) == fingerprint(sources[1]) + + +@pytest.mark.parametrize("role", ROLES) +def test_record_ids_follow_the_key_order(role: str) -> None: + """Two or three key columns, of the tag and the data role, in several orders.""" + by_keys = {} + for keys in (("t1", "t2", "t3"), ("t3", "t1", "t2"), ("d3", "t1"), ("t1", "d3")): + source = ArrowTableSource( + _table(), + tag_columns=list(MAIN.tags), + source_id="s", + record_id_columns=list(keys), + tag_only_columns=_tag_only(role), + ) + ids = _record_ids(source) + # The canonical JSON of the key columns, in key order. + assert all(list(json.loads(record_id)) == list(keys) for record_id in ids) + by_keys[keys] = source + first = by_keys[("t1", "t2", "t3")] + for keys, source in by_keys.items(): + # The keys name the rows: one pipeline hash and one source id ... + assert source.pipeline_hash() == first.pipeline_hash() + assert source.source_id == first.source_id + # ... and another key order is another source. + if keys != ("t1", "t2", "t3"): + assert source.content_hash() != first.content_hash() + assert _record_ids(source) != _record_ids(first) + # Same keys, same order: the same source. + again = ArrowTableSource( + _table(), + tag_columns=list(MAIN.tags), + source_id="s", + record_id_columns=("t3", "t1", "t2"), + tag_only_columns=_tag_only(role), + ) + assert again.content_hash() == by_keys[("t3", "t1", "t2")].content_hash() + + +def test_record_id_columns_as_a_set_is_refused() -> None: + with pytest.raises(TypeError): + ArrowTableSource( + _table(), + tag_columns=list(MAIN.tags), + source_id="s", + record_id_columns={"t1", "t2"}, + ) + + +@pytest.mark.parametrize("role", ROLES) +def test_tags_as_data_order_is_the_order_of_the_new_columns_only(role: str) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + forward = DerivedSource(source, tags_as_data={"t1": "c1", "t3": "c3"}) + backward = DerivedSource(source, tags_as_data={"t3": "c3", "t1": "c1"}) + assert presentation(forward)[1][-2:] == ("c1", "c3") + assert presentation(backward)[1][-2:] == ("c3", "c1") + assert forward.source_id == backward.source_id + assert fingerprint(forward) == fingerprint(backward) + # Each new column holds its own tag's values. + for tag, data in forward.iter_data(): + assert (data["c1"], data["c3"]) == (tag["t1"], tag["t3"]) + + +@pytest.mark.parametrize("role", ROLES) +def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> None: + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + arrangement = Arrangement(order, layout, role) + database = InMemoryArrowDatabase() + cached = CachedSource( + make_source(MAIN, arrangement), cache_database=database + ) + assert fingerprint(cached) == fingerprint(make_source(MAIN, arrangement)) + # A second CachedSource reads the stored rows back. + again = CachedSource( + make_source(MAIN, arrangement), cache_database=database + ) + assert presentation(again) == presentation(make_source(MAIN, arrangement)) + assert fingerprint(again) == fingerprint(cached) diff --git a/tests/test_pipeline/test_bind_several_items.py b/tests/test_pipeline/test_bind_several_items.py new file mode 100644 index 000000000..3ddfc28c0 --- /dev/null +++ b/tests/test_pipeline/test_bind_several_items.py @@ -0,0 +1,152 @@ +"""Binding several sources, and sources in another column arrangement (ITL-1236). + +``PipelineJob.bind(sources=...)`` takes several sources in one call. Every source is +validated before anything changes, so one refused source leaves every binding as it was +(the ``bind`` docstring). A source whose columns are the slot's in another order (tag +order, table layout) is accepted, and the bind changes no node's hash or columns +(CLAUDE.md, "Important implementation details", ITL-1112). +""" + +from __future__ import annotations + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +from ..test_core.operators.several_items import ( + DATA_TAGS, + LAYOUTS, + MAIN, + MIXED, + PLAIN, + TAGS_FIRST, + Arrangement, + P, + make_source, + presentation, +) + + +@function_pod(output_keys="total") +def total(d1: int, p1: int) -> int: + return d1 + p1 + + +def _job(role: str) -> PipelineJob: + """A job over slots ``s`` (``MAIN``) and ``p`` (``P``), joined by ``total``.""" + job = PipelineJob(name="bind", database=InMemoryArrowDatabase()) + with job: + total.pod( + make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)), + make_source(P, Arrangement(P.tags, TAGS_FIRST, role)), + label="total", + ) + return job + + +def _slot_names(job: PipelineJob) -> tuple[str, str]: + return ("s", "p") if {"s", "p"} <= set(job.nodes) else ("", "") + + +def _snapshot(job: PipelineJob) -> dict[str, object]: + return { + name: (node.pipeline_hash(), presentation(node)) + for name, node in job.nodes.items() + } + + +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +@pytest.mark.parametrize("role", [PLAIN, DATA_TAGS, MIXED]) +def test_several_sources_bound_in_one_call(role: str, orchestrator: type) -> None: + job = _job(role) + assert _slot_names(job) == ("s", "p") + before = _snapshot(job) + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + job.bind( + sources={ + "s": make_source(MAIN, arrangement, source_id="s_new", rows=[0, 1]), + "p": make_source( + P, Arrangement(P.tags, TAGS_FIRST, role), source_id="p_new" + ), + } + ) + assert _snapshot(job) == before + job.run(orchestrator=orchestrator()) + rows = sorted( + (tag["t1"], tag["t2"], data["total"]) + for tag, data in job.nodes["total"].iter_data() + ) + # Rows 0 and 1 of MAIN: (b, 2, d1=10) with p1=6; (a, 1, d1=20) with p1=9. + assert rows == [("a", 1, 29), ("b", 2, 16)] + + +@pytest.mark.parametrize("refused", ["s", "p"]) +def test_one_refused_source_leaves_every_binding_as_it_was(refused: str) -> None: + """Two sources in one call, one of them refused: neither is bound.""" + job = _job(PLAIN) + bound_before = {name: job.nodes[name].bound_source for name in ("s", "p")} + sources: dict[str, StreamProtocol] = { + "s": make_source(MAIN, source_id="s_new"), + "p": make_source(P, source_id="p_new"), + } + # The other slot's source does not fit: other column names. + sources[refused] = make_source(P if refused == "s" else MAIN, source_id="wrong") + with pytest.raises(SourceSpecMismatchError): + job.bind(sources=sources) + assert {name: job.nodes[name].bound_source for name in ("s", "p")} == bound_before + + +def _bind_in(role: str, order: tuple[str, ...], layout: str) -> None: + """Bind ``MAIN`` in another arrangement to slot ``s``: nothing moves, all reads.""" + job = _job(role) + before = _snapshot(job) + # The columns of the table, not their order: CLAUDE.md promises the declared names, + # roles and types; the column order within the table is a convenience only. + table_before = sorted(job.nodes["s"].as_table().column_names) + job.bind( + sources={ + "s": make_source(MAIN, Arrangement(order, layout, role), source_id="s_new") + } + ) + assert _snapshot(job) == before, (order, layout) + assert sorted(job.nodes["s"].as_table().column_names) == table_before, ( + order, + layout, + ) + job.run() + assert len(job.nodes["total"].as_table()) == 6 + + +@pytest.mark.parametrize("role", [PLAIN, DATA_TAGS]) +def test_a_source_in_another_arrangement_is_bound_as_the_slot_declares( + role: str, +) -> None: + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + _bind_in(role, order, layout) + + +BIND_MIXED_ISSUE = "ITL-1236 defect 5, issue pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{BIND_MIXED_ISSUE}: a mixed-role source in another column order is " + + "accepted, then the source node's table repeats a column", + raises=KeyError, +) +def test_a_mixed_role_source_in_another_arrangement_is_bound_as_the_slot_declares() -> ( + None +): + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + _bind_in(MIXED, order, layout) From e8ae6217c5a4b37ca1430e47ee545a37a0e94400 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:19:08 -0800 Subject: [PATCH 03/10] test(operators): async rows equal sync rows in every several-input case; MergeJoin and the restricted SemiJoin in every layout (ITL-1236) Addresses the Copilot review of #442. Co-Authored-By: Claude Opus 5.5 --- .../test_core/operators/test_several_items.py | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index 48432323b..250346a0a 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -59,6 +59,7 @@ from .several_items import ( DATA_TAGS, + LAYOUTS, MAIN, MIXED, P, @@ -141,7 +142,17 @@ def _unary_params() -> list[object]: def _assert_all_equal(results: dict[str, dict[str, object]]) -> None: - """Assert every arrangement's fingerprint equals the first one's, naming the field.""" + """Assert every arrangement's fingerprint equals the first one's, naming the field. + + A result that holds ``async_rows`` must also have them equal to its own sync + ``rows``: equal across arrangements alone would pass an async path that is wrong + the same way every time. + """ + for result_id, result in results.items(): + if "async_rows" in result: + assert result["async_rows"] == result["rows"], ( + f"async rows differ from sync rows in {result_id}" + ) (first_id, first), *others = results.items() for arrangement_id, result in others: for field, value in first.items(): @@ -472,7 +483,7 @@ def test_join_result_does_not_depend_on_the_arrangement_of_its_inputs( specs = JOIN_INPUTS[n] results: dict[str, dict[str, object]] = {} for order_index in range(3): - for layout in ("tags_first", "data_first_reversed", "interleaved"): + for layout in LAYOUTS: sources = _sources(specs, order_index, layout, role) stream = Join()(*sources) results[f"{order_index}/{layout}"] = { @@ -530,26 +541,27 @@ def test_join_of_four_inputs_matches_on_every_shared_tag() -> None: def test_merge_join_result_does_not_depend_on_the_order_of_its_inputs( role: str, ) -> None: - """Three colliding data columns: either input order, any tag order. + """Three colliding data columns: either input order, any tag order and layout. Swapping the inputs changes nothing, not even the column order. Another tag order - of the inputs changes only the column order. + or layout of the inputs changes only the column order. """ results: dict[str, dict[str, object]] = {} for order_index in range(3): - left, right = _sources( - (MAIN, MAIN_OTHER), order_index, TAGS_FIRST, role, rows=NON_NULL_ROWS - ) - ab, ba = MergeJoin()(left, right), MergeJoin()(right, left) - assert presentation(ab) == presentation(ba) - for label, inputs, stream in ( - ("ab", (left, right), ab), - ("ba", (right, left), ba), - ): - results[f"{order_index}/{label}"] = { - **fingerprint(stream), - "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), - } + for layout in LAYOUTS: + left, right = _sources( + (MAIN, MAIN_OTHER), order_index, layout, role, rows=NON_NULL_ROWS + ) + ab, ba = MergeJoin()(left, right), MergeJoin()(right, left) + assert presentation(ab) == presentation(ba) + for label, inputs, stream in ( + ("ab", (left, right), ab), + ("ba", (right, left), ba), + ): + results[f"{order_index}/{layout}/{label}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), + } _assert_all_equal(results) @@ -594,7 +606,7 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N """The right input's tag order and layout never change which rows are kept.""" results: dict[str, dict[str, object]] = {} for order_index in range(3): - for layout in ("tags_first", "data_first_reversed", "interleaved"): + for layout in LAYOUTS: left, right = _sources((MAIN, P), order_index, layout, role) stream = SemiJoin()(left, right) results[f"{order_index}/{layout}"] = { @@ -602,9 +614,14 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N "async_rows": rows_fingerprint(async_rows(SemiJoin(), left, right)), } _assert_all_equal(results) - left, right = _sources((MAIN, P), 0, TAGS_FIRST, role) - kept = SemiJoin()( - left, make_source(P, Arrangement(P.tags, TAGS_FIRST, role), rows=[0, 1]) - ) - # P's first two rows are (a, 2) and (b, 1): matched on t1 AND t2. - assert _sorted_values(kept, "d1") == [30, 40] + # A right input without every (t1, t2) pair tells a match on both keys from a + # match on one, or none: P's first two rows are (a, 2) and (b, 1). + for order_index in range(3): + for layout in LAYOUTS: + left, _ = _sources((MAIN, P), order_index, layout, role) + [right] = _sources((P,), order_index, layout, role, rows=[0, 1]) + kept = SemiJoin()(left, right) + assert _sorted_values(kept, "d1") == [30, 40] + assert rows_fingerprint(async_rows(SemiJoin(), left, right)) == ( + rows_fingerprint(list(kept.iter_data())) + ) From 8129cfa1e3624642a90c7c5db75bacae0d6ba473 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:20:27 -0800 Subject: [PATCH 04/10] test(operators): issue ids for the defects found (ITL-1243, ITL-1247); predicates in every order and role, sync and async (ITL-1236) Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 22 ++++----- .../test_core/operators/test_several_items.py | 49 ++++++++++++++----- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index ffd91122e..aa67452d9 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -50,7 +50,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | `Join` inputs, 4+ | async only: `tests/test_channels/test_native_async_operators.py::TestJoinNativeAsync::test_four_way_join` | no → NEW (all 24 permutations) | no → NEW (plain, data-tags, mixed) | no → NEW | no sync comparison → NEW | | `Join` shared keys, several | schema only: `tests/test_pipeline/test_zero_row_column_order.py` step `join` | no (keys never in another order in one input) → NEW (tag orders per input) | plain, data-tag | no → NEW | schema only → NEW | | `MergeJoin` colliding columns | yes: `ops/test_merge_join.py::TestMergeJoinBasic::test_values_sorted_independently_per_column`; ASE `merge_source` | left/right only, distinct values → NEW `test_merge_join_result_does_not_depend_on_the_order_of_its_inputs`; ties: **defect ITL-1241** | yes: ASE `merge_join`, `ref_merge_join_*` | no → NEW | yes: ASE | -| `MergeJoin` nullable colliding column with a null | no | n/a | n/a | n/a | n/a: **defect 3** (raises `TypeError`) → NEW xfail | +| `MergeJoin` nullable colliding column with a null | no | n/a | n/a | n/a | n/a: **ITL-1243** (raises `TypeError`) → NEW xfail | | `SemiJoin` match keys | rows with 1 key only (`ops/test_semijoin_tag_keys.py::test_semijoin_keys_on_shared_tags_only`) | no → NEW `test_semijoin_matches_on_every_shared_tag_in_any_arrangement` | yes: plain, `ref_semijoin`, mixed | no → NEW | 1 key: `test_semijoin_tag_keys.py::test_async_rows_equal_sync_rows` → NEW (2 keys) | | `Batch` (tags and data become lists) | yes: ASE `batch_2` (2 tags, 9 data) | n/a (no column argument); member order is identity: `WHF::TestBatchGroupByFold::test_member_order_is_identity` | yes: `dt/test_batch.py` | no → NEW `test_unary_result_does_not_depend_on_the_input_arrangement[batch*]` | yes: ASE | | `GroupBy` `by`, member order | yes: `ops/test_group_by.py::TestGroupByShape::test_one_row_per_distinct_key` | keys only on `main` | yes | no on `main` | 1 key on `main` | @@ -66,7 +66,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | `MapTags` name_map | **almost none**: one chained rename with `tag_only` (`dt/test_operators.py::test_map_tags_tag_only_keeps_a_chained_rename_of_another_tag`) → NEW | **no** → NEW | yes; a swap of data-tags is refused: **defect ITL-1242** → NEW xfail | no → NEW | single rename only → NEW | | `MapData` name_map | yes: `dt/test_operators.py::test_map_data_chained_rename_does_not_reach_the_data_tag_copy` | **no** → NEW (incl. a swap) | yes | no → NEW | single rename → NEW | | `MapColumns` name_map | yes: `ops/test_mappers_strict.py::test_map_columns_renames_tags_and_data`; ASE `map_columns` | **no** → NEW | plain; refused with data-tags (ITL-1022): `dt/test_operators.py::test_uncovered_operators_raise` | no → NEW | yes | -| `PolarsFilter` predicates | **no** (always one) → NEW | **no**; the order IS in the identity (`filters.py` hashes an ordered tuple): **question Q1** | data | no → NEW | yes | +| `PolarsFilter` predicates | **no** (always one) → NEW | **no**; the order IS in the identity (`filters.py` hashes an ordered tuple): **ITL-1247** | data | no → NEW | yes | | `PolarsFilter` constraints | **no** (never two keys) → NEW | **no** → NEW (not in identity) | tag, data, data-tag | no → NEW | yes | | `stream.polars_filter(*predicates, constraint_map, **constraints)` | **no** | no | — | no | sync only | | `DropNulls` columns | yes: `ops/test_drop_nulls.py::test_tag_and_data_columns_in_one_call` | hashes: `test_identity_is_the_set_of_names`; rows → NEW | tag, data; a nullable data-tag holding a null: **no** | one layout → NEW | yes: ASE `drop_nulls_many` | @@ -84,10 +84,10 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | `CachedSource` | 1 tag | no | **plain only** (data-tag read-back untested) | no | no | | `DerivedSource(tags_as_data=)` | yes: `sources/test_derived_source_tags_as_data.py::TestColumns::test_several_copies_follow_map_order` | content hash only: `TestIdentity::test_mapping_order_does_not_change_content_hash` | plain; data-tag origin `dt/test_derived_source.py::test_tags_as_data_over_a_data_tag_node` | one arrangement | yes | | function pod, several inputs | 2: `function_pod/test_function_pod_auto_select.py::test_call_selects_once_after_the_join`; 3: `dt/test_edi_end_to_end.py` | **no**: `pod(b, a)` never compared with `pod(a, b)` | data-tag parameter (EDI) | no | yes | -| `output_keys`, several | yes: `data_function/test_data_function.py::test_multiple_output_keys` | **no**; order is not in the identity: **defect or decision 4** | — | — | value only | +| `output_keys`, several | yes: `data_function/test_data_function.py::test_multiple_output_keys` | **no**; order is not in the identity: **ITL-1244** | — | — | value only | | automatic select of several extra columns | yes: `test_dropping_logs_one_info_message_naming_the_columns` | no (kept list sorted) | one data-tag falls through | no | yes | | `PipelineJob.bind(sources=)`, several | yes: `test_pipeline/test_pipeline_job.py::TestPipelineJobBind::test_bind_sources_mutates_job` (membership) | n/a (dict order unused) | plain only | — | — | -| bind a source in another column order | yes: `test_bind_type_check.py::test_columns_match_by_name_within_each_group_whatever_their_order` (plain, run ok only) | yes (plain) | **mixed roles: defect 5** (accepted, then the node's table breaks) | partly | async read in `test_bind_narrower_source.py::test_columns_follow_the_declared_order` | +| bind a source in another column order | yes: `test_bind_type_check.py::test_columns_match_by_name_within_each_group_whatever_their_order` (plain, run ok only) | yes (plain) | **mixed roles: ITL-1245** (accepted, then the node's table breaks) | partly | async read in `test_bind_narrower_source.py::test_columns_follow_the_declared_order` | | `ColumnConfig` groups | all_info: `tests/test_types_column_config_all_info.py`; each group alone: `test_zero_row_column_order.py` | n/a | data-tag variants | one arrangement | yes | ## Defects and questions found @@ -96,23 +96,23 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest |---|---|---|---| | 1 | `MergeJoin(A, B)` and `MergeJoin(B, A)` give the source-info lists of tied values in argument order | stored source info (operator cache, downstream tables); not the hashes | ITL-1241 | | 2 | `MapTags` refuses a swap or cycle of data-tags that it accepts for plain tags (`MapData` is right: a data-only move) | error only | ITL-1242 | -| 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | pending | -| 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | pending (decision) | -| 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | pending | -| Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | question | +| 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | ITL-1243 | +| 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | ITL-1244 (rule undecided) | +| 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | ITL-1245 | +| Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | ITL-1247 (rule undecided) | ## Gaps, ranked Order- or role-dependent behaviour that reaches a hash or a stored table first. -1. `output_keys` order: not in identity, stored values served under swapped names (defect 4). +1. `output_keys` order: not in identity, stored values served under swapped names (ITL-1244). 2. `MergeJoin` tied values: source-info order follows the inputs (ITL-1241). -3. `PolarsFilter` predicate order in identity, constraint order not; never tested with two (Q1). +3. `PolarsFilter` predicate order in identity, constraint order not; never tested with two (ITL-1247). 4. `record_id_columns` key order: moves record ids, tokens and the content hash; never compared. 5. `Join` with 3 and 4 inputs: content and pipeline hash under every permutation; 4 inputs never run sync. Closed by NEW. 6. `MapTags` / `MapData` / `MapColumns`: several renames, mapping order, swaps, sync and async; `MapTags` has no golden entry. Closed by NEW (except golden: out of scope). 7. Column arrangement (d) never varied for any operator but `GroupBy` (PR #434). Closed by NEW for every operator. -8. Bind of a mixed-role source in another column order (defect 5); several sources with one refused. +8. Bind of a mixed-role source in another column order (ITL-1245); several sources with one refused. 9. `SelectTagColumns` / `DropTagColumns` with two or more names, in two orders. Closed by NEW. 10. `function_pod` over several inputs: `pod(b, a)` against `pod(a, b)` (hashes, system-tag names, stored rows). 11. `SemiJoin` with two or more keys, rows sync and async. Closed by NEW. diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index 250346a0a..b7576135b 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -269,18 +269,45 @@ def test_the_order_of_a_set_argument_changes_nothing( _assert_all_equal(results) -def test_polars_filter_predicates_in_any_order_choose_the_same_rows() -> None: - """Two predicates in either order keep the same rows (identity: see ITL-1236 doc).""" - predicates = (pl.col("d1") > 10, pl.col("t3") == 1, pl.col("d3") != "y") - rows = { - repr(order): rows_fingerprint( - list(PolarsFilter(predicates=list(order))(make_source(MAIN)).iter_data()) - ) - for order in itertools.permutations(predicates) - } +PREDICATES = (pl.col("d1") > 10, pl.col("t3") == 1, pl.col("d3") != "y") + + +@pytest.mark.parametrize("role", ROLES) +def test_polars_filter_predicates_in_any_order_choose_the_same_rows( + role: str, +) -> None: + """Three predicates over a data column, a tag and a data column, in every order.""" + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + rows = {} + for order in itertools.permutations(PREDICATES): + op = PolarsFilter(predicates=list(order)) + stream = op(source) + rows[repr(order)] = rows_fingerprint(list(stream.iter_data())) + assert rows_fingerprint(async_rows(op, source)) == rows[repr(order)] + assert _sorted_values(stream, "d1") == [50] assert len({repr(v) for v in rows.values()}) == 1 +@pytest.mark.xfail( + strict=True, + reason="ITL-1247 (rule undecided): the order of PolarsFilter's predicates is in " + + "its identity; proposed: predicates are AND-ed, so their order is not", +) +@pytest.mark.parametrize("role", ROLES) +def test_polars_filter_predicate_order_is_not_identity(role: str) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + results = { + repr(order): { + "operator_identity": PolarsFilter(predicates=list(order)) + .content_hash() + .to_string(), + **fingerprint(PolarsFilter(predicates=list(order))(source)), + } + for order in itertools.permutations(PREDICATES) + } + _assert_all_equal(results) + + @pytest.mark.parametrize("role", ROLES) def test_add_tags_order_is_the_order_of_the_new_tags_and_its_identity( role: str, @@ -521,7 +548,7 @@ def test_join_of_four_inputs_matches_on_every_shared_tag() -> None: # MergeJoin: two inputs with the same columns; every data column collides. The rows # of ``MAIN`` without a null (``NON_NULL_ROWS``), and other values in every data column, -# so no merged pair ties: a tie is ITL-1241, a null ``MERGE_NULL_ISSUE``. +# so no merged pair ties: a tie is ITL-1241, a null ITL-1243. NON_NULL_ROWS = [0, 2, 3, 4] MAIN_OTHER = Spec( source_id="s2", @@ -582,7 +609,7 @@ def test_merge_join_of_tied_values_does_not_depend_on_the_order_of_its_inputs( ) -MERGE_NULL_ISSUE = "ITL-1236 defect 3, issue pending" +MERGE_NULL_ISSUE = "ITL-1243" @pytest.mark.xfail( From ec9c140f7b61018fa43c74f3c32d4517b25fff88 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:24:29 -0800 Subject: [PATCH 05/10] test(operators): type the results helper by Mapping (ITL-1236) Co-Authored-By: Claude Opus 5.5 --- tests/test_core/operators/test_several_items.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index b7576135b..90de77f39 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -25,7 +25,7 @@ from __future__ import annotations import itertools -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Any @@ -141,7 +141,7 @@ def _unary_params() -> list[object]: ] -def _assert_all_equal(results: dict[str, dict[str, object]]) -> None: +def _assert_all_equal(results: Mapping[str, Mapping[str, object]]) -> None: """Assert every arrangement's fingerprint equals the first one's, naming the field. A result that holds ``async_rows`` must also have them equal to its own sync From aa7fa71d6f2428cd2c7fd0c05d4604056056aabb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:30:55 -0800 Subject: [PATCH 06/10] test(operators): column order of sync rows, async rows and keys, per arrangement; defect 7 as xfail (ITL-1236) Addresses the second Copilot review of #442: sync and async rows are compared with their column order, system-tag order and Arrow schemas within each arrangement, and a result's keys with its rows' keys. Both fail on inputs whose table layout differs from the tag order (reported, issue pending); those combinations are xfail(strict=True), the others are asserted. Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 1 + tests/test_core/operators/several_items.py | 21 +++ .../test_core/operators/test_several_items.py | 152 +++++++++++++++++- 3 files changed, 166 insertions(+), 8 deletions(-) diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index aa67452d9..6b01de4c0 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -99,6 +99,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | ITL-1243 | | 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | ITL-1244 (rule undecided) | | 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | ITL-1245 | +| 7 | Input whose table layout differs from its tag order: every unary operator's result keys list the tags in the physical order while its rows use the tag order; Join, SemiJoin, MergeJoin (and, with data-tags, the tag select/drop and `MapTags` forms) give the async rows another column order than the sync rows | column order only; no hash | pending | | Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | ITL-1247 (rule undecided) | ## Gaps, ranked diff --git a/tests/test_core/operators/several_items.py b/tests/test_core/operators/several_items.py index 49642b438..e42215024 100644 --- a/tests/test_core/operators/several_items.py +++ b/tests/test_core/operators/several_items.py @@ -235,6 +235,27 @@ def rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: return sorted(row_fingerprint(tag, data) for tag, data in rows) +def ordered_rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: + """``rows_fingerprint`` plus each row's column order, system-tag order and Arrow schemas. + + For comparing the async rows with the sync rows of one arrangement, where the + column order must be the same too (as ``assert_equivalent`` in + ``test_async_sync_equivalence.py``); across arrangements only the order-free + ``rows_fingerprint`` must be. + """ + return sorted( + ( + *row_fingerprint(tag, data), + repr(list(tag.keys())), + repr(list(data.keys())), + repr(list(tag.system_tags())), + str(tag.arrow_schema(columns={"system_tags": True})), + str(data.arrow_schema(columns={"source": True})), + ) + for tag, data in rows + ) + + def fingerprint(stream: StreamProtocol) -> dict[str, Any]: """What must not depend on the input's arrangement or a set argument's order. diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index 90de77f39..ece00e0f8 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -74,6 +74,7 @@ async_rows, fingerprint, make_source, + ordered_rows_fingerprint, presentation, rows_fingerprint, ) @@ -144,18 +145,21 @@ def _unary_params() -> list[object]: def _assert_all_equal(results: Mapping[str, Mapping[str, object]]) -> None: """Assert every arrangement's fingerprint equals the first one's, naming the field. - A result that holds ``async_rows`` must also have them equal to its own sync - ``rows``: equal across arrangements alone would pass an async path that is wrong - the same way every time. + A result that holds ``async_rows`` must also have them equal to its own + ``sync_rows``: equal across arrangements alone would pass an async path that is + wrong the same way every time. (Column order is compared separately, in + ``test_async_column_order_equals_sync_in_every_arrangement``.) """ for result_id, result in results.items(): if "async_rows" in result: - assert result["async_rows"] == result["rows"], ( + assert result["async_rows"] == result["sync_rows"], ( f"async rows differ from sync rows in {result_id}" ) (first_id, first), *others = results.items() for arrangement_id, result in others: for field, value in first.items(): + if field in ("async_rows", "sync_rows"): + continue assert result[field] == value, ( f"{field} differs between {first_id} and {arrangement_id}" ) @@ -187,6 +191,93 @@ def test_unary_async_rows_equal_sync_rows_in_every_arrangement( ) +COLUMN_ORDER_ISSUE = "ITL-1236 defect 7, issue pending" +KEYS_XFAIL = pytest.mark.xfail( + strict=True, + reason=f"{COLUMN_ORDER_ISSUE}: with a table layout other than the tag order, the " + + "result's keys list the tags in the input table's physical order, its rows in " + + "tag order", +) +ASYNC_ORDER_XFAIL = pytest.mark.xfail( + strict=True, + reason=f"{COLUMN_ORDER_ISSUE}: with a table layout other than the tag order, the " + + "async rows' column order differs from the sync rows'", +) + + +def assert_keys_equal_row_keys(stream: StreamProtocol) -> None: + """Every row of ``stream`` lists its columns in the order of ``stream.keys()``.""" + keys = stream.keys() + for tag, data in stream.iter_data(): + assert (tuple(tag.keys()), tuple(data.keys())) == keys + + +def assert_async_column_order_equals_sync( + op: Callable[[], StaticOutputOperatorPod], *sources: StreamProtocol +) -> None: + """Async rows equal sync rows with column order, system-tag order and Arrow schemas.""" + sync = list(op()(*sources).iter_data()) + assert ordered_rows_fingerprint(async_rows(op(), *sources)) == ( + ordered_rows_fingerprint(sync) + ) + + +# (operator, role) pairs that fail today (defect 7). Every unary case but one fails +# the keys check; DropTagColumns(["t3", "t1"]) leaves one tag, so nothing to order. +KEYS_PASS = {("drop_tag_columns", PLAIN)} +ASYNC_ORDER_FAILS = { + ("select_tag_columns", DATA_TAGS), + ("drop_tag_columns", DATA_TAGS), + ("drop_tag_columns", MIXED), + ("map_tags_drop_unmapped", DATA_TAGS), + ("map_tags_tag_only", DATA_TAGS), + ("map_tags_tag_only", MIXED), +} + + +def _marked_unary_params( + fails: Callable[[str, str], bool], mark: pytest.MarkDecorator +) -> list[Any]: + return [ + pytest.param( + case, + role, + id=f"{case.label}/{role}", + marks=[mark] if fails(case.label, role) else [], + ) + for case in UNARY_CASES + for role in case.roles + ] + + +@pytest.mark.parametrize( + ("case", "role"), + _marked_unary_params( + lambda label, role: (label, role) not in KEYS_PASS, KEYS_XFAIL + ), +) +def test_unary_result_keys_equal_its_row_keys_in_every_arrangement( + case: UnaryCase, role: str +) -> None: + for arrangement in arrangements(MAIN, roles=(role,)): + assert_keys_equal_row_keys(case.make_op()(make_source(MAIN, arrangement))) + + +@pytest.mark.parametrize( + ("case", "role"), + _marked_unary_params( + lambda label, role: (label, role) in ASYNC_ORDER_FAILS, ASYNC_ORDER_XFAIL + ), +) +def test_unary_async_column_order_equals_sync_in_every_arrangement( + case: UnaryCase, role: str +) -> None: + for arrangement in arrangements(MAIN, roles=(role,)): + assert_async_column_order_equals_sync( + case.make_op, make_source(MAIN, arrangement) + ) + + # ── order of a set argument ───────────────────────────────────────────────── @@ -262,6 +353,7 @@ def test_the_order_of_a_set_argument_changes_nothing( **fingerprint(stream), "presentation": presentation(stream), "operator_identity": op.content_hash().to_string(), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint( async_rows(op, make_source(MAIN, arrangement)) ), @@ -283,7 +375,9 @@ def test_polars_filter_predicates_in_any_order_choose_the_same_rows( op = PolarsFilter(predicates=list(order)) stream = op(source) rows[repr(order)] = rows_fingerprint(list(stream.iter_data())) - assert rows_fingerprint(async_rows(op, source)) == rows[repr(order)] + assert ordered_rows_fingerprint(async_rows(op, source)) == ( + ordered_rows_fingerprint(list(stream.iter_data())) + ) assert _sorted_values(stream, "d1") == [50] assert len({repr(v) for v in rows.values()}) == 1 @@ -405,9 +499,9 @@ def test_names_of_several_roles_in_one_call(case: RoleCase) -> None: source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) result = case.make_op()(source) assert result.keys() == case.keys - assert rows_fingerprint(async_rows(case.make_op(), source)) == rows_fingerprint( - list(result.iter_data()) - ) + assert ordered_rows_fingerprint( + async_rows(case.make_op(), source) + ) == ordered_rows_fingerprint(list(result.iter_data())) def _sorted_values(stream: StreamProtocol, name: str, role: str = "data") -> list[Any]: @@ -498,6 +592,7 @@ def test_join_result_does_not_depend_on_the_order_of_its_inputs( **fingerprint(stream), "presentation": presentation(stream), "system_tag_order": stream.keys(columns={"system_tags": True})[0], + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(Join(), *ordered)), } _assert_all_equal(results) @@ -515,6 +610,7 @@ def test_join_result_does_not_depend_on_the_arrangement_of_its_inputs( stream = Join()(*sources) results[f"{order_index}/{layout}"] = { **fingerprint(stream), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(Join(), *sources)), } _assert_all_equal(results) @@ -587,6 +683,7 @@ def test_merge_join_result_does_not_depend_on_the_order_of_its_inputs( ): results[f"{order_index}/{layout}/{label}"] = { **fingerprint(stream), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), } _assert_all_equal(results) @@ -638,6 +735,7 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N stream = SemiJoin()(left, right) results[f"{order_index}/{layout}"] = { **fingerprint(stream), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(SemiJoin(), left, right)), } _assert_all_equal(results) @@ -652,3 +750,41 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N assert rows_fingerprint(async_rows(SemiJoin(), left, right)) == ( rows_fingerprint(list(kept.iter_data())) ) + + +SEVERAL_INPUT_CASES: dict[ + str, tuple[Callable[[], StaticOutputOperatorPod], tuple[Spec, ...]] +] = { + "join_2": (Join, (MAIN, P)), + "join_3": (Join, (MAIN, P, Q)), + "join_4": (Join, (MAIN, P, Q, R)), + "semijoin": (SemiJoin, (MAIN, P)), + "merge_join": (MergeJoin, (MAIN, MAIN_OTHER)), +} + + +def _several_input_sources( + label: str, order_index: int, layout: str, role: str +) -> tuple[Callable[[], StaticOutputOperatorPod], list[ArrowTableSource]]: + op, specs = SEVERAL_INPUT_CASES[label] + rows = NON_NULL_ROWS if label == "merge_join" else None + return op, _sources(specs, order_index, layout, role, rows=rows) + + +@pytest.mark.parametrize("role", ROLES) +@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) +def test_several_input_result_keys_equal_its_row_keys(label: str, role: str) -> None: + for order_index in range(3): + for layout in LAYOUTS: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_keys_equal_row_keys(op()(*sources)) + + +@ASYNC_ORDER_XFAIL +@pytest.mark.parametrize("role", ROLES) +@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) +def test_several_input_async_column_order_equals_sync(label: str, role: str) -> None: + for order_index in range(3): + for layout in LAYOUTS: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_async_column_order_equals_sync(op, *sources) From ea4814f8e0da9889f43124342ddc5e0f94283351 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:32:03 -0800 Subject: [PATCH 07/10] test(sources, pipeline): compare key orders with each other; check both bound sources (ITL-1236) Addresses the Copilot review of #443. Co-Authored-By: Claude Opus 5.5 --- tests/test_core/sources/test_several_items.py | 8 ++++++ .../test_pipeline/test_bind_several_items.py | 25 +++++++++++-------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index ecfc017f7..9dfc9ad5e 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -116,6 +116,14 @@ def test_record_ids_follow_the_key_order(role: str) -> None: if keys != ("t1", "t2", "t3"): assert source.content_hash() != first.content_hash() assert _record_ids(source) != _record_ids(first) + # The same key columns in another order, compared with each other. + for one, other in ( + (("t1", "t2", "t3"), ("t3", "t1", "t2")), + (("d3", "t1"), ("t1", "d3")), + ): + assert by_keys[one].content_hash() != by_keys[other].content_hash() + assert by_keys[one].pipeline_hash() == by_keys[other].pipeline_hash() + assert _record_ids(by_keys[one]) != _record_ids(by_keys[other]) # Same keys, same order: the same source. again = ArrowTableSource( _table(), diff --git a/tests/test_pipeline/test_bind_several_items.py b/tests/test_pipeline/test_bind_several_items.py index 342630ffb..2a3580297 100644 --- a/tests/test_pipeline/test_bind_several_items.py +++ b/tests/test_pipeline/test_bind_several_items.py @@ -71,22 +71,27 @@ def test_several_sources_bound_in_one_call(role: str, orchestrator: type) -> Non assert _slot_names(job) == ("s", "p") before = _snapshot(job) arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) - job.bind( - sources={ - "s": make_source(MAIN, arrangement, source_id="s_new", rows=[0, 1]), - "p": make_source( - P, Arrangement(P.tags, TAGS_FIRST, role), source_id="p_new" - ), - } - ) + # Both replacements hold other rows than the slots' sources, so the result shows + # that both were bound. + sources: dict[str, StreamProtocol] = { + "s": make_source(MAIN, arrangement, source_id="s_new", rows=[0, 1]), + "p": make_source( + P, Arrangement(P.tags, TAGS_FIRST, role), source_id="p_new", rows=[1, 3] + ), + } + job.bind(sources=sources) + for name, source in sources.items(): + assert job.nodes[name].bound_source is source assert _snapshot(job) == before job.run(orchestrator=orchestrator()) rows = sorted( (tag["t1"], tag["t2"], data["total"]) for tag, data in job.nodes["total"].iter_data() ) - # Rows 0 and 1 of MAIN: (b, 2, d1=10) with p1=6; (a, 1, d1=20) with p1=9. - assert rows == [("a", 1, 29), ("b", 2, 16)] + # MAIN's rows 0 and 1 are (b, 2, d1=10) and (a, 1, d1=20); P's rows 1 and 3 are + # (b, 1, p1=8) and (b, 2, p1=6). Only (b, 2) matches: with all of P, (a, 1) would + # match too. + assert rows == [("b", 2, 16)] @pytest.mark.parametrize("refused", ["s", "p"]) From d896a1887bf6d2572adf258d9a9db4108e81e4f6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:34:54 -0800 Subject: [PATCH 08/10] test(operators): ITL-1249 for the column order xfails; an operator node stores one column order on both orchestrators (ITL-1236) Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 2 +- .../test_core/operators/test_several_items.py | 47 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index 6b01de4c0..37561f578 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -99,7 +99,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | ITL-1243 | | 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | ITL-1244 (rule undecided) | | 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | ITL-1245 | -| 7 | Input whose table layout differs from its tag order: every unary operator's result keys list the tags in the physical order while its rows use the tag order; Join, SemiJoin, MergeJoin (and, with data-tags, the tag select/drop and `MapTags` forms) give the async rows another column order than the sync rows | column order only; no hash | pending | +| 7 | Input whose table layout differs from its tag order: every unary operator's result keys list the tags in the physical order while its rows use the tag order; Join, SemiJoin, MergeJoin (and, with data-tags, the tag select/drop and `MapTags` forms) give the async rows another column order than the sync rows. In a job, an operator node stores one column order on both orchestrators (measured, tested) | column order only; no hash, no stored table | ITL-1249 | | Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | ITL-1247 (rule undecided) | ## Gaps, ranked diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index ece00e0f8..d590b8eec 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -27,6 +27,7 @@ import itertools from collections.abc import Callable, Mapping from dataclasses import dataclass +from pathlib import Path from typing import Any import polars as pl @@ -52,12 +53,18 @@ SemiJoin, TagBy, ) +from orcapod import PipelineJob from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import InputValidationError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.types import CacheMode from orcapod.protocols.core_protocols import StreamProtocol from .several_items import ( + DATA_FIRST_REVERSED, DATA_TAGS, LAYOUTS, MAIN, @@ -191,7 +198,7 @@ def test_unary_async_rows_equal_sync_rows_in_every_arrangement( ) -COLUMN_ORDER_ISSUE = "ITL-1236 defect 7, issue pending" +COLUMN_ORDER_ISSUE = "ITL-1249" KEYS_XFAIL = pytest.mark.xfail( strict=True, reason=f"{COLUMN_ORDER_ISSUE}: with a table layout other than the tag order, the " @@ -788,3 +795,41 @@ def test_several_input_async_column_order_equals_sync(label: str, role: str) -> for layout in LAYOUTS: op, sources = _several_input_sources(label, order_index, layout, role) assert_async_column_order_equals_sync(op, *sources) + + +@pytest.mark.parametrize( + "make_op", + [Join, lambda: SelectTagColumns(["t3", "t1"])], + ids=["join", "select_tag_columns"], +) +@pytest.mark.parametrize("role", ROLES) +def test_an_operator_node_stores_one_column_order_on_both_orchestrators( + make_op: Callable[[], StaticOutputOperatorPod], role: str, tmp_path: Path +) -> None: + """A job run by each orchestrator over one store: one column order, rows once. + + The input's layout is not its tag order (ITL-1249 concerns the bare streams); in a + job, an operator node keeps its output in row order on both paths (ITL-1133). + """ + for database in (InMemoryArrowDatabase(), DeltaTableDatabase(tmp_path / role)): + tables = [] + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="order", database=database) + sources = _sources((MAIN, P), 0, DATA_FIRST_REVERSED, role) + with job: + make_op()(*sources[: 2 if make_op is Join else 1], label="op") + job.nodes["op"].attach_databases( + pipeline_database=database.at("order"), cache_mode=CacheMode.LOG + ) + job.run(orchestrator=orchestrator()) + tables.append(job.nodes["op"].as_table(all_info=True)) + assert tables[0].column_names == tables[1].column_names + # The tags first, in the input's tag order. + tags = ["t1", "t2", "t3"] if make_op is Join else ["t1", "t3"] + assert tables[0].column_names[: len(tags)] == tags + stored = [ + database.get_all_records(path) for path in database.list_table_paths() + ] + stored = [table for table in stored if table is not None] + assert len(stored) == 1 + assert stored[0].num_rows == tables[0].num_rows From e1800ebd962bb42b7520d06316588cd3afa80baa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:50:29 -0800 Subject: [PATCH 09/10] test(operators): one column-order test per arrangement, xfail only where ITL-1249 reproduces (ITL-1236) Addresses the third Copilot review of #442. Co-Authored-By: Claude Opus 5.5 --- .../test_core/operators/test_several_items.py | 114 +++++++++++------- 1 file changed, 72 insertions(+), 42 deletions(-) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index d590b8eec..83af880f8 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -66,6 +66,7 @@ from .several_items import ( DATA_FIRST_REVERSED, DATA_TAGS, + INTERLEAVED, LAYOUTS, MAIN, MIXED, @@ -229,10 +230,13 @@ def assert_async_column_order_equals_sync( ) -# (operator, role) pairs that fail today (defect 7). Every unary case but one fails -# the keys check; DropTagColumns(["t3", "t1"]) leaves one tag, so nothing to order. +# Where the column order fails today (ITL-1249), per arrangement. In the reversed layout +# the tags sit in another order than the tag order: every unary result's keys follow +# the table (but ``DropTagColumns(["t3", "t1"])`` of plain tags leaves one tag). With +# data-tags, a tag demoted to data sits elsewhere in the sync rows than in the async +# rows, in the reversed and the interleaved layout. KEYS_PASS = {("drop_tag_columns", PLAIN)} -ASYNC_ORDER_FAILS = { +DEMOTED_DATA_TAG_CASES = { ("select_tag_columns", DATA_TAGS), ("drop_tag_columns", DATA_TAGS), ("drop_tag_columns", MIXED), @@ -242,47 +246,50 @@ def assert_async_column_order_equals_sync( } -def _marked_unary_params( - fails: Callable[[str, str], bool], mark: pytest.MarkDecorator +def _keys_fail(label: str, role: str, layout: str) -> bool: + if layout == DATA_FIRST_REVERSED: + return (label, role) not in KEYS_PASS + return layout == INTERLEAVED and (label, role) in DEMOTED_DATA_TAG_CASES + + +def _async_order_fails(label: str, role: str, layout: str) -> bool: + return layout != TAGS_FIRST and (label, role) in DEMOTED_DATA_TAG_CASES + + +def _arrangement_params( + fails: Callable[[str, str, str], bool], mark: pytest.MarkDecorator ) -> list[Any]: + """One parameter per unary case, role and arrangement; ``mark`` where it fails.""" return [ pytest.param( case, - role, - id=f"{case.label}/{role}", - marks=[mark] if fails(case.label, role) else [], + arrangement, + id=f"{case.label}/{arrangement.id}", + marks=[mark] + if fails(case.label, arrangement.roles, arrangement.layout) + else [], ) for case in UNARY_CASES - for role in case.roles + for arrangement in arrangements(MAIN, roles=case.roles) ] @pytest.mark.parametrize( - ("case", "role"), - _marked_unary_params( - lambda label, role: (label, role) not in KEYS_PASS, KEYS_XFAIL - ), + ("case", "arrangement"), _arrangement_params(_keys_fail, KEYS_XFAIL) ) -def test_unary_result_keys_equal_its_row_keys_in_every_arrangement( - case: UnaryCase, role: str +def test_unary_result_keys_equal_its_row_keys( + case: UnaryCase, arrangement: Arrangement ) -> None: - for arrangement in arrangements(MAIN, roles=(role,)): - assert_keys_equal_row_keys(case.make_op()(make_source(MAIN, arrangement))) + assert_keys_equal_row_keys(case.make_op()(make_source(MAIN, arrangement))) @pytest.mark.parametrize( - ("case", "role"), - _marked_unary_params( - lambda label, role: (label, role) in ASYNC_ORDER_FAILS, ASYNC_ORDER_XFAIL - ), + ("case", "arrangement"), _arrangement_params(_async_order_fails, ASYNC_ORDER_XFAIL) ) -def test_unary_async_column_order_equals_sync_in_every_arrangement( - case: UnaryCase, role: str +def test_unary_async_column_order_equals_sync( + case: UnaryCase, arrangement: Arrangement ) -> None: - for arrangement in arrangements(MAIN, roles=(role,)): - assert_async_column_order_equals_sync( - case.make_op, make_source(MAIN, arrangement) - ) + assert_async_column_order_equals_sync(case.make_op, make_source(MAIN, arrangement)) # ── order of a set argument ───────────────────────────────────────────────── @@ -778,23 +785,46 @@ def _several_input_sources( return op, _sources(specs, order_index, layout, role, rows=rows) -@pytest.mark.parametrize("role", ROLES) -@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) -def test_several_input_result_keys_equal_its_row_keys(label: str, role: str) -> None: - for order_index in range(3): - for layout in LAYOUTS: - op, sources = _several_input_sources(label, order_index, layout, role) - assert_keys_equal_row_keys(op()(*sources)) +def _several_input_params(fails_in_reversed_layout: bool) -> list[Any]: + """One parameter per several-input case, role, tag order and layout.""" + return [ + pytest.param( + label, + role, + order_index, + layout, + id=f"{label}/{role}/{order_index}/{layout}", + marks=[ASYNC_ORDER_XFAIL] + if fails_in_reversed_layout and layout == DATA_FIRST_REVERSED + else [], + ) + for label in SEVERAL_INPUT_CASES + for role in ROLES + for order_index in range(3) + for layout in LAYOUTS + ] -@ASYNC_ORDER_XFAIL -@pytest.mark.parametrize("role", ROLES) -@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) -def test_several_input_async_column_order_equals_sync(label: str, role: str) -> None: - for order_index in range(3): - for layout in LAYOUTS: - op, sources = _several_input_sources(label, order_index, layout, role) - assert_async_column_order_equals_sync(op, *sources) +@pytest.mark.parametrize( + ("label", "role", "order_index", "layout"), _several_input_params(False) +) +def test_several_input_result_keys_equal_its_row_keys( + label: str, role: str, order_index: int, layout: str +) -> None: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_keys_equal_row_keys(op()(*sources)) + + +# In the reversed layout, Join, SemiJoin and MergeJoin give the sync rows the table's +# tag order and the async rows the tag order (ITL-1249). +@pytest.mark.parametrize( + ("label", "role", "order_index", "layout"), _several_input_params(True) +) +def test_several_input_async_column_order_equals_sync( + label: str, role: str, order_index: int, layout: str +) -> None: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_async_column_order_equals_sync(op, *sources) @pytest.mark.parametrize( From 3702e659c3bfac310ecd35af68872a1ad5d67f6f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:52:20 -0800 Subject: [PATCH 10/10] test(function_pod, sources, pipeline): weighted outputs, async arrangements, cache-only readback, bind on both orchestrators (ITL-1236) Addresses the second Copilot review of #443. Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_several_items.py | 45 +++++++++++++++---- tests/test_core/sources/test_several_items.py | 17 ++++--- .../test_pipeline/test_bind_several_items.py | 45 +++++++++++++++---- 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/tests/test_core/function_pod/test_several_items.py b/tests/test_core/function_pod/test_several_items.py index 5f287c2b6..53cccd60c 100644 --- a/tests/test_core/function_pod/test_several_items.py +++ b/tests/test_core/function_pod/test_several_items.py @@ -46,7 +46,8 @@ # Parameters in another order than the columns: q2 is in Q, d1 in MAIN, p1 in P. @function_pod(output_keys=["total", "product"]) def combine(q2: int, d1: int, p1: int) -> tuple[int, int]: - return q2 + d1 + p1, d1 * p1 + # Weighted: no two parameters can be swapped without changing ``total``. + return q2 + 2 * d1 + 3 * p1, d1 * p1 def _inputs(role: str) -> list[ArrowTableSource]: @@ -87,14 +88,14 @@ def test_values_are_matched_by_parameter_name(role: str) -> None: (tag["t1"], tag["t2"], tag["t3"], data["total"], data["product"]) for tag, data in stream.iter_data() ) - # MAIN x P on (t1, t2) x Q on (t2, t3); total = q2 + d1 + p1, product = d1 * p1. + # MAIN x P on (t1, t2) x Q on (t2, t3); total = q2 + 2 d1 + 3 p1, product = d1 p1. assert rows == [ - ("a", 1, 1, 400 + 50 + 9, 50 * 9), - ("a", 1, 2, 100 + 20 + 9, 20 * 9), - ("a", 2, 1, 200 + 30 + 7, 30 * 7), - ("b", 1, 2, 100 + 40 + 8, 40 * 8), - ("b", 2, 1, 200 + 10 + 6, 10 * 6), - ("b", 2, 2, 300 + 60 + 6, 60 * 6), + ("a", 1, 1, 400 + 2 * 50 + 3 * 9, 50 * 9), + ("a", 1, 2, 100 + 2 * 20 + 3 * 9, 20 * 9), + ("a", 2, 1, 200 + 2 * 30 + 3 * 7, 30 * 7), + ("b", 1, 2, 100 + 2 * 40 + 3 * 8, 40 * 8), + ("b", 2, 1, 200 + 2 * 10 + 3 * 6, 10 * 6), + ("b", 2, 2, 300 + 2 * 60 + 3 * 6, 60 * 6), ] @@ -152,6 +153,34 @@ def test_a_function_reads_several_data_tags_and_data_in_any_arrangement( assert all(result == first for result in results.values()) +@pytest.mark.parametrize("layout", LAYOUTS) +@pytest.mark.parametrize("order", MAIN.tag_orders(), ids="-".join) +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_a_function_over_several_data_tags_gives_the_same_rows_on_both_orchestrators( + role: str, order: tuple[str, ...], layout: str +) -> None: + """In a job, sync and async: same rows, column order and stored results.""" + tables = [] + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="labels", database=InMemoryArrowDatabase()) + with job: + label_rows.pod( + make_source(MAIN, Arrangement(order, layout, role)), label="labels" + ) + job.run(orchestrator=orchestrator()) + node = job.nodes["labels"] + tables.append( + ( + presentation(node), + [(tuple(t.keys()), tuple(d.keys())) for t, d in node.iter_data()], + rows_fingerprint(list(node.iter_data()), source_info=False), + sorted(str(data["labelled"]) for _, data in node.iter_data()), + ) + ) + assert tables[0] == tables[1] + assert tables[0][3] == ["a1v", "a1y", "a2x", "b1w", "b2u", "b2z"] + + @pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) def test_a_layout_of_the_input_does_not_change_a_fall_through_pods_hashes( role: str, diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index 9dfc9ad5e..50aae4d59 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -160,7 +160,9 @@ def test_tags_as_data_order_is_the_order_of_the_new_columns_only(role: str) -> N @pytest.mark.parametrize("role", ROLES) -def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> None: +def test_a_cached_source_reads_back_every_role_and_arrangement( + role: str, monkeypatch: pytest.MonkeyPatch +) -> None: for order in MAIN.tag_orders(): for layout in LAYOUTS: arrangement = Arrangement(order, layout, role) @@ -169,9 +171,14 @@ def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> Non make_source(MAIN, arrangement), cache_database=database ) assert fingerprint(cached) == fingerprint(make_source(MAIN, arrangement)) - # A second CachedSource reads the stored rows back. - again = CachedSource( - make_source(MAIN, arrangement), cache_database=database - ) + # A second CachedSource over the same source (same schema and identity) + # that cannot provide rows: it serves the stored rows only. + unavailable = make_source(MAIN, arrangement) + + def no_rows(*args: object, **kwargs: object) -> pa.Table: + raise NotImplementedError("rows are not available") + + monkeypatch.setattr(unavailable, "as_table", no_rows) + again = CachedSource(unavailable, cache_database=database) assert presentation(again) == presentation(make_source(MAIN, arrangement)) assert fingerprint(again) == fingerprint(cached) diff --git a/tests/test_pipeline/test_bind_several_items.py b/tests/test_pipeline/test_bind_several_items.py index 2a3580297..b3ee295d2 100644 --- a/tests/test_pipeline/test_bind_several_items.py +++ b/tests/test_pipeline/test_bind_several_items.py @@ -29,6 +29,7 @@ P, make_source, presentation, + rows_fingerprint, ) @@ -110,8 +111,29 @@ def test_one_refused_source_leaves_every_binding_as_it_was(refused: str) -> None assert {name: job.nodes[name].bound_source for name in ("s", "p")} == bound_before -def _bind_in(role: str, order: tuple[str, ...], layout: str) -> None: - """Bind ``MAIN`` in another arrangement to slot ``s``: nothing moves, all reads.""" +def _total_rows(job: PipelineJob) -> list[tuple[str, ...]]: + return rows_fingerprint(list(job.nodes["total"].iter_data()), source_info=False) + + +def _total_values(job: PipelineJob) -> list[tuple[str, str]]: + """The rows' tag and data values only: the bound source has another source id.""" + return sorted( + (repr(sorted(tag.as_dict().items())), repr(sorted(data.as_dict().items()))) + for tag, data in job.nodes["total"].iter_data() + ) + + +def _bind_in( + role: str, + order: tuple[str, ...], + layout: str, + orchestrator: type = SyncPipelineOrchestrator, +) -> PipelineJob: + """Bind ``MAIN`` in another arrangement to slot ``s``: nothing moves, all reads. + + Returns: + The job, run on ``orchestrator``. + """ job = _job(role) before = _snapshot(job) # The columns of the table, not their order: CLAUDE.md promises the declared names, @@ -127,17 +149,24 @@ def _bind_in(role: str, order: tuple[str, ...], layout: str) -> None: order, layout, ) - job.run() - assert len(job.nodes["total"].as_table()) == 6 + job.run(orchestrator=orchestrator()) + return job +@pytest.mark.parametrize("layout", LAYOUTS) +@pytest.mark.parametrize("order", MAIN.tag_orders(), ids="-".join) @pytest.mark.parametrize("role", [PLAIN, DATA_TAGS]) def test_a_source_in_another_arrangement_is_bound_as_the_slot_declares( - role: str, + role: str, order: tuple[str, ...], layout: str ) -> None: - for order in MAIN.tag_orders(): - for layout in LAYOUTS: - _bind_in(role, order, layout) + """Both orchestrators: the same rows, with the values of the slot's own source.""" + reference = _job(role) + reference.run() + sync = _bind_in(role, order, layout, SyncPipelineOrchestrator) + asynchronous = _bind_in(role, order, layout, AsyncPipelineOrchestrator) + assert _total_rows(asynchronous) == _total_rows(sync) + assert _total_values(sync) == _total_values(reference) + assert len(_total_values(sync)) == 6 BIND_MIXED_ISSUE = "ITL-1245"