From 0bf5e72665a4c18d5dacf028e2e997db0cb930bc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:12:29 -0800 Subject: [PATCH 01/13] test(operators): several columns, names, mappings and inputs, in every order, role and arrangement (ITL-1236) A shared family of inputs with three tags and three data columns, in several tag orders, table layouts and role settings (plain, data-tags, mixed), and tests of every operator over it, sync and async: the result does not depend on the arrangement of the input; the order of a set argument (names, rename mappings, constraints) changes nothing; Join over 2, 3 and 4 inputs and MergeJoin are commutative; SemiJoin matches on every shared tag. Defects found are xfail(strict=True): ITL-1241 (MergeJoin ties), ITL-1242 (MapTags swap of data-tags), MergeJoin null crash (issue pending). The inventory of every several-item argument, with the evidence and the ranked gaps, is docs/dev/specs/itl-1236-several-item-test-audit.md. CLAUDE.md and .zed/rules state the rule in "Testing conventions". Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 12 + CLAUDE.md | 12 + .../specs/itl-1236-several-item-test-audit.md | 129 ++++ tests/test_core/operators/several_items.py | 283 ++++++++ .../test_core/operators/test_several_items.py | 610 ++++++++++++++++++ 5 files changed, 1046 insertions(+) create mode 100644 docs/dev/specs/itl-1236-several-item-test-audit.md create mode 100644 tests/test_core/operators/several_items.py create mode 100644 tests/test_core/operators/test_several_items.py diff --git a/.zed/rules b/.zed/rules index 915a08c69..7d1dd6ed3 100644 --- a/.zed/rules +++ b/.zed/rules @@ -582,6 +582,18 @@ unless the test is about the default (developer, November 16, 2026, ITL-1159). T sources that take the option (the table, file and database sources); `ListSource`, `DerivedSource`, `CachedSource` and `PollingSource` do not take it. +Every argument or input that can take several items (column names, a mapping, keywords, +input streams, sources to bind, output keys) is tested (developer, November 17, 2026, +ITL-1236): with two or more items; in more than one order, asserting what the order does or +must not change (rows, column order, content hash, pipeline hash, system-tag column names); +in each role it accepts (plain tag, data, data-tag: build the role you test, whatever the +`tag_only_columns` default above says); on inputs with two or more tag columns AND two or +more data columns, in more than one arrangement (tag order, table layout); and on the sync +and the async path. A test whose streams have one tag and one data column does not test an +operator. The shared family of such inputs is `tests/test_core/operators/several_items.py`; +the inventory is `docs/dev/specs/itl-1236-several-item-test-audit.md`. The docs show the +several-item form of such an argument, not only the one-item form. + --- ## Project layout diff --git a/CLAUDE.md b/CLAUDE.md index a06981d07..e4b82cbc3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -593,6 +593,18 @@ unless the test is about the default (developer, November 16, 2026, ITL-1159). T sources that take the option (the table, file and database sources); `ListSource`, `DerivedSource`, `CachedSource` and `PollingSource` do not take it. +Every argument or input that can take several items (column names, a mapping, keywords, +input streams, sources to bind, output keys) is tested (developer, November 17, 2026, +ITL-1236): with two or more items; in more than one order, asserting what the order does or +must not change (rows, column order, content hash, pipeline hash, system-tag column names); +in each role it accepts (plain tag, data, data-tag: build the role you test, whatever the +`tag_only_columns` default above says); on inputs with two or more tag columns AND two or +more data columns, in more than one arrangement (tag order, table layout); and on the sync +and the async path. A test whose streams have one tag and one data column does not test an +operator. The shared family of such inputs is `tests/test_core/operators/several_items.py`; +the inventory is `docs/dev/specs/itl-1236-several-item-test-audit.md`. The docs show the +several-item form of such an argument, not only the one-item form. + --- ## Project layout diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md new file mode 100644 index 000000000..ffd91122e --- /dev/null +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -0,0 +1,129 @@ +# ITL-1236: tests of every argument that takes several items + +Status: inventory of 2026-11-17 (phase 1), on `main` at 4da42681. Kept up to date as the +gaps are closed. + +## Why + +`GroupBy` was tested with one tag and one data column. That hid a result that depended on +the order of the input's tag columns (ITL-1226). The same pattern can hide a defect in any +argument that takes several items: column names, a mapping, several input streams. + +## The rule + +Every argument or input that can take several items is tested: + +- **(a)** with two or more items; +- **(b)** with the items in more than one order, with an assertion of what the order does or + must not do (rows, output schema and column order, content hash, pipeline hash, + system-tag column names); +- **(c)** in each role the argument accepts: plain tag, data, data-tag; +- **(d)** on inputs with two or more tag columns AND two or more data columns, in more than + one arrangement (tag order and table layout); +- **(e)** on the sync and the async path. + +A test whose streams have one tag and one data column does not test an operator. + +## How the inventory was made + +1. Every test that uses the argument was found by searching for the class and the stream + method (`grep -rn` over `tests/`), and read. +2. A probe ran every operator on one stream, three tags and three data columns, in 18 + arrangements (6 tag orders x 3 layouts), for plain tags, data-tags and a mix. It compared + rows, row hashes, system tags, source info, content and pipeline hashes. A second probe + compared each argument given in two orders. On `main` only `GroupBy` depended on the + arrangement (ITL-1226, PR #434). The other differences found are the defects below. + +Legend: **yes** (with the best test as evidence), **no** (with what the closest test does), +**n/a**. Paths: `ops/` = `tests/test_core/operators/`, `dt/` = `tests/test_core/data_tags/`, +`ASE` = `ops/test_async_sync_equivalence.py`, `WHF` = `tests/test_hashing/test_wrong_hash_findings.py`, +`NEW` = `ops/test_several_items.py` (this issue). + +## Inventory + +### Operators + +| Argument | (a) several | (b) order | (c) roles | (d) 2+ tags × 2+ data, arrangements | (e) sync / async | +|---|---|---|---|---|---| +| `Join` inputs, 2 | yes: `ops/test_operators.py::TestJoinBehavior::test_join_combines_streams_on_shared_tags` | yes: `ops/test_join_tag_order.py::test_operand_order_does_not_change_columns`; hashes `TestArgumentSymmetryIdentity::test_join_swapped_inputs_same_*` | yes: ASE `join_2`, `ref_join_2`, `ref_join_2_plain_left/right` | no (one arrangement) → NEW `test_join_result_does_not_depend_on_the_arrangement_of_its_inputs` | yes: ASE `join_2`, `join_2_reversed` | +| `Join` inputs, 3 | yes: `TestJoinSystemTagCanonicalOrdering::test_three_way_join_produces_six_system_tag_columns` | partial: system-tag names for 3 of 6 permutations; `dt/test_merged_source_info.py::test_join_token_does_not_depend_on_input_order`. No content/pipeline hash → NEW `test_join_result_does_not_depend_on_the_order_of_its_inputs[join_3/*]` (all 6) | yes: ASE `join_3`, `ref_join_3` | no → NEW | yes: ASE `join_3`, `join_3_rotated` | +| `Join` inputs, 4+ | async only: `tests/test_channels/test_native_async_operators.py::TestJoinNativeAsync::test_four_way_join` | no → NEW (all 24 permutations) | no → NEW (plain, data-tags, mixed) | no → NEW | no sync comparison → NEW | +| `Join` shared keys, several | schema only: `tests/test_pipeline/test_zero_row_column_order.py` step `join` | no (keys never in another order in one input) → NEW (tag orders per input) | plain, data-tag | no → NEW | schema only → NEW | +| `MergeJoin` colliding columns | yes: `ops/test_merge_join.py::TestMergeJoinBasic::test_values_sorted_independently_per_column`; ASE `merge_source` | left/right only, distinct values → NEW `test_merge_join_result_does_not_depend_on_the_order_of_its_inputs`; ties: **defect ITL-1241** | yes: ASE `merge_join`, `ref_merge_join_*` | no → NEW | yes: ASE | +| `MergeJoin` nullable colliding column with a null | no | n/a | n/a | n/a | n/a: **defect 3** (raises `TypeError`) → NEW xfail | +| `SemiJoin` match keys | rows with 1 key only (`ops/test_semijoin_tag_keys.py::test_semijoin_keys_on_shared_tags_only`) | no → NEW `test_semijoin_matches_on_every_shared_tag_in_any_arrangement` | yes: plain, `ref_semijoin`, mixed | no → NEW | 1 key: `test_semijoin_tag_keys.py::test_async_rows_equal_sync_rows` → NEW (2 keys) | +| `Batch` (tags and data become lists) | yes: ASE `batch_2` (2 tags, 9 data) | n/a (no column argument); member order is identity: `WHF::TestBatchGroupByFold::test_member_order_is_identity` | yes: `dt/test_batch.py` | no → NEW `test_unary_result_does_not_depend_on_the_input_arrangement[batch*]` | yes: ASE | +| `GroupBy` `by`, member order | yes: `ops/test_group_by.py::TestGroupByShape::test_one_row_per_distinct_key` | keys only on `main` | yes | no on `main` | 1 key on `main` | +| ↳ covered by PR #434 (ITL-1226) | | `test_group_by_member_order.py::test_the_order_of_by_is_not_identity` | plain, data-tag, mixed | `test_no_arrangement_of_the_input_changes_the_result` (24 × 4, source and raw stream) | `test_a_job_gives_the_same_rows_sync_and_async_for_any_arrangement` | +| `TagBy` columns | yes: `dt/test_tag_by.py::test_named_data_columns_become_data_tags_in_the_order_given` | yes: `test_the_order_of_the_names_is_part_of_the_identity` (PR #434 reverses this decision) | n/a / yes / refused: `test_an_existing_data_tag_is_refused` | no (1 tag) → NEW `[tag_by/*]` | yes: ASE `tag_by`, `tag_by_reversed` | +| `AddTags` tags | yes: `ops/test_add_tags.py::TestValues::test_arrow_type_of_each_accepted_value_type` | operator hash and keys: `TestIdentity::test_order_is_part_of_the_identity`; stream hashes → NEW `test_add_tags_order_is_the_order_of_the_new_tags_and_its_identity` | yes: `dt/test_add_tags.py` | no → NEW | yes: ASE `add_tags`, `ref_add_tags` | +| `SelectTagColumns` columns | **no** (only one name, or all) → NEW | **no** (`WHF::test_tag_column_spellings_share_identity` uses one name) → NEW | yes: `dt/test_roles.py::test_each_operator_acts_by_role` | no → NEW | yes | +| `DropTagColumns` columns | **no** → NEW | **no** → NEW | yes | no → NEW | yes | +| `SelectDataColumns` columns | yes: ASE `select_data_columns` | hash: `WHF::TestColumnSelectionIdentity`; keys, rows → NEW | yes | no → NEW | yes | +| `DropDataColumns` columns | yes: ASE `drop_data_columns` | hash only → NEW | yes | no → NEW | yes | +| `SelectColumns` columns | yes: `ops/test_tag_and_data_columns.py::test_select_columns_keeps_only_the_named_columns` | yes (keys and pipeline hash) | yes: `dt/test_operators.py::test_select_and_drop_columns_act_on_the_one_column_in_both_roles` | no → NEW | yes | +| `DropColumns` columns | yes: `test_drop_columns_drops_tags_and_data` | hash only → NEW | yes | no → NEW | yes | +| `MapTags` name_map | **almost none**: one chained rename with `tag_only` (`dt/test_operators.py::test_map_tags_tag_only_keeps_a_chained_rename_of_another_tag`) → NEW | **no** → NEW | yes; a swap of data-tags is refused: **defect ITL-1242** → NEW xfail | no → NEW | single rename only → NEW | +| `MapData` name_map | yes: `dt/test_operators.py::test_map_data_chained_rename_does_not_reach_the_data_tag_copy` | **no** → NEW (incl. a swap) | yes | no → NEW | single rename → NEW | +| `MapColumns` name_map | yes: `ops/test_mappers_strict.py::test_map_columns_renames_tags_and_data`; ASE `map_columns` | **no** → NEW | plain; refused with data-tags (ITL-1022): `dt/test_operators.py::test_uncovered_operators_raise` | no → NEW | yes | +| `PolarsFilter` predicates | **no** (always one) → NEW | **no**; the order IS in the identity (`filters.py` hashes an ordered tuple): **question Q1** | data | no → NEW | yes | +| `PolarsFilter` constraints | **no** (never two keys) → NEW | **no** → NEW (not in identity) | tag, data, data-tag | no → NEW | yes | +| `stream.polars_filter(*predicates, constraint_map, **constraints)` | **no** | no | — | no | sync only | +| `DropNulls` columns | yes: `ops/test_drop_nulls.py::test_tag_and_data_columns_in_one_call` | hashes: `test_identity_is_the_set_of_names`; rows → NEW | tag, data; a nullable data-tag holding a null: **no** | one layout → NEW | yes: ASE `drop_nulls_many` | +| `Pick` / `Index` | one item each (n/a); inputs with 2+ tags and 2+ data: `ops/test_index_pick_arrow_types.py::test_materialized_schema_matches_prediction` | n/a | data-tag with `out=`: **no**; ASE has no `ref_pick` | no | yes | +| Stream methods (`select_*`, `drop_*`, `map_*`, `join`, `semi_join`, `drop_nulls`) | partly (`test_stream_convenience_methods.py`, plain only) | no | plain only | no | sync only | + +### Sources, function pods, binding + +| Argument | (a) | (b) | (c) | (d) | (e) | +|---|---|---|---|---|---| +| `tag_columns` | yes: `sources/test_source_protocol_conformance.py::test_dict_source_multiple_tag_columns` (membership) | **no** source test (raw stream keys only: `dt/test_data_tag_order.py::test_data_tags_follow_the_tag_order`) | plain, data-tag | no | sync | +| `tag_only_columns` | True: `sources/test_tag_only_columns.py` (9 kinds, 1 tag); a list: `dt/test_error_messages.py::test_the_named_remedy_binds` | no (list order) | mixed: `dt/test_add_tags.py::test_a_mixed_stream_keeps_its_plain_tags_plain` | no | `dt/test_persistence.py` | +| `record_id_columns` | one composite key: `sources/test_stream_builder.py::TestKeyRowIdentifiers::test_record_id_and_token[composite]` | **no**: two key orders never compared (the order is in the identity, `sources/base.py`) | tag key only | no | no | +| schema arguments (`data_schema`, `schema`) | yes: `sources/test_large_normalization_sources.py` | **no**: schema always in the table's order | plain | no | no | +| `CachedSource` | 1 tag | no | **plain only** (data-tag read-back untested) | no | no | +| `DerivedSource(tags_as_data=)` | yes: `sources/test_derived_source_tags_as_data.py::TestColumns::test_several_copies_follow_map_order` | content hash only: `TestIdentity::test_mapping_order_does_not_change_content_hash` | plain; data-tag origin `dt/test_derived_source.py::test_tags_as_data_over_a_data_tag_node` | one arrangement | yes | +| function pod, several inputs | 2: `function_pod/test_function_pod_auto_select.py::test_call_selects_once_after_the_join`; 3: `dt/test_edi_end_to_end.py` | **no**: `pod(b, a)` never compared with `pod(a, b)` | data-tag parameter (EDI) | no | yes | +| `output_keys`, several | yes: `data_function/test_data_function.py::test_multiple_output_keys` | **no**; order is not in the identity: **defect or decision 4** | — | — | value only | +| automatic select of several extra columns | yes: `test_dropping_logs_one_info_message_naming_the_columns` | no (kept list sorted) | one data-tag falls through | no | yes | +| `PipelineJob.bind(sources=)`, several | yes: `test_pipeline/test_pipeline_job.py::TestPipelineJobBind::test_bind_sources_mutates_job` (membership) | n/a (dict order unused) | plain only | — | — | +| bind a source in another column order | yes: `test_bind_type_check.py::test_columns_match_by_name_within_each_group_whatever_their_order` (plain, run ok only) | yes (plain) | **mixed roles: defect 5** (accepted, then the node's table breaks) | partly | async read in `test_bind_narrower_source.py::test_columns_follow_the_declared_order` | +| `ColumnConfig` groups | all_info: `tests/test_types_column_config_all_info.py`; each group alone: `test_zero_row_column_order.py` | n/a | data-tag variants | one arrangement | yes | + +## Defects and questions found + +| # | What | Reaches | Issue | +|---|---|---|---| +| 1 | `MergeJoin(A, B)` and `MergeJoin(B, A)` give the source-info lists of tied values in argument order | stored source info (operator cache, downstream tables); not the hashes | ITL-1241 | +| 2 | `MapTags` refuses a swap or cycle of data-tags that it accepts for plain tags (`MapData` is right: a data-only move) | error only | ITL-1242 | +| 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | pending | +| 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | pending (decision) | +| 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | pending | +| Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | question | + +## Gaps, ranked + +Order- or role-dependent behaviour that reaches a hash or a stored table first. + +1. `output_keys` order: not in identity, stored values served under swapped names (defect 4). +2. `MergeJoin` tied values: source-info order follows the inputs (ITL-1241). +3. `PolarsFilter` predicate order in identity, constraint order not; never tested with two (Q1). +4. `record_id_columns` key order: moves record ids, tokens and the content hash; never compared. +5. `Join` with 3 and 4 inputs: content and pipeline hash under every permutation; 4 inputs never run sync. Closed by NEW. +6. `MapTags` / `MapData` / `MapColumns`: several renames, mapping order, swaps, sync and async; `MapTags` has no golden entry. Closed by NEW (except golden: out of scope). +7. Column arrangement (d) never varied for any operator but `GroupBy` (PR #434). Closed by NEW for every operator. +8. Bind of a mixed-role source in another column order (defect 5); several sources with one refused. +9. `SelectTagColumns` / `DropTagColumns` with two or more names, in two orders. Closed by NEW. +10. `function_pod` over several inputs: `pod(b, a)` against `pod(a, b)` (hashes, system-tag names, stored rows). +11. `SemiJoin` with two or more keys, rows sync and async. Closed by NEW. +12. `DropNulls` on a nullable data-tag holding a null, sync and async. +13. `tag_columns` order on a source: keys follow, hashes and system tags must not. +14. `DerivedSource(tags_as_data=)` reversed: default `source_id` and pipeline hash. +15. `CachedSource` over a data-tag source; `Pick` / `Index` from a data-tag with `out=`; schema arguments in another field order; the stream methods' several-argument forms (`polars_filter` with `constraint_map` and keywords). + +## Tests added + +`tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`, +`t2`, `t3`; data `d1`, `d2`, `d3`) and join partners `P`, `Q`, `R`, each in any tag order, +three layouts and three role settings. `ops/test_several_items.py` uses it for every +operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). diff --git a/tests/test_core/operators/several_items.py b/tests/test_core/operators/several_items.py new file mode 100644 index 000000000..49642b438 --- /dev/null +++ b/tests/test_core/operators/several_items.py @@ -0,0 +1,283 @@ +"""Streams with several tag and several data columns, in several arrangements (ITL-1236). + +An argument that takes several items (column names, a mapping, input streams) must be +tested with two or more items, in more than one order, in each role a column can have +(plain tag, data, data-tag), on inputs with two or more tags AND two or more data columns, +on the sync and the async path. A stream with one tag and one data column cannot show a +result that depends on column order or role: ``GroupBy``'s did (ITL-1226). + +This module is the shared family of such inputs. ``main_source`` has three tags +(``t1``, ``t2``, ``t3``) and three data columns (``d1``, ``d2``, ``d3``); the join +partners ``P``, ``Q`` and ``R`` share some of those tags and add data columns of their +own. Each source can be built in any ``Arrangement``: + +* the order of its tag columns (``tag_order``); +* the physical order of its table's columns (``layout``: tags first, data first and + reversed, or interleaved); +* the role of its tags (``roles``): all plain (``tag_only_columns=True``), all data-tags + (the default of a source), or mixed (one plain tag, the others data-tags). + +``fingerprint`` is what must not depend on an arrangement or on the order of an argument +that names a set: the rows (values, system tags, row hashes, source info), the stream's +content and pipeline hashes, and the column names of each group. ``presentation`` is the +column order, which follows the input as for every operator and enters no hash. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any, TypeAlias + +import pyarrow as pa + +from orcapod.channels import Channel +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.core.sources import ArrowTableSource +from orcapod.protocols.core_protocols import ( + DataProtocol, + StreamProtocol, + TagProtocol, +) + +Row: TypeAlias = tuple[TagProtocol, DataProtocol] + +# Roles of a source's tags. +PLAIN = "plain" +DATA_TAGS = "data_tags" +MIXED = "mixed" +ROLES = (PLAIN, DATA_TAGS, MIXED) + +# Layouts of a table: where the tag columns sit among the data columns. +TAGS_FIRST = "tags_first" +DATA_FIRST_REVERSED = "data_first_reversed" +INTERLEAVED = "interleaved" +LAYOUTS = (TAGS_FIRST, DATA_FIRST_REVERSED, INTERLEAVED) + + +@dataclass(frozen=True) +class Spec: + """The columns of one source: tags, data, values and the plain tag of ``MIXED``.""" + + source_id: str + tags: tuple[str, ...] + data: tuple[str, ...] + columns: dict[str, "pa.Array[Any]"] + mixed_plain: str + + def tag_orders(self) -> tuple[tuple[str, ...], ...]: + """The declared order, its reverse, and (with three tags) a rotation.""" + orders = [self.tags, tuple(reversed(self.tags))] + if len(self.tags) > 2: + orders.append(self.tags[1:] + self.tags[:1]) + return tuple(orders) + + +def _array(values: list[Any], arrow_type: pa.DataType) -> "pa.Array[Any]": + return pa.array(values, arrow_type) + + +# Six rows, each tag tuple once. ``t1`` and ``t2`` repeat on their own; ``d2`` holds a +# null (nullable); the other data columns hold none and are non-nullable. +MAIN = Spec( + source_id="s", + tags=("t1", "t2", "t3"), + data=("d1", "d2", "d3"), + columns={ + "t1": _array(["b", "a", "a", "b", "a", "b"], pa.large_string()), + "t2": _array([2, 1, 2, 1, 1, 2], pa.int64()), + "t3": _array([1, 2, 1, 2, 1, 2], pa.int64()), + "d1": _array([10, 20, 30, 40, 50, 60], pa.int64()), + "d2": _array([0.5, None, 2.5, 3.5, 4.5, None], pa.float64()), + "d3": _array(["w", "x", "y", "z", "v", "u"], pa.large_string()), + }, + mixed_plain="t2", +) + +# Join partners. Each shares tags with ``MAIN`` and with each other, never a data name. +P = Spec( + source_id="p", + tags=("t1", "t2"), + data=("p1", "p2"), + columns={ + "t1": _array(["a", "b", "a", "b"], pa.large_string()), + "t2": _array([2, 1, 1, 2], pa.int64()), + "p1": _array([7, 8, 9, 6], pa.int64()), + "p2": _array(["m", "n", "o", "q"], pa.large_string()), + }, + mixed_plain="t1", +) +Q = Spec( + source_id="q", + tags=("t2", "t3"), + data=("q1", "q2"), + columns={ + "t2": _array([1, 2, 2, 1], pa.int64()), + "t3": _array([2, 1, 2, 1], pa.int64()), + "q1": _array([0.25, 0.75, 1.25, 1.75], pa.float64()), + "q2": _array([100, 200, 300, 400], pa.int64()), + }, + mixed_plain="t3", +) +R = Spec( + source_id="r", + tags=("t1",), + data=("r1", "r2"), + columns={ + "t1": _array(["b", "a"], pa.large_string()), + "r1": _array([-1, -2], pa.int64()), + "r2": _array(["rb", "ra"], pa.large_string()), + }, + mixed_plain="t1", +) + + +@dataclass(frozen=True) +class Arrangement: + """How a source presents its columns: tag order, table layout and tag roles.""" + + tag_order: tuple[str, ...] + layout: str + roles: str + + @property + def id(self) -> str: + return f"{'-'.join(self.tag_order)}/{self.layout}/{self.roles}" + + +def arrangements(spec: Spec, roles: Sequence[str] = ROLES) -> list[Arrangement]: + """Every tag order of ``spec.tag_orders()`` in every layout, for each role.""" + return [ + Arrangement(order, layout, role) + for role in roles + for order in spec.tag_orders() + for layout in LAYOUTS + ] + + +def _layout(spec: Spec, tag_order: tuple[str, ...], layout: str) -> list[str]: + """The physical column order of the table.""" + if layout == TAGS_FIRST: + return [*tag_order, *spec.data] + if layout == DATA_FIRST_REVERSED: + return [*reversed(spec.data), *reversed(tag_order)] + if layout == INTERLEAVED: + names: list[str] = [] + for i in range(max(len(tag_order), len(spec.data))): + if i < len(spec.data): + names.append(spec.data[i]) + if i < len(tag_order): + names.append(tag_order[i]) + return names + raise ValueError(f"unknown layout {layout!r}") + + +def make_source( + spec: Spec, + arrangement: Arrangement | None = None, + *, + source_id: str | None = None, + rows: Sequence[int] | None = None, +) -> ArrowTableSource: + """Build ``spec`` as an ``ArrowTableSource`` in ``arrangement``. + + Args: + spec: The columns and values. + arrangement: Tag order, layout and roles; default: declared order, tags + first, plain tags. + source_id: The source id; default ``spec.source_id``. + rows: The row positions to keep, in this order; default every row. + + Returns: + The source. A column is nullable exactly when it holds a null. + """ + arrangement = arrangement or Arrangement(spec.tags, TAGS_FIRST, PLAIN) + names = _layout(spec, arrangement.tag_order, arrangement.layout) + columns = {name: spec.columns[name] for name in names} + if rows is not None: + columns = {name: column.take(list(rows)) for name, column in columns.items()} + schema = pa.schema( + pa.field(name, column.type, nullable=column.null_count > 0) + for name, column in columns.items() + ) + table = pa.table(list(columns.values()), schema=schema) + if arrangement.roles == PLAIN: + tag_only: bool | list[str] = True + elif arrangement.roles == DATA_TAGS: + tag_only = [] + elif arrangement.roles == MIXED: + tag_only = [spec.mixed_plain] + else: + raise ValueError(f"unknown roles {arrangement.roles!r}") + return ArrowTableSource( + table, + tag_columns=list(arrangement.tag_order), + source_id=source_id or spec.source_id, + tag_only_columns=tag_only, + ) + + +def row_fingerprint(tag: TagProtocol, data: DataProtocol) -> tuple[str, ...]: + """What identifies one row regardless of column order: values, system tags, hashes.""" + return ( + repr(sorted(tag.as_dict().items())), + repr(sorted(data.as_dict().items())), + repr(sorted(tag.system_tags().items())), + tag.content_hash().to_string(), + data.content_hash().to_string(), + repr(sorted(data.source_info().items())), + ) + + +def rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: + """The rows' fingerprints, sorted: row order is not part of it.""" + return sorted(row_fingerprint(tag, data) for tag, data in rows) + + +def fingerprint(stream: StreamProtocol) -> dict[str, Any]: + """What must not depend on the input's arrangement or a set argument's order. + + Args: + stream: An operator's result. + + Returns: + The rows (``rows_fingerprint``), the content and pipeline hashes, and the + sorted column names of the tag group (system tags included) and the data group. + """ + tag_keys, data_keys = stream.keys(columns={"system_tags": True}) + return { + "rows": rows_fingerprint(list(stream.iter_data())), + "content_hash": stream.content_hash().to_string(), + "pipeline_hash": stream.pipeline_hash().to_string(), + "tag_names": sorted(tag_keys), + "data_names": sorted(data_keys), + } + + +def presentation(stream: StreamProtocol) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The column order of each group: follows the input, enters no hash.""" + tag_keys, data_keys = stream.keys() + return tuple(tag_keys), tuple(data_keys) + + +async def _run_async( + op: StaticOutputOperatorPod, sources: Sequence[StreamProtocol] +) -> list[Row]: + channels: list[Channel[Row]] = [Channel(buffer_size=64) for _ in sources] + output: Channel[Row] = Channel(buffer_size=64) + for stream, channel in zip(sources, channels): + for row in stream.iter_data(): + await channel.writer.send(row) + await channel.writer.close() + await op.async_execute( + [channel.reader for channel in channels], + output.writer, + input_pipeline_hashes=[s.pipeline_hash() for s in sources], + ) + return await output.reader.collect() + + +def async_rows(op: StaticOutputOperatorPod, *sources: StreamProtocol) -> list[Row]: + """The rows of the async path, ``op.async_execute`` over channels.""" + return asyncio.run(_run_async(op, sources)) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py new file mode 100644 index 000000000..48432323b --- /dev/null +++ b/tests/test_core/operators/test_several_items.py @@ -0,0 +1,610 @@ +"""Operators with several items: several columns, names, mappings and inputs (ITL-1236). + +Every argument that takes several items is tested here with two or more, in more than +one order, in each role (plain tag, data, data-tag), on inputs with three tags and three +data columns in several arrangements (``several_items``), on the sync and the async path. + +What is asserted: + +* **Arrangement.** An operator's result does not depend on the arrangement of its input: + the order of the tag columns, the physical layout of the table, for each role. The + rows (values, system tags, row hashes, source info), the stream's content and pipeline + hashes and the column names are the same (``fingerprint``); only the column order + follows the input. +* **Order of a set argument.** Column names to select, drop or narrow, and the entries + of a rename mapping, name a set: their order changes nothing, not even the column + order of the result, nor the operator's identity. +* **Order of the inputs of a commutative operator** (``Join``, ``MergeJoin``) changes + nothing, with two, three and four inputs. + +``GroupBy`` is not here: its arrangement and ``by`` order are tested by ITL-1226 +(``test_group_by_member_order.py``). The order of ``AddTags``'s names is the order of +the new tags and part of its identity, and is tested as such below. +""" + +from __future__ import annotations + +import itertools +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import polars as pl +import pyarrow as pa +import pytest + +from orcapod.core.operators import ( + AddTags, + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + Join, + MapColumns, + MapData, + MapTags, + MergeJoin, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, + TagBy, +) +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.core.sources import ArrowTableSource +from orcapod.errors import InputValidationError +from orcapod.protocols.core_protocols import StreamProtocol + +from .several_items import ( + DATA_TAGS, + MAIN, + MIXED, + P, + PLAIN, + Q, + R, + ROLES, + TAGS_FIRST, + Arrangement, + Spec, + arrangements, + async_rows, + fingerprint, + make_source, + presentation, + rows_fingerprint, +) + +# Selecting or dropping the data role of a data-tag warns that the tag is materialized; +# that warning is tested in the data-tag tests, not here. +pytestmark = pytest.mark.filterwarnings( + "ignore::orcapod.core.data_tags.DataTagMaterializationWarning" +) + + +@dataclass(frozen=True) +class UnaryCase: + """An operator applied to ``MAIN``, and the roles it supports.""" + + label: str + make_op: Callable[[], StaticOutputOperatorPod] + roles: tuple[str, ...] = ROLES + + +UNARY_CASES = [ + UnaryCase("select_tag_columns", lambda: SelectTagColumns(["t3", "t1"])), + UnaryCase("select_data_columns", lambda: SelectDataColumns(["d3", "d1"])), + UnaryCase("drop_tag_columns", lambda: DropTagColumns(["t3", "t1"])), + UnaryCase("drop_data_columns", lambda: DropDataColumns(["d3", "d1"])), + UnaryCase("select_columns", lambda: SelectColumns(["d3", "t2", "d1", "t1"])), + UnaryCase("drop_columns", lambda: DropColumns(["d3", "t2"])), + UnaryCase("map_tags", lambda: MapTags({"t3": "u3", "t1": "u1"})), + UnaryCase( + "map_tags_drop_unmapped", + lambda: MapTags({"t3": "u3", "t1": "u1"}, drop_unmapped=True), + ), + UnaryCase("map_tags_tag_only", lambda: MapTags({"t3": "u3"}, tag_only=True)), + UnaryCase("map_data", lambda: MapData({"d3": "e3", "d1": "e1"})), + UnaryCase( + "map_data_drop_unmapped", + lambda: MapData({"d3": "e3", "d1": "e1"}, drop_unmapped=True), + ), + # MapColumns is refused with data-tags (ITL-1022). + UnaryCase( + "map_columns", lambda: MapColumns({"d3": "e3", "t1": "u1"}), roles=(PLAIN,) + ), + UnaryCase( + "polars_filter_constraints", + lambda: PolarsFilter(constraints={"t1": "a", "t2": 1}), + ), + UnaryCase( + "polars_filter_predicates", + lambda: PolarsFilter(predicates=[pl.col("d1") > 10, pl.col("t3") == 1]), + ), + UnaryCase("drop_nulls_named", lambda: DropNulls(["d2", "t2"])), + UnaryCase("drop_nulls_all", DropNulls), + UnaryCase("batch", lambda: Batch(batch_size=4)), + UnaryCase("batch_all", Batch), + UnaryCase("add_tags", lambda: AddTags({"x": 1, "y": "z"})), + UnaryCase("tag_by", lambda: TagBy(["d3", "d1"])), +] + + +def _unary_params() -> list[object]: + return [ + pytest.param(case, role, id=f"{case.label}/{role}") + for case in UNARY_CASES + for role in case.roles + ] + + +def _assert_all_equal(results: dict[str, dict[str, object]]) -> None: + """Assert every arrangement's fingerprint equals the first one's, naming the field.""" + (first_id, first), *others = results.items() + for arrangement_id, result in others: + for field, value in first.items(): + assert result[field] == value, ( + f"{field} differs between {first_id} and {arrangement_id}" + ) + + +# ── arrangement: one input ────────────────────────────────────────────────── + + +@pytest.mark.parametrize(("case", "role"), _unary_params()) +def test_unary_result_does_not_depend_on_the_input_arrangement( + case: UnaryCase, role: str +) -> None: + results = { + arrangement.id: fingerprint(case.make_op()(make_source(MAIN, arrangement))) + for arrangement in arrangements(MAIN, roles=(role,)) + } + _assert_all_equal(results) + + +@pytest.mark.parametrize(("case", "role"), _unary_params()) +def test_unary_async_rows_equal_sync_rows_in_every_arrangement( + case: UnaryCase, role: str +) -> None: + for arrangement in arrangements(MAIN, roles=(role,)): + source = make_source(MAIN, arrangement) + sync = rows_fingerprint(list(case.make_op()(source).iter_data())) + assert rows_fingerprint(async_rows(case.make_op(), source)) == sync, ( + arrangement.id + ) + + +# ── order of a set argument ───────────────────────────────────────────────── + + +@dataclass(frozen=True) +class OrderCase: + """An operator built from each permutation of its names or mapping entries.""" + + label: str + items: tuple[Any, ...] + make_op: Callable[[Any], StaticOutputOperatorPod] + roles: tuple[str, ...] = ROLES + + +ORDER_CASES = [ + OrderCase("select_tag_columns", ("t1", "t2", "t3"), SelectTagColumns), + OrderCase("select_data_columns", ("d1", "d2", "d3"), SelectDataColumns), + OrderCase("drop_tag_columns", ("t1", "t3"), DropTagColumns), + OrderCase("drop_data_columns", ("d1", "d3"), DropDataColumns), + # Tag and data names in one call. + OrderCase("select_columns", ("t1", "d3", "t3", "d1"), SelectColumns), + OrderCase("drop_columns", ("t1", "d3", "d1"), DropColumns), + OrderCase("drop_nulls", ("d2", "t2", "d1"), DropNulls), + OrderCase("map_tags", (("t1", "u1"), ("t2", "u2"), ("t3", "u3")), MapTags), + # A swap: each target is another entry's source. + OrderCase("map_tags_swap", (("t1", "t3"), ("t3", "t1")), MapTags, roles=(PLAIN,)), + OrderCase("map_data", (("d1", "e1"), ("d2", "e2"), ("d3", "e3")), MapData), + OrderCase("map_data_swap", (("d1", "d3"), ("d3", "d1")), MapData), + OrderCase( + "map_data_drop_unmapped", + (("d1", "e1"), ("d3", "e3")), + lambda items: MapData(items, drop_unmapped=True), + ), + OrderCase( + "map_columns", + (("t1", "u1"), ("d2", "e2"), ("t3", "u3")), + MapColumns, + roles=(PLAIN,), + ), + OrderCase( + "polars_filter_constraints", + (("t1", "a"), ("t2", 1), ("t3", 1)), + lambda items: PolarsFilter(constraints=dict(items)), + ), +] + + +def _make(case: OrderCase, items: tuple[Any, ...]) -> StaticOutputOperatorPod: + """Build the operator from ``items``: a dict of pairs, or a list of names.""" + if isinstance(case.items[0], tuple): + return case.make_op(dict(items)) + return case.make_op(list(items)) + + +def _order_params() -> list[object]: + return [ + pytest.param(case, role, id=f"{case.label}/{role}") + for case in ORDER_CASES + for role in case.roles + ] + + +@pytest.mark.parametrize(("case", "role"), _order_params()) +def test_the_order_of_a_set_argument_changes_nothing( + case: OrderCase, role: str +) -> None: + """Every permutation: same rows, hashes, column names AND column order.""" + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + results: dict[str, dict[str, object]] = {} + for items in itertools.permutations(case.items): + op = _make(case, items) + stream = op(make_source(MAIN, arrangement)) + results[repr(items)] = { + **fingerprint(stream), + "presentation": presentation(stream), + "operator_identity": op.content_hash().to_string(), + "async_rows": rows_fingerprint( + async_rows(op, make_source(MAIN, arrangement)) + ), + } + _assert_all_equal(results) + + +def test_polars_filter_predicates_in_any_order_choose_the_same_rows() -> None: + """Two predicates in either order keep the same rows (identity: see ITL-1236 doc).""" + predicates = (pl.col("d1") > 10, pl.col("t3") == 1, pl.col("d3") != "y") + rows = { + repr(order): rows_fingerprint( + list(PolarsFilter(predicates=list(order))(make_source(MAIN)).iter_data()) + ) + for order in itertools.permutations(predicates) + } + assert len({repr(v) for v in rows.values()}) == 1 + + +@pytest.mark.parametrize("role", ROLES) +def test_add_tags_order_is_the_order_of_the_new_tags_and_its_identity( + role: str, +) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + xy = AddTags({"x": 1, "y": "z"})(source) + yx = AddTags({"y": "z", "x": 1})(source) + assert xy.keys()[0] == ("t1", "t2", "t3", "x", "y") + assert yx.keys()[0] == ("t1", "t2", "t3", "y", "x") + assert xy.content_hash() != yx.content_hash() + assert xy.pipeline_hash() != yx.pipeline_hash() + # Same rows: the values of each row do not depend on the order. + assert rows_fingerprint(list(xy.iter_data())) == rows_fingerprint( + list(yx.iter_data()) + ) + + +# ── roles: names of several roles in one call ─────────────────────────────── + + +@dataclass(frozen=True) +class RoleCase: + """An operator naming columns of several roles, on ``MIXED``, and its result keys. + + In ``MIXED`` ``t2`` is a plain tag and ``t1``, ``t3`` are data-tags. + """ + + label: str + make_op: Callable[[], StaticOutputOperatorPod] + keys: tuple[tuple[str, ...], tuple[str, ...]] + + +ROLE_CASES = [ + # Dropping a data-tag's tag role keeps it as data; a plain tag is removed. + RoleCase( + "drop_tag_columns", + lambda: DropTagColumns(["t3", "t2"]), + (("t1",), ("t1", "t3", "d1", "d2", "d3")), + ), + # Dropping a data-tag's data role leaves a plain tag. + RoleCase( + "drop_data_columns", + lambda: DropDataColumns(["t3", "d1"]), + (("t1", "t2", "t3"), ("t1", "d2", "d3")), + ), + RoleCase( + "select_tag_columns", + lambda: SelectTagColumns(["t3", "t2"]), + # t1 lost its tag role: plain data, after the data-tag t3 (ITL-1205). + (("t2", "t3"), ("t3", "t1", "d1", "d2", "d3")), + ), + RoleCase( + "select_data_columns", + lambda: SelectDataColumns(["d2", "t3"]), + (("t1", "t2", "t3"), ("t3", "d2")), + ), + # Both roles at once: a data-tag kept or removed whole, a plain tag, data. + RoleCase( + "select_columns", + lambda: SelectColumns(["d2", "t3", "t2"]), + (("t2", "t3"), ("t3", "d2")), + ), + RoleCase( + "drop_columns", + lambda: DropColumns(["t3", "t2", "d1"]), + (("t1",), ("t1", "d2", "d3")), + ), + # A data-tag renamed in both roles; a plain tag renamed. + RoleCase( + "map_tags", + lambda: MapTags({"t3": "u3", "t2": "u2"}), + (("t1", "u2", "u3"), ("t1", "u3", "d1", "d2", "d3")), + ), + # A data-only move of a data-tag: the tag keeps its name, plain. + RoleCase( + "map_data", + lambda: MapData({"t3": "e3", "d1": "e1"}), + (("t1", "t2", "t3"), ("t1", "e3", "e1", "d2", "d3")), + ), + RoleCase( + "drop_nulls", + lambda: DropNulls(["t3", "t2", "d2"]), + (("t1", "t2", "t3"), ("t1", "t3", "d1", "d2", "d3")), + ), + RoleCase( + "polars_filter", + lambda: PolarsFilter(constraints={"t3": 1, "t2": 2, "d3": "w"}), + (("t1", "t2", "t3"), ("t1", "t3", "d1", "d2", "d3")), + ), +] + + +@pytest.mark.parametrize("case", ROLE_CASES, ids=lambda case: case.label) +def test_names_of_several_roles_in_one_call(case: RoleCase) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) + result = case.make_op()(source) + assert result.keys() == case.keys + assert rows_fingerprint(async_rows(case.make_op(), source)) == rows_fingerprint( + list(result.iter_data()) + ) + + +def _sorted_values(stream: StreamProtocol, name: str, role: str = "data") -> list[Any]: + """The values of column ``name`` of every row, sorted.""" + index = 0 if role == "tag" else 1 + values: list[Any] = [row[index][name] for row in stream.iter_data()] + return sorted(values) + + +def test_drop_nulls_of_several_roles_drops_a_row_with_a_null_in_any() -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) + kept = DropNulls(["t3", "t2", "d2"])(source) + assert _sorted_values(kept, "d1") == [10, 30, 40, 50] + + +def test_polars_filter_constraints_of_several_roles_are_all_applied() -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) + kept = PolarsFilter(constraints={"t1": "a", "t2": 1, "t3": 1})(source) + assert [data["d1"] for _, data in kept.iter_data()] == [50] + + +@pytest.mark.xfail( + strict=True, + reason="ITL-1242: MapTags refuses a swap of data-tags that it accepts for plain tags", + raises=InputValidationError, +) +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_map_tags_swaps_data_tags_as_it_swaps_plain_tags(role: str) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + swapped = MapTags({"t1": "t3", "t3": "t1"})(source) + plain = MapTags({"t1": "t3", "t3": "t1"})( + make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, PLAIN)) + ) + assert swapped.keys()[0] == plain.keys()[0] + assert _sorted_values(swapped, "t1", "tag") == _sorted_values(plain, "t1", "tag") + + +# ── several inputs: Join, MergeJoin, SemiJoin ─────────────────────────────── + + +JOIN_INPUTS: dict[str, tuple[Spec, ...]] = { + "2": (MAIN, P), + "3": (MAIN, P, Q), + "4": (MAIN, P, Q, R), +} + + +def _join_params() -> list[object]: + return [ + pytest.param(n, role, id=f"join_{n}/{role}") + for n in JOIN_INPUTS + for role in ROLES + ] + + +def _sources( + specs: tuple[Spec, ...], + order_index: int, + layout: str, + role: str, + rows: list[int] | None = None, +) -> list[ArrowTableSource]: + """Each spec in its ``order_index``-th tag order, in ``layout`` and ``role``.""" + return [ + make_source( + spec, + Arrangement( + spec.tag_orders()[order_index % len(spec.tag_orders())], layout, role + ), + rows=rows, + ) + for spec in specs + ] + + +@pytest.mark.parametrize(("n", "role"), _join_params()) +def test_join_result_does_not_depend_on_the_order_of_its_inputs( + n: str, role: str +) -> None: + """Every permutation of the inputs: same rows, hashes, system-tag names, order.""" + specs = JOIN_INPUTS[n] + results: dict[str, dict[str, object]] = {} + for permutation in itertools.permutations(range(len(specs))): + sources = _sources(specs, 0, TAGS_FIRST, role) + ordered = [sources[i] for i in permutation] + stream = Join()(*ordered) + results[repr(permutation)] = { + **fingerprint(stream), + "presentation": presentation(stream), + "system_tag_order": stream.keys(columns={"system_tags": True})[0], + "async_rows": rows_fingerprint(async_rows(Join(), *ordered)), + } + _assert_all_equal(results) + + +@pytest.mark.parametrize(("n", "role"), _join_params()) +def test_join_result_does_not_depend_on_the_arrangement_of_its_inputs( + n: str, role: str +) -> None: + specs = JOIN_INPUTS[n] + results: dict[str, dict[str, object]] = {} + for order_index in range(3): + for layout in ("tags_first", "data_first_reversed", "interleaved"): + sources = _sources(specs, order_index, layout, role) + stream = Join()(*sources) + results[f"{order_index}/{layout}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(Join(), *sources)), + } + _assert_all_equal(results) + + +def test_join_of_four_inputs_matches_on_every_shared_tag() -> None: + sources = _sources(JOIN_INPUTS["4"], 0, TAGS_FIRST, PLAIN) + joined = Join()(*sources) + rows = sorted( + ( + tag["t1"], + tag["t2"], + tag["t3"], + data["d1"], + data["p1"], + data["q2"], + data["r1"], + ) + for tag, data in joined.iter_data() + ) + # MAIN x P on (t1, t2), x Q on (t2, t3), x R on t1: every MAIN row has one match. + assert rows == [ + ("a", 1, 1, 50, 9, 400, -2), + ("a", 1, 2, 20, 9, 100, -2), + ("a", 2, 1, 30, 7, 200, -2), + ("b", 1, 2, 40, 8, 100, -1), + ("b", 2, 1, 10, 6, 200, -1), + ("b", 2, 2, 60, 6, 300, -1), + ] + + +# MergeJoin: two inputs with the same columns; every data column collides. The rows +# of ``MAIN`` without a null (``NON_NULL_ROWS``), and other values in every data column, +# so no merged pair ties: a tie is ITL-1241, a null ``MERGE_NULL_ISSUE``. +NON_NULL_ROWS = [0, 2, 3, 4] +MAIN_OTHER = Spec( + source_id="s2", + tags=MAIN.tags, + data=MAIN.data, + columns={ + **{tag: MAIN.columns[tag] for tag in MAIN.tags}, + "d1": pa.array([60, 55, 45, 35, 25, 15], pa.int64()), + "d2": pa.array([9.5, None, 7.5, 6.5, 5.5, 1.5], pa.float64()), + "d3": pa.array(["a", "b", "c", "d", "e", "f"], pa.large_string()), + }, + mixed_plain=MAIN.mixed_plain, +) + + +@pytest.mark.parametrize("role", ROLES) +def test_merge_join_result_does_not_depend_on_the_order_of_its_inputs( + role: str, +) -> None: + """Three colliding data columns: either input order, any tag order. + + Swapping the inputs changes nothing, not even the column order. Another tag order + of the inputs changes only the column order. + """ + results: dict[str, dict[str, object]] = {} + for order_index in range(3): + left, right = _sources( + (MAIN, MAIN_OTHER), order_index, TAGS_FIRST, role, rows=NON_NULL_ROWS + ) + ab, ba = MergeJoin()(left, right), MergeJoin()(right, left) + assert presentation(ab) == presentation(ba) + for label, inputs, stream in ( + ("ab", (left, right), ab), + ("ba", (right, left), ba), + ): + results[f"{order_index}/{label}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), + } + _assert_all_equal(results) + + +@pytest.mark.xfail( + strict=True, + reason="ITL-1241: MergeJoin orders tied values' source info by input position", +) +@pytest.mark.parametrize("role", ROLES) +def test_merge_join_of_tied_values_does_not_depend_on_the_order_of_its_inputs( + role: str, +) -> None: + """Two inputs with equal values in three colliding columns (ITL-1241).""" + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + left = make_source(MAIN, arrangement, source_id="s", rows=NON_NULL_ROWS) + right = make_source(MAIN, arrangement, source_id="s2", rows=NON_NULL_ROWS) + assert fingerprint(MergeJoin()(left, right)) == fingerprint( + MergeJoin()(right, left) + ) + + +MERGE_NULL_ISSUE = "ITL-1236 defect 3, issue pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{MERGE_NULL_ISSUE}: MergeJoin raises TypeError on a null in a merged column", + raises=TypeError, +) +@pytest.mark.parametrize("role", ROLES) +def test_merge_join_of_a_nullable_column_holding_nulls(role: str) -> None: + """``d2`` holds a null in rows 1 and 5 of ``MAIN``; the merge must not crash.""" + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + left = make_source(MAIN, arrangement) + right = make_source(MAIN_OTHER, arrangement) + assert fingerprint(MergeJoin()(left, right)) == fingerprint( + MergeJoin()(right, left) + ) + + +@pytest.mark.parametrize("role", ROLES) +def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> None: + """The right input's tag order and layout never change which rows are kept.""" + results: dict[str, dict[str, object]] = {} + for order_index in range(3): + for layout in ("tags_first", "data_first_reversed", "interleaved"): + left, right = _sources((MAIN, P), order_index, layout, role) + stream = SemiJoin()(left, right) + results[f"{order_index}/{layout}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(SemiJoin(), left, right)), + } + _assert_all_equal(results) + left, right = _sources((MAIN, P), 0, TAGS_FIRST, role) + kept = SemiJoin()( + left, make_source(P, Arrangement(P.tags, TAGS_FIRST, role), rows=[0, 1]) + ) + # P's first two rows are (a, 2) and (b, 1): matched on t1 AND t2. + assert _sorted_values(kept, "d1") == [30, 40] From 03f4a26ac4c14afd3aec34d0557ea05852223bd4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:17:56 -0800 Subject: [PATCH 02/13] test(sources, function_pod, pipeline): several tags, keys, inputs and bound sources, in every order and role (ITL-1236) Sources: tag order and layout enter no hash; tag_only_columns order changes nothing; record ids follow the key order (content hash moves, pipeline hash and source id do not); tags_as_data order is the new columns' order only; CachedSource reads back every role and arrangement. Function pods: every order of three inputs gives the same rows, hashes and columns, and a job over another order finds every result stored, sync and async; parameters match by name; a function reads several data-tags in any arrangement. Bind: two sources in one call, one refused leaves both unbound; a source in another arrangement binds without moving anything. Defects found are xfail(strict=True) with the issue pending: the fall-through tags' order in a function pod's identity (defect 6), swapped output_keys keep the uri (defect 4), a mixed-role source in another order breaks the bound node's table (defect 5). Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 21 +- .../function_pod/test_several_items.py | 215 ++++++++++++++++++ tests/test_core/operators/several_items.py | 29 ++- tests/test_core/sources/test_several_items.py | 169 ++++++++++++++ .../test_pipeline/test_bind_several_items.py | 152 +++++++++++++ 5 files changed, 578 insertions(+), 8 deletions(-) create mode 100644 tests/test_core/function_pod/test_several_items.py create mode 100644 tests/test_core/sources/test_several_items.py create mode 100644 tests/test_pipeline/test_bind_several_items.py diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index ffd91122e..ae870753b 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -99,6 +99,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | pending | | 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | pending (decision) | | 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | pending | +| 6 | A function pod over data-tags (`FallThroughPod`) hashes its fall-through tags in the input's tag order: two sources that differ only in tag order give the function node another content and pipeline hash | pipeline hash, pipeline table path | pending | | Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | question | ## Gaps, ranked @@ -121,9 +122,27 @@ Order- or role-dependent behaviour that reaches a hash or a stored table first. 14. `DerivedSource(tags_as_data=)` reversed: default `source_id` and pipeline hash. 15. `CachedSource` over a data-tag source; `Pick` / `Index` from a data-tag with `out=`; schema arguments in another field order; the stream methods' several-argument forms (`polars_filter` with `constraint_map` and keywords). +## Checked and found as documented (part 2) + +- A source's `tag_columns` order and table layout: keys follow the tag order; rows, record + ids, system tags, content and pipeline hashes do not change (plain, data-tags, mixed). +- `tag_only_columns` given in two orders: no change. +- `record_id_columns` in another key order: record ids follow the key order; the content + hash changes, the pipeline hash and the source id do not; a set is refused. +- `DerivedSource(tags_as_data=)` reversed: only the new columns' order changes. +- `CachedSource` over a source of each role and arrangement: reads back as the source. +- `pod(a, b, c)` in every input order: same rows, hashes and columns; a job over the inputs + in another order finds every result stored (sync and async). The in-memory output's + source info names a fresh random record per computation, so comparisons leave it out. +- `bind(sources=)` with two sources: both bound, nothing moves; one refused, neither bound. + A plain or data-tag source in any arrangement binds and reads; a mixed one is defect 5. + ## Tests added `tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`, `t2`, `t3`; data `d1`, `d2`, `d3`) and join partners `P`, `Q`, `R`, each in any tag order, three layouts and three role settings. `ops/test_several_items.py` uses it for every -operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). +operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). Part 2: +`tests/test_core/sources/test_several_items.py` (gaps 4, 13, 14, 15: CachedSource), +`tests/test_core/function_pod/test_several_items.py` (gaps 1, 10; defect 6) and +`tests/test_pipeline/test_bind_several_items.py` (gap 8; defect 5). diff --git a/tests/test_core/function_pod/test_several_items.py b/tests/test_core/function_pod/test_several_items.py new file mode 100644 index 000000000..24ed87f0b --- /dev/null +++ b/tests/test_core/function_pod/test_several_items.py @@ -0,0 +1,215 @@ +"""Function pods over several inputs, reading several parameters, with several outputs (ITL-1236). + +The rules tested: + +* ``pod(a, b, c)`` joins its inputs as ``Join`` does, so the order of the inputs changes + nothing: rows, hashes, system-tag names and column order, in memory and in a job (a + second job over the inputs in another order finds every result stored). +* Data columns that the function does not read are dropped by the automatic select + (ITL-807), whatever their number and position. +* The function's parameters are matched by name, never by position. +* A data-tag is a data column a function may read (ITL-1023). +""" + +from __future__ import annotations + +import itertools + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +from ..operators.several_items import ( + DATA_TAGS, + LAYOUTS, + MAIN, + MIXED, + ROLES, + TAGS_FIRST, + Arrangement, + P, + Q, + Spec, + fingerprint, + make_source, + presentation, + rows_fingerprint, +) + +INPUTS: tuple[Spec, ...] = (MAIN, P, Q) + + +# Parameters in another order than the columns: q2 is in Q, d1 in MAIN, p1 in P. +@function_pod(output_keys=["total", "product"]) +def combine(q2: int, d1: int, p1: int) -> tuple[int, int]: + return q2 + d1 + p1, d1 * p1 + + +def _inputs(role: str) -> list[ArrowTableSource]: + return [ + make_source(spec, Arrangement(spec.tags, TAGS_FIRST, role)) for spec in INPUTS + ] + + +@pytest.mark.parametrize("role", ROLES) +def test_the_order_of_the_inputs_changes_nothing(role: str) -> None: + """All six orders of three inputs: same rows, hashes and columns. + + Each in-memory computation names a fresh record in its outputs' source info, so the + rows are compared without it. + """ + results = {} + for permutation in itertools.permutations(range(len(INPUTS))): + inputs = _inputs(role) + stream = combine.pod(*(inputs[i] for i in permutation)) + results[permutation] = { + **fingerprint(stream, source_info=False), + "presentation": presentation(stream), + } + first = results[(0, 1, 2)] + for permutation, result in results.items(): + assert result == first, permutation + # The extra data columns (d2, d3, p2, q1) were dropped; every tag stays. + tag_keys, data_keys = first["presentation"] + assert sorted(tag_keys) == ["t1", "t2", "t3"] + assert {"d2", "d3", "p2", "q1"}.isdisjoint(data_keys) + assert {"total", "product"} <= set(data_keys) + + +@pytest.mark.parametrize("role", ROLES) +def test_values_are_matched_by_parameter_name(role: str) -> None: + stream = combine.pod(*_inputs(role)) + rows = sorted( + (tag["t1"], tag["t2"], tag["t3"], data["total"], data["product"]) + for tag, data in stream.iter_data() + ) + # MAIN x P on (t1, t2) x Q on (t2, t3); total = q2 + d1 + p1, product = d1 * p1. + assert rows == [ + ("a", 1, 1, 400 + 50 + 9, 50 * 9), + ("a", 1, 2, 100 + 20 + 9, 20 * 9), + ("a", 2, 1, 200 + 30 + 7, 30 * 7), + ("b", 1, 2, 100 + 40 + 8, 40 * 8), + ("b", 2, 1, 200 + 10 + 6, 10 * 6), + ("b", 2, 2, 300 + 60 + 6, 60 * 6), + ] + + +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +@pytest.mark.parametrize("role", ROLES) +def test_a_job_over_the_inputs_in_another_order_finds_every_result_stored( + role: str, orchestrator: type +) -> None: + database = InMemoryArrowDatabase() + stored = {} + for permutation in ((0, 1, 2), (2, 0, 1), (1, 2, 0)): + inputs = _inputs(role) + job = PipelineJob(name="several", database=database) + with job: + combine.pod(*(inputs[i] for i in permutation), label="combine") + summary = job.run(orchestrator=orchestrator()) + node = job.nodes["combine"] + stored[permutation] = ( + rows_fingerprint(list(node.iter_data()), source_info=False), + node.pipeline_hash(), + node.content_hash(), + presentation(node), + ) + if permutation != (0, 1, 2): + assert summary.nodes["combine"].computed == 0, permutation + assert summary.nodes["combine"].reused == 6, permutation + first = stored[(0, 1, 2)] + assert all(result == first for result in stored.values()) + + +@function_pod(output_keys="labelled") +def label_rows(t1: str, d3: str, t3: int) -> str: + return f"{t1}{t3}{d3}" + + +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_a_function_reads_several_data_tags_and_data_in_any_arrangement( + role: str, +) -> None: + """t1 and t3 are data-tags; the tag order and layout change no row.""" + results = {} + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + stream = label_rows.pod(make_source(MAIN, Arrangement(order, layout, role))) + results[(order, layout)] = rows_fingerprint( + list(stream.iter_data()), source_info=False + ) + values = sorted(str(data["labelled"]) for _, data in stream.iter_data()) + assert values == ["a1v", "a1y", "a2x", "b1w", "b2u", "b2z"] + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_a_layout_of_the_input_does_not_change_a_fall_through_pods_hashes( + role: str, +) -> None: + hashes = { + layout: fingerprint( + label_rows.pod(make_source(MAIN, Arrangement(MAIN.tags, layout, role))), + source_info=False, + ) + for layout in LAYOUTS + } + first = next(iter(hashes.values())) + assert all(result == first for result in hashes.values()) + + +FALL_THROUGH_ORDER_ISSUE = "ITL-1236 defect 6, issue pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{FALL_THROUGH_ORDER_ISSUE}: the fall-through tags enter the pod's " + + "identity in the input's tag order", +) +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_the_tag_order_of_the_input_does_not_change_a_fall_through_pods_hashes( + role: str, +) -> None: + results = { + order: fingerprint( + label_rows.pod(make_source(MAIN, Arrangement(order, TAGS_FIRST, role))), + source_info=False, + ) + for order in MAIN.tag_orders() + } + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +OUTPUT_KEYS_ISSUE = "ITL-1236 defect 4, decision pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{OUTPUT_KEYS_ISSUE}: the order of output_keys is not in the identity", +) +def test_swapped_output_keys_are_another_function() -> None: + """Swapped output keys map the results to other names: not the same function. + + Today both have one uri, so a job serves one's stored values under the other's + names. + """ + + def make(keys: list[str]): + @function_pod(output_keys=keys) + def stats(d1: int, t2: int) -> tuple[int, int]: + return d1 + t2, d1 * t2 + + return stats + + forward, backward = make(["sum", "product"]), make(["product", "sum"]) + assert forward.pod.uri != backward.pod.uri diff --git a/tests/test_core/operators/several_items.py b/tests/test_core/operators/several_items.py index 49642b438..c405ab0c8 100644 --- a/tests/test_core/operators/several_items.py +++ b/tests/test_core/operators/several_items.py @@ -218,28 +218,43 @@ def make_source( ) -def row_fingerprint(tag: TagProtocol, data: DataProtocol) -> tuple[str, ...]: - """What identifies one row regardless of column order: values, system tags, hashes.""" +def row_fingerprint( + tag: TagProtocol, data: DataProtocol, *, source_info: bool = True +) -> tuple[str, ...]: + """What identifies one row regardless of column order: values, system tags, hashes. + + Args: + tag: The row's tag. + data: The row's data. + source_info: Include the data's source info. A function's in-memory output + names a fresh random record per computation, so a comparison of two + computations leaves it out. + """ return ( repr(sorted(tag.as_dict().items())), repr(sorted(data.as_dict().items())), repr(sorted(tag.system_tags().items())), tag.content_hash().to_string(), data.content_hash().to_string(), - repr(sorted(data.source_info().items())), + repr(sorted(data.source_info().items())) if source_info else "", ) -def rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: +def rows_fingerprint( + rows: Sequence[Row], *, source_info: bool = True +) -> list[tuple[str, ...]]: """The rows' fingerprints, sorted: row order is not part of it.""" - return sorted(row_fingerprint(tag, data) for tag, data in rows) + return sorted( + row_fingerprint(tag, data, source_info=source_info) for tag, data in rows + ) -def fingerprint(stream: StreamProtocol) -> dict[str, Any]: +def fingerprint(stream: StreamProtocol, *, source_info: bool = True) -> dict[str, Any]: """What must not depend on the input's arrangement or a set argument's order. Args: stream: An operator's result. + source_info: Include each row's source info (see ``row_fingerprint``). Returns: The rows (``rows_fingerprint``), the content and pipeline hashes, and the @@ -247,7 +262,7 @@ def fingerprint(stream: StreamProtocol) -> dict[str, Any]: """ tag_keys, data_keys = stream.keys(columns={"system_tags": True}) return { - "rows": rows_fingerprint(list(stream.iter_data())), + "rows": rows_fingerprint(list(stream.iter_data()), source_info=source_info), "content_hash": stream.content_hash().to_string(), "pipeline_hash": stream.pipeline_hash().to_string(), "tag_names": sorted(tag_keys), diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py new file mode 100644 index 000000000..ecfc017f7 --- /dev/null +++ b/tests/test_core/sources/test_several_items.py @@ -0,0 +1,169 @@ +"""Sources with several tag, key and data columns, in several orders and roles (ITL-1236). + +The rules tested (CLAUDE.md, "Important implementation details" and "Two identity +chains"): + +* The order of ``tag_columns`` is the order of the tags (and of the data-tags among the + data columns); it enters no hash, no record id and no system-tag column name. +* ``tag_only_columns`` names a set: its order changes nothing. +* ``record_id_columns`` is in key order: a record id is the canonical JSON of the key + columns in key order, so another key order is another source with the same pipeline + hash and source id, another content hash. An unordered collection is refused. +* ``DerivedSource(tags_as_data=...)``: the entries' order is the order of the new data + columns and nothing else. +* ``CachedSource`` reads back a source of each role as the source itself. +""" + +from __future__ import annotations + +import itertools +import json + +import pyarrow as pa +import pytest + +from orcapod.core.sources import ArrowTableSource, CachedSource, DerivedSource +from orcapod.databases import InMemoryArrowDatabase + +from ..operators.several_items import ( + DATA_TAGS, + LAYOUTS, + MAIN, + MIXED, + PLAIN, + ROLES, + TAGS_FIRST, + Arrangement, + fingerprint, + make_source, + presentation, +) + + +def _tag_only(role: str) -> bool | list[str]: + return {PLAIN: True, DATA_TAGS: [], MIXED: [MAIN.mixed_plain]}[role] + + +def _table() -> pa.Table: + return pa.table({name: MAIN.columns[name] for name in (*MAIN.tags, *MAIN.data)}) + + +def _record_ids(source: ArrowTableSource) -> list[str]: + ids: list[str] = [] + for tag, _ in source.iter_data(): + [record_id] = [ + value for name, value in tag.system_tags().items() if "record_id" in name + ] + ids.append(str(record_id)) + return sorted(ids) + + +@pytest.mark.parametrize("role", ROLES) +def test_tag_order_is_the_order_of_the_tags_and_enters_no_hash(role: str) -> None: + """Every tag order and layout: same rows, hashes and system tags; keys follow.""" + results = {} + for order in itertools.permutations(MAIN.tags): + for layout in LAYOUTS: + source = make_source(MAIN, Arrangement(order, layout, role)) + tag_keys, data_keys = presentation(source) + assert tag_keys == order + # Data-tags come first among the data columns, in tag order (ITL-1205). + data_tags = [t for t in order if role != PLAIN and t != _plain_tag(role)] + assert list(data_keys[: len(data_tags)]) == data_tags + results[(order, layout)] = fingerprint(source) + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +def _plain_tag(role: str) -> str | None: + return MAIN.mixed_plain if role == MIXED else None + + +def test_the_order_of_tag_only_columns_changes_nothing() -> None: + sources = [ + ArrowTableSource( + _table(), tag_columns=list(MAIN.tags), source_id="s", tag_only_columns=names + ) + for names in (["t1", "t3"], ["t3", "t1"]) + ] + assert presentation(sources[0]) == presentation(sources[1]) + assert presentation(sources[0]) == (("t1", "t2", "t3"), ("t2", "d1", "d2", "d3")) + assert fingerprint(sources[0]) == fingerprint(sources[1]) + + +@pytest.mark.parametrize("role", ROLES) +def test_record_ids_follow_the_key_order(role: str) -> None: + """Two or three key columns, of the tag and the data role, in several orders.""" + by_keys = {} + for keys in (("t1", "t2", "t3"), ("t3", "t1", "t2"), ("d3", "t1"), ("t1", "d3")): + source = ArrowTableSource( + _table(), + tag_columns=list(MAIN.tags), + source_id="s", + record_id_columns=list(keys), + tag_only_columns=_tag_only(role), + ) + ids = _record_ids(source) + # The canonical JSON of the key columns, in key order. + assert all(list(json.loads(record_id)) == list(keys) for record_id in ids) + by_keys[keys] = source + first = by_keys[("t1", "t2", "t3")] + for keys, source in by_keys.items(): + # The keys name the rows: one pipeline hash and one source id ... + assert source.pipeline_hash() == first.pipeline_hash() + assert source.source_id == first.source_id + # ... and another key order is another source. + if keys != ("t1", "t2", "t3"): + assert source.content_hash() != first.content_hash() + assert _record_ids(source) != _record_ids(first) + # Same keys, same order: the same source. + again = ArrowTableSource( + _table(), + tag_columns=list(MAIN.tags), + source_id="s", + record_id_columns=("t3", "t1", "t2"), + tag_only_columns=_tag_only(role), + ) + assert again.content_hash() == by_keys[("t3", "t1", "t2")].content_hash() + + +def test_record_id_columns_as_a_set_is_refused() -> None: + with pytest.raises(TypeError): + ArrowTableSource( + _table(), + tag_columns=list(MAIN.tags), + source_id="s", + record_id_columns={"t1", "t2"}, + ) + + +@pytest.mark.parametrize("role", ROLES) +def test_tags_as_data_order_is_the_order_of_the_new_columns_only(role: str) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + forward = DerivedSource(source, tags_as_data={"t1": "c1", "t3": "c3"}) + backward = DerivedSource(source, tags_as_data={"t3": "c3", "t1": "c1"}) + assert presentation(forward)[1][-2:] == ("c1", "c3") + assert presentation(backward)[1][-2:] == ("c3", "c1") + assert forward.source_id == backward.source_id + assert fingerprint(forward) == fingerprint(backward) + # Each new column holds its own tag's values. + for tag, data in forward.iter_data(): + assert (data["c1"], data["c3"]) == (tag["t1"], tag["t3"]) + + +@pytest.mark.parametrize("role", ROLES) +def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> None: + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + arrangement = Arrangement(order, layout, role) + database = InMemoryArrowDatabase() + cached = CachedSource( + make_source(MAIN, arrangement), cache_database=database + ) + assert fingerprint(cached) == fingerprint(make_source(MAIN, arrangement)) + # A second CachedSource reads the stored rows back. + again = CachedSource( + make_source(MAIN, arrangement), cache_database=database + ) + assert presentation(again) == presentation(make_source(MAIN, arrangement)) + assert fingerprint(again) == fingerprint(cached) diff --git a/tests/test_pipeline/test_bind_several_items.py b/tests/test_pipeline/test_bind_several_items.py new file mode 100644 index 000000000..3ddfc28c0 --- /dev/null +++ b/tests/test_pipeline/test_bind_several_items.py @@ -0,0 +1,152 @@ +"""Binding several sources, and sources in another column arrangement (ITL-1236). + +``PipelineJob.bind(sources=...)`` takes several sources in one call. Every source is +validated before anything changes, so one refused source leaves every binding as it was +(the ``bind`` docstring). A source whose columns are the slot's in another order (tag +order, table layout) is accepted, and the bind changes no node's hash or columns +(CLAUDE.md, "Important implementation details", ITL-1112). +""" + +from __future__ import annotations + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import SourceSpecMismatchError +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +from ..test_core.operators.several_items import ( + DATA_TAGS, + LAYOUTS, + MAIN, + MIXED, + PLAIN, + TAGS_FIRST, + Arrangement, + P, + make_source, + presentation, +) + + +@function_pod(output_keys="total") +def total(d1: int, p1: int) -> int: + return d1 + p1 + + +def _job(role: str) -> PipelineJob: + """A job over slots ``s`` (``MAIN``) and ``p`` (``P``), joined by ``total``.""" + job = PipelineJob(name="bind", database=InMemoryArrowDatabase()) + with job: + total.pod( + make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)), + make_source(P, Arrangement(P.tags, TAGS_FIRST, role)), + label="total", + ) + return job + + +def _slot_names(job: PipelineJob) -> tuple[str, str]: + return ("s", "p") if {"s", "p"} <= set(job.nodes) else ("", "") + + +def _snapshot(job: PipelineJob) -> dict[str, object]: + return { + name: (node.pipeline_hash(), presentation(node)) + for name, node in job.nodes.items() + } + + +@pytest.mark.parametrize( + "orchestrator", + [SyncPipelineOrchestrator, AsyncPipelineOrchestrator], + ids=["sync", "async"], +) +@pytest.mark.parametrize("role", [PLAIN, DATA_TAGS, MIXED]) +def test_several_sources_bound_in_one_call(role: str, orchestrator: type) -> None: + job = _job(role) + assert _slot_names(job) == ("s", "p") + before = _snapshot(job) + arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) + job.bind( + sources={ + "s": make_source(MAIN, arrangement, source_id="s_new", rows=[0, 1]), + "p": make_source( + P, Arrangement(P.tags, TAGS_FIRST, role), source_id="p_new" + ), + } + ) + assert _snapshot(job) == before + job.run(orchestrator=orchestrator()) + rows = sorted( + (tag["t1"], tag["t2"], data["total"]) + for tag, data in job.nodes["total"].iter_data() + ) + # Rows 0 and 1 of MAIN: (b, 2, d1=10) with p1=6; (a, 1, d1=20) with p1=9. + assert rows == [("a", 1, 29), ("b", 2, 16)] + + +@pytest.mark.parametrize("refused", ["s", "p"]) +def test_one_refused_source_leaves_every_binding_as_it_was(refused: str) -> None: + """Two sources in one call, one of them refused: neither is bound.""" + job = _job(PLAIN) + bound_before = {name: job.nodes[name].bound_source for name in ("s", "p")} + sources: dict[str, StreamProtocol] = { + "s": make_source(MAIN, source_id="s_new"), + "p": make_source(P, source_id="p_new"), + } + # The other slot's source does not fit: other column names. + sources[refused] = make_source(P if refused == "s" else MAIN, source_id="wrong") + with pytest.raises(SourceSpecMismatchError): + job.bind(sources=sources) + assert {name: job.nodes[name].bound_source for name in ("s", "p")} == bound_before + + +def _bind_in(role: str, order: tuple[str, ...], layout: str) -> None: + """Bind ``MAIN`` in another arrangement to slot ``s``: nothing moves, all reads.""" + job = _job(role) + before = _snapshot(job) + # The columns of the table, not their order: CLAUDE.md promises the declared names, + # roles and types; the column order within the table is a convenience only. + table_before = sorted(job.nodes["s"].as_table().column_names) + job.bind( + sources={ + "s": make_source(MAIN, Arrangement(order, layout, role), source_id="s_new") + } + ) + assert _snapshot(job) == before, (order, layout) + assert sorted(job.nodes["s"].as_table().column_names) == table_before, ( + order, + layout, + ) + job.run() + assert len(job.nodes["total"].as_table()) == 6 + + +@pytest.mark.parametrize("role", [PLAIN, DATA_TAGS]) +def test_a_source_in_another_arrangement_is_bound_as_the_slot_declares( + role: str, +) -> None: + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + _bind_in(role, order, layout) + + +BIND_MIXED_ISSUE = "ITL-1236 defect 5, issue pending" + + +@pytest.mark.xfail( + strict=True, + reason=f"{BIND_MIXED_ISSUE}: a mixed-role source in another column order is " + + "accepted, then the source node's table repeats a column", + raises=KeyError, +) +def test_a_mixed_role_source_in_another_arrangement_is_bound_as_the_slot_declares() -> ( + None +): + for order in MAIN.tag_orders(): + for layout in LAYOUTS: + _bind_in(MIXED, order, layout) From e8ae6217c5a4b37ca1430e47ee545a37a0e94400 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:19:08 -0800 Subject: [PATCH 03/13] test(operators): async rows equal sync rows in every several-input case; MergeJoin and the restricted SemiJoin in every layout (ITL-1236) Addresses the Copilot review of #442. Co-Authored-By: Claude Opus 5.5 --- .../test_core/operators/test_several_items.py | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index 48432323b..250346a0a 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -59,6 +59,7 @@ from .several_items import ( DATA_TAGS, + LAYOUTS, MAIN, MIXED, P, @@ -141,7 +142,17 @@ def _unary_params() -> list[object]: def _assert_all_equal(results: dict[str, dict[str, object]]) -> None: - """Assert every arrangement's fingerprint equals the first one's, naming the field.""" + """Assert every arrangement's fingerprint equals the first one's, naming the field. + + A result that holds ``async_rows`` must also have them equal to its own sync + ``rows``: equal across arrangements alone would pass an async path that is wrong + the same way every time. + """ + for result_id, result in results.items(): + if "async_rows" in result: + assert result["async_rows"] == result["rows"], ( + f"async rows differ from sync rows in {result_id}" + ) (first_id, first), *others = results.items() for arrangement_id, result in others: for field, value in first.items(): @@ -472,7 +483,7 @@ def test_join_result_does_not_depend_on_the_arrangement_of_its_inputs( specs = JOIN_INPUTS[n] results: dict[str, dict[str, object]] = {} for order_index in range(3): - for layout in ("tags_first", "data_first_reversed", "interleaved"): + for layout in LAYOUTS: sources = _sources(specs, order_index, layout, role) stream = Join()(*sources) results[f"{order_index}/{layout}"] = { @@ -530,26 +541,27 @@ def test_join_of_four_inputs_matches_on_every_shared_tag() -> None: def test_merge_join_result_does_not_depend_on_the_order_of_its_inputs( role: str, ) -> None: - """Three colliding data columns: either input order, any tag order. + """Three colliding data columns: either input order, any tag order and layout. Swapping the inputs changes nothing, not even the column order. Another tag order - of the inputs changes only the column order. + or layout of the inputs changes only the column order. """ results: dict[str, dict[str, object]] = {} for order_index in range(3): - left, right = _sources( - (MAIN, MAIN_OTHER), order_index, TAGS_FIRST, role, rows=NON_NULL_ROWS - ) - ab, ba = MergeJoin()(left, right), MergeJoin()(right, left) - assert presentation(ab) == presentation(ba) - for label, inputs, stream in ( - ("ab", (left, right), ab), - ("ba", (right, left), ba), - ): - results[f"{order_index}/{label}"] = { - **fingerprint(stream), - "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), - } + for layout in LAYOUTS: + left, right = _sources( + (MAIN, MAIN_OTHER), order_index, layout, role, rows=NON_NULL_ROWS + ) + ab, ba = MergeJoin()(left, right), MergeJoin()(right, left) + assert presentation(ab) == presentation(ba) + for label, inputs, stream in ( + ("ab", (left, right), ab), + ("ba", (right, left), ba), + ): + results[f"{order_index}/{layout}/{label}"] = { + **fingerprint(stream), + "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), + } _assert_all_equal(results) @@ -594,7 +606,7 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N """The right input's tag order and layout never change which rows are kept.""" results: dict[str, dict[str, object]] = {} for order_index in range(3): - for layout in ("tags_first", "data_first_reversed", "interleaved"): + for layout in LAYOUTS: left, right = _sources((MAIN, P), order_index, layout, role) stream = SemiJoin()(left, right) results[f"{order_index}/{layout}"] = { @@ -602,9 +614,14 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N "async_rows": rows_fingerprint(async_rows(SemiJoin(), left, right)), } _assert_all_equal(results) - left, right = _sources((MAIN, P), 0, TAGS_FIRST, role) - kept = SemiJoin()( - left, make_source(P, Arrangement(P.tags, TAGS_FIRST, role), rows=[0, 1]) - ) - # P's first two rows are (a, 2) and (b, 1): matched on t1 AND t2. - assert _sorted_values(kept, "d1") == [30, 40] + # A right input without every (t1, t2) pair tells a match on both keys from a + # match on one, or none: P's first two rows are (a, 2) and (b, 1). + for order_index in range(3): + for layout in LAYOUTS: + left, _ = _sources((MAIN, P), order_index, layout, role) + [right] = _sources((P,), order_index, layout, role, rows=[0, 1]) + kept = SemiJoin()(left, right) + assert _sorted_values(kept, "d1") == [30, 40] + assert rows_fingerprint(async_rows(SemiJoin(), left, right)) == ( + rows_fingerprint(list(kept.iter_data())) + ) From 8129cfa1e3624642a90c7c5db75bacae0d6ba473 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:20:27 -0800 Subject: [PATCH 04/13] test(operators): issue ids for the defects found (ITL-1243, ITL-1247); predicates in every order and role, sync and async (ITL-1236) Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 22 ++++----- .../test_core/operators/test_several_items.py | 49 ++++++++++++++----- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index ffd91122e..aa67452d9 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -50,7 +50,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | `Join` inputs, 4+ | async only: `tests/test_channels/test_native_async_operators.py::TestJoinNativeAsync::test_four_way_join` | no → NEW (all 24 permutations) | no → NEW (plain, data-tags, mixed) | no → NEW | no sync comparison → NEW | | `Join` shared keys, several | schema only: `tests/test_pipeline/test_zero_row_column_order.py` step `join` | no (keys never in another order in one input) → NEW (tag orders per input) | plain, data-tag | no → NEW | schema only → NEW | | `MergeJoin` colliding columns | yes: `ops/test_merge_join.py::TestMergeJoinBasic::test_values_sorted_independently_per_column`; ASE `merge_source` | left/right only, distinct values → NEW `test_merge_join_result_does_not_depend_on_the_order_of_its_inputs`; ties: **defect ITL-1241** | yes: ASE `merge_join`, `ref_merge_join_*` | no → NEW | yes: ASE | -| `MergeJoin` nullable colliding column with a null | no | n/a | n/a | n/a | n/a: **defect 3** (raises `TypeError`) → NEW xfail | +| `MergeJoin` nullable colliding column with a null | no | n/a | n/a | n/a | n/a: **ITL-1243** (raises `TypeError`) → NEW xfail | | `SemiJoin` match keys | rows with 1 key only (`ops/test_semijoin_tag_keys.py::test_semijoin_keys_on_shared_tags_only`) | no → NEW `test_semijoin_matches_on_every_shared_tag_in_any_arrangement` | yes: plain, `ref_semijoin`, mixed | no → NEW | 1 key: `test_semijoin_tag_keys.py::test_async_rows_equal_sync_rows` → NEW (2 keys) | | `Batch` (tags and data become lists) | yes: ASE `batch_2` (2 tags, 9 data) | n/a (no column argument); member order is identity: `WHF::TestBatchGroupByFold::test_member_order_is_identity` | yes: `dt/test_batch.py` | no → NEW `test_unary_result_does_not_depend_on_the_input_arrangement[batch*]` | yes: ASE | | `GroupBy` `by`, member order | yes: `ops/test_group_by.py::TestGroupByShape::test_one_row_per_distinct_key` | keys only on `main` | yes | no on `main` | 1 key on `main` | @@ -66,7 +66,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | `MapTags` name_map | **almost none**: one chained rename with `tag_only` (`dt/test_operators.py::test_map_tags_tag_only_keeps_a_chained_rename_of_another_tag`) → NEW | **no** → NEW | yes; a swap of data-tags is refused: **defect ITL-1242** → NEW xfail | no → NEW | single rename only → NEW | | `MapData` name_map | yes: `dt/test_operators.py::test_map_data_chained_rename_does_not_reach_the_data_tag_copy` | **no** → NEW (incl. a swap) | yes | no → NEW | single rename → NEW | | `MapColumns` name_map | yes: `ops/test_mappers_strict.py::test_map_columns_renames_tags_and_data`; ASE `map_columns` | **no** → NEW | plain; refused with data-tags (ITL-1022): `dt/test_operators.py::test_uncovered_operators_raise` | no → NEW | yes | -| `PolarsFilter` predicates | **no** (always one) → NEW | **no**; the order IS in the identity (`filters.py` hashes an ordered tuple): **question Q1** | data | no → NEW | yes | +| `PolarsFilter` predicates | **no** (always one) → NEW | **no**; the order IS in the identity (`filters.py` hashes an ordered tuple): **ITL-1247** | data | no → NEW | yes | | `PolarsFilter` constraints | **no** (never two keys) → NEW | **no** → NEW (not in identity) | tag, data, data-tag | no → NEW | yes | | `stream.polars_filter(*predicates, constraint_map, **constraints)` | **no** | no | — | no | sync only | | `DropNulls` columns | yes: `ops/test_drop_nulls.py::test_tag_and_data_columns_in_one_call` | hashes: `test_identity_is_the_set_of_names`; rows → NEW | tag, data; a nullable data-tag holding a null: **no** | one layout → NEW | yes: ASE `drop_nulls_many` | @@ -84,10 +84,10 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | `CachedSource` | 1 tag | no | **plain only** (data-tag read-back untested) | no | no | | `DerivedSource(tags_as_data=)` | yes: `sources/test_derived_source_tags_as_data.py::TestColumns::test_several_copies_follow_map_order` | content hash only: `TestIdentity::test_mapping_order_does_not_change_content_hash` | plain; data-tag origin `dt/test_derived_source.py::test_tags_as_data_over_a_data_tag_node` | one arrangement | yes | | function pod, several inputs | 2: `function_pod/test_function_pod_auto_select.py::test_call_selects_once_after_the_join`; 3: `dt/test_edi_end_to_end.py` | **no**: `pod(b, a)` never compared with `pod(a, b)` | data-tag parameter (EDI) | no | yes | -| `output_keys`, several | yes: `data_function/test_data_function.py::test_multiple_output_keys` | **no**; order is not in the identity: **defect or decision 4** | — | — | value only | +| `output_keys`, several | yes: `data_function/test_data_function.py::test_multiple_output_keys` | **no**; order is not in the identity: **ITL-1244** | — | — | value only | | automatic select of several extra columns | yes: `test_dropping_logs_one_info_message_naming_the_columns` | no (kept list sorted) | one data-tag falls through | no | yes | | `PipelineJob.bind(sources=)`, several | yes: `test_pipeline/test_pipeline_job.py::TestPipelineJobBind::test_bind_sources_mutates_job` (membership) | n/a (dict order unused) | plain only | — | — | -| bind a source in another column order | yes: `test_bind_type_check.py::test_columns_match_by_name_within_each_group_whatever_their_order` (plain, run ok only) | yes (plain) | **mixed roles: defect 5** (accepted, then the node's table breaks) | partly | async read in `test_bind_narrower_source.py::test_columns_follow_the_declared_order` | +| bind a source in another column order | yes: `test_bind_type_check.py::test_columns_match_by_name_within_each_group_whatever_their_order` (plain, run ok only) | yes (plain) | **mixed roles: ITL-1245** (accepted, then the node's table breaks) | partly | async read in `test_bind_narrower_source.py::test_columns_follow_the_declared_order` | | `ColumnConfig` groups | all_info: `tests/test_types_column_config_all_info.py`; each group alone: `test_zero_row_column_order.py` | n/a | data-tag variants | one arrangement | yes | ## Defects and questions found @@ -96,23 +96,23 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest |---|---|---|---| | 1 | `MergeJoin(A, B)` and `MergeJoin(B, A)` give the source-info lists of tied values in argument order | stored source info (operator cache, downstream tables); not the hashes | ITL-1241 | | 2 | `MapTags` refuses a swap or cycle of data-tags that it accepts for plain tags (`MapData` is right: a data-only move) | error only | ITL-1242 | -| 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | pending | -| 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | pending (decision) | -| 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | pending | -| Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | question | +| 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | ITL-1243 | +| 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | ITL-1244 (rule undecided) | +| 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | ITL-1245 | +| Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | ITL-1247 (rule undecided) | ## Gaps, ranked Order- or role-dependent behaviour that reaches a hash or a stored table first. -1. `output_keys` order: not in identity, stored values served under swapped names (defect 4). +1. `output_keys` order: not in identity, stored values served under swapped names (ITL-1244). 2. `MergeJoin` tied values: source-info order follows the inputs (ITL-1241). -3. `PolarsFilter` predicate order in identity, constraint order not; never tested with two (Q1). +3. `PolarsFilter` predicate order in identity, constraint order not; never tested with two (ITL-1247). 4. `record_id_columns` key order: moves record ids, tokens and the content hash; never compared. 5. `Join` with 3 and 4 inputs: content and pipeline hash under every permutation; 4 inputs never run sync. Closed by NEW. 6. `MapTags` / `MapData` / `MapColumns`: several renames, mapping order, swaps, sync and async; `MapTags` has no golden entry. Closed by NEW (except golden: out of scope). 7. Column arrangement (d) never varied for any operator but `GroupBy` (PR #434). Closed by NEW for every operator. -8. Bind of a mixed-role source in another column order (defect 5); several sources with one refused. +8. Bind of a mixed-role source in another column order (ITL-1245); several sources with one refused. 9. `SelectTagColumns` / `DropTagColumns` with two or more names, in two orders. Closed by NEW. 10. `function_pod` over several inputs: `pod(b, a)` against `pod(a, b)` (hashes, system-tag names, stored rows). 11. `SemiJoin` with two or more keys, rows sync and async. Closed by NEW. diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index 250346a0a..b7576135b 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -269,18 +269,45 @@ def test_the_order_of_a_set_argument_changes_nothing( _assert_all_equal(results) -def test_polars_filter_predicates_in_any_order_choose_the_same_rows() -> None: - """Two predicates in either order keep the same rows (identity: see ITL-1236 doc).""" - predicates = (pl.col("d1") > 10, pl.col("t3") == 1, pl.col("d3") != "y") - rows = { - repr(order): rows_fingerprint( - list(PolarsFilter(predicates=list(order))(make_source(MAIN)).iter_data()) - ) - for order in itertools.permutations(predicates) - } +PREDICATES = (pl.col("d1") > 10, pl.col("t3") == 1, pl.col("d3") != "y") + + +@pytest.mark.parametrize("role", ROLES) +def test_polars_filter_predicates_in_any_order_choose_the_same_rows( + role: str, +) -> None: + """Three predicates over a data column, a tag and a data column, in every order.""" + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + rows = {} + for order in itertools.permutations(PREDICATES): + op = PolarsFilter(predicates=list(order)) + stream = op(source) + rows[repr(order)] = rows_fingerprint(list(stream.iter_data())) + assert rows_fingerprint(async_rows(op, source)) == rows[repr(order)] + assert _sorted_values(stream, "d1") == [50] assert len({repr(v) for v in rows.values()}) == 1 +@pytest.mark.xfail( + strict=True, + reason="ITL-1247 (rule undecided): the order of PolarsFilter's predicates is in " + + "its identity; proposed: predicates are AND-ed, so their order is not", +) +@pytest.mark.parametrize("role", ROLES) +def test_polars_filter_predicate_order_is_not_identity(role: str) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + results = { + repr(order): { + "operator_identity": PolarsFilter(predicates=list(order)) + .content_hash() + .to_string(), + **fingerprint(PolarsFilter(predicates=list(order))(source)), + } + for order in itertools.permutations(PREDICATES) + } + _assert_all_equal(results) + + @pytest.mark.parametrize("role", ROLES) def test_add_tags_order_is_the_order_of_the_new_tags_and_its_identity( role: str, @@ -521,7 +548,7 @@ def test_join_of_four_inputs_matches_on_every_shared_tag() -> None: # MergeJoin: two inputs with the same columns; every data column collides. The rows # of ``MAIN`` without a null (``NON_NULL_ROWS``), and other values in every data column, -# so no merged pair ties: a tie is ITL-1241, a null ``MERGE_NULL_ISSUE``. +# so no merged pair ties: a tie is ITL-1241, a null ITL-1243. NON_NULL_ROWS = [0, 2, 3, 4] MAIN_OTHER = Spec( source_id="s2", @@ -582,7 +609,7 @@ def test_merge_join_of_tied_values_does_not_depend_on_the_order_of_its_inputs( ) -MERGE_NULL_ISSUE = "ITL-1236 defect 3, issue pending" +MERGE_NULL_ISSUE = "ITL-1243" @pytest.mark.xfail( From ec9c140f7b61018fa43c74f3c32d4517b25fff88 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:24:29 -0800 Subject: [PATCH 05/13] test(operators): type the results helper by Mapping (ITL-1236) Co-Authored-By: Claude Opus 5.5 --- tests/test_core/operators/test_several_items.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index b7576135b..90de77f39 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -25,7 +25,7 @@ from __future__ import annotations import itertools -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Any @@ -141,7 +141,7 @@ def _unary_params() -> list[object]: ] -def _assert_all_equal(results: dict[str, dict[str, object]]) -> None: +def _assert_all_equal(results: Mapping[str, Mapping[str, object]]) -> None: """Assert every arrangement's fingerprint equals the first one's, naming the field. A result that holds ``async_rows`` must also have them equal to its own sync From 56213e863d95ba4ad9aee540ac5766041c7ebd46 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:25:17 -0800 Subject: [PATCH 06/13] test(data_tags, streams, sources): several nullable data-tags, list-valued data-tags, stream methods and column groups (ITL-1236) DropNulls naming plain tags, data-tags holding nulls and data in every order; Index with out= over a list-valued data-tag; a DictSource schema in another field order; every stream method given several items equals its operator (polars_filter keywords win over constraint_map); every combination of ColumnConfig groups keeps the all_info column order. Sync and async compared where an async path exists. All as documented. Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 23 ++- .../test_core/data_tags/test_several_items.py | 98 +++++++++ tests/test_core/sources/test_several_items.py | 43 +++- .../streams/test_several_items_methods.py | 192 ++++++++++++++++++ 4 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 tests/test_core/data_tags/test_several_items.py create mode 100644 tests/test_core/streams/test_several_items_methods.py diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index 46d94cd15..c31d7cb2d 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -137,6 +137,24 @@ Order- or role-dependent behaviour that reaches a hash or a stored table first. - `bind(sources=)` with two sources: both bound, nothing moves; one refused, neither bound. A plain or data-tag source in any arrangement binds and reads; a mixed one is ITL-1245. +## Checked and found as documented (part 3) + +- `DropNulls` naming plain tags, data-tags holding nulls and data, in every order: one row + set and schema, each named column non-nullable (a data-tag in both roles), sync equal to + async. +- `Index` with `out=` over a list-valued data-tag (after `Batch`): reads it as a data + column; the data-tag stays; sync equal to async. +- A `DictSource` `data_schema` in another field order: the same source (rows, hashes). +- Every stream method given several items equals its operator (rows, hashes, column + order); `polars_filter` keywords win over `constraint_map`. +- Every combination of `ColumnConfig` groups gives the `all_info` columns of those groups + in the same order, for a source, a join and a batch, in every role and layout. + +## Remaining + +- `MapTags` has no golden entry (adding golden entries is out of scope for this issue). +- The defects and the two undecided rules above: ITL-1241 to ITL-1247. + ## Tests added `tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`, @@ -145,4 +163,7 @@ three layouts and three role settings. `ops/test_several_items.py` uses it for e operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). Part 2: `tests/test_core/sources/test_several_items.py` (gaps 4, 13, 14, 15: CachedSource), `tests/test_core/function_pod/test_several_items.py` (gaps 1, 10; ITL-1244, ITL-1246) and -`tests/test_pipeline/test_bind_several_items.py` (gap 8; ITL-1245). +`tests/test_pipeline/test_bind_several_items.py` (gap 8; ITL-1245). Part 3: +`tests/test_core/data_tags/test_several_items.py` (gaps 12, 15: `Index`), +`tests/test_core/streams/test_several_items_methods.py` (gap 15: stream methods; +`ColumnConfig` groups) and a schema-order test in the sources file (gap 15). diff --git a/tests/test_core/data_tags/test_several_items.py b/tests/test_core/data_tags/test_several_items.py new file mode 100644 index 000000000..30ee628b4 --- /dev/null +++ b/tests/test_core/data_tags/test_several_items.py @@ -0,0 +1,98 @@ +"""Data-tags among several named columns: DropNulls and Index (ITL-1236). + +* ``DropNulls`` naming several columns, plain tags, data-tags holding nulls and data, + in several orders: one row set, one schema, the same on the async path; each named + column, in both roles for a data-tag, becomes non-nullable. +* ``Index`` with ``out=`` reads a list-valued data-tag (after ``Batch``) as it reads a + data column, on both paths. +""" + +from __future__ import annotations + +import itertools + +import pyarrow as pa +import pytest + +from orcapod.core.operators import Batch, DropNulls, Index +from orcapod.core.sources import ArrowTableSource + +from ..operators.several_items import ( + DATA_TAGS, + MAIN, + MIXED, + TAGS_FIRST, + Arrangement, + async_rows, + fingerprint, + make_source, + presentation, + rows_fingerprint, +) + +# Tags t1 and t2 and data d1, each null in one row: only row 0 has no null. +NULLS = pa.table( + { + "t1": pa.array(["a", None, "b", "c"], pa.large_string()), + "t2": pa.array([1, 2, None, 4], pa.int64()), + "d1": pa.array([1, 2, 3, None], pa.int64()), + "d2": pa.array([5, 6, 7, 8], pa.int64()), + } +) + + +def _nullable_source(tag_only: list[str]) -> ArrowTableSource: + return ArrowTableSource( + NULLS, tag_columns=["t1", "t2"], source_id="n", tag_only_columns=tag_only + ) + + +@pytest.mark.parametrize("tag_only", [[], ["t2"]], ids=["data_tags", "mixed"]) +def test_drop_nulls_of_several_columns_with_nullable_data_tags( + tag_only: list[str], +) -> None: + source = _nullable_source(tag_only) + results = {} + for names in itertools.permutations(["t1", "t2", "d1"]): + op = DropNulls(list(names)) + kept = op(source) + tag_schema, data_schema = kept.output_schema() + assert tag_schema["t1"] is str and tag_schema["t2"] is int + assert data_schema["d1"] is int + if "t1" not in tag_only: + # A data-tag is one column: narrowed in both roles. + assert data_schema["t1"] is str + rows = rows_fingerprint(list(kept.iter_data())) + assert rows_fingerprint(async_rows(DropNulls(list(names)), source)) == rows + results[names] = {**fingerprint(kept), "presentation": presentation(kept)} + assert [data["d2"] for _, data in kept.iter_data()] == [5] + first = next(iter(results.values())) + assert all(result == first for result in results.values()) + + +@pytest.mark.parametrize("tag_only", [[], ["t2"]], ids=["data_tags", "mixed"]) +def test_drop_nulls_of_one_data_tag_keeps_the_rows_where_it_has_a_value( + tag_only: list[str], +) -> None: + kept = DropNulls(["t1"])(_nullable_source(tag_only)) + assert sorted(data["d2"] for _, data in kept.iter_data()) == [5, 7, 8] # pyright: ignore[reportArgumentType] + + +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +@pytest.mark.parametrize("column", ["t1", "t3", "d1"]) +def test_index_reads_a_list_valued_data_tag_as_a_data_column( + role: str, column: str +) -> None: + """After ``Batch``, t1 and t3 are list-valued data-tags (t2 is plain in MIXED).""" + batched = Batch(batch_size=3)( + make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + ) + op = Index(column, 0, out=f"first_{column}") + indexed = op(batched) + first = [data[f"first_{column}"] for _, data in indexed.iter_data()] + assert first == {"t1": ["b", "b"], "t3": [1, 2], "d1": [10, 40]}[column] + # The data-tag itself is unchanged: still a tag and a data column. + assert presentation(indexed)[0] == ("t1", "t2", "t3") + assert rows_fingerprint( + async_rows(Index(column, 0, out=f"first_{column}"), batched) + ) == rows_fingerprint(list(indexed.iter_data())) diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index ecfc017f7..d1847a2f9 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -22,7 +22,12 @@ import pyarrow as pa import pytest -from orcapod.core.sources import ArrowTableSource, CachedSource, DerivedSource +from orcapod.core.sources import ( + ArrowTableSource, + CachedSource, + DerivedSource, + DictSource, +) from orcapod.databases import InMemoryArrowDatabase from ..operators.several_items import ( @@ -167,3 +172,39 @@ def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> Non ) assert presentation(again) == presentation(make_source(MAIN, arrangement)) assert fingerprint(again) == fingerprint(cached) + + +@pytest.mark.parametrize( + "data_schema", + [ + {"t1": str, "t2": int, "d1": int, "d3": str}, + {"d3": str, "t2": int, "d1": int, "t1": str}, + ], + ids=["table_order", "another_order"], +) +@pytest.mark.parametrize("role", ROLES) +def test_a_schema_in_another_field_order_gives_the_same_source( + data_schema: dict[str, type], role: str +) -> None: + """A ``DictSource`` schema with several fields, in the rows' order or another.""" + rows = [ + {"t1": "a", "t2": 1, "d1": 10, "d3": "x"}, + {"t1": "b", "t2": 2, "d1": 20, "d3": "y"}, + ] + tag_only = {PLAIN: True, DATA_TAGS: [], MIXED: ["t2"]}[role] + source = DictSource( + rows, + tag_columns=["t1", "t2"], + source_id="d", + data_schema=data_schema, + tag_only_columns=tag_only, + ) + reference = DictSource( + rows, + tag_columns=["t1", "t2"], + source_id="d", + data_schema={"t1": str, "t2": int, "d1": int, "d3": str}, + tag_only_columns=tag_only, + ) + assert fingerprint(source) == fingerprint(reference) + assert presentation(source)[0] == ("t1", "t2") diff --git a/tests/test_core/streams/test_several_items_methods.py b/tests/test_core/streams/test_several_items_methods.py new file mode 100644 index 000000000..47d119a4f --- /dev/null +++ b/tests/test_core/streams/test_several_items_methods.py @@ -0,0 +1,192 @@ +"""Stream methods given several items equal their operators; several column groups (ITL-1236). + +Every stream method that builds an operator from several items (names, a mapping, +keywords, predicates) gives the stream that operator gives: same rows, hashes and +column order, for plain tags, data-tags and a mix, on a stream with three tags and three +data columns. ``polars_filter`` merges ``constraint_map`` and keyword constraints, the +keywords winning. Several ``ColumnConfig`` groups together give the ``all_info`` +columns of those groups, in the same order. +""" + +from __future__ import annotations + +import itertools +from collections.abc import Callable +from dataclasses import dataclass + +import polars as pl +import pytest + +from orcapod.core.operators import ( + AddTags, + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + Join, + MapData, + MapTags, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, + TagBy, +) +from orcapod.core.sources import ArrowTableSource +from orcapod.protocols.core_protocols import StreamProtocol + +from ..operators.several_items import ( + LAYOUTS, + MAIN, + P, + ROLES, + TAGS_FIRST, + Arrangement, + fingerprint, + make_source, + presentation, +) + +pytestmark = pytest.mark.filterwarnings( + "ignore::orcapod.core.data_tags.DataTagMaterializationWarning" +) + +# The stream methods are on the stream classes, not on ``StreamProtocol``. +Build = Callable[[ArrowTableSource], StreamProtocol] + + +@dataclass(frozen=True) +class MethodCase: + """A stream method call and the operator call it must equal.""" + + label: str + method: Build + operator: Build + + +CASES = [ + MethodCase( + "select_tag_columns", + lambda s: s.select_tag_columns(["t3", "t1"]), + lambda s: SelectTagColumns(["t3", "t1"])(s), + ), + MethodCase( + "select_data_columns", + lambda s: s.select_data_columns(["d3", "d1"]), + lambda s: SelectDataColumns(["d3", "d1"])(s), + ), + MethodCase( + "drop_tag_columns", + lambda s: s.drop_tag_columns(["t3", "t1"]), + lambda s: DropTagColumns(["t3", "t1"])(s), + ), + MethodCase( + "drop_data_columns", + lambda s: s.drop_data_columns(["d3", "d1"]), + lambda s: DropDataColumns(["d3", "d1"])(s), + ), + MethodCase( + "select_columns_positional", + lambda s: s.select_columns("d3", "t2", "d1"), + lambda s: SelectColumns(["d3", "t2", "d1"])(s), + ), + MethodCase( + "drop_columns_positional", + lambda s: s.drop_columns("d3", "t2"), + lambda s: DropColumns(["d3", "t2"])(s), + ), + MethodCase( + "map_tags", + lambda s: s.map_tags({"t3": "u3", "t1": "u1"}), + lambda s: MapTags({"t3": "u3", "t1": "u1"})(s), + ), + MethodCase( + "map_data", + lambda s: s.map_data({"d3": "e3", "d1": "e1"}), + lambda s: MapData({"d3": "e3", "d1": "e1"})(s), + ), + MethodCase( + "drop_nulls", + lambda s: s.drop_nulls(["d2", "t2"]), + lambda s: DropNulls(["d2", "t2"])(s), + ), + MethodCase( + "polars_filter_predicates", + lambda s: s.polars_filter(pl.col("d1") > 10, pl.col("t3") == 1), + lambda s: PolarsFilter(predicates=[pl.col("d1") > 10, pl.col("t3") == 1])(s), + ), + MethodCase( + "polars_filter_constraints", + lambda s: s.polars_filter(t1="a", t2=1), + lambda s: PolarsFilter(constraints={"t1": "a", "t2": 1})(s), + ), + # Keyword constraints win over the map's value for the same name. + MethodCase( + "polars_filter_constraint_map_and_keywords", + lambda s: s.polars_filter(constraint_map={"t1": "a", "t2": 2}, t2=1), + lambda s: PolarsFilter(constraints={"t1": "a", "t2": 1})(s), + ), + MethodCase( + "add_tags", + lambda s: s.add_tags(x=1, y="z"), + lambda s: AddTags({"x": 1, "y": "z"})(s), + ), + MethodCase( + "tag_by", + lambda s: s.tag_by(["d3", "d1"]), + lambda s: TagBy(["d3", "d1"])(s), + ), +] + + +@pytest.mark.parametrize("role", ROLES) +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.label) +def test_a_stream_method_with_several_items_equals_its_operator( + case: MethodCase, role: str +) -> None: + source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role)) + by_method, by_operator = case.method(source), case.operator(source) + assert presentation(by_method) == presentation(by_operator) + assert fingerprint(by_method) == fingerprint(by_operator) + + +@pytest.mark.parametrize("role", ROLES) +def test_join_and_semi_join_methods_equal_their_operators(role: str) -> None: + def build(spec): + return make_source(spec, Arrangement(spec.tags, TAGS_FIRST, role)) + + main, partner = build(MAIN), build(P) + assert fingerprint(main.join(partner)) == fingerprint(Join()(main, partner)) + assert fingerprint(partner.join(main)) == fingerprint(Join()(main, partner)) + assert fingerprint(main.semi_join(partner)) == fingerprint( + SemiJoin()(main, partner) + ) + + +def test_polars_filter_keywords_win_over_the_constraint_map() -> None: + kept = make_source(MAIN).polars_filter(constraint_map={"t1": "a", "t2": 2}, t2=1) + assert sorted(data["d1"] for _, data in kept.iter_data()) == [20, 50] # pyright: ignore[reportArgumentType] + + +COLUMN_GROUPS = ("meta", "context", "source", "system_tags", "content_hash") + + +@pytest.mark.parametrize("layout", LAYOUTS) +@pytest.mark.parametrize("role", ROLES) +def test_several_column_groups_keep_the_all_info_column_order( + role: str, layout: str +) -> None: + """Every combination of ``ColumnConfig`` groups: the all_info columns, filtered.""" + source = make_source(MAIN, Arrangement(MAIN.tags, layout, role)) + partner = make_source(P, Arrangement(P.tags, layout, role)) + for stream in (source, Join()(source, partner), Batch(batch_size=2)(source)): + full = stream.as_table(all_info=True).column_names + for n in range(1, len(COLUMN_GROUPS) + 1): + for groups in itertools.combinations(COLUMN_GROUPS, n): + columns = stream.as_table( + columns=dict.fromkeys(groups, True) + ).column_names + assert len(set(columns)) == len(columns), groups + assert columns == [c for c in full if c in columns], groups From aa7fa71d6f2428cd2c7fd0c05d4604056056aabb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:30:55 -0800 Subject: [PATCH 07/13] test(operators): column order of sync rows, async rows and keys, per arrangement; defect 7 as xfail (ITL-1236) Addresses the second Copilot review of #442: sync and async rows are compared with their column order, system-tag order and Arrow schemas within each arrangement, and a result's keys with its rows' keys. Both fail on inputs whose table layout differs from the tag order (reported, issue pending); those combinations are xfail(strict=True), the others are asserted. Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 1 + tests/test_core/operators/several_items.py | 21 +++ .../test_core/operators/test_several_items.py | 152 +++++++++++++++++- 3 files changed, 166 insertions(+), 8 deletions(-) diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index aa67452d9..6b01de4c0 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -99,6 +99,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | ITL-1243 | | 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | ITL-1244 (rule undecided) | | 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | ITL-1245 | +| 7 | Input whose table layout differs from its tag order: every unary operator's result keys list the tags in the physical order while its rows use the tag order; Join, SemiJoin, MergeJoin (and, with data-tags, the tag select/drop and `MapTags` forms) give the async rows another column order than the sync rows | column order only; no hash | pending | | Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | ITL-1247 (rule undecided) | ## Gaps, ranked diff --git a/tests/test_core/operators/several_items.py b/tests/test_core/operators/several_items.py index 49642b438..e42215024 100644 --- a/tests/test_core/operators/several_items.py +++ b/tests/test_core/operators/several_items.py @@ -235,6 +235,27 @@ def rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: return sorted(row_fingerprint(tag, data) for tag, data in rows) +def ordered_rows_fingerprint(rows: Sequence[Row]) -> list[tuple[str, ...]]: + """``rows_fingerprint`` plus each row's column order, system-tag order and Arrow schemas. + + For comparing the async rows with the sync rows of one arrangement, where the + column order must be the same too (as ``assert_equivalent`` in + ``test_async_sync_equivalence.py``); across arrangements only the order-free + ``rows_fingerprint`` must be. + """ + return sorted( + ( + *row_fingerprint(tag, data), + repr(list(tag.keys())), + repr(list(data.keys())), + repr(list(tag.system_tags())), + str(tag.arrow_schema(columns={"system_tags": True})), + str(data.arrow_schema(columns={"source": True})), + ) + for tag, data in rows + ) + + def fingerprint(stream: StreamProtocol) -> dict[str, Any]: """What must not depend on the input's arrangement or a set argument's order. diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index 90de77f39..ece00e0f8 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -74,6 +74,7 @@ async_rows, fingerprint, make_source, + ordered_rows_fingerprint, presentation, rows_fingerprint, ) @@ -144,18 +145,21 @@ def _unary_params() -> list[object]: def _assert_all_equal(results: Mapping[str, Mapping[str, object]]) -> None: """Assert every arrangement's fingerprint equals the first one's, naming the field. - A result that holds ``async_rows`` must also have them equal to its own sync - ``rows``: equal across arrangements alone would pass an async path that is wrong - the same way every time. + A result that holds ``async_rows`` must also have them equal to its own + ``sync_rows``: equal across arrangements alone would pass an async path that is + wrong the same way every time. (Column order is compared separately, in + ``test_async_column_order_equals_sync_in_every_arrangement``.) """ for result_id, result in results.items(): if "async_rows" in result: - assert result["async_rows"] == result["rows"], ( + assert result["async_rows"] == result["sync_rows"], ( f"async rows differ from sync rows in {result_id}" ) (first_id, first), *others = results.items() for arrangement_id, result in others: for field, value in first.items(): + if field in ("async_rows", "sync_rows"): + continue assert result[field] == value, ( f"{field} differs between {first_id} and {arrangement_id}" ) @@ -187,6 +191,93 @@ def test_unary_async_rows_equal_sync_rows_in_every_arrangement( ) +COLUMN_ORDER_ISSUE = "ITL-1236 defect 7, issue pending" +KEYS_XFAIL = pytest.mark.xfail( + strict=True, + reason=f"{COLUMN_ORDER_ISSUE}: with a table layout other than the tag order, the " + + "result's keys list the tags in the input table's physical order, its rows in " + + "tag order", +) +ASYNC_ORDER_XFAIL = pytest.mark.xfail( + strict=True, + reason=f"{COLUMN_ORDER_ISSUE}: with a table layout other than the tag order, the " + + "async rows' column order differs from the sync rows'", +) + + +def assert_keys_equal_row_keys(stream: StreamProtocol) -> None: + """Every row of ``stream`` lists its columns in the order of ``stream.keys()``.""" + keys = stream.keys() + for tag, data in stream.iter_data(): + assert (tuple(tag.keys()), tuple(data.keys())) == keys + + +def assert_async_column_order_equals_sync( + op: Callable[[], StaticOutputOperatorPod], *sources: StreamProtocol +) -> None: + """Async rows equal sync rows with column order, system-tag order and Arrow schemas.""" + sync = list(op()(*sources).iter_data()) + assert ordered_rows_fingerprint(async_rows(op(), *sources)) == ( + ordered_rows_fingerprint(sync) + ) + + +# (operator, role) pairs that fail today (defect 7). Every unary case but one fails +# the keys check; DropTagColumns(["t3", "t1"]) leaves one tag, so nothing to order. +KEYS_PASS = {("drop_tag_columns", PLAIN)} +ASYNC_ORDER_FAILS = { + ("select_tag_columns", DATA_TAGS), + ("drop_tag_columns", DATA_TAGS), + ("drop_tag_columns", MIXED), + ("map_tags_drop_unmapped", DATA_TAGS), + ("map_tags_tag_only", DATA_TAGS), + ("map_tags_tag_only", MIXED), +} + + +def _marked_unary_params( + fails: Callable[[str, str], bool], mark: pytest.MarkDecorator +) -> list[Any]: + return [ + pytest.param( + case, + role, + id=f"{case.label}/{role}", + marks=[mark] if fails(case.label, role) else [], + ) + for case in UNARY_CASES + for role in case.roles + ] + + +@pytest.mark.parametrize( + ("case", "role"), + _marked_unary_params( + lambda label, role: (label, role) not in KEYS_PASS, KEYS_XFAIL + ), +) +def test_unary_result_keys_equal_its_row_keys_in_every_arrangement( + case: UnaryCase, role: str +) -> None: + for arrangement in arrangements(MAIN, roles=(role,)): + assert_keys_equal_row_keys(case.make_op()(make_source(MAIN, arrangement))) + + +@pytest.mark.parametrize( + ("case", "role"), + _marked_unary_params( + lambda label, role: (label, role) in ASYNC_ORDER_FAILS, ASYNC_ORDER_XFAIL + ), +) +def test_unary_async_column_order_equals_sync_in_every_arrangement( + case: UnaryCase, role: str +) -> None: + for arrangement in arrangements(MAIN, roles=(role,)): + assert_async_column_order_equals_sync( + case.make_op, make_source(MAIN, arrangement) + ) + + # ── order of a set argument ───────────────────────────────────────────────── @@ -262,6 +353,7 @@ def test_the_order_of_a_set_argument_changes_nothing( **fingerprint(stream), "presentation": presentation(stream), "operator_identity": op.content_hash().to_string(), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint( async_rows(op, make_source(MAIN, arrangement)) ), @@ -283,7 +375,9 @@ def test_polars_filter_predicates_in_any_order_choose_the_same_rows( op = PolarsFilter(predicates=list(order)) stream = op(source) rows[repr(order)] = rows_fingerprint(list(stream.iter_data())) - assert rows_fingerprint(async_rows(op, source)) == rows[repr(order)] + assert ordered_rows_fingerprint(async_rows(op, source)) == ( + ordered_rows_fingerprint(list(stream.iter_data())) + ) assert _sorted_values(stream, "d1") == [50] assert len({repr(v) for v in rows.values()}) == 1 @@ -405,9 +499,9 @@ def test_names_of_several_roles_in_one_call(case: RoleCase) -> None: source = make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, MIXED)) result = case.make_op()(source) assert result.keys() == case.keys - assert rows_fingerprint(async_rows(case.make_op(), source)) == rows_fingerprint( - list(result.iter_data()) - ) + assert ordered_rows_fingerprint( + async_rows(case.make_op(), source) + ) == ordered_rows_fingerprint(list(result.iter_data())) def _sorted_values(stream: StreamProtocol, name: str, role: str = "data") -> list[Any]: @@ -498,6 +592,7 @@ def test_join_result_does_not_depend_on_the_order_of_its_inputs( **fingerprint(stream), "presentation": presentation(stream), "system_tag_order": stream.keys(columns={"system_tags": True})[0], + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(Join(), *ordered)), } _assert_all_equal(results) @@ -515,6 +610,7 @@ def test_join_result_does_not_depend_on_the_arrangement_of_its_inputs( stream = Join()(*sources) results[f"{order_index}/{layout}"] = { **fingerprint(stream), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(Join(), *sources)), } _assert_all_equal(results) @@ -587,6 +683,7 @@ def test_merge_join_result_does_not_depend_on_the_order_of_its_inputs( ): results[f"{order_index}/{layout}/{label}"] = { **fingerprint(stream), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(MergeJoin(), *inputs)), } _assert_all_equal(results) @@ -638,6 +735,7 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N stream = SemiJoin()(left, right) results[f"{order_index}/{layout}"] = { **fingerprint(stream), + "sync_rows": rows_fingerprint(list(stream.iter_data())), "async_rows": rows_fingerprint(async_rows(SemiJoin(), left, right)), } _assert_all_equal(results) @@ -652,3 +750,41 @@ def test_semijoin_matches_on_every_shared_tag_in_any_arrangement(role: str) -> N assert rows_fingerprint(async_rows(SemiJoin(), left, right)) == ( rows_fingerprint(list(kept.iter_data())) ) + + +SEVERAL_INPUT_CASES: dict[ + str, tuple[Callable[[], StaticOutputOperatorPod], tuple[Spec, ...]] +] = { + "join_2": (Join, (MAIN, P)), + "join_3": (Join, (MAIN, P, Q)), + "join_4": (Join, (MAIN, P, Q, R)), + "semijoin": (SemiJoin, (MAIN, P)), + "merge_join": (MergeJoin, (MAIN, MAIN_OTHER)), +} + + +def _several_input_sources( + label: str, order_index: int, layout: str, role: str +) -> tuple[Callable[[], StaticOutputOperatorPod], list[ArrowTableSource]]: + op, specs = SEVERAL_INPUT_CASES[label] + rows = NON_NULL_ROWS if label == "merge_join" else None + return op, _sources(specs, order_index, layout, role, rows=rows) + + +@pytest.mark.parametrize("role", ROLES) +@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) +def test_several_input_result_keys_equal_its_row_keys(label: str, role: str) -> None: + for order_index in range(3): + for layout in LAYOUTS: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_keys_equal_row_keys(op()(*sources)) + + +@ASYNC_ORDER_XFAIL +@pytest.mark.parametrize("role", ROLES) +@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) +def test_several_input_async_column_order_equals_sync(label: str, role: str) -> None: + for order_index in range(3): + for layout in LAYOUTS: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_async_column_order_equals_sync(op, *sources) From ea4814f8e0da9889f43124342ddc5e0f94283351 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:32:03 -0800 Subject: [PATCH 08/13] test(sources, pipeline): compare key orders with each other; check both bound sources (ITL-1236) Addresses the Copilot review of #443. Co-Authored-By: Claude Opus 5.5 --- tests/test_core/sources/test_several_items.py | 8 ++++++ .../test_pipeline/test_bind_several_items.py | 25 +++++++++++-------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index ecfc017f7..9dfc9ad5e 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -116,6 +116,14 @@ def test_record_ids_follow_the_key_order(role: str) -> None: if keys != ("t1", "t2", "t3"): assert source.content_hash() != first.content_hash() assert _record_ids(source) != _record_ids(first) + # The same key columns in another order, compared with each other. + for one, other in ( + (("t1", "t2", "t3"), ("t3", "t1", "t2")), + (("d3", "t1"), ("t1", "d3")), + ): + assert by_keys[one].content_hash() != by_keys[other].content_hash() + assert by_keys[one].pipeline_hash() == by_keys[other].pipeline_hash() + assert _record_ids(by_keys[one]) != _record_ids(by_keys[other]) # Same keys, same order: the same source. again = ArrowTableSource( _table(), diff --git a/tests/test_pipeline/test_bind_several_items.py b/tests/test_pipeline/test_bind_several_items.py index 342630ffb..2a3580297 100644 --- a/tests/test_pipeline/test_bind_several_items.py +++ b/tests/test_pipeline/test_bind_several_items.py @@ -71,22 +71,27 @@ def test_several_sources_bound_in_one_call(role: str, orchestrator: type) -> Non assert _slot_names(job) == ("s", "p") before = _snapshot(job) arrangement = Arrangement(MAIN.tags, TAGS_FIRST, role) - job.bind( - sources={ - "s": make_source(MAIN, arrangement, source_id="s_new", rows=[0, 1]), - "p": make_source( - P, Arrangement(P.tags, TAGS_FIRST, role), source_id="p_new" - ), - } - ) + # Both replacements hold other rows than the slots' sources, so the result shows + # that both were bound. + sources: dict[str, StreamProtocol] = { + "s": make_source(MAIN, arrangement, source_id="s_new", rows=[0, 1]), + "p": make_source( + P, Arrangement(P.tags, TAGS_FIRST, role), source_id="p_new", rows=[1, 3] + ), + } + job.bind(sources=sources) + for name, source in sources.items(): + assert job.nodes[name].bound_source is source assert _snapshot(job) == before job.run(orchestrator=orchestrator()) rows = sorted( (tag["t1"], tag["t2"], data["total"]) for tag, data in job.nodes["total"].iter_data() ) - # Rows 0 and 1 of MAIN: (b, 2, d1=10) with p1=6; (a, 1, d1=20) with p1=9. - assert rows == [("a", 1, 29), ("b", 2, 16)] + # MAIN's rows 0 and 1 are (b, 2, d1=10) and (a, 1, d1=20); P's rows 1 and 3 are + # (b, 1, p1=8) and (b, 2, p1=6). Only (b, 2) matches: with all of P, (a, 1) would + # match too. + assert rows == [("b", 2, 16)] @pytest.mark.parametrize("refused", ["s", "p"]) From d896a1887bf6d2572adf258d9a9db4108e81e4f6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:34:54 -0800 Subject: [PATCH 09/13] test(operators): ITL-1249 for the column order xfails; an operator node stores one column order on both orchestrators (ITL-1236) Co-Authored-By: Claude Opus 5.5 --- .../specs/itl-1236-several-item-test-audit.md | 2 +- .../test_core/operators/test_several_items.py | 47 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/dev/specs/itl-1236-several-item-test-audit.md b/docs/dev/specs/itl-1236-several-item-test-audit.md index 6b01de4c0..37561f578 100644 --- a/docs/dev/specs/itl-1236-several-item-test-audit.md +++ b/docs/dev/specs/itl-1236-several-item-test-audit.md @@ -99,7 +99,7 @@ Legend: **yes** (with the best test as evidence), **no** (with what the closest | 3 | `MergeJoin` raises `TypeError` on a null in a colliding column | error only | ITL-1243 | | 4 | Swapping `output_keys` keeps the function's uri, so a job serves stored values under swapped names | result cache: wrong values served | ITL-1244 (rule undecided) | | 5 | Binding a source with mixed tag roles in another column order is accepted, then the source node's `as_table` raises | error on read; the run is right | ITL-1245 | -| 7 | Input whose table layout differs from its tag order: every unary operator's result keys list the tags in the physical order while its rows use the tag order; Join, SemiJoin, MergeJoin (and, with data-tags, the tag select/drop and `MapTags` forms) give the async rows another column order than the sync rows | column order only; no hash | pending | +| 7 | Input whose table layout differs from its tag order: every unary operator's result keys list the tags in the physical order while its rows use the tag order; Join, SemiJoin, MergeJoin (and, with data-tags, the tag select/drop and `MapTags` forms) give the async rows another column order than the sync rows. In a job, an operator node stores one column order on both orchestrators (measured, tested) | column order only; no hash, no stored table | ITL-1249 | | Q1 | `PolarsFilter`: the order of the predicates is in the identity (and the stored path); the order of the constraints is not | pipeline hash | ITL-1247 (rule undecided) | ## Gaps, ranked diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index ece00e0f8..d590b8eec 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -27,6 +27,7 @@ import itertools from collections.abc import Callable, Mapping from dataclasses import dataclass +from pathlib import Path from typing import Any import polars as pl @@ -52,12 +53,18 @@ SemiJoin, TagBy, ) +from orcapod import PipelineJob from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import InputValidationError +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.types import CacheMode from orcapod.protocols.core_protocols import StreamProtocol from .several_items import ( + DATA_FIRST_REVERSED, DATA_TAGS, LAYOUTS, MAIN, @@ -191,7 +198,7 @@ def test_unary_async_rows_equal_sync_rows_in_every_arrangement( ) -COLUMN_ORDER_ISSUE = "ITL-1236 defect 7, issue pending" +COLUMN_ORDER_ISSUE = "ITL-1249" KEYS_XFAIL = pytest.mark.xfail( strict=True, reason=f"{COLUMN_ORDER_ISSUE}: with a table layout other than the tag order, the " @@ -788,3 +795,41 @@ def test_several_input_async_column_order_equals_sync(label: str, role: str) -> for layout in LAYOUTS: op, sources = _several_input_sources(label, order_index, layout, role) assert_async_column_order_equals_sync(op, *sources) + + +@pytest.mark.parametrize( + "make_op", + [Join, lambda: SelectTagColumns(["t3", "t1"])], + ids=["join", "select_tag_columns"], +) +@pytest.mark.parametrize("role", ROLES) +def test_an_operator_node_stores_one_column_order_on_both_orchestrators( + make_op: Callable[[], StaticOutputOperatorPod], role: str, tmp_path: Path +) -> None: + """A job run by each orchestrator over one store: one column order, rows once. + + The input's layout is not its tag order (ITL-1249 concerns the bare streams); in a + job, an operator node keeps its output in row order on both paths (ITL-1133). + """ + for database in (InMemoryArrowDatabase(), DeltaTableDatabase(tmp_path / role)): + tables = [] + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="order", database=database) + sources = _sources((MAIN, P), 0, DATA_FIRST_REVERSED, role) + with job: + make_op()(*sources[: 2 if make_op is Join else 1], label="op") + job.nodes["op"].attach_databases( + pipeline_database=database.at("order"), cache_mode=CacheMode.LOG + ) + job.run(orchestrator=orchestrator()) + tables.append(job.nodes["op"].as_table(all_info=True)) + assert tables[0].column_names == tables[1].column_names + # The tags first, in the input's tag order. + tags = ["t1", "t2", "t3"] if make_op is Join else ["t1", "t3"] + assert tables[0].column_names[: len(tags)] == tags + stored = [ + database.get_all_records(path) for path in database.list_table_paths() + ] + stored = [table for table in stored if table is not None] + assert len(stored) == 1 + assert stored[0].num_rows == tables[0].num_rows From e1800ebd962bb42b7520d06316588cd3afa80baa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:50:29 -0800 Subject: [PATCH 10/13] test(operators): one column-order test per arrangement, xfail only where ITL-1249 reproduces (ITL-1236) Addresses the third Copilot review of #442. Co-Authored-By: Claude Opus 5.5 --- .../test_core/operators/test_several_items.py | 114 +++++++++++------- 1 file changed, 72 insertions(+), 42 deletions(-) diff --git a/tests/test_core/operators/test_several_items.py b/tests/test_core/operators/test_several_items.py index d590b8eec..83af880f8 100644 --- a/tests/test_core/operators/test_several_items.py +++ b/tests/test_core/operators/test_several_items.py @@ -66,6 +66,7 @@ from .several_items import ( DATA_FIRST_REVERSED, DATA_TAGS, + INTERLEAVED, LAYOUTS, MAIN, MIXED, @@ -229,10 +230,13 @@ def assert_async_column_order_equals_sync( ) -# (operator, role) pairs that fail today (defect 7). Every unary case but one fails -# the keys check; DropTagColumns(["t3", "t1"]) leaves one tag, so nothing to order. +# Where the column order fails today (ITL-1249), per arrangement. In the reversed layout +# the tags sit in another order than the tag order: every unary result's keys follow +# the table (but ``DropTagColumns(["t3", "t1"])`` of plain tags leaves one tag). With +# data-tags, a tag demoted to data sits elsewhere in the sync rows than in the async +# rows, in the reversed and the interleaved layout. KEYS_PASS = {("drop_tag_columns", PLAIN)} -ASYNC_ORDER_FAILS = { +DEMOTED_DATA_TAG_CASES = { ("select_tag_columns", DATA_TAGS), ("drop_tag_columns", DATA_TAGS), ("drop_tag_columns", MIXED), @@ -242,47 +246,50 @@ def assert_async_column_order_equals_sync( } -def _marked_unary_params( - fails: Callable[[str, str], bool], mark: pytest.MarkDecorator +def _keys_fail(label: str, role: str, layout: str) -> bool: + if layout == DATA_FIRST_REVERSED: + return (label, role) not in KEYS_PASS + return layout == INTERLEAVED and (label, role) in DEMOTED_DATA_TAG_CASES + + +def _async_order_fails(label: str, role: str, layout: str) -> bool: + return layout != TAGS_FIRST and (label, role) in DEMOTED_DATA_TAG_CASES + + +def _arrangement_params( + fails: Callable[[str, str, str], bool], mark: pytest.MarkDecorator ) -> list[Any]: + """One parameter per unary case, role and arrangement; ``mark`` where it fails.""" return [ pytest.param( case, - role, - id=f"{case.label}/{role}", - marks=[mark] if fails(case.label, role) else [], + arrangement, + id=f"{case.label}/{arrangement.id}", + marks=[mark] + if fails(case.label, arrangement.roles, arrangement.layout) + else [], ) for case in UNARY_CASES - for role in case.roles + for arrangement in arrangements(MAIN, roles=case.roles) ] @pytest.mark.parametrize( - ("case", "role"), - _marked_unary_params( - lambda label, role: (label, role) not in KEYS_PASS, KEYS_XFAIL - ), + ("case", "arrangement"), _arrangement_params(_keys_fail, KEYS_XFAIL) ) -def test_unary_result_keys_equal_its_row_keys_in_every_arrangement( - case: UnaryCase, role: str +def test_unary_result_keys_equal_its_row_keys( + case: UnaryCase, arrangement: Arrangement ) -> None: - for arrangement in arrangements(MAIN, roles=(role,)): - assert_keys_equal_row_keys(case.make_op()(make_source(MAIN, arrangement))) + assert_keys_equal_row_keys(case.make_op()(make_source(MAIN, arrangement))) @pytest.mark.parametrize( - ("case", "role"), - _marked_unary_params( - lambda label, role: (label, role) in ASYNC_ORDER_FAILS, ASYNC_ORDER_XFAIL - ), + ("case", "arrangement"), _arrangement_params(_async_order_fails, ASYNC_ORDER_XFAIL) ) -def test_unary_async_column_order_equals_sync_in_every_arrangement( - case: UnaryCase, role: str +def test_unary_async_column_order_equals_sync( + case: UnaryCase, arrangement: Arrangement ) -> None: - for arrangement in arrangements(MAIN, roles=(role,)): - assert_async_column_order_equals_sync( - case.make_op, make_source(MAIN, arrangement) - ) + assert_async_column_order_equals_sync(case.make_op, make_source(MAIN, arrangement)) # ── order of a set argument ───────────────────────────────────────────────── @@ -778,23 +785,46 @@ def _several_input_sources( return op, _sources(specs, order_index, layout, role, rows=rows) -@pytest.mark.parametrize("role", ROLES) -@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) -def test_several_input_result_keys_equal_its_row_keys(label: str, role: str) -> None: - for order_index in range(3): - for layout in LAYOUTS: - op, sources = _several_input_sources(label, order_index, layout, role) - assert_keys_equal_row_keys(op()(*sources)) +def _several_input_params(fails_in_reversed_layout: bool) -> list[Any]: + """One parameter per several-input case, role, tag order and layout.""" + return [ + pytest.param( + label, + role, + order_index, + layout, + id=f"{label}/{role}/{order_index}/{layout}", + marks=[ASYNC_ORDER_XFAIL] + if fails_in_reversed_layout and layout == DATA_FIRST_REVERSED + else [], + ) + for label in SEVERAL_INPUT_CASES + for role in ROLES + for order_index in range(3) + for layout in LAYOUTS + ] -@ASYNC_ORDER_XFAIL -@pytest.mark.parametrize("role", ROLES) -@pytest.mark.parametrize("label", list(SEVERAL_INPUT_CASES)) -def test_several_input_async_column_order_equals_sync(label: str, role: str) -> None: - for order_index in range(3): - for layout in LAYOUTS: - op, sources = _several_input_sources(label, order_index, layout, role) - assert_async_column_order_equals_sync(op, *sources) +@pytest.mark.parametrize( + ("label", "role", "order_index", "layout"), _several_input_params(False) +) +def test_several_input_result_keys_equal_its_row_keys( + label: str, role: str, order_index: int, layout: str +) -> None: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_keys_equal_row_keys(op()(*sources)) + + +# In the reversed layout, Join, SemiJoin and MergeJoin give the sync rows the table's +# tag order and the async rows the tag order (ITL-1249). +@pytest.mark.parametrize( + ("label", "role", "order_index", "layout"), _several_input_params(True) +) +def test_several_input_async_column_order_equals_sync( + label: str, role: str, order_index: int, layout: str +) -> None: + op, sources = _several_input_sources(label, order_index, layout, role) + assert_async_column_order_equals_sync(op, *sources) @pytest.mark.parametrize( From 3702e659c3bfac310ecd35af68872a1ad5d67f6f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:52:20 -0800 Subject: [PATCH 11/13] test(function_pod, sources, pipeline): weighted outputs, async arrangements, cache-only readback, bind on both orchestrators (ITL-1236) Addresses the second Copilot review of #443. Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_several_items.py | 45 +++++++++++++++---- tests/test_core/sources/test_several_items.py | 17 ++++--- .../test_pipeline/test_bind_several_items.py | 45 +++++++++++++++---- 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/tests/test_core/function_pod/test_several_items.py b/tests/test_core/function_pod/test_several_items.py index 5f287c2b6..53cccd60c 100644 --- a/tests/test_core/function_pod/test_several_items.py +++ b/tests/test_core/function_pod/test_several_items.py @@ -46,7 +46,8 @@ # Parameters in another order than the columns: q2 is in Q, d1 in MAIN, p1 in P. @function_pod(output_keys=["total", "product"]) def combine(q2: int, d1: int, p1: int) -> tuple[int, int]: - return q2 + d1 + p1, d1 * p1 + # Weighted: no two parameters can be swapped without changing ``total``. + return q2 + 2 * d1 + 3 * p1, d1 * p1 def _inputs(role: str) -> list[ArrowTableSource]: @@ -87,14 +88,14 @@ def test_values_are_matched_by_parameter_name(role: str) -> None: (tag["t1"], tag["t2"], tag["t3"], data["total"], data["product"]) for tag, data in stream.iter_data() ) - # MAIN x P on (t1, t2) x Q on (t2, t3); total = q2 + d1 + p1, product = d1 * p1. + # MAIN x P on (t1, t2) x Q on (t2, t3); total = q2 + 2 d1 + 3 p1, product = d1 p1. assert rows == [ - ("a", 1, 1, 400 + 50 + 9, 50 * 9), - ("a", 1, 2, 100 + 20 + 9, 20 * 9), - ("a", 2, 1, 200 + 30 + 7, 30 * 7), - ("b", 1, 2, 100 + 40 + 8, 40 * 8), - ("b", 2, 1, 200 + 10 + 6, 10 * 6), - ("b", 2, 2, 300 + 60 + 6, 60 * 6), + ("a", 1, 1, 400 + 2 * 50 + 3 * 9, 50 * 9), + ("a", 1, 2, 100 + 2 * 20 + 3 * 9, 20 * 9), + ("a", 2, 1, 200 + 2 * 30 + 3 * 7, 30 * 7), + ("b", 1, 2, 100 + 2 * 40 + 3 * 8, 40 * 8), + ("b", 2, 1, 200 + 2 * 10 + 3 * 6, 10 * 6), + ("b", 2, 2, 300 + 2 * 60 + 3 * 6, 60 * 6), ] @@ -152,6 +153,34 @@ def test_a_function_reads_several_data_tags_and_data_in_any_arrangement( assert all(result == first for result in results.values()) +@pytest.mark.parametrize("layout", LAYOUTS) +@pytest.mark.parametrize("order", MAIN.tag_orders(), ids="-".join) +@pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) +def test_a_function_over_several_data_tags_gives_the_same_rows_on_both_orchestrators( + role: str, order: tuple[str, ...], layout: str +) -> None: + """In a job, sync and async: same rows, column order and stored results.""" + tables = [] + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="labels", database=InMemoryArrowDatabase()) + with job: + label_rows.pod( + make_source(MAIN, Arrangement(order, layout, role)), label="labels" + ) + job.run(orchestrator=orchestrator()) + node = job.nodes["labels"] + tables.append( + ( + presentation(node), + [(tuple(t.keys()), tuple(d.keys())) for t, d in node.iter_data()], + rows_fingerprint(list(node.iter_data()), source_info=False), + sorted(str(data["labelled"]) for _, data in node.iter_data()), + ) + ) + assert tables[0] == tables[1] + assert tables[0][3] == ["a1v", "a1y", "a2x", "b1w", "b2u", "b2z"] + + @pytest.mark.parametrize("role", [DATA_TAGS, MIXED]) def test_a_layout_of_the_input_does_not_change_a_fall_through_pods_hashes( role: str, diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index 9dfc9ad5e..50aae4d59 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -160,7 +160,9 @@ def test_tags_as_data_order_is_the_order_of_the_new_columns_only(role: str) -> N @pytest.mark.parametrize("role", ROLES) -def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> None: +def test_a_cached_source_reads_back_every_role_and_arrangement( + role: str, monkeypatch: pytest.MonkeyPatch +) -> None: for order in MAIN.tag_orders(): for layout in LAYOUTS: arrangement = Arrangement(order, layout, role) @@ -169,9 +171,14 @@ def test_a_cached_source_reads_back_every_role_and_arrangement(role: str) -> Non make_source(MAIN, arrangement), cache_database=database ) assert fingerprint(cached) == fingerprint(make_source(MAIN, arrangement)) - # A second CachedSource reads the stored rows back. - again = CachedSource( - make_source(MAIN, arrangement), cache_database=database - ) + # A second CachedSource over the same source (same schema and identity) + # that cannot provide rows: it serves the stored rows only. + unavailable = make_source(MAIN, arrangement) + + def no_rows(*args: object, **kwargs: object) -> pa.Table: + raise NotImplementedError("rows are not available") + + monkeypatch.setattr(unavailable, "as_table", no_rows) + again = CachedSource(unavailable, cache_database=database) assert presentation(again) == presentation(make_source(MAIN, arrangement)) assert fingerprint(again) == fingerprint(cached) diff --git a/tests/test_pipeline/test_bind_several_items.py b/tests/test_pipeline/test_bind_several_items.py index 2a3580297..b3ee295d2 100644 --- a/tests/test_pipeline/test_bind_several_items.py +++ b/tests/test_pipeline/test_bind_several_items.py @@ -29,6 +29,7 @@ P, make_source, presentation, + rows_fingerprint, ) @@ -110,8 +111,29 @@ def test_one_refused_source_leaves_every_binding_as_it_was(refused: str) -> None assert {name: job.nodes[name].bound_source for name in ("s", "p")} == bound_before -def _bind_in(role: str, order: tuple[str, ...], layout: str) -> None: - """Bind ``MAIN`` in another arrangement to slot ``s``: nothing moves, all reads.""" +def _total_rows(job: PipelineJob) -> list[tuple[str, ...]]: + return rows_fingerprint(list(job.nodes["total"].iter_data()), source_info=False) + + +def _total_values(job: PipelineJob) -> list[tuple[str, str]]: + """The rows' tag and data values only: the bound source has another source id.""" + return sorted( + (repr(sorted(tag.as_dict().items())), repr(sorted(data.as_dict().items()))) + for tag, data in job.nodes["total"].iter_data() + ) + + +def _bind_in( + role: str, + order: tuple[str, ...], + layout: str, + orchestrator: type = SyncPipelineOrchestrator, +) -> PipelineJob: + """Bind ``MAIN`` in another arrangement to slot ``s``: nothing moves, all reads. + + Returns: + The job, run on ``orchestrator``. + """ job = _job(role) before = _snapshot(job) # The columns of the table, not their order: CLAUDE.md promises the declared names, @@ -127,17 +149,24 @@ def _bind_in(role: str, order: tuple[str, ...], layout: str) -> None: order, layout, ) - job.run() - assert len(job.nodes["total"].as_table()) == 6 + job.run(orchestrator=orchestrator()) + return job +@pytest.mark.parametrize("layout", LAYOUTS) +@pytest.mark.parametrize("order", MAIN.tag_orders(), ids="-".join) @pytest.mark.parametrize("role", [PLAIN, DATA_TAGS]) def test_a_source_in_another_arrangement_is_bound_as_the_slot_declares( - role: str, + role: str, order: tuple[str, ...], layout: str ) -> None: - for order in MAIN.tag_orders(): - for layout in LAYOUTS: - _bind_in(role, order, layout) + """Both orchestrators: the same rows, with the values of the slot's own source.""" + reference = _job(role) + reference.run() + sync = _bind_in(role, order, layout, SyncPipelineOrchestrator) + asynchronous = _bind_in(role, order, layout, AsyncPipelineOrchestrator) + assert _total_rows(asynchronous) == _total_rows(sync) + assert _total_values(sync) == _total_values(reference) + assert len(_total_values(sync)) == 6 BIND_MIXED_ISSUE = "ITL-1245" From 88f810b6f553bc3733502ce157b70da38d9a8ced Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:53:13 -0800 Subject: [PATCH 12/13] test(data_tags, streams): narrowing checked against the computed schema; Index keeps the data-tag's lists; column groups checked against their own columns (ITL-1236) Addresses the Copilot review of #444. Co-Authored-By: Claude Opus 5.5 --- .../test_core/data_tags/test_several_items.py | 31 ++++++++++++-- .../streams/test_several_items_methods.py | 40 +++++++++++++++++-- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/tests/test_core/data_tags/test_several_items.py b/tests/test_core/data_tags/test_several_items.py index 30ee628b4..f59035b20 100644 --- a/tests/test_core/data_tags/test_several_items.py +++ b/tests/test_core/data_tags/test_several_items.py @@ -59,9 +59,26 @@ def test_drop_nulls_of_several_columns_with_nullable_data_tags( tag_schema, data_schema = kept.output_schema() assert tag_schema["t1"] is str and tag_schema["t2"] is int assert data_schema["d1"] is int - if "t1" not in tag_only: - # A data-tag is one column: narrowed in both roles. - assert data_schema["t1"] is str + # A data-tag is one column: narrowed in both roles, each data-tag. + for name, python_type in (("t1", str), ("t2", int)): + if name not in tag_only: + assert data_schema[name] is python_type + else: + assert name not in data_schema + # The prediction equals the computed stream's schema (a prediction is + # verified against the static result, CLAUDE.md), with system tags and + # source info too. + for config in ( + {}, + {"system_tags": True}, + {"source": True, "system_tags": True}, + ): + computed = op.static_process(source).output_schema(columns=config) + assert kept.output_schema(columns=config) == computed + # The computed rows hold no null in a named column, in either role. + table = kept.as_table() + assert all(table.column(name).null_count == 0 for name in names) + assert not any(table.schema.field(name).nullable for name in names) rows = rows_fingerprint(list(kept.iter_data())) assert rows_fingerprint(async_rows(DropNulls(list(names)), source)) == rows results[names] = {**fingerprint(kept), "presentation": presentation(kept)} @@ -91,8 +108,14 @@ def test_index_reads_a_list_valued_data_tag_as_a_data_column( indexed = op(batched) first = [data[f"first_{column}"] for _, data in indexed.iter_data()] assert first == {"t1": ["b", "b"], "t3": [1, 2], "d1": [10, 40]}[column] - # The data-tag itself is unchanged: still a tag and a data column. + # The data-tag itself is unchanged: still a tag and a data column, with its lists. assert presentation(indexed)[0] == ("t1", "t2", "t3") + assert column in presentation(indexed)[1] + lists = {"t1": [["b", "a", "a"], ["b", "a", "b"]], "t3": [[1, 2, 1], [2, 1, 2]]} + lists["d1"] = [[10, 20, 30], [40, 50, 60]] + assert [data[column] for _, data in indexed.iter_data()] == lists[column] + if column != "d1": + assert [tag[column] for tag, _ in indexed.iter_data()] == lists[column] assert rows_fingerprint( async_rows(Index(column, 0, out=f"first_{column}"), batched) ) == rows_fingerprint(list(indexed.iter_data())) diff --git a/tests/test_core/streams/test_several_items_methods.py b/tests/test_core/streams/test_several_items_methods.py index 47d119a4f..fa31c955f 100644 --- a/tests/test_core/streams/test_several_items_methods.py +++ b/tests/test_core/streams/test_several_items_methods.py @@ -36,6 +36,7 @@ ) from orcapod.core.sources import ArrowTableSource from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.system_constants import constants from ..operators.several_items import ( LAYOUTS, @@ -171,22 +172,53 @@ def test_polars_filter_keywords_win_over_the_constraint_map() -> None: COLUMN_GROUPS = ("meta", "context", "source", "system_tags", "content_hash") +CONTENT_HASH_COLUMN = "_content_hash" + + +def _group_of(name: str) -> str | None: + """The ``ColumnConfig`` group of a column name, or ``None`` for a user column. + + Derived from the column naming convention (CLAUDE.md, "Column naming + conventions"), not from the stream under test. + """ + if name.startswith(constants.META_PREFIX): + return "meta" + if name == constants.CONTEXT_KEY: + return "context" + if name.startswith(constants.SOURCE_PREFIX): + return "source" + if name.startswith(constants.SYSTEM_TAG_PREFIX): + return "system_tags" + if name == CONTENT_HASH_COLUMN: + return "content_hash" + return None @pytest.mark.parametrize("layout", LAYOUTS) @pytest.mark.parametrize("role", ROLES) -def test_several_column_groups_keep_the_all_info_column_order( +def test_several_column_groups_give_exactly_their_columns_in_the_all_info_order( role: str, layout: str ) -> None: - """Every combination of ``ColumnConfig`` groups: the all_info columns, filtered.""" + """Every combination of groups: the user columns and those groups' columns, in order. + + The expected list is the ``all_info`` table's columns kept by their group, so a + missing requested column and an unrequested group both fail. + """ source = make_source(MAIN, Arrangement(MAIN.tags, layout, role)) partner = make_source(P, Arrangement(P.tags, layout, role)) for stream in (source, Join()(source, partner), Batch(batch_size=2)(source)): full = stream.as_table(all_info=True).column_names + present = {_group_of(name) for name in full} - {None} + # Every group but meta shows up in all_info for these streams. + assert present == {"context", "source", "system_tags", "content_hash"} for n in range(1, len(COLUMN_GROUPS) + 1): for groups in itertools.combinations(COLUMN_GROUPS, n): + expected = [ + name + for name in full + if _group_of(name) is None or _group_of(name) in groups + ] columns = stream.as_table( columns=dict.fromkeys(groups, True) ).column_names - assert len(set(columns)) == len(columns), groups - assert columns == [c for c in full if c in columns], groups + assert columns == expected, groups From 54708fe4fb889c766ed8768abbaac17c67022a67 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 18 Nov 2026 14:19:42 -0800 Subject: [PATCH 13/13] test(sources): compare whole record ids with canonical JSON built from the input values (ITL-1236) The record-id test checked only the JSON key names. It now compares each source's record ids, for every key order, with the canonical JSON of each row's key values in that order, built in the test from the input columns, and pins the format with two literals (Copilot, #443). Co-Authored-By: Claude Opus 5.5 --- tests/test_core/sources/test_several_items.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/test_core/sources/test_several_items.py b/tests/test_core/sources/test_several_items.py index 50aae4d59..ff7ee7c8a 100644 --- a/tests/test_core/sources/test_several_items.py +++ b/tests/test_core/sources/test_several_items.py @@ -58,6 +58,23 @@ def _record_ids(source: ArrowTableSource) -> list[str]: return sorted(ids) +def _expected_record_ids(keys: tuple[str, ...]) -> list[str]: + """Each row's record id for ``keys``, built from the input values, sorted. + + The canonical JSON of the key columns in key order: compact separators, the values + as written (``{"t1":"b","t2":2}``). + """ + rows = len(MAIN.columns[keys[0]]) + return sorted( + json.dumps( + {key: MAIN.columns[key][row].as_py() for key in keys}, + separators=(",", ":"), + ensure_ascii=False, + ) + for row in range(rows) + ) + + @pytest.mark.parametrize("role", ROLES) def test_tag_order_is_the_order_of_the_tags_and_enters_no_hash(role: str) -> None: """Every tag order and layout: same rows, hashes and system tags; keys follow.""" @@ -103,10 +120,13 @@ def test_record_ids_follow_the_key_order(role: str) -> None: record_id_columns=list(keys), tag_only_columns=_tag_only(role), ) - ids = _record_ids(source) - # The canonical JSON of the key columns, in key order. - assert all(list(json.loads(record_id)) == list(keys) for record_id in ids) + # The whole record id: the canonical JSON of each row's key values, in key + # order, built here from the input columns (Copilot, #443). + assert _record_ids(source) == _expected_record_ids(keys) by_keys[keys] = source + # The format itself, for the first row (t1="b", t2=2, t3=1, d3="w"). + assert '{"t1":"b","t2":2,"t3":1}' in _expected_record_ids(("t1", "t2", "t3")) + assert '{"d3":"w","t1":"b"}' in _expected_record_ids(("d3", "t1")) first = by_keys[("t1", "t2", "t3")] for keys, source in by_keys.items(): # The keys name the rows: one pipeline hash and one source id ...