Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
0bf5e72
test(operators): several columns, names, mappings and inputs, in ever…
Nov 18, 2026
03f4a26
test(sources, function_pod, pipeline): several tags, keys, inputs and…
Nov 18, 2026
e8ae621
test(operators): async rows equal sync rows in every several-input ca…
Nov 18, 2026
f40e1af
Merge branch 'eywalker/itl-1236-tests-audit-every-argument-that-takes…
Nov 18, 2026
8129cfa
test(operators): issue ids for the defects found (ITL-1243, ITL-1247)…
Nov 18, 2026
86a99fc
Merge branch 'eywalker/itl-1236-tests-audit-every-argument-that-takes…
Nov 18, 2026
ec9c140
test(operators): type the results helper by Mapping (ITL-1236)
Nov 18, 2026
7217f5c
Merge branch 'eywalker/itl-1236-tests-audit-every-argument-that-takes…
Nov 18, 2026
56213e8
test(data_tags, streams, sources): several nullable data-tags, list-v…
Nov 18, 2026
aa7fa71
test(operators): column order of sync rows, async rows and keys, per …
Nov 18, 2026
b66ee60
Merge branch 'eywalker/itl-1236-tests-audit-every-argument-that-takes…
Nov 18, 2026
ea4814f
test(sources, pipeline): compare key orders with each other; check bo…
Nov 18, 2026
86ed772
Merge branch 'eywalker/itl-1236-part2-sources-pods-bind' into eywalke…
Nov 18, 2026
d896a18
test(operators): ITL-1249 for the column order xfails; an operator no…
Nov 18, 2026
fb75b54
Merge branch 'eywalker/itl-1236-tests-audit-every-argument-that-takes…
Nov 18, 2026
74cef31
Merge branch 'eywalker/itl-1236-part2-sources-pods-bind' into eywalke…
Nov 18, 2026
e1800eb
test(operators): one column-order test per arrangement, xfail only wh…
Nov 18, 2026
4bf0e18
Merge branch 'eywalker/itl-1236-tests-audit-every-argument-that-takes…
Nov 18, 2026
3702e65
test(function_pod, sources, pipeline): weighted outputs, async arrang…
Nov 18, 2026
adf6650
Merge branch 'eywalker/itl-1236-part2-sources-pods-bind' into eywalke…
Nov 18, 2026
88f810b
test(data_tags, streams): narrowing checked against the computed sche…
Nov 18, 2026
91c49ec
Merge remote-tracking branch 'origin/main' into eywalker/itl-1236-par…
Nov 18, 2026
54708fe
test(sources): compare whole record ids with canonical JSON built fro…
Nov 18, 2026
c05922f
Merge remote-tracking branch 'origin/eywalker/itl-1236-part2-sources-…
Nov 19, 2026
766d5ee
Merge remote-tracking branch 'origin/main' into HEAD
Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion docs/dev/specs/itl-1236-several-item-test-audit.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,24 @@ Order- or role-dependent behaviour that reaches a hash or a stored table first.
- `bind(sources=)` with two sources: both bound, nothing moves; one refused, neither bound.
A plain or data-tag source in any arrangement binds and reads; a mixed one is ITL-1245.

## Checked and found as documented (part 3)

- `DropNulls` naming plain tags, data-tags holding nulls and data, in every order: one row
set and schema, each named column non-nullable (a data-tag in both roles), sync equal to
async.
- `Index` with `out=` over a list-valued data-tag (after `Batch`): reads it as a data
column; the data-tag stays; sync equal to async.
- A `DictSource` `data_schema` in another field order: the same source (rows, hashes).
- Every stream method given several items equals its operator (rows, hashes, column
order); `polars_filter` keywords win over `constraint_map`.
- Every combination of `ColumnConfig` groups gives the `all_info` columns of those groups
in the same order, for a source, a join and a batch, in every role and layout.

## Remaining

- `MapTags` has no golden entry (adding golden entries is out of scope for this issue).
- The defects and the two undecided rules above: ITL-1241 to ITL-1247.

## Tests added

`tests/test_core/operators/several_items.py` is the shared family: `MAIN` (tags `t1`,
Expand All @@ -146,4 +164,7 @@ three layouts and three role settings. `ops/test_several_items.py` uses it for e
operator (gaps 5, 6, 7, 9, 11 and the operator half of 1, 2, 3). Part 2:
`tests/test_core/sources/test_several_items.py` (gaps 4, 13, 14, 15: CachedSource),
`tests/test_core/function_pod/test_several_items.py` (gaps 1, 10; ITL-1244, ITL-1246) and
`tests/test_pipeline/test_bind_several_items.py` (gap 8; ITL-1245).
`tests/test_pipeline/test_bind_several_items.py` (gap 8; ITL-1245). Part 3:
`tests/test_core/data_tags/test_several_items.py` (gaps 12, 15: `Index`),
`tests/test_core/streams/test_several_items_methods.py` (gap 15: stream methods;
`ColumnConfig` groups) and a schema-order test in the sources file (gap 15).
121 changes: 121 additions & 0 deletions tests/test_core/data_tags/test_several_items.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
"""Data-tags among several named columns: DropNulls and Index (ITL-1236).

* ``DropNulls`` naming several columns, plain tags, data-tags holding nulls and data,
in several orders: one row set, one schema, the same on the async path; each named
column, in both roles for a data-tag, becomes non-nullable.
* ``Index`` with ``out=`` reads a list-valued data-tag (after ``Batch``) as it reads a
data column, on both paths.
"""

from __future__ import annotations

import itertools

import pyarrow as pa
import pytest

from orcapod.core.operators import Batch, DropNulls, Index
from orcapod.core.sources import ArrowTableSource

from ..operators.several_items import (
DATA_TAGS,
MAIN,
MIXED,
TAGS_FIRST,
Arrangement,
async_rows,
fingerprint,
make_source,
presentation,
rows_fingerprint,
)

# Tags t1 and t2 and data d1, each null in one row: only row 0 has no null.
NULLS = pa.table(
{
"t1": pa.array(["a", None, "b", "c"], pa.large_string()),
"t2": pa.array([1, 2, None, 4], pa.int64()),
"d1": pa.array([1, 2, 3, None], pa.int64()),
"d2": pa.array([5, 6, 7, 8], pa.int64()),
}
)


def _nullable_source(tag_only: list[str]) -> ArrowTableSource:
return ArrowTableSource(
NULLS, tag_columns=["t1", "t2"], source_id="n", tag_only_columns=tag_only
)


@pytest.mark.parametrize("tag_only", [[], ["t2"]], ids=["data_tags", "mixed"])
def test_drop_nulls_of_several_columns_with_nullable_data_tags(
tag_only: list[str],
) -> None:
source = _nullable_source(tag_only)
results = {}
for names in itertools.permutations(["t1", "t2", "d1"]):
op = DropNulls(list(names))
kept = op(source)
tag_schema, data_schema = kept.output_schema()
assert tag_schema["t1"] is str and tag_schema["t2"] is int
assert data_schema["d1"] is int
# A data-tag is one column: narrowed in both roles, each data-tag.
for name, python_type in (("t1", str), ("t2", int)):
if name not in tag_only:
assert data_schema[name] is python_type
else:
assert name not in data_schema
# The prediction equals the computed stream's schema (a prediction is
# verified against the static result, CLAUDE.md), with system tags and
# source info too.
for config in (
{},
{"system_tags": True},
{"source": True, "system_tags": True},
):
computed = op.static_process(source).output_schema(columns=config)
assert kept.output_schema(columns=config) == computed
# The computed rows hold no null in a named column, in either role.
table = kept.as_table()
assert all(table.column(name).null_count == 0 for name in names)
assert not any(table.schema.field(name).nullable for name in names)
rows = rows_fingerprint(list(kept.iter_data()))
assert rows_fingerprint(async_rows(DropNulls(list(names)), source)) == rows
results[names] = {**fingerprint(kept), "presentation": presentation(kept)}
assert [data["d2"] for _, data in kept.iter_data()] == [5]
first = next(iter(results.values()))
assert all(result == first for result in results.values())


@pytest.mark.parametrize("tag_only", [[], ["t2"]], ids=["data_tags", "mixed"])
def test_drop_nulls_of_one_data_tag_keeps_the_rows_where_it_has_a_value(
tag_only: list[str],
) -> None:
kept = DropNulls(["t1"])(_nullable_source(tag_only))
assert sorted(data["d2"] for _, data in kept.iter_data()) == [5, 7, 8] # pyright: ignore[reportArgumentType]


@pytest.mark.parametrize("role", [DATA_TAGS, MIXED])
@pytest.mark.parametrize("column", ["t1", "t3", "d1"])
def test_index_reads_a_list_valued_data_tag_as_a_data_column(
role: str, column: str
) -> None:
"""After ``Batch``, t1 and t3 are list-valued data-tags (t2 is plain in MIXED)."""
batched = Batch(batch_size=3)(
make_source(MAIN, Arrangement(MAIN.tags, TAGS_FIRST, role))
)
op = Index(column, 0, out=f"first_{column}")
indexed = op(batched)
first = [data[f"first_{column}"] for _, data in indexed.iter_data()]
assert first == {"t1": ["b", "b"], "t3": [1, 2], "d1": [10, 40]}[column]
# The data-tag itself is unchanged: still a tag and a data column, with its lists.
assert presentation(indexed)[0] == ("t1", "t2", "t3")
assert column in presentation(indexed)[1]
lists = {"t1": [["b", "a", "a"], ["b", "a", "b"]], "t3": [[1, 2, 1], [2, 1, 2]]}
lists["d1"] = [[10, 20, 30], [40, 50, 60]]
assert [data[column] for _, data in indexed.iter_data()] == lists[column]
if column != "d1":
assert [tag[column] for tag, _ in indexed.iter_data()] == lists[column]
assert rows_fingerprint(
async_rows(Index(column, 0, out=f"first_{column}"), batched)
) == rows_fingerprint(list(indexed.iter_data()))
43 changes: 42 additions & 1 deletion tests/test_core/sources/test_several_items.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@
import pyarrow as pa
import pytest

from orcapod.core.sources import ArrowTableSource, CachedSource, DerivedSource
from orcapod.core.sources import (
ArrowTableSource,
CachedSource,
DerivedSource,
DictSource,
)
from orcapod.databases import InMemoryArrowDatabase

from ..operators.several_items import (
Expand Down Expand Up @@ -202,3 +207,39 @@ def no_rows(*args: object, **kwargs: object) -> pa.Table:
again = CachedSource(unavailable, cache_database=database)
assert presentation(again) == presentation(make_source(MAIN, arrangement))
assert fingerprint(again) == fingerprint(cached)


@pytest.mark.parametrize(
"data_schema",
[
{"t1": str, "t2": int, "d1": int, "d3": str},
{"d3": str, "t2": int, "d1": int, "t1": str},
],
ids=["table_order", "another_order"],
)
@pytest.mark.parametrize("role", ROLES)
def test_a_schema_in_another_field_order_gives_the_same_source(
data_schema: dict[str, type], role: str
) -> None:
"""A ``DictSource`` schema with several fields, in the rows' order or another."""
rows = [
{"t1": "a", "t2": 1, "d1": 10, "d3": "x"},
{"t1": "b", "t2": 2, "d1": 20, "d3": "y"},
]
tag_only = {PLAIN: True, DATA_TAGS: [], MIXED: ["t2"]}[role]
source = DictSource(
rows,
tag_columns=["t1", "t2"],
source_id="d",
data_schema=data_schema,
tag_only_columns=tag_only,
)
reference = DictSource(
rows,
tag_columns=["t1", "t2"],
source_id="d",
data_schema={"t1": str, "t2": int, "d1": int, "d3": str},
tag_only_columns=tag_only,
)
assert fingerprint(source) == fingerprint(reference)
assert presentation(source)[0] == ("t1", "t2")
Loading
Loading