Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog.d/ITL-1029.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- A job node's table gave rows the content hash of other rows (ITL-1029). `node.as_table(all_info=True)`, or any `columns` that asked for `content_hash` together with `sort_by_tags`, sorted the rows by tag and then attached the `_content_hash` column in the order the node had computed its rows. Whenever that order differed from tag order, rows carried another row's hash: on the sync path when the node's input was not in tag order, and on the async path whenever rows finished out of order, which happens at random. The hashes are now attached before the sort, so each row keeps its own. Stored records, record ids and every stored or golden hash were never affected; only this in-memory table was. A function pod's own stream (`FunctionPodStream`) and operator nodes were already correct.
21 changes: 21 additions & 0 deletions design_issues/ITL-1029-FN2.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
## `src/orcapod/core/nodes/function_node.py`

### FN2 — a job node's sorted table attached content hashes to the wrong rows
**Status:** resolved
**Severity:** critical
**Issue:** ITL-1029

`FunctionNodeBase._shape_table` sorted the rows of a function node's table by tag
(`arrow_utils.apply_column_config` with `sort_by_tags`, which `all_info=True` turns on) and
only then appended the `_content_hash` column, built from `iter_data()` in the order the node
held its rows: input order on the sync path, completion order on the async path. Whenever
that order was not tag order, `node.as_table(all_info=True)` gave rows the content hash of
other rows. On the async path rows finish out of order at random, which made two tests that
compare a script run with a run inside an event loop fail now and then on CI (ITL-1001); on
the sync path an input not in tag order showed it every time. Stored records, record ids and
stored hashes were correct; no library code reads that column.

**Fix:** `_shape_table` selects the column groups, appends the hash column, and only then
sorts (`arrow_utils.sort_by_tag_columns`), as `FunctionPodStream.as_table` and
`ArrowTableStream.as_table` already did. `tests/test_pipeline/test_job_node_row_hashes.py`
forces both orders deterministically (reversed input; an async row held back by an event).
13 changes: 10 additions & 3 deletions src/orcapod/core/nodes/function_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import asyncio
import contextlib
import dataclasses
import logging
import uuid
from abc import abstractmethod
Expand Down Expand Up @@ -587,14 +588,18 @@ def _shape_table(
) -> "pa.Table":
"""Return ``table`` with the column groups of ``column_config``.

The content hashes are attached before the rows are sorted by tag
(``sort_by_tags``), so every row keeps its own hash (ITL-1029).

Args:
table: Output rows with every column group (``_table_from_pairs``).
column_config: The column groups to keep.
content_hashes: Returns the content hash of each row of ``table``; called
only when ``column_config`` asks for them.
content_hashes: Returns the content hash of each row of ``table``, in the
order of ``table``; called only when ``column_config`` asks for them.
"""
tag_keys = self.keys()[0]
output_table = arrow_utils.apply_column_config(
table, column_config, self.keys()[0]
table, dataclasses.replace(column_config, sort_by_tags=False), tag_keys
)
if column_config.content_hash:
hash_column_name = (
Expand All @@ -605,6 +610,8 @@ def _shape_table(
output_table = output_table.append_column(
hash_column_name, content_hashes()
)
if column_config.sort_by_tags:
output_table = arrow_utils.sort_by_tag_columns(output_table, tag_keys)
return output_table

def head(
Expand Down
31 changes: 22 additions & 9 deletions src/orcapod/utils/arrow_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1833,17 +1833,30 @@ def apply_column_config(
)
output_table = table.drop([c for c in drop_columns if c in table.column_names])
if column_config.sort_by_tags:
import polars as pl

output_table_schema = output_table.schema
output_table = (
pl.DataFrame(output_table)
.sort(by=list(tag_keys), descending=False)
.to_arrow()
)
output_table = restore_schema_nullability(output_table, output_table_schema)
output_table = sort_by_tag_columns(output_table, tag_keys)
return output_table


def sort_by_tag_columns(table: "pa.Table", tag_keys: tuple[str, ...]) -> "pa.Table":
"""Return ``table`` with its rows sorted ascending by the tag columns ``tag_keys``.

Whole rows move together: a column added to ``table`` before the sort stays with
its row. Column types and nullability are those of ``table``.

Args:
table: The table to sort.
tag_keys: Names of the regular (non-system) tag columns, in sort priority.

Returns:
A new table with the same columns, rows in tag order.
"""
import polars as pl

sorted_table = (
pl.DataFrame(table).sort(by=list(tag_keys), descending=False).to_arrow()
)
return restore_schema_nullability(sorted_table, table.schema)


if __name__ == "__main__":
test_schema_normalization()
71 changes: 71 additions & 0 deletions superpowers/plans/2026-11-12-itl-1001-flaky-async-run.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# ITL-1001 / ITL-1029 — flaky async-run tests

Branch `eywalker/itl-1001-flaky-timing-test`. PR against `main`: `Fixes ITL-1029`, `Refs ITL-1001`
(coordinator, November 12, 2026: ITL-1001's original goal, `test_concurrent_with_io_bound_work`
and the wall-clock sweep, stays open).

## Findings

### Tests 1 and 2: library bug (ITL-1029)

`test_run_inside_running_loop_stores_what_a_script_run_stores[in_memory]` and
`test_await_run_async_stores_what_a_script_run_stores[in_memory]`.

CI logs (runs 43369849949 / job 131702869649 and 43356121100 / job 131658688707): the stored
records are equal. The difference is in `node.as_table(all_info=True)`: the `_content_hash`
column of the *script* run is rotated against its rows (r1 carries r2's output hash, ...).

Mechanism: `FunctionNodeBase._shape_table` (`src/orcapod/core/nodes/function_node.py:596-607`)
calls `arrow_utils.apply_column_config`, which sorts the rows by tag when `sort_by_tags` is on
(`all_info=True` turns it on), and only then appends the hash column, built from
`iter_data()` in `_cached_output_datas` order. The async node fills that dict in completion
order (`record_and_forward`), so when rows finish out of order the hashes land on other rows.
The sync path computes in input order, which is tag order in these tests, so it never shows.
`FunctionPodStream.as_table` and `ArrowTableStream.as_table` append the hash before sorting.

Deterministic repro: one row that sleeps makes it finish last: misaligned every time.

Other table-shaping paths checked: `OperatorNode.as_table` delegates to
`ArrowTableStream.as_table` (correct order); node `head()` refuses `sort_by_tags`, so its
hashes follow its rows; the run summary builds no table.

### Test 3: test assumption

`test_crash_mid_batch_keeps_committed_batches[async]`, CI `assert (100 == 200)` on `records`.

The async pod creates one task per input with no limit (`function_pod.py:644-648`,
`max_concurrency` unlimited by default), and each task hands the sync function to the thread
pool at once (`data_function.py:864`). The node records rows and commits every 100 in a
separate stage (`record_and_forward`, `_row_processed`, `function_node.py:1694`), on the loop
thread. So the thread pool can compute row 234 while fewer than 200 rows are recorded, most
easily while the loop is blocked in the 100-row Delta commit. The committed batches are intact
(100 records, 100+ results); nothing is lost. The library never promised that row x starts only
after the batches before it are committed.

Measured (scratch probe, uninterrupted async runs, 250 rows, 100 per commit): rows started
when the second node commit begins (flush 3): 223, 223, 223, 223, 239. So the thread pool
runs 24 to 40 rows ahead of recording (16-row result channel plus rows in flight); 234 is
inside that range. Natural rate on macOS: 0 / 510 under 3 processes (the CI failures were on
Linux). A slow first commit (scratch adversary, 0.3 s) did not change it (0 / 60): what matters
is the lag at the second commit, not the time the loop is blocked.

## Plan

1. Regression test for ITL-1029 (public interface): a job whose function makes the first row
finish last, deterministically (threading events, no sleep), run with the async
orchestrator; `node.as_table(all_info=True)` each row's `_content_hash` equals the hash
of its own data (compare with `iter_data()` pairs keyed by tag). Show it fails.
2. Fix `_shape_table`: append the hash column to the full table, then `apply_column_config`.
3. Test 3: in `JOB_SCRIPT`, the function at the crash row waits on a `threading.Event` set by
`CrashOnFlush.flush` after the inner flush returns, once the store has finished
`1 + x // every` flushes (the run's first flush plus one per committed batch). Bounded wait;
on timeout exit with a distinct code, which `run_job` rejects (test fails, not passes).
Keep `records == 200` and `200 <= results < 234`.
4. Before/after rates: loop each test >= 500 times under load (<= 4 processes); for test 3 also
an adversarial slow first commit (scratch only) to show the old form fails and the new does not.
5. design_issues entry `ITL-1029-FN2.md`, changelog fragment `changelog.d/ITL-1029.md`.
6. Full suite, ruff, basedpyright; PR; CodeRabbit.

## Status notes

- Setting ITL-1029 In Progress was denied by the auto-mode classifier; reported to coordinator.
125 changes: 125 additions & 0 deletions tests/test_pipeline/test_job_node_row_hashes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
"""A job node's table gives every row its own content hash, in any row order (ITL-1029).

``node.as_table`` sorts its rows by tag when asked to (``all_info=True`` asks). The content
hash of a row must stay with that row. A node holds its rows in the order they were
computed: input order on the sync path, completion order on the async path. Both orders can
differ from tag order, and these tests force each of them to.
"""

from __future__ import annotations

import asyncio
from typing import Any

import pytest

from orcapod import PipelineJob, function_pod
from orcapod.databases import InMemoryArrowDatabase
from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator
from orcapod.sources import DictSource

N_ROWS = 5

# Set by the last row of ``first_row_last`` once it is computed; the first row waits for it.
# Bound to the running loop on its first wait, so each async run gets a fresh one.
last_row_computed: asyncio.Event | None = None


@function_pod(output_keys="y")
def double(x: int) -> int:
"""Return twice ``x``."""
return 2 * x


@function_pod(output_keys="y")
async def first_row_last(x: int) -> int:
"""Return twice ``x``; row 0 finishes only after the last row is computed.

The wait is bounded: if the last row never comes, row 0 fails and the test sees a
missing row instead of hanging.
"""
assert last_row_computed is not None
if x == 0:
await asyncio.wait_for(last_row_computed.wait(), timeout=60)
elif x == N_ROWS - 1:
last_row_computed.set()
return 2 * x


def run_job(pod: Any, rows: list[dict[str, Any]], orchestrator: Any) -> Any:
"""Run ``pod`` over ``rows`` in a fresh job; return its node."""
job = PipelineJob(name="row_hashes", store=InMemoryArrowDatabase())
with job:
pod.pod(
DictSource(data=rows, tag_columns=["k"], source_id="rows"), label="node"
)
job.run(orchestrator=orchestrator)
return job.nodes["node"]


def sync_with_input_out_of_tag_order() -> Any:
"""A sync run whose input rows come in reverse tag order."""
rows = [{"k": f"r{i}", "x": i} for i in reversed(range(N_ROWS))]
return run_job(double, rows, SyncPipelineOrchestrator())


def async_with_first_row_finishing_last() -> Any:
"""An async run whose input is in tag order but whose first row finishes last."""
global last_row_computed
last_row_computed = asyncio.Event()
rows = [{"k": f"r{i}", "x": i} for i in range(N_ROWS)]
return run_job(first_row_last, rows, AsyncPipelineOrchestrator())


RUNS = {
"sync-input-out-of-tag-order": sync_with_input_out_of_tag_order,
"async-completion-out-of-tag-order": async_with_first_row_finishing_last,
}

SORTED_WITH_HASHES = {
"all_info": {"all_info": True},
"content_hash-sort_by_tags": {
"columns": {"content_hash": True, "sort_by_tags": True}
},
"custom-name-sort_by_tags": {
"columns": {"content_hash": "row_hash", "sort_by_tags": True}
},
}


@pytest.mark.parametrize("run", list(RUNS))
@pytest.mark.parametrize("table_args", list(SORTED_WITH_HASHES))
def test_each_row_of_a_sorted_table_has_its_own_content_hash(
run: str, table_args: str
) -> None:
"""Every row of the tag-sorted table carries the content hash of its own data."""
node = RUNS[run]()
own_hash = {
tag.as_dict()["k"]: data.content_hash().to_string()
for tag, data in node.iter_data()
}
# The run must really hold its rows out of tag order, or the test proves nothing.
assert len(own_hash) == N_ROWS
assert list(own_hash) != sorted(own_hash)

table = node.as_table(**SORTED_WITH_HASHES[table_args])
hash_column = "row_hash" if table_args.startswith("custom") else "_content_hash"
tags = table.column("k").to_pylist()
assert tags == sorted(own_hash)
assert table.column(hash_column).to_pylist() == [own_hash[k] for k in tags]
# The rest of each row stays with its tag as well.
assert table.column("y").to_pylist() == [2 * int(k[1:]) for k in tags]


@pytest.mark.parametrize("run", list(RUNS))
def test_unsorted_table_keeps_hashes_with_rows(run: str) -> None:
"""Without sorting, the table and its hashes are both in the node's own row order."""
node = RUNS[run]()
pairs = list(node.iter_data())

table = node.as_table(columns={"content_hash": True})

assert table.column("k").to_pylist() == [tag.as_dict()["k"] for tag, _ in pairs]
assert table.column("_content_hash").to_pylist() == [
data.content_hash().to_string() for _, data in pairs
]
Loading