diff --git a/changelog.d/ITL-1029.md b/changelog.d/ITL-1029.md new file mode 100644 index 000000000..1083de994 --- /dev/null +++ b/changelog.d/ITL-1029.md @@ -0,0 +1,3 @@ +### Fixed + +- A job node's table gave rows the content hash of other rows (ITL-1029). `node.as_table(all_info=True)`, or any `columns` that asked for `content_hash` together with `sort_by_tags`, sorted the rows by tag and then attached the `_content_hash` column in the order the node had computed its rows. Whenever that order differed from tag order, rows carried another row's hash: on the sync path when the node's input was not in tag order, and on the async path whenever rows finished out of order, which happens at random. The hashes are now attached before the sort, so each row keeps its own. Stored records, record ids and every stored or golden hash were never affected; only this in-memory table was. A function pod's own stream (`FunctionPodStream`) and operator nodes were already correct. diff --git a/design_issues/ITL-1029-FN2.md b/design_issues/ITL-1029-FN2.md new file mode 100644 index 000000000..691d0cc31 --- /dev/null +++ b/design_issues/ITL-1029-FN2.md @@ -0,0 +1,21 @@ +## `src/orcapod/core/nodes/function_node.py` + +### FN2 — a job node's sorted table attached content hashes to the wrong rows +**Status:** resolved +**Severity:** critical +**Issue:** ITL-1029 + +`FunctionNodeBase._shape_table` sorted the rows of a function node's table by tag +(`arrow_utils.apply_column_config` with `sort_by_tags`, which `all_info=True` turns on) and +only then appended the `_content_hash` column, built from `iter_data()` in the order the node +held its rows: input order on the sync path, completion order on the async path. Whenever +that order was not tag order, `node.as_table(all_info=True)` gave rows the content hash of +other rows. On the async path rows finish out of order at random, which made two tests that +compare a script run with a run inside an event loop fail now and then on CI (ITL-1001); on +the sync path an input not in tag order showed it every time. Stored records, record ids and +stored hashes were correct; no library code reads that column. + +**Fix:** `_shape_table` selects the column groups, appends the hash column, and only then +sorts (`arrow_utils.sort_by_tag_columns`), as `FunctionPodStream.as_table` and +`ArrowTableStream.as_table` already did. `tests/test_pipeline/test_job_node_row_hashes.py` +forces both orders deterministically (reversed input; an async row held back by an event). diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 6148314b4..8fda8cf78 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -18,6 +18,7 @@ import asyncio import contextlib +import dataclasses import logging import uuid from abc import abstractmethod @@ -587,14 +588,18 @@ def _shape_table( ) -> "pa.Table": """Return ``table`` with the column groups of ``column_config``. + The content hashes are attached before the rows are sorted by tag + (``sort_by_tags``), so every row keeps its own hash (ITL-1029). + Args: table: Output rows with every column group (``_table_from_pairs``). column_config: The column groups to keep. - content_hashes: Returns the content hash of each row of ``table``; called - only when ``column_config`` asks for them. + content_hashes: Returns the content hash of each row of ``table``, in the + order of ``table``; called only when ``column_config`` asks for them. """ + tag_keys = self.keys()[0] output_table = arrow_utils.apply_column_config( - table, column_config, self.keys()[0] + table, dataclasses.replace(column_config, sort_by_tags=False), tag_keys ) if column_config.content_hash: hash_column_name = ( @@ -605,6 +610,8 @@ def _shape_table( output_table = output_table.append_column( hash_column_name, content_hashes() ) + if column_config.sort_by_tags: + output_table = arrow_utils.sort_by_tag_columns(output_table, tag_keys) return output_table def head( diff --git a/src/orcapod/utils/arrow_utils.py b/src/orcapod/utils/arrow_utils.py index e12c61ac0..d85e9a75c 100644 --- a/src/orcapod/utils/arrow_utils.py +++ b/src/orcapod/utils/arrow_utils.py @@ -1833,17 +1833,30 @@ def apply_column_config( ) output_table = table.drop([c for c in drop_columns if c in table.column_names]) if column_config.sort_by_tags: - import polars as pl - - output_table_schema = output_table.schema - output_table = ( - pl.DataFrame(output_table) - .sort(by=list(tag_keys), descending=False) - .to_arrow() - ) - output_table = restore_schema_nullability(output_table, output_table_schema) + output_table = sort_by_tag_columns(output_table, tag_keys) return output_table +def sort_by_tag_columns(table: "pa.Table", tag_keys: tuple[str, ...]) -> "pa.Table": + """Return ``table`` with its rows sorted ascending by the tag columns ``tag_keys``. + + Whole rows move together: a column added to ``table`` before the sort stays with + its row. Column types and nullability are those of ``table``. + + Args: + table: The table to sort. + tag_keys: Names of the regular (non-system) tag columns, in sort priority. + + Returns: + A new table with the same columns, rows in tag order. + """ + import polars as pl + + sorted_table = ( + pl.DataFrame(table).sort(by=list(tag_keys), descending=False).to_arrow() + ) + return restore_schema_nullability(sorted_table, table.schema) + + if __name__ == "__main__": test_schema_normalization() diff --git a/superpowers/plans/2026-11-12-itl-1001-flaky-async-run.md b/superpowers/plans/2026-11-12-itl-1001-flaky-async-run.md new file mode 100644 index 000000000..e8533d8b0 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-1001-flaky-async-run.md @@ -0,0 +1,71 @@ +# ITL-1001 / ITL-1029 — flaky async-run tests + +Branch `eywalker/itl-1001-flaky-timing-test`. PR against `main`: `Fixes ITL-1029`, `Refs ITL-1001` +(coordinator, November 12, 2026: ITL-1001's original goal, `test_concurrent_with_io_bound_work` +and the wall-clock sweep, stays open). + +## Findings + +### Tests 1 and 2: library bug (ITL-1029) + +`test_run_inside_running_loop_stores_what_a_script_run_stores[in_memory]` and +`test_await_run_async_stores_what_a_script_run_stores[in_memory]`. + +CI logs (runs 43369849949 / job 131702869649 and 43356121100 / job 131658688707): the stored +records are equal. The difference is in `node.as_table(all_info=True)`: the `_content_hash` +column of the *script* run is rotated against its rows (r1 carries r2's output hash, ...). + +Mechanism: `FunctionNodeBase._shape_table` (`src/orcapod/core/nodes/function_node.py:596-607`) +calls `arrow_utils.apply_column_config`, which sorts the rows by tag when `sort_by_tags` is on +(`all_info=True` turns it on), and only then appends the hash column, built from +`iter_data()` in `_cached_output_datas` order. The async node fills that dict in completion +order (`record_and_forward`), so when rows finish out of order the hashes land on other rows. +The sync path computes in input order, which is tag order in these tests, so it never shows. +`FunctionPodStream.as_table` and `ArrowTableStream.as_table` append the hash before sorting. + +Deterministic repro: one row that sleeps makes it finish last: misaligned every time. + +Other table-shaping paths checked: `OperatorNode.as_table` delegates to +`ArrowTableStream.as_table` (correct order); node `head()` refuses `sort_by_tags`, so its +hashes follow its rows; the run summary builds no table. + +### Test 3: test assumption + +`test_crash_mid_batch_keeps_committed_batches[async]`, CI `assert (100 == 200)` on `records`. + +The async pod creates one task per input with no limit (`function_pod.py:644-648`, +`max_concurrency` unlimited by default), and each task hands the sync function to the thread +pool at once (`data_function.py:864`). The node records rows and commits every 100 in a +separate stage (`record_and_forward`, `_row_processed`, `function_node.py:1694`), on the loop +thread. So the thread pool can compute row 234 while fewer than 200 rows are recorded, most +easily while the loop is blocked in the 100-row Delta commit. The committed batches are intact +(100 records, 100+ results); nothing is lost. The library never promised that row x starts only +after the batches before it are committed. + +Measured (scratch probe, uninterrupted async runs, 250 rows, 100 per commit): rows started +when the second node commit begins (flush 3): 223, 223, 223, 223, 239. So the thread pool +runs 24 to 40 rows ahead of recording (16-row result channel plus rows in flight); 234 is +inside that range. Natural rate on macOS: 0 / 510 under 3 processes (the CI failures were on +Linux). A slow first commit (scratch adversary, 0.3 s) did not change it (0 / 60): what matters +is the lag at the second commit, not the time the loop is blocked. + +## Plan + +1. Regression test for ITL-1029 (public interface): a job whose function makes the first row + finish last, deterministically (threading events, no sleep), run with the async + orchestrator; `node.as_table(all_info=True)` each row's `_content_hash` equals the hash + of its own data (compare with `iter_data()` pairs keyed by tag). Show it fails. +2. Fix `_shape_table`: append the hash column to the full table, then `apply_column_config`. +3. Test 3: in `JOB_SCRIPT`, the function at the crash row waits on a `threading.Event` set by + `CrashOnFlush.flush` after the inner flush returns, once the store has finished + `1 + x // every` flushes (the run's first flush plus one per committed batch). Bounded wait; + on timeout exit with a distinct code, which `run_job` rejects (test fails, not passes). + Keep `records == 200` and `200 <= results < 234`. +4. Before/after rates: loop each test >= 500 times under load (<= 4 processes); for test 3 also + an adversarial slow first commit (scratch only) to show the old form fails and the new does not. +5. design_issues entry `ITL-1029-FN2.md`, changelog fragment `changelog.d/ITL-1029.md`. +6. Full suite, ruff, basedpyright; PR; CodeRabbit. + +## Status notes + +- Setting ITL-1029 In Progress was denied by the auto-mode classifier; reported to coordinator. diff --git a/tests/test_pipeline/test_job_node_row_hashes.py b/tests/test_pipeline/test_job_node_row_hashes.py new file mode 100644 index 000000000..dc9150fe8 --- /dev/null +++ b/tests/test_pipeline/test_job_node_row_hashes.py @@ -0,0 +1,125 @@ +"""A job node's table gives every row its own content hash, in any row order (ITL-1029). + +``node.as_table`` sorts its rows by tag when asked to (``all_info=True`` asks). The content +hash of a row must stay with that row. A node holds its rows in the order they were +computed: input order on the sync path, completion order on the async path. Both orders can +differ from tag order, and these tests force each of them to. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + +N_ROWS = 5 + +# Set by the last row of ``first_row_last`` once it is computed; the first row waits for it. +# Bound to the running loop on its first wait, so each async run gets a fresh one. +last_row_computed: asyncio.Event | None = None + + +@function_pod(output_keys="y") +def double(x: int) -> int: + """Return twice ``x``.""" + return 2 * x + + +@function_pod(output_keys="y") +async def first_row_last(x: int) -> int: + """Return twice ``x``; row 0 finishes only after the last row is computed. + + The wait is bounded: if the last row never comes, row 0 fails and the test sees a + missing row instead of hanging. + """ + assert last_row_computed is not None + if x == 0: + await asyncio.wait_for(last_row_computed.wait(), timeout=60) + elif x == N_ROWS - 1: + last_row_computed.set() + return 2 * x + + +def run_job(pod: Any, rows: list[dict[str, Any]], orchestrator: Any) -> Any: + """Run ``pod`` over ``rows`` in a fresh job; return its node.""" + job = PipelineJob(name="row_hashes", store=InMemoryArrowDatabase()) + with job: + pod.pod( + DictSource(data=rows, tag_columns=["k"], source_id="rows"), label="node" + ) + job.run(orchestrator=orchestrator) + return job.nodes["node"] + + +def sync_with_input_out_of_tag_order() -> Any: + """A sync run whose input rows come in reverse tag order.""" + rows = [{"k": f"r{i}", "x": i} for i in reversed(range(N_ROWS))] + return run_job(double, rows, SyncPipelineOrchestrator()) + + +def async_with_first_row_finishing_last() -> Any: + """An async run whose input is in tag order but whose first row finishes last.""" + global last_row_computed + last_row_computed = asyncio.Event() + rows = [{"k": f"r{i}", "x": i} for i in range(N_ROWS)] + return run_job(first_row_last, rows, AsyncPipelineOrchestrator()) + + +RUNS = { + "sync-input-out-of-tag-order": sync_with_input_out_of_tag_order, + "async-completion-out-of-tag-order": async_with_first_row_finishing_last, +} + +SORTED_WITH_HASHES = { + "all_info": {"all_info": True}, + "content_hash-sort_by_tags": { + "columns": {"content_hash": True, "sort_by_tags": True} + }, + "custom-name-sort_by_tags": { + "columns": {"content_hash": "row_hash", "sort_by_tags": True} + }, +} + + +@pytest.mark.parametrize("run", list(RUNS)) +@pytest.mark.parametrize("table_args", list(SORTED_WITH_HASHES)) +def test_each_row_of_a_sorted_table_has_its_own_content_hash( + run: str, table_args: str +) -> None: + """Every row of the tag-sorted table carries the content hash of its own data.""" + node = RUNS[run]() + own_hash = { + tag.as_dict()["k"]: data.content_hash().to_string() + for tag, data in node.iter_data() + } + # The run must really hold its rows out of tag order, or the test proves nothing. + assert len(own_hash) == N_ROWS + assert list(own_hash) != sorted(own_hash) + + table = node.as_table(**SORTED_WITH_HASHES[table_args]) + hash_column = "row_hash" if table_args.startswith("custom") else "_content_hash" + tags = table.column("k").to_pylist() + assert tags == sorted(own_hash) + assert table.column(hash_column).to_pylist() == [own_hash[k] for k in tags] + # The rest of each row stays with its tag as well. + assert table.column("y").to_pylist() == [2 * int(k[1:]) for k in tags] + + +@pytest.mark.parametrize("run", list(RUNS)) +def test_unsorted_table_keeps_hashes_with_rows(run: str) -> None: + """Without sorting, the table and its hashes are both in the node's own row order.""" + node = RUNS[run]() + pairs = list(node.iter_data()) + + table = node.as_table(columns={"content_hash": True}) + + assert table.column("k").to_pylist() == [tag.as_dict()["k"] for tag, _ in pairs] + assert table.column("_content_hash").to_pylist() == [ + data.content_hash().to_string() for _, data in pairs + ]