From 0703724323986e2249663ea3d5aeb2ca9ea9ee53 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 01:30:20 -0800 Subject: [PATCH 01/21] docs(itl-895): root-cause findings and evidence scripts for float-to-int truncation Step 1 of ITL-895: why a pod declared -> int stores 1.5 as 1, with one reproduction script per claim, captured outputs, and fix options. Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-11-itl-895-float-to-int.md | 12 + .../01_pyarrow_minimal.py | 163 +++++++++ .../itl-895-float-to-int/02_call_chain.py | 138 ++++++++ .../itl-895-float-to-int/03_split_datagram.py | 69 ++++ .../itl-895-float-to-int/04_input_sites.py | 186 +++++++++++ .../itl-895-float-to-int/05_pod_matrix.py | 146 ++++++++ .../06_released_versions.py | 31 ++ .../itl-895-float-to-int/07_blast_radius.py | 79 +++++ .../itl-895-float-to-int/08_none_output.py | 93 ++++++ .../specs/itl-895-float-to-int/FINDINGS.md | 315 ++++++++++++++++++ .../specs/itl-895-float-to-int/_tracer.py | 167 ++++++++++ 11 files changed, 1399 insertions(+) create mode 100644 superpowers/plans/2026-11-11-itl-895-float-to-int.md create mode 100644 superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py create mode 100644 superpowers/specs/itl-895-float-to-int/02_call_chain.py create mode 100644 superpowers/specs/itl-895-float-to-int/03_split_datagram.py create mode 100644 superpowers/specs/itl-895-float-to-int/04_input_sites.py create mode 100644 superpowers/specs/itl-895-float-to-int/05_pod_matrix.py create mode 100644 superpowers/specs/itl-895-float-to-int/06_released_versions.py create mode 100644 superpowers/specs/itl-895-float-to-int/07_blast_radius.py create mode 100644 superpowers/specs/itl-895-float-to-int/08_none_output.py create mode 100644 superpowers/specs/itl-895-float-to-int/FINDINGS.md create mode 100644 superpowers/specs/itl-895-float-to-int/_tracer.py diff --git a/superpowers/plans/2026-11-11-itl-895-float-to-int.md b/superpowers/plans/2026-11-11-itl-895-float-to-int.md new file mode 100644 index 000000000..4bc822266 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-895-float-to-int.md @@ -0,0 +1,12 @@ +# ITL-895: function output values are not checked against the declared type + +Status log (for a successor picking this up): + +- 2026-11-11 — Step 1 (root cause) done. Findings and one evidence script per claim are in + `superpowers/specs/itl-895-float-to-int/` (`FINDINGS.md`, script outputs in `outputs/`). + The findings were sent to the coordinator with `orca orchestration ask`. **Waiting for the + coordinator's choice of fix option ((a) pod check, (b) strict converter, (c) both on one + rule) and the conformance decisions D1–D7 in FINDINGS section 11.** No library code has + been changed yet. + +The implementation plan (writing-plans) is written here after the coordinator replies. diff --git a/superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py b/superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py new file mode 100644 index 000000000..b57638cb1 --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py @@ -0,0 +1,163 @@ +"""Claim 1: pyarrow's Python-to-Arrow conversion itself truncates float -> integer. + +Pure pyarrow (plus numpy / stdlib values); no orcapod import. Shows: + +1. The minimal reproduction of the exact orcapod call: + ``pa.Table.from_pylist([{"y": 1.5}], schema=pa.schema([("y", pa.int64())]))``. +2. A matrix of Python value x declared Arrow type through the same call, so the + asymmetry (float -> int truncates, the other mismatches raise) is visible. +3. The same call through ``pa.array(values, type=...)``, and the explicit safe cast + (``pc.cast``), which does refuse 1.5 -> int64. + +Why (from pyarrow's own C++ source, shipped in the wheel under +``pyarrow/src/arrow/python``): + +* ``python_to_arrow.cc`` ``PyValue::Convert(IntegerType)`` calls ``CIntFromPython``. +* ``helpers.cc`` ``CIntFromPython`` rejects ``bool`` explicitly, then + ``CIntFromPythonImpl``: if the object is not a ``PyLong`` it calls ``PyObjectToPyInt``, + which tries ``__index__`` and, when that fails, FALLS BACK TO ``__int__`` + (``nb_int``). ``float.__int__`` truncates toward zero, so 1.5 -> 1, -1.5 -> -1. + Range is checked afterwards (overflow raises). Anything with ``__int__`` is accepted + the same way: ``Decimal``, ``Fraction``, ``numpy.float64``. +* ``Convert(DoubleType)`` accepts float, numpy float, and ints only through + ``IntegerScalarToDoubleSafe`` (|v| <= 2**53, else raises). +* ``Convert(FloatType)`` (float32) does ``static_cast(double)``: rounds silently. +* ``Convert(BooleanType)`` accepts only ``True``/``False``/``numpy.bool_``. +* String/binary converters reject non-str/bytes. +* Timestamp/date/time/duration converters fall through to ``CIntFromPython`` for + non-datetime objects, so a float there truncates too (1.5 -> 1 tick). + +No conversion option changes this: ``pa.array``/``from_pylist`` take no ``safe=`` for +Python sequences (``safe`` only applies to pandas/numpy inputs), and ``from_pandas`` +does not change the integer converter. + +Run: uv run python superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py + uv run --no-project --with pyarrow==20.0.0 --with numpy \ + python superpowers/specs/itl-895-float-to-int/01_pyarrow_minimal.py +""" + +from __future__ import annotations + +import datetime as dt +from decimal import Decimal +from fractions import Fraction + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import warnings + +warnings.simplefilter("ignore", DeprecationWarning) + +print(f"pyarrow {pa.__version__}, numpy {np.__version__}\n") + +print("1. minimal reproduction of the orcapod call") +t = pa.Table.from_pylist([{"y": 1.5}], schema=pa.schema([("y", pa.int64())])) +print( + f" pa.Table.from_pylist([{{'y': 1.5}}], schema=int64) -> {t['y'].to_pylist()}\n" +) + + +def through(value, arrow_type) -> str: + try: + tbl = pa.Table.from_pylist( + [{"v": value}], schema=pa.schema([("v", arrow_type)]) + ) + out = tbl["v"].to_pylist()[0] + if (pa.types.is_temporal(arrow_type)) and not isinstance( + value, (dt.date, dt.timedelta) + ): + tag = "REINTERPRETED" # a number taken as ticks/days since the epoch + elif out == value and type(out) is type(value): + tag = "kept" + elif out == value: + tag = "kept, type changed" + else: + tag = "LOSSY" + return f"{tag:18} -> {out!r}" + except Exception as e: # noqa: BLE001 - we want every error class + return f"{'raises':18} {type(e).__name__}: {str(e)[:70]}" + + +CASES = [ + # (label, value, arrow type) + ("float 1.5 -> int64", 1.5, pa.int64()), + ("float 2.9 -> int64", 2.9, pa.int64()), + ("float -1.5 -> int64", -1.5, pa.int64()), + ("float 1.5 -> int8", 1.5, pa.int8()), + ("float 1.5 -> uint32", 1.5, pa.uint32()), + ("float 2.0 -> int64", 2.0, pa.int64()), + ("float nan -> int64", float("nan"), pa.int64()), + ("float inf -> int64", float("inf"), pa.int64()), + ("float 1e30 -> int64", 1e30, pa.int64()), + ("np.float64(1.5) -> int64", np.float64(1.5), pa.int64()), + ("np.float32(1.5) -> int64", np.float32(1.5), pa.int64()), + ("Decimal('1.5') -> int64", Decimal("1.5"), pa.int64()), + ("Fraction(3, 2) -> int64", Fraction(3, 2), pa.int64()), + ("np.int64(3) -> int64", np.int64(3), pa.int64()), + ("np.int64(3) -> int8", np.int64(3), pa.int8()), + ("True -> int64", True, pa.int64()), + ("np.bool_(True) -> int64", np.bool_(True), pa.int64()), + ("'1' -> int64", "1", pa.int64()), + ("300 -> int8", 300, pa.int8()), + ("-1 -> uint8", -1, pa.uint8()), + ("1 -> bool", 1, pa.bool_()), + ("1.0 -> bool", 1.0, pa.bool_()), + ("np.bool_(True) -> bool", np.bool_(True), pa.bool_()), + ("1 -> float64", 1, pa.float64()), + ("2**53+1 -> float64", 2**53 + 1, pa.float64()), + ("True -> float64", True, pa.float64()), + ("Decimal('1.5') -> float64", Decimal("1.5"), pa.float64()), + ("np.int64(3) -> float64", np.int64(3), pa.float64()), + ("np.float32(0.1) -> float64", np.float32(0.1), pa.float64()), + ("0.1 -> float32", 0.1, pa.float32()), + ("2**24+1 -> float32", 2**24 + 1, pa.float32()), + ("1 -> large_string", 1, pa.large_string()), + ("b'x' -> large_string", b"x", pa.large_string()), + ("'x' -> large_binary", "x", pa.large_binary()), + ("1.5 -> timestamp[us]", 1.5, pa.timestamp("us")), + ("1 -> timestamp[us]", 1, pa.timestamp("us")), + ("1.5 -> date32", 1.5, pa.date32()), + ("1.5 -> duration[us]", 1.5, pa.duration("us")), + ("date(2020,1,1) -> timestamp[us]", dt.date(2020, 1, 1), pa.timestamp("us")), + ( + "datetime naive -> timestamp[us,UTC]", + dt.datetime(2020, 1, 1, 12), + pa.timestamp("us", "UTC"), + ), + ( + "datetime aware -> timestamp[us]", + dt.datetime(2020, 1, 1, 12, tzinfo=dt.timezone(dt.timedelta(hours=2))), + pa.timestamp("us"), + ), + ("datetime -> date32", dt.datetime(2020, 1, 1, 12), pa.date32()), + ("1.5 -> decimal128(5,1)", 1.5, pa.decimal128(5, 1)), + ("Decimal('1.25') -> decimal128(5,1)", Decimal("1.25"), pa.decimal128(5, 1)), + ("1 -> decimal128(5,1)", 1, pa.decimal128(5, 1)), + ("[1.5] -> large_list", [1.5], pa.large_list(pa.int64())), + ( + "{'a': 1.5} -> struct", + {"a": 1.5}, + pa.struct([("a", pa.int64())]), + ), +] + +print("2. Table.from_pylist(rows, schema=...) for each (value, declared Arrow type)") +for label, value, typ in CASES: + print(f" {label:40} {through(value, typ)}") + +print("\n3. other entry points") +print( + f" pa.array([1.5], type=int64) -> {pa.array([1.5], type=pa.int64()).to_pylist()}" +) +print( + f" pa.scalar(1.5, type=int64) -> {pa.scalar(1.5, type=pa.int64()).as_py()}" +) +try: + pc.cast(pa.array([1.5]), pa.int64()) +except pa.ArrowInvalid as e: + print(f" pc.cast(pa.array([1.5]), int64) [safe] -> raises ArrowInvalid: {e}") +print( + f" pc.cast(..., safe=False) -> {pc.cast(pa.array([1.5]), pa.int64(), safe=False).to_pylist()}" +) +print(f" int(1.5) (what nb_int does) -> {int(1.5)}") diff --git a/superpowers/specs/itl-895-float-to-int/02_call_chain.py b/superpowers/specs/itl-895-float-to-int/02_call_chain.py new file mode 100644 index 000000000..1ccea1602 --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/02_call_chain.py @@ -0,0 +1,138 @@ +"""Claim 2: the orcapod call chain from the function's return value to the stored value. + +For every execution path, run a pod declared ``-> int`` that returns ``1.5`` with the +tracer installed, and print (a) each pyarrow call that received a non-int value for an +integer field, with the orcapod frames that led to it, and (b) what ends up stored / +yielded. + +Run: uv run python superpowers/specs/itl-895-float-to-int/02_call_chain.py +""" + +from __future__ import annotations + +import importlib + +import asyncio +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +_tracer = importlib.import_module("_tracer") # the sibling helper, via sys.path + +from orcapod import PipelineJob, function_pod # noqa: E402 +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase # noqa: E402 +from orcapod.pipeline import AsyncPipelineOrchestrator # noqa: E402 +from orcapod.sources import DictSource # noqa: E402 + +_tracer.install() + + +def make_src() -> DictSource: + return DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="src") + + +def banner(title: str) -> None: + print(f"\n=== {title} ===") + _tracer.HITS.clear() + + +def show_iter(stream) -> None: + for _tag, data in stream.iter_data(): + print( + f" iter_data: as_dict()={data.as_dict()} as_table()={data.as_table().to_pylist()}" + f" arrow type={data.arrow_schema().field('y').type}" + ) + + +# --- A. standalone FunctionPod stream, sync function -------------------------------- +@function_pod(output_keys="y") +def f_sync(x: int) -> int: + return x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +banner("A1 standalone FunctionPod, iter_data()") +show_iter(f_sync.pod(make_src())) + +banner("A2 standalone FunctionPod, as_table()") +t = f_sync.pod(make_src()).as_table() +print(f" as_table: y={t['y'].to_pylist()} type={t.schema.field('y').type}") + + +# --- B. standalone FunctionPod stream, async function ------------------------------- +@function_pod(output_keys="y") +async def f_async(x: int) -> int: + return x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +banner("B1 standalone FunctionPod with an async function, iter_data()") +show_iter(f_async.pod(make_src())) + +banner("B2 standalone FunctionPod with an async function, as_table()") +t = f_async.pod(make_src()).as_table() +print(f" as_table: y={t['y'].to_pylist()} type={t.schema.field('y').type}") + +# --- C. cached function pod: result_database (CachedDataFunction) ------------------- +db_c = InMemoryArrowDatabase() + + +@function_pod(output_keys="y", result_database=db_c) +def f_cached(x: int) -> int: + return x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +banner("C1 function_pod(result_database=...) -> CachedDataFunction, as_table()") +t = f_cached.pod(make_src()).as_table() +print(f" as_table: y={t['y'].to_pylist()} type={t.schema.field('y').type}") +banner("C2 CachedDataFunction, second run (served from the store)") +show_iter(f_cached.pod(make_src())) + +# --- D. cached function pod: pod_cache_database (CachedFunctionPod) ----------------- +db_d = InMemoryArrowDatabase() + + +@function_pod(output_keys="y", pod_cache_database=db_d) +def f_podcached(x: int) -> int: + return x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +banner("D1 function_pod(pod_cache_database=...) -> CachedFunctionPod, as_table()") +t = f_podcached.pod(make_src()).as_table() +print(f" as_table: y={t['y'].to_pylist()} type={t.schema.field('y').type}") + + +# --- E. FunctionNode in a PipelineJob ----------------------------------------------- +def run_job(store, orchestrator=None, fn=f_sync) -> None: + job = PipelineJob(name="itl895", store=store) + with job: + fn.pod(make_src(), label="f") + job.run(orchestrator=orchestrator) + node = job.nodes["f"] + t = node.as_table() + print(f" node.as_table: y={t['y'].to_pylist()} type={t.schema.field('y').type}") + recs = node.get_all_records() + if recs is not None: + print( + f" stored record y={recs['y'].to_pylist()} type={recs.schema.field('y').type}" + ) + + +banner("E1 PipelineJob sync, InMemoryArrowDatabase") +run_job(InMemoryArrowDatabase()) + +with tempfile.TemporaryDirectory() as d: + banner("E2 PipelineJob sync, DeltaTableDatabase") + run_job(DeltaTableDatabase(Path(d) / "sync")) + +banner("E3 PipelineJob async orchestrator, InMemoryArrowDatabase") +run_job(InMemoryArrowDatabase(), AsyncPipelineOrchestrator()) + +with tempfile.TemporaryDirectory() as d: + banner("E4 PipelineJob async orchestrator, DeltaTableDatabase") + run_job(DeltaTableDatabase(Path(d) / "async"), AsyncPipelineOrchestrator()) + +banner("E5 PipelineJob async orchestrator, async function, InMemoryArrowDatabase") +run_job(InMemoryArrowDatabase(), AsyncPipelineOrchestrator(), fn=f_async) + +_ = asyncio # imported for readers running parts interactively diff --git a/superpowers/specs/itl-895-float-to-int/03_split_datagram.py b/superpowers/specs/itl-895-float-to-int/03_split_datagram.py new file mode 100644 index 000000000..28b5b44ad --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/03_split_datagram.py @@ -0,0 +1,69 @@ +"""Claim 3: the output Data is internally inconsistent, and downstream pods see 1.5. + +``PythonDataFunction._build_output_data`` builds ``Data({"y": 1.5}, python_schema={"y": int})`` +without checking the value. The Data keeps the raw dict; its Arrow table (used for +``content_hash()`` and for storage) is built lazily and truncates to 1. So: + +* ``data.as_dict()`` says 1.5, ``data.as_table()`` / the stored record / the hash say 1. +* A downstream pod consumes ``as_dict()``. Chained standalone it receives 1.5; in a + PipelineJob it receives 1 on the sync orchestrator and 1.5 on the async one. The same + graph gives z=15 or z=10 depending on how it runs. In the standalone chain the upstream + Data is never converted to Arrow before the downstream function runs. + +Run: uv run python superpowers/specs/itl-895-float-to-int/03_split_datagram.py +""" + +from __future__ import annotations + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.sources import DictSource + +seen: list[object] = [] + + +@function_pod(output_keys="y") +def up(x: int) -> int: + return x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +@function_pod(output_keys="z") +def down(y: int) -> int: + seen.append(y) + return y * 10 + + +def src() -> DictSource: + return DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="src") + + +print("=== one output Data, two answers ===") +for _tag, data in up.pod(src()).iter_data(): + print(f" as_dict() -> {data.as_dict()}") + print(f" as_table() -> {data.as_table().to_pylist()}") + print(f" python_schema-> {dict(data.schema())}") + print(f" content_hash -> {data.content_hash().to_hex()[:16]}...") + from orcapod.core.datagrams import Data + + same_as_int = Data({"y": 1}, python_schema={"y": int}).content_hash().to_hex()[:16] + print( + f" content_hash of Data({{'y': 1}}) -> {same_as_int}... (identical: hash is of the truncated value)" + ) + +print("\n=== standalone chain: what does the downstream function receive? ===") +seen.clear() +t = down.pod(up.pod(src())).as_table() +print(f" down received y={seen!r}; z={t['z'].to_pylist()}") + +for label, orch in [("sync", None), ("async", AsyncPipelineOrchestrator())]: + seen.clear() + job = PipelineJob(name=f"itl895_chain_{label}", store=InMemoryArrowDatabase()) + with job: + down.pod(up.pod(src(), label="up"), label="down") + job.run(orchestrator=orch) + print(f"\n=== PipelineJob ({label}) ===") + print( + f" down received y={seen!r}; z={job.nodes['down'].as_table()['z'].to_pylist()}; " + f"up stored y={job.nodes['up'].as_table()['y'].to_pylist()}" + ) diff --git a/superpowers/specs/itl-895-float-to-int/04_input_sites.py b/superpowers/specs/itl-895-float-to-int/04_input_sites.py new file mode 100644 index 000000000..4d318e2aa --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/04_input_sites.py @@ -0,0 +1,186 @@ +"""Claim 4: the same truncation is reachable for INPUT data, not only function outputs. + +Every place orcapod turns Python values into Arrow against a declared type goes through +``UniversalTypeConverter.python_dicts_to_arrow_table`` (``pa.Table.from_pylist`` with a +schema, ``universal_converter.py:1106``), whose primitive converter is a passthrough +(``universal_converter.py:1586-1588``). Paths that start from Arrow (Arrow/Delta/CSV/ +DataFrame sources, operators) use ``Table.cast``, which is a SAFE cast and raises. + +Each probe below prints the stored value and type, or the error, and the tracer's site. + +Run: uv run python superpowers/specs/itl-895-float-to-int/04_input_sites.py +""" + +from __future__ import annotations + +import importlib + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +_tracer = importlib.import_module("_tracer") # the sibling helper, via sys.path + +import polars as pl # noqa: E402 +import pyarrow as pa # noqa: E402 + +from orcapod.core.datagrams import Data, Datagram, Tag # noqa: E402 +from orcapod.operators import Batch, PolarsFilter # noqa: E402 +from orcapod.sources import ( # noqa: E402 + ArrowTableSource, + DataFrameSource, + DictSource, + ListSource, +) + +_tracer.install() + + +def probe(label: str, fn) -> None: + _tracer.HITS.clear() + try: + result = fn() + verdict = "TRUNCATED" if _tracer.HITS else "no truncation" + print(f"- {label}\n {verdict}: {result}") + except Exception as e: # noqa: BLE001 + print(f"- {label}\n raises {type(e).__name__}: {str(e)[:110]}") + for site in _tracer.summary(): + print(f" site: {site}") + + +def col(stream, name): + t = stream.as_table() + return f"{name}={t[name].to_pylist()} ({t.schema.field(name).type})" + + +print("== sources built from Python values ==") +probe( + "DictSource, data_schema={'k': str, 'x': int}, x=1.5", + lambda: col( + DictSource( + data=[{"k": "a", "x": 1.5}], + tag_columns=["k"], + data_schema={"k": str, "x": int}, + ), + "x", + ), +) +probe( + "DictSource, data_schema=pa.schema(x: int64), x=1.5", + lambda: col( + DictSource( + data=[{"k": "a", "x": 1.5}], + tag_columns=["k"], + data_schema=pa.schema([("k", pa.large_string()), ("x", pa.int64())]), + ), + "x", + ), +) +probe( + "DictSource, TAG column declared int, k=1.5", + lambda: col( + DictSource( + data=[{"k": 1.5, "x": 1}], + tag_columns=["k"], + data_schema={"k": int, "x": int}, + ), + "k", + ), +) +probe( + "DictSource, no schema, x = 1 then 1.5 (inferred)", + lambda: col( + DictSource(data=[{"k": "a", "x": 1}, {"k": "b", "x": 1.5}], tag_columns=["k"]), + "x", + ), +) +probe( + "ListSource, data_schema={'v': int}, element 1.5", + lambda: col( + ListSource(name="v", data=[1.5], data_schema={"v": int, "element_index": int}), + "v", + ), +) + +print("\n== sources built from Arrow / DataFrames (Table.cast, safe) ==") +probe( + "DataFrameSource, float column, schema=int64", + lambda: col( + DataFrameSource( + pl.DataFrame({"k": ["a"], "x": [1.5]}), + tag_columns=["k"], + schema=pa.schema([("k", pa.large_string()), ("x", pa.int64())]), + ), + "x", + ), +) +probe( + "ArrowTableSource, float64 column (type kept, no declared type to disagree with)", + lambda: col( + ArrowTableSource(pa.table({"k": ["a"], "x": [1.5]}), tag_columns=["k"]), "x" + ), +) + +print("\n== datagrams built from Python values ==") +probe( + "Datagram({'x': 1.5}, python_schema={'x': int}).as_table()", + lambda: Datagram({"x": 1.5}, python_schema={"x": int}).as_table().to_pylist(), +) +probe( + "Data({'x': 1.5}, python_schema={'x': int}).as_table()", + lambda: Data({"x": 1.5}, python_schema={"x": int}).as_table().to_pylist(), +) +probe( + "Tag({'k': 1.5}, python_schema={'k': int}).as_table()", + lambda: Tag({"k": 1.5}, python_schema={"k": int}).as_table().to_pylist(), +) + + +def _update_arrow_backed(): + d = Datagram({"x": 1}, python_schema={"x": int}) + d.as_table() # Arrow table loaded: update() takes its Arrow path + return d.update(x=1.5).as_table().to_pylist() + + +def _update_dict_backed(): + d = Datagram({"x": 1}, python_schema={"x": int}) + return d.update(x=1.5).as_table().to_pylist() + + +probe("Datagram.update(x=1.5) on an int column, Arrow-backed", _update_arrow_backed) +probe("Datagram.update(x=1.5) on an int column, dict-backed", _update_dict_backed) +probe( + "Datagram.with_columns(column_types={'z': int}, z=1.5)", + lambda: ( + Datagram({"x": 1}, python_schema={"x": int}) + .with_columns(column_types={"z": int}, z=1.5) + .as_table() + .to_pylist() + ), +) +probe( + "Datagram.with_columns(column_types={'z': pa.int64()}, z=1.5)", + lambda: ( + Datagram({"x": 1}, python_schema={"x": int}) + .with_columns(column_types={"z": pa.int64()}, z=1.5) + .as_table() + .to_pylist() + ), +) +probe( + "Data source_info value 1.5 (type derived from the value, no declared type)", + lambda: ( + Data({"y": 1}, source_info={"y": 1.5}) # pyright: ignore[reportArgumentType] # deliberate wrong value: probes the conversion + .as_table(columns={"source": True}) + .to_pylist() + ), +) + +print("\n== operators (Arrow in, Arrow out) ==") +src = DictSource(data=[{"k": "a", "x": 1}, {"k": "b", "x": 2}], tag_columns=["k"]) +probe("Batch(batch_size=2) over int column", lambda: col(Batch(batch_size=2)(src), "x")) +probe( + "PolarsFilter(x > 1) over int column", + lambda: col(PolarsFilter([pl.col("x") > 1])(src), "x"), +) diff --git a/superpowers/specs/itl-895-float-to-int/05_pod_matrix.py b/superpowers/specs/itl-895-float-to-int/05_pod_matrix.py new file mode 100644 index 000000000..a5a2dd48f --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/05_pod_matrix.py @@ -0,0 +1,146 @@ +"""Claim 5: declared output type x returned value, through a real pod, today. + +For each pair, one pod is built and run two ways: + +* ``standalone``: ``f.pod(src).as_table()`` (no store) +* ``job``: ``PipelineJob`` (sync orchestrator, in-memory store), reading the stored record + +Each cell is ``kept``, ``LOSSY`` (stored value differs from the returned one), +``converted`` (equal value, different Python type), or ``raises ``. The last +section shows the standalone ``as_table()`` "Complex unions" error from the issue. + +Run: uv run python superpowers/specs/itl-895-float-to-int/05_pod_matrix.py +""" + +from __future__ import annotations + +import datetime as dt +import logging +import warnings +from decimal import Decimal +from typing import Any + +import numpy as np + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import DictSource + +warnings.simplefilter("ignore") +logging.disable(logging.CRITICAL) + +UTC = dt.timezone.utc +PLUS2 = dt.timezone(dt.timedelta(hours=2)) + +CASES: list[tuple[Any, Any]] = [ + (int, 1.5), + (int, 2.0), + (int, np.float64(1.5)), + (int, Decimal("1.5")), + (int, np.int64(3)), + (int, True), + (int, np.bool_(True)), + (int, "1"), + (int, None), + (int, 2**63), + (np.int8, 1.5), + (np.int8, 300), + (float, 1), + (float, True), + (float, 2**53 + 1), + (float, np.float32(0.1)), + (float, Decimal("1.5")), + (float, "1.0"), + (float, None), + (bool, 1), + (bool, np.bool_(True)), + (bool, 1.0), + (str, 1), + (str, b"x"), + (bytes, "x"), + (dt.datetime, dt.datetime(2020, 1, 1, 12, tzinfo=PLUS2)), + (dt.datetime, dt.datetime(2020, 1, 1, 12)), + (dt.datetime, dt.date(2020, 1, 1)), + (dt.datetime, 1.5), + (dt.date, dt.datetime(2020, 1, 1, 12)), + (dt.date, 1.5), + (list[int], [1.5]), + (list[int], (1, 2)), + (int | None, 1.5), +] + + +def make_pod(declared, value, label): + def fn(x: int): + return value + + fn.__annotations__["return"] = declared + fn.__name__ = fn.__qualname__ = label + return function_pod(output_keys="y")(fn) + + +def classify(stored: Any, value: Any) -> str: + if stored == value and type(stored) is type(value): + return f"kept {stored!r}" + try: + equal = stored == value + if isinstance(equal, np.ndarray): + equal = bool(equal.all()) + except Exception: # noqa: BLE001 + equal = False + if equal and not isinstance(value, dt.datetime): + return f"converted {stored!r}" + return f"LOSSY {stored!r}" + + +def run(declared, value, i) -> tuple[str, str]: + f = make_pod(declared, value, f"f{i}") + src = DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="src") + out = [] + try: + out.append(classify(f.pod(src).as_table()["y"].to_pylist()[0], value)) + except Exception as e: # noqa: BLE001 + out.append(f"raises {type(e).__name__}") + try: + job = PipelineJob(name=f"m{i}", store=InMemoryArrowDatabase()) + with job: + f.pod(src, label="f") + job.run() + recs = job.nodes["f"].get_all_records() + out.append( + classify(recs["y"].to_pylist()[0], value) + if recs is not None and recs.num_rows + else "no row stored" + ) + except Exception as e: # noqa: BLE001 + out.append(f"raises {type(e).__name__}") + return out[0], out[1] + + +def name(t) -> str: + return getattr(t, "__name__", None) or str(t) + + +print( + f"{'declared':10} {'returned':42} {'standalone as_table':34} {'PipelineJob stored'}" +) +for i, (declared, value) in enumerate(CASES): + s, j = run(declared, value, i) + print(f"{name(declared):10} {repr(value)[:40]:42} {s[:34]:34} {j[:40]}") + +print( + "\n== the issue's 'Complex unions' error: standalone as_table() over mixed results ==" +) + + +@function_pod(output_keys="y") +def g(x: int) -> int: + return x if x == 1 else x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +src2 = DictSource(data=[{"k": "a", "x": 1}, {"k": "b", "x": 2}], tag_columns=["k"]) +print(" iter_data:", [d.as_dict() for _, d in g.pod(src2).iter_data()]) +try: + g.pod(src2).as_table() +except Exception as e: # noqa: BLE001 + print(f" as_table raises {type(e).__name__}: {e}") diff --git a/superpowers/specs/itl-895-float-to-int/06_released_versions.py b/superpowers/specs/itl-895-float-to-int/06_released_versions.py new file mode 100644 index 000000000..04eca1d8d --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/06_released_versions.py @@ -0,0 +1,31 @@ +"""Claim 6: released versions have the bug (version-agnostic reproduction). + +Runs the issue's reproduction against whatever ``orcapod`` is importable, using only the +API every release since v0.1.0 shares (``function_pod``, ``DictSource``, ``.pod(...)``, +``as_table()``). + +Run against a release without touching this worktree: + + git archive v0.1.4 | tar -x -C /v0.1.4 + uv run --no-project --python 3.12 --with /v0.1.4 \ + python -I superpowers/specs/itl-895-float-to-int/06_released_versions.py +""" + +from __future__ import annotations + +from importlib.metadata import version + +from orcapod import function_pod +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def f(x: int) -> int: + return x + 0.5 # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +src = DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"]) +t = f.pod(src).as_table() +print( + f"orcapod {version('orcapod')}: y={t['y'].to_pylist()} type={t.schema.field('y').type}" +) diff --git a/superpowers/specs/itl-895-float-to-int/07_blast_radius.py b/superpowers/specs/itl-895-float-to-int/07_blast_radius.py new file mode 100644 index 000000000..139cb1744 --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/07_blast_radius.py @@ -0,0 +1,79 @@ +"""Claim 7: blast radius for existing stores, and how to find affected rows. + +1. A truncated stored row is byte-identical to a genuine one: the stored value, the + output data hash and the record are those of ``1``; nothing records that the function + returned ``1.5``. Stored data alone cannot tell the two apart. +2. Later runs reuse the stored row without calling the function (Phase 1 of + ``FunctionNode``). A fix that rejects floats therefore does NOT touch rows already + stored: they keep being served until the user removes them or bumps the pod version. +3. Recipe to find affected rows: for each pod whose declared output contains an integer + type (or another type pyarrow coerces: ``date``/``datetime``, ``str``/``bytes``, + ``float32``, lists/structs of these), call the raw function again on the node's stored + inputs and flag rows whose returned value is not exactly the declared type. + +Run: uv run python superpowers/specs/itl-895-float-to-int/07_blast_radius.py +""" + +from __future__ import annotations + +import logging + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import DictSource + +logging.disable(logging.CRITICAL) +calls: list[int] = [] + + +def raw_half(x: int) -> int: + calls.append(x) + return ( # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + x + 0.5 if x % 2 else x + ) # odd inputs return a float + + +half = function_pod(output_keys="y")(raw_half) + +rows = [{"k": f"r{i}", "x": i} for i in range(4)] +db = InMemoryArrowDatabase() + + +def build() -> PipelineJob: + job = PipelineJob(name="blast", store=db) + with job: + src = DictSource(data=rows, tag_columns=["k"], source_id="src") + half.pod(src, label="half") + return job + + +job = build() +job.run() +recs = job.nodes["half"].get_all_records(columns={"meta": True}) +print("1. stored rows (odd x returned x + 0.5):") +for k, y in zip(recs["k"].to_pylist(), recs["y"].to_pylist()): + x = int(k[1:]) + print(f" {k}: x={x} stored y={y!r}") +print( + f" stored column type: {recs.schema.field('y').type}; meta columns: " + f"{[c for c in recs.column_names if c.startswith('__')][:4]} ..." +) + +calls.clear() +job2 = build() +job2.run() +print( + f"\n2. second run over the same store: function calls={len(calls)}; " + f"y={job2.nodes['half'].as_table()['y'].to_pylist()} (served from the store)" +) + +print( + "\n3. finding affected rows: re-call the raw function on the node's stored inputs" +) +src_stream = job2.nodes["src"] +affected = [] +for tag, data in src_stream.iter_data(): + returned = raw_half(**data.as_dict()) + if type(returned) is not int: + affected.append((tag.as_dict()["k"], returned)) +print(f" affected rows (tag, value the function returns): {affected}") diff --git a/superpowers/specs/itl-895-float-to-int/08_none_output.py b/superpowers/specs/itl-895-float-to-int/08_none_output.py new file mode 100644 index 000000000..d08d988db --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/08_none_output.py @@ -0,0 +1,93 @@ +"""Claim 8: ``None`` returned for a non-optional output (coordinator follow-up from PR #289). + +Same family as float -> int: the returned value is never checked against the declared +output type. ``-> int`` maps to ``int64 not null``; ``pa.Table.from_pylist`` writes a +NULL into a ``not null`` field without error, and ``Table.validate(full=True)`` does not +check field nullability either. What happens next depends on the path: + +* standalone ``as_table()``: returns a table whose ``not null`` field holds a NULL; +* in-memory store: the run completes; the NULL is stored; reading the node back fails + in ``restore_schema_nullability`` (``Table.cast``: "Casting field 'y' with null values + to non-nullable"); +* Delta store: the Delta writer refuses the batch at flush ("declared as non-nullable + but contains null values"); today that is logged as a row failure and the final flush + raises ``RuntimeError`` (ITL-878 / PR #287 turns it into ``StoreWriteError``). + +Run: uv run python superpowers/specs/itl-895-float-to-int/08_none_output.py +""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.datagrams import Data +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.sources import DictSource + +logging.disable(logging.CRITICAL) + +print("== pyarrow only ==") +t = pa.Table.from_pylist( + [{"y": None}], schema=pa.schema([pa.field("y", pa.int64(), nullable=False)]) +) +print( + f" from_pylist(None, 'y: int64 not null') -> {t.to_pylist()} field={t.schema.field('y')} " + f"null_count={t['y'].null_count}" +) +t.validate(full=True) +print(" Table.validate(full=True) -> passes (Arrow does not check field nullability)") + +print("\n== orcapod Data ==") +d = Data({"y": None}, python_schema={"y": int}) +print( + f" Data({{'y': None}}, python_schema={{'y': int}}).as_table() -> {d.as_table().to_pylist()} " + f"field={d.as_table().schema.field('y')}" +) + + +@function_pod(output_keys="y") +def f(x: int) -> int: + return None # pyright: ignore[reportReturnType] # deliberate wrong return type: reproduces ITL-895 + + +def src() -> DictSource: + return DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="s") + + +print("\n== standalone pod ==") +st = f.pod(src()).as_table() +print(f" as_table() -> y={st['y'].to_pylist()} field={st.schema.field('y')}") + +print("\n== PipelineJob ==") +for label, kind, orch in [ + ("in-memory, sync", "mem", None), + ("in-memory, async", "mem", AsyncPipelineOrchestrator()), + ("Delta, sync", "delta", None), + ("Delta, async", "delta", AsyncPipelineOrchestrator()), +]: + with tempfile.TemporaryDirectory() as tmp: + store = ( + DeltaTableDatabase(Path(tmp) / "db") + if kind == "delta" + else InMemoryArrowDatabase() + ) + job = PipelineJob(name="none_out", store=store) + with job: + f.pod(src(), label="f") + try: + job.run(orchestrator=orch) + run = "run completes" + except Exception as e: # noqa: BLE001 + run = f"run raises {type(e).__name__}: {str(e)[:80]}" + try: + recs = job.nodes["f"].get_all_records() + read = "no rows" if recs is None else f"stored y={recs['y'].to_pylist()}" + except Exception as e: # noqa: BLE001 + read = f"read raises {type(e).__name__}: {str(e)[:70]}" + print(f" {label:18} {run}\n {'':18} {read}") diff --git a/superpowers/specs/itl-895-float-to-int/FINDINGS.md b/superpowers/specs/itl-895-float-to-int/FINDINGS.md new file mode 100644 index 000000000..0f811d81b --- /dev/null +++ b/superpowers/specs/itl-895-float-to-int/FINDINGS.md @@ -0,0 +1,315 @@ +# ITL-895 root cause: a pod declared `-> int` stores `1.5` as `1` + +Investigated on `b197544e` (origin/main, November 11, 2026). Each claim has a script in this +directory, and its captured output is in `outputs/`. Run any script with +`uv run python superpowers/specs/itl-895-float-to-int/