Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions python/ray/data/_internal/arrow_ops/transform_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,6 @@ def sort(table: "pyarrow.Table", sort_key: "SortKey") -> "pyarrow.Table":
return take_table(table, indices)


def _create_empty_table(schema: "pyarrow.Schema"):
import pyarrow as pa

arrays = [pa.array([], type=t) for t in schema.types]

return pa.table(arrays, schema=schema)


def _has_unhashable_pandas_types(schema: "pyarrow.Schema") -> bool:
"""Check if any column type becomes unhashable after to_pandas() conversion.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
_is_local_scheme,
_resolve_custom_scheme,
)
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import BlockMetadata
from ray.data.context import DataContext
from ray.data.datasource.datasource import Datasource, ReadTask
Expand Down Expand Up @@ -482,7 +483,7 @@ def _raise_on_pickle_object_meta_parquet(local_meta_dir: str, root_uri: str) ->
try:
# Unpickling untrusted data can execute arbitrary code. Reject object
# columns unless the user has explicitly opted in.
raise_on_pickle_object_columns(schema.empty_table())
raise_on_pickle_object_columns(_create_empty_table(schema))
except ValueError as e:
rel = path.relative_to(meta_dir).as_posix()
raise ValueError(f"{root_uri!r}: meta/{rel}: {e}") from e
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
_is_local_scheme,
iterate_with_retry,
)
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import Block, BlockAccessor, BlockMetadata
from ray.data.context import DataContext
from ray.data.datasource import Datasource
Expand Down Expand Up @@ -936,7 +937,7 @@ def _derive_schema(
# Try to infer dataset schema by passing dummy table through UDF.
try:
# An empty table with extensions will fail for pyarrow==9.0.0
dummy_table = target_schema.empty_table()
dummy_table = _create_empty_table(target_schema)
target_schema = _block_udf(dummy_table).schema.with_metadata(
target_schema.metadata
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,7 @@
from ray._raylet import StreamingGeneratorStats
from ray.actor import ActorHandle
from ray.data._internal.arrow_block import ArrowBlockBuilder
from ray.data._internal.arrow_ops.transform_pyarrow import (
_create_empty_table,
hash_partition,
)
from ray.data._internal.arrow_ops.transform_pyarrow import hash_partition
from ray.data._internal.execution.interfaces import (
BlockEntry,
ExecutionOptions,
Expand All @@ -63,6 +60,7 @@
from ray.data._internal.stats import OpRuntimeMetrics
from ray.data._internal.table_block import TableBlockAccessor
from ray.data._internal.util import GiB, MiB
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import (
Block,
BlockAccessor,
Expand Down
15 changes: 8 additions & 7 deletions python/ray/data/_internal/execution/operators/join.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
)
from ray.data._internal.logical.operators import JoinType
from ray.data._internal.util import GiB, MiB
from ray.data._internal.utils.transform_pyarrow import _is_pa_extension_type
from ray.data._internal.utils.transform_pyarrow import (
_create_empty_table,
_is_pa_extension_type,
)
from ray.data.block import Block, Schema
from ray.data.context import DataContext

Expand Down Expand Up @@ -143,19 +146,17 @@ def _side_table(
if tables:
return _combine(tables)
if isinstance(schema, pa.Schema):
return schema.empty_table()
return _create_empty_table(schema)
return None

def _empty_table_with_key_schema(
table: "pa.Table",
source_key_col_names: Tuple[str, ...],
target_key_col_names: Tuple[str, ...],
) -> "pa.Table":
return (
table.select(list(source_key_col_names))
.schema.empty_table()
.rename_columns(list(target_key_col_names))
)
return _create_empty_table(
table.select(list(source_key_col_names)).schema
).rename_columns(list(target_key_col_names))

def _reduce(
partition_id: int, tables_by_input: List[List[Block]]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
)
from ray.data._internal.execution.util import merge_label_selector
from ray.data._internal.logical.interfaces.partitioning import HashPartitioning
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import (
BlockAccessor,
BlockMetadata,
Expand Down Expand Up @@ -432,7 +433,7 @@ def _emit_empty_partition(self, partition_id: int, schema: pa.Schema) -> None:
build the empty block from the schema the mappers reported and queue it
as this partition's single output block.
"""
empty_block = schema.empty_table()
empty_block = _create_empty_table(schema)
block_meta = BlockAccessor.for_block(empty_block).get_metadata()
out_bundle = RefBundle(
(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
OutputBlockSizeOption,
)
from ray.data._internal.table_block import TableBlockAccessor
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import (
Block,
BlockAccessor,
Expand Down Expand Up @@ -322,7 +323,7 @@ def _with_typed_empty_inputs(
out: List[List[pa.Table]] = []
for tables, schema in zip(tables_by_input, schemas_by_input):
if not tables and schema is not None:
tables = [schema.empty_table()]
tables = [_create_empty_table(schema)]
out.append(tables)
return out

Expand Down Expand Up @@ -377,7 +378,7 @@ def _no_input_blocks():
if not emit_empty_partition:
return
assert output_schema is not None
yield output_schema.empty_table()
yield _create_empty_table(output_schema)

else:

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
)
from ray.data._internal.execution.util import merge_label_selector
from ray.data._internal.logical.interfaces.partitioning import HashPartitioning
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import BlockAccessor, BlockStats, TaskExecWorkerStats, to_stats
from ray.data.context import DataContext

Expand Down Expand Up @@ -363,7 +364,7 @@ def _emit_empty_partition(
reached for schema-preserving reduces (``should_emit_empty_partitions=True``);
aggregation disables this path instead of emitting a partial-schema block.
"""
empty_block = schema.empty_table()
empty_block = _create_empty_table(schema)
block_meta = BlockAccessor.for_block(empty_block).get_metadata()
out_bundle = RefBundle(
(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from ray.data._internal.execution.util import yield_block_with_stats
from ray.data._internal.output_buffer import BlockOutputBuffer, OutputBlockSizeOption
from ray.data._internal.table_block import TableBlockAccessor
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import (
Block,
BlockAccessor,
Expand Down Expand Up @@ -141,7 +142,9 @@ def _shuffle_map_task(
output_schema = TableBlockAccessor.try_convert_block_type(
shard_source_blocks[0], block_type=BlockType.ARROW
).schema
empty_shard = _encode_partition_ipc(output_schema.empty_table(), ipc_write_options)
empty_shard = _encode_partition_ipc(
_create_empty_table(output_schema), ipc_write_options
)

shard_sizes: Dict[int, Tuple[int, int]] = {}
partition_bufs: List[pa.Buffer] = []
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from ray.data._internal.remote_fn import cached_remote_fn
from ray.data._internal.stats import StatsDict
from ray.data._internal.table_block import TableBlockAccessor
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import (
Block,
BlockAccessor,
Expand Down Expand Up @@ -446,7 +447,7 @@ def _empty_block_from_schema(schema: Schema) -> Block:
}
)
assert isinstance(schema, pa.Schema), type(schema)
return schema.empty_table()
return _create_empty_table(schema)


def _final_top_n_task(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
PredicatePassThroughBehavior,
)
from ray.data._internal.logical.operators.n_ary_operator import NAry
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.expressions import BinaryExpr, ColumnExpr, Expr, Operation

if TYPE_CHECKING:
Expand Down Expand Up @@ -460,8 +461,8 @@ def infer_schema(self) -> Optional["Schema"]:
return None
try:
joined = join_tables(
left_schema.empty_table(),
right_schema.empty_table(),
_create_empty_table(left_schema),
_create_empty_table(right_schema),
join_type=join_type_enum,
left_key_col_names=self.left_key_columns,
right_key_col_names=self.right_key_columns,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
LogicalOperatorUnifiesInputSchemas,
PredicatePassThroughBehavior,
)
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.util.annotations import PublicAPI

if TYPE_CHECKING:
Expand Down Expand Up @@ -120,9 +121,9 @@ def infer_schema(self) -> Optional["Schema"]:
):
return None
try:
combined = input_schemas[0].empty_table()
combined = _create_empty_table(input_schemas[0])
for s in input_schemas[1:]:
combined = BlockAccessor.for_block(combined).zip(s.empty_table())
combined = BlockAccessor.for_block(combined).zip(_create_empty_table(s))
except (pa.ArrowTypeError, pa.ArrowInvalid):
return None
return combined.schema
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from ray.data._internal.planner.plan_expression.expression_visitors import (
get_column_references,
)
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.block import (
Block,
BlockMetadata,
Expand Down Expand Up @@ -319,7 +320,7 @@ def infer_schema(self) -> "pa.Schema":
# probe fails — the UDF may require a non-empty input.
if self.block_udf is not None:
try:
transformed = self.block_udf(schema.empty_table()).schema
transformed = self.block_udf(_create_empty_table(schema)).schema
schema = transformed.with_metadata(schema.metadata)
except Exception:
pass
Expand Down
16 changes: 16 additions & 0 deletions python/ray/data/_internal/utils/transform_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,22 @@ def _is_pa_extension_type(pa_type: "pyarrow.lib.DataType") -> bool:
return isinstance(pa_type, pyarrow.BaseExtensionType)


def _create_empty_table(schema: "pyarrow.Schema") -> "pyarrow.Table":
"""Create an empty table with the given schema.

Use this instead of ``Schema.empty_table()``, which fails for some extension
columns: on pyarrow 25+ for native ``fixed_shape_tensor`` columns with a
``permutation`` (pyarrow adds one in ``from_numpy_ndarray``), and on every
supported version for extension types nested in a struct or list.
``pyarrow.nulls`` builds each column without that conversion. The result
matches ``Schema.empty_table()``: one empty chunk per column and the schema
metadata kept.
"""
return pyarrow.Table.from_arrays(
[pyarrow.nulls(0, type=field.type) for field in schema], schema=schema
)
Comment thread
rayo5o marked this conversation as resolved.
Resolved


def _is_multi_chunk_extension_column(column: "pyarrow.ChunkedArray") -> bool:
"""Return whether a column needs multi-chunk extension-array handling."""
return column.num_chunks > 1 and _is_pa_extension_type(column.type)
Expand Down
9 changes: 6 additions & 3 deletions python/ray/data/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@
get_compute_strategy,
merge_resources_to_ray_remote_args,
)
from ray.data._internal.utils.transform_pyarrow import _create_empty_table
from ray.data.aggregate import (
AggregateFn,
AggregateFnV2,
Expand Down Expand Up @@ -7677,7 +7678,7 @@ def block_to_df(block_ref: ObjectRef[Block]) -> pd.DataFrame:
}
)
else:
meta = schema.empty_table().to_pandas()
meta = _create_empty_table(schema).to_pandas()

dfs = []
for ref_bundle in self.iter_internal_ref_bundles():
Expand Down Expand Up @@ -7717,7 +7718,7 @@ def to_mars(self) -> "mars.dataframe.DataFrame":
if isinstance(schema, PandasBlockSchema):
dtypes = pd.Series(schema.types, index=schema.names)
elif isinstance(schema, pa.Schema):
dtypes = schema.empty_table().to_pandas().dtypes
dtypes = _create_empty_table(schema).to_pandas().dtypes
else:
raise NotImplementedError(f"Unsupported format of schema {schema}")
index_value = parse_index(pd.RangeIndex(-1))
Expand Down Expand Up @@ -7846,7 +7847,9 @@ def to_pandas(self, limit: Optional[int] = None) -> "pandas.DataFrame":
# `BlockAccessor.to_pandas()` path used for non-empty blocks
# so the two agree on column types (types_mapper, tensor
# casting, etc.).
df = BlockAccessor.for_block(base_schema.empty_table()).to_pandas()
df = BlockAccessor.for_block(
_create_empty_table(base_schema)
).to_pandas()
else:
import pandas

Expand Down
39 changes: 39 additions & 0 deletions python/ray/data/tests/test_tensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,45 @@ def test_tensors_sort(ray_start_regular_shared, tensor_format_context):
)


@pytest.mark.parametrize(
"op", ["map_groups", "repartition_keys", "repartition_shuffle", "join"]
)
def test_tensors_shuffle_and_join(ray_start_regular_shared, tensor_format_context, op):
# Shuffle and join tasks create empty blocks from the schema, which failed
# for native tensor columns on pyarrow 25+.
arr = np.arange(64).reshape((4, 4, 4))
ds = ray.data.from_arrow(pa.table({"a": TensorArray(arr), "k": [0, 1, 0, 1]}))
if op == "map_groups":
out = ds.groupby("k").map_groups(lambda group: group)
elif op == "repartition_keys":
out = ds.repartition(4, keys=["k"])
elif op == "repartition_shuffle":
out = ds.repartition(4, shuffle=True)
else:
other = ray.data.from_items([{"k": 0, "v": "x"}, {"k": 1, "v": "y"}])
out = ds.join(other, join_type="inner", num_partitions=4, on=("k",))

rows = sorted(out.take_all(), key=lambda row: np.asarray(row["a"]).min())
np.testing.assert_equal([np.asarray(row["a"]) for row in rows], list(arr))


def test_tensors_top_n_empty_result(ray_start_regular_shared, tensor_format_context):
# The Top-N merge builds an empty block from the schema when no rows are
# left, which failed for native tensor columns on pyarrow 25+.
from ray.data._internal.logical.rules.top_n_rewrite import (
TOP_N_REWRITE_ENABLED_CONFIG_KEY,
)
from ray.data.expressions import col

ray.data.DataContext.get_current().set_config(
TOP_N_REWRITE_ENABLED_CONFIG_KEY, True
)
arr = np.arange(64).reshape((4, 4, 4))
ds = ray.data.from_arrow(pa.table({"a": TensorArray(arr), "k": [0, 1, 0, 1]}))
out = ds.filter(expr=col("k") < 0).sort("k").limit(2)
assert out.take_all() == []


def test_tensors_inferred_from_map(ray_start_regular_shared, tensor_format_context):
tensor_format = tensor_format_context
# With tensor_format_context, ARROW_NATIVE only runs when supported,
Expand Down
Loading
Loading