diff --git a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py index e3de5b0cd128..3a58d9199824 100644 --- a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py +++ b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py @@ -65,14 +65,6 @@ def sort(table: "pyarrow.Table", sort_key: "SortKey") -> "pyarrow.Table": return take_table(table, indices) -def _create_empty_table(schema: "pyarrow.Schema"): - import pyarrow as pa - - arrays = [pa.array([], type=t) for t in schema.types] - - return pa.table(arrays, schema=schema) - - def _has_unhashable_pandas_types(schema: "pyarrow.Schema") -> bool: """Check if any column type becomes unhashable after to_pandas() conversion. diff --git a/python/ray/data/_internal/datasource/lerobot_datasource.py b/python/ray/data/_internal/datasource/lerobot_datasource.py index 9f6443417cf6..523cdb8740ac 100644 --- a/python/ray/data/_internal/datasource/lerobot_datasource.py +++ b/python/ray/data/_internal/datasource/lerobot_datasource.py @@ -38,6 +38,7 @@ _is_local_scheme, _resolve_custom_scheme, ) +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import BlockMetadata from ray.data.context import DataContext from ray.data.datasource.datasource import Datasource, ReadTask @@ -482,7 +483,7 @@ def _raise_on_pickle_object_meta_parquet(local_meta_dir: str, root_uri: str) -> try: # Unpickling untrusted data can execute arbitrary code. Reject object # columns unless the user has explicitly opted in. - raise_on_pickle_object_columns(schema.empty_table()) + raise_on_pickle_object_columns(_create_empty_table(schema)) except ValueError as e: rel = path.relative_to(meta_dir).as_posix() raise ValueError(f"{root_uri!r}: meta/{rel}: {e}") from e diff --git a/python/ray/data/_internal/datasource/parquet_datasource.py b/python/ray/data/_internal/datasource/parquet_datasource.py index 08a9adf4d166..05d662726475 100644 --- a/python/ray/data/_internal/datasource/parquet_datasource.py +++ b/python/ray/data/_internal/datasource/parquet_datasource.py @@ -49,6 +49,7 @@ _is_local_scheme, iterate_with_retry, ) +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import Block, BlockAccessor, BlockMetadata from ray.data.context import DataContext from ray.data.datasource import Datasource @@ -936,7 +937,7 @@ def _derive_schema( # Try to infer dataset schema by passing dummy table through UDF. try: # An empty table with extensions will fail for pyarrow==9.0.0 - dummy_table = target_schema.empty_table() + dummy_table = _create_empty_table(target_schema) target_schema = _block_udf(dummy_table).schema.with_metadata( target_schema.metadata ) diff --git a/python/ray/data/_internal/execution/operators/hash_shuffle.py b/python/ray/data/_internal/execution/operators/hash_shuffle.py index bd8553552679..8631527f6925 100644 --- a/python/ray/data/_internal/execution/operators/hash_shuffle.py +++ b/python/ray/data/_internal/execution/operators/hash_shuffle.py @@ -38,10 +38,7 @@ from ray._raylet import StreamingGeneratorStats from ray.actor import ActorHandle from ray.data._internal.arrow_block import ArrowBlockBuilder -from ray.data._internal.arrow_ops.transform_pyarrow import ( - _create_empty_table, - hash_partition, -) +from ray.data._internal.arrow_ops.transform_pyarrow import hash_partition from ray.data._internal.execution.interfaces import ( BlockEntry, ExecutionOptions, @@ -63,6 +60,7 @@ from ray.data._internal.stats import OpRuntimeMetrics from ray.data._internal.table_block import TableBlockAccessor from ray.data._internal.util import GiB, MiB +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import ( Block, BlockAccessor, diff --git a/python/ray/data/_internal/execution/operators/join.py b/python/ray/data/_internal/execution/operators/join.py index 839efbb291fa..76bd3a94305f 100644 --- a/python/ray/data/_internal/execution/operators/join.py +++ b/python/ray/data/_internal/execution/operators/join.py @@ -15,7 +15,10 @@ ) from ray.data._internal.logical.operators import JoinType from ray.data._internal.util import GiB, MiB -from ray.data._internal.utils.transform_pyarrow import _is_pa_extension_type +from ray.data._internal.utils.transform_pyarrow import ( + _create_empty_table, + _is_pa_extension_type, +) from ray.data.block import Block, Schema from ray.data.context import DataContext @@ -143,7 +146,7 @@ def _side_table( if tables: return _combine(tables) if isinstance(schema, pa.Schema): - return schema.empty_table() + return _create_empty_table(schema) return None def _empty_table_with_key_schema( @@ -151,11 +154,9 @@ def _empty_table_with_key_schema( source_key_col_names: Tuple[str, ...], target_key_col_names: Tuple[str, ...], ) -> "pa.Table": - return ( - table.select(list(source_key_col_names)) - .schema.empty_table() - .rename_columns(list(target_key_col_names)) - ) + return _create_empty_table( + table.select(list(source_key_col_names)).schema + ).rename_columns(list(target_key_col_names)) def _reduce( partition_id: int, tables_by_input: List[List[Block]] diff --git a/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_reduce_operator.py b/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_reduce_operator.py index 0fbdcced2c93..94d6f27d9ea8 100644 --- a/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_reduce_operator.py +++ b/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_reduce_operator.py @@ -53,6 +53,7 @@ ) from ray.data._internal.execution.util import merge_label_selector from ray.data._internal.logical.interfaces.partitioning import HashPartitioning +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import ( BlockAccessor, BlockMetadata, @@ -432,7 +433,7 @@ def _emit_empty_partition(self, partition_id: int, schema: pa.Schema) -> None: build the empty block from the schema the mappers reported and queue it as this partition's single output block. """ - empty_block = schema.empty_table() + empty_block = _create_empty_table(schema) block_meta = BlockAccessor.for_block(empty_block).get_metadata() out_bundle = RefBundle( ( diff --git a/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_tasks.py b/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_tasks.py index 6f5dab6a9eff..cd0aa777355f 100644 --- a/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_tasks.py +++ b/python/ray/data/_internal/execution/operators/shuffle_operators/disk_shuffle_tasks.py @@ -54,6 +54,7 @@ OutputBlockSizeOption, ) from ray.data._internal.table_block import TableBlockAccessor +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import ( Block, BlockAccessor, @@ -322,7 +323,7 @@ def _with_typed_empty_inputs( out: List[List[pa.Table]] = [] for tables, schema in zip(tables_by_input, schemas_by_input): if not tables and schema is not None: - tables = [schema.empty_table()] + tables = [_create_empty_table(schema)] out.append(tables) return out @@ -377,7 +378,7 @@ def _no_input_blocks(): if not emit_empty_partition: return assert output_schema is not None - yield output_schema.empty_table() + yield _create_empty_table(output_schema) else: diff --git a/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_reduce_operator.py b/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_reduce_operator.py index 9db46e6cac5f..487e0f1769d3 100644 --- a/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_reduce_operator.py +++ b/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_reduce_operator.py @@ -37,6 +37,7 @@ ) from ray.data._internal.execution.util import merge_label_selector from ray.data._internal.logical.interfaces.partitioning import HashPartitioning +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import BlockAccessor, BlockStats, TaskExecWorkerStats, to_stats from ray.data.context import DataContext @@ -363,7 +364,7 @@ def _emit_empty_partition( reached for schema-preserving reduces (``should_emit_empty_partitions=True``); aggregation disables this path instead of emitting a partial-schema block. """ - empty_block = schema.empty_table() + empty_block = _create_empty_table(schema) block_meta = BlockAccessor.for_block(empty_block).get_metadata() out_bundle = RefBundle( ( diff --git a/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_tasks.py b/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_tasks.py index ebcc8a6670b3..b094f901ef55 100644 --- a/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_tasks.py +++ b/python/ray/data/_internal/execution/operators/shuffle_operators/shuffle_tasks.py @@ -16,6 +16,7 @@ from ray.data._internal.execution.util import yield_block_with_stats from ray.data._internal.output_buffer import BlockOutputBuffer, OutputBlockSizeOption from ray.data._internal.table_block import TableBlockAccessor +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import ( Block, BlockAccessor, @@ -141,7 +142,9 @@ def _shuffle_map_task( output_schema = TableBlockAccessor.try_convert_block_type( shard_source_blocks[0], block_type=BlockType.ARROW ).schema - empty_shard = _encode_partition_ipc(output_schema.empty_table(), ipc_write_options) + empty_shard = _encode_partition_ipc( + _create_empty_table(output_schema), ipc_write_options + ) shard_sizes: Dict[int, Tuple[int, int]] = {} partition_bufs: List[pa.Buffer] = [] diff --git a/python/ray/data/_internal/execution/operators/top_n_operator.py b/python/ray/data/_internal/execution/operators/top_n_operator.py index f94814ce9b75..5a0a1bf8b965 100644 --- a/python/ray/data/_internal/execution/operators/top_n_operator.py +++ b/python/ray/data/_internal/execution/operators/top_n_operator.py @@ -39,6 +39,7 @@ from ray.data._internal.remote_fn import cached_remote_fn from ray.data._internal.stats import StatsDict from ray.data._internal.table_block import TableBlockAccessor +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import ( Block, BlockAccessor, @@ -446,7 +447,7 @@ def _empty_block_from_schema(schema: Schema) -> Block: } ) assert isinstance(schema, pa.Schema), type(schema) - return schema.empty_table() + return _create_empty_table(schema) def _final_top_n_task( diff --git a/python/ray/data/_internal/logical/operators/join_operator.py b/python/ray/data/_internal/logical/operators/join_operator.py index 21bdb3991883..82209cf8f155 100644 --- a/python/ray/data/_internal/logical/operators/join_operator.py +++ b/python/ray/data/_internal/logical/operators/join_operator.py @@ -8,6 +8,7 @@ PredicatePassThroughBehavior, ) from ray.data._internal.logical.operators.n_ary_operator import NAry +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.expressions import BinaryExpr, ColumnExpr, Expr, Operation if TYPE_CHECKING: @@ -460,8 +461,8 @@ def infer_schema(self) -> Optional["Schema"]: return None try: joined = join_tables( - left_schema.empty_table(), - right_schema.empty_table(), + _create_empty_table(left_schema), + _create_empty_table(right_schema), join_type=join_type_enum, left_key_col_names=self.left_key_columns, right_key_col_names=self.right_key_columns, diff --git a/python/ray/data/_internal/logical/operators/n_ary_operator.py b/python/ray/data/_internal/logical/operators/n_ary_operator.py index eab23a32530e..404176060b78 100644 --- a/python/ray/data/_internal/logical/operators/n_ary_operator.py +++ b/python/ray/data/_internal/logical/operators/n_ary_operator.py @@ -8,6 +8,7 @@ LogicalOperatorUnifiesInputSchemas, PredicatePassThroughBehavior, ) +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.util.annotations import PublicAPI if TYPE_CHECKING: @@ -120,9 +121,9 @@ def infer_schema(self) -> Optional["Schema"]: ): return None try: - combined = input_schemas[0].empty_table() + combined = _create_empty_table(input_schemas[0]) for s in input_schemas[1:]: - combined = BlockAccessor.for_block(combined).zip(s.empty_table()) + combined = BlockAccessor.for_block(combined).zip(_create_empty_table(s)) except (pa.ArrowTypeError, pa.ArrowInvalid): return None return combined.schema diff --git a/python/ray/data/_internal/logical/operators/read_operator.py b/python/ray/data/_internal/logical/operators/read_operator.py index d35d751cd122..5597076a2784 100644 --- a/python/ray/data/_internal/logical/operators/read_operator.py +++ b/python/ray/data/_internal/logical/operators/read_operator.py @@ -22,6 +22,7 @@ from ray.data._internal.planner.plan_expression.expression_visitors import ( get_column_references, ) +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import ( Block, BlockMetadata, @@ -319,7 +320,7 @@ def infer_schema(self) -> "pa.Schema": # probe fails — the UDF may require a non-empty input. if self.block_udf is not None: try: - transformed = self.block_udf(schema.empty_table()).schema + transformed = self.block_udf(_create_empty_table(schema)).schema schema = transformed.with_metadata(schema.metadata) except Exception: pass diff --git a/python/ray/data/_internal/utils/transform_pyarrow.py b/python/ray/data/_internal/utils/transform_pyarrow.py index 1cc0e49ab5f4..8179a25b87b7 100644 --- a/python/ray/data/_internal/utils/transform_pyarrow.py +++ b/python/ray/data/_internal/utils/transform_pyarrow.py @@ -9,6 +9,22 @@ def _is_pa_extension_type(pa_type: "pyarrow.lib.DataType") -> bool: return isinstance(pa_type, pyarrow.BaseExtensionType) +def _create_empty_table(schema: "pyarrow.Schema") -> "pyarrow.Table": + """Create an empty table with the given schema. + + Use this instead of ``Schema.empty_table()``, which fails for some extension + columns: on pyarrow 25+ for native ``fixed_shape_tensor`` columns with a + ``permutation`` (pyarrow adds one in ``from_numpy_ndarray``), and on every + supported version for extension types nested in a struct or list. + ``pyarrow.nulls`` builds each column without that conversion. The result + matches ``Schema.empty_table()``: one empty chunk per column and the schema + metadata kept. + """ + return pyarrow.Table.from_arrays( + [pyarrow.nulls(0, type=field.type) for field in schema], schema=schema + ) + + def _is_multi_chunk_extension_column(column: "pyarrow.ChunkedArray") -> bool: """Return whether a column needs multi-chunk extension-array handling.""" return column.num_chunks > 1 and _is_pa_extension_type(column.type) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 883ab2376190..1e0cf1a76767 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -115,6 +115,7 @@ get_compute_strategy, merge_resources_to_ray_remote_args, ) +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.aggregate import ( AggregateFn, AggregateFnV2, @@ -7677,7 +7678,7 @@ def block_to_df(block_ref: ObjectRef[Block]) -> pd.DataFrame: } ) else: - meta = schema.empty_table().to_pandas() + meta = _create_empty_table(schema).to_pandas() dfs = [] for ref_bundle in self.iter_internal_ref_bundles(): @@ -7717,7 +7718,7 @@ def to_mars(self) -> "mars.dataframe.DataFrame": if isinstance(schema, PandasBlockSchema): dtypes = pd.Series(schema.types, index=schema.names) elif isinstance(schema, pa.Schema): - dtypes = schema.empty_table().to_pandas().dtypes + dtypes = _create_empty_table(schema).to_pandas().dtypes else: raise NotImplementedError(f"Unsupported format of schema {schema}") index_value = parse_index(pd.RangeIndex(-1)) @@ -7846,7 +7847,9 @@ def to_pandas(self, limit: Optional[int] = None) -> "pandas.DataFrame": # `BlockAccessor.to_pandas()` path used for non-empty blocks # so the two agree on column types (types_mapper, tensor # casting, etc.). - df = BlockAccessor.for_block(base_schema.empty_table()).to_pandas() + df = BlockAccessor.for_block( + _create_empty_table(base_schema) + ).to_pandas() else: import pandas diff --git a/python/ray/data/tests/test_tensor.py b/python/ray/data/tests/test_tensor.py index e5a51bbca9ac..cba4e90428a3 100644 --- a/python/ray/data/tests/test_tensor.py +++ b/python/ray/data/tests/test_tensor.py @@ -305,6 +305,45 @@ def test_tensors_sort(ray_start_regular_shared, tensor_format_context): ) +@pytest.mark.parametrize( + "op", ["map_groups", "repartition_keys", "repartition_shuffle", "join"] +) +def test_tensors_shuffle_and_join(ray_start_regular_shared, tensor_format_context, op): + # Shuffle and join tasks create empty blocks from the schema, which failed + # for native tensor columns on pyarrow 25+. + arr = np.arange(64).reshape((4, 4, 4)) + ds = ray.data.from_arrow(pa.table({"a": TensorArray(arr), "k": [0, 1, 0, 1]})) + if op == "map_groups": + out = ds.groupby("k").map_groups(lambda group: group) + elif op == "repartition_keys": + out = ds.repartition(4, keys=["k"]) + elif op == "repartition_shuffle": + out = ds.repartition(4, shuffle=True) + else: + other = ray.data.from_items([{"k": 0, "v": "x"}, {"k": 1, "v": "y"}]) + out = ds.join(other, join_type="inner", num_partitions=4, on=("k",)) + + rows = sorted(out.take_all(), key=lambda row: np.asarray(row["a"]).min()) + np.testing.assert_equal([np.asarray(row["a"]) for row in rows], list(arr)) + + +def test_tensors_top_n_empty_result(ray_start_regular_shared, tensor_format_context): + # The Top-N merge builds an empty block from the schema when no rows are + # left, which failed for native tensor columns on pyarrow 25+. + from ray.data._internal.logical.rules.top_n_rewrite import ( + TOP_N_REWRITE_ENABLED_CONFIG_KEY, + ) + from ray.data.expressions import col + + ray.data.DataContext.get_current().set_config( + TOP_N_REWRITE_ENABLED_CONFIG_KEY, True + ) + arr = np.arange(64).reshape((4, 4, 4)) + ds = ray.data.from_arrow(pa.table({"a": TensorArray(arr), "k": [0, 1, 0, 1]})) + out = ds.filter(expr=col("k") < 0).sort("k").limit(2) + assert out.take_all() == [] + + def test_tensors_inferred_from_map(ray_start_regular_shared, tensor_format_context): tensor_format = tensor_format_context # With tensor_format_context, ARROW_NATIVE only runs when supported, diff --git a/python/ray/data/tests/unit/test_transform_pyarrow.py b/python/ray/data/tests/unit/test_transform_pyarrow.py index 45d5a1606945..0cf185d1d6b8 100644 --- a/python/ray/data/tests/unit/test_transform_pyarrow.py +++ b/python/ray/data/tests/unit/test_transform_pyarrow.py @@ -24,6 +24,7 @@ ArrowTensorTypeV2, create_arrow_fixed_shape_tensor_type, ) +from ray.data._internal.utils.transform_pyarrow import _create_empty_table from ray.data.block import BlockAccessor from ray.data.extensions import ( ArrowConversionError, @@ -385,6 +386,33 @@ def test_shuffle(): ) +@pytest.mark.parametrize( + "pa_type", + [ + pa.int64(), + pa.string(), + pa.struct([("a", pa.int64())]), + ArrowTensorType((2, 2), pa.float32()), + ArrowTensorTypeV2((2, 2), pa.float32()), + ArrowVariableShapedTensorType(pa.float32(), 2), + ArrowPythonObjectType(), + # `from_numpy_ndarray` adds `permutation=[0, 1]`, which + # `Schema.empty_table()` rejects on pyarrow 25+. + pa.FixedShapeTensorArray.from_numpy_ndarray(np.zeros((1, 2, 2))).type, + # `Schema.empty_table()` rejects extension types nested in a struct. + pa.struct([("x", ArrowTensorTypeV2((2,), pa.int64()))]), + ], +) +def test_create_empty_table(pa_type): + schema = pa.schema([("c", pa_type), ("i", pa.int64())], metadata={b"k": b"v"}) + + table = _create_empty_table(schema) + + assert table.num_rows == 0 + assert table.schema.equals(schema, check_metadata=True) + assert [column.num_chunks for column in table.columns] == [1, 1] + + def test_arrow_concat_empty(simple_concat_data): # Test empty. assert concat(simple_concat_data["empty"]) == pa.table([])