diff --git a/packages/evo-objects/README.md b/packages/evo-objects/README.md index ab93e4d37..c2b70de5e 100644 --- a/packages/evo-objects/README.md +++ b/packages/evo-objects/README.md @@ -25,6 +25,17 @@ data, empowering our users to build responsive modern workflows. pip install evo-objects ``` +The base installation supports object operations and low-level data transfers without PyArrow. +`ObjectAPIClient.prepare_data_upload()` and `prepare_data_download()` return contexts for transferring existing Parquet files. You must create or read the files yourself and provide matching table descriptions when creating objects. + +The optional `ObjectDataClient` provides higher-level operations on PyArrow tables without requiring pandas. Its +DataFrame methods and other pandas-based helpers additionally require pandas. Install the `utils` extra for these +dependencies: + +```shell +pip install "evo-objects[utils]" +``` + ## Usage See [the evo-sdk-common documentation](https://github.com/SeequentEvo/evo-python-sdk/blob/main/packages/evo-sdk-common/README.md) diff --git a/packages/evo-objects/src/evo/objects/client/object_client.py b/packages/evo-objects/src/evo/objects/client/object_client.py index 7a5ce42a0..ac5112cc5 100644 --- a/packages/evo-objects/src/evo/objects/client/object_client.py +++ b/packages/evo-objects/src/evo/objects/client/object_client.py @@ -40,6 +40,7 @@ ParquetLoader, TableInfo, ) + from ..utils.tables import table_to_dataframe except ImportError as e: print(e) _LOADER_AVAILABLE = False @@ -486,10 +487,11 @@ async def download_dataframe( nan_description structure. :param column_names: An optional list of column names for the table, instead of those from the Parquet file. - :returns: A Pandas DataFrame containing the downloaded data. + :returns: A Pandas DataFrame containing the downloaded data. Column dtypes are derived from the + Geoscience Object column types; integer and boolean columns use pandas nullable dtypes. """ table = await self.download_table(table_info, fb=fb, nan_values=nan_values, column_names=column_names) - return table.to_pandas() + return table_to_dataframe(table) async def download_category_dataframe( self, diff --git a/packages/evo-objects/src/evo/objects/parquet/loader.py b/packages/evo-objects/src/evo/objects/parquet/loader.py index f4f529af6..80921a00a 100644 --- a/packages/evo-objects/src/evo/objects/parquet/loader.py +++ b/packages/evo-objects/src/evo/objects/parquet/loader.py @@ -28,6 +28,7 @@ from ..exceptions import SchemaValidationError, TableFormatError from ..utils import ArrowTableFormat, KnownTableFormat, TableInfo +from ..utils.tables import table_to_dataframe try: import pandas as pd @@ -140,9 +141,13 @@ def load_as_table(self) -> pa.Table: # Optional support for pandas dataframes def load_as_dataframe(self) -> pd.DataFrame: - """Load the Parquet data as a Pandas DataFrame.""" + """Load the Parquet data as a Pandas DataFrame. + + Column dtypes are derived from the Geoscience Object column types, never from pandas metadata in the file. + Integer and boolean columns use pandas nullable dtypes (e.g. `Int64`, `boolean`). + """ table = self.load_as_table() - return table.to_pandas() + return table_to_dataframe(table) if _NP_AVAILABLE: # Optional support for numpy arrays diff --git a/packages/evo-objects/src/evo/objects/utils/__init__.py b/packages/evo-objects/src/evo/objects/utils/__init__.py index af0879537..fc02bce8d 100644 --- a/packages/evo-objects/src/evo/objects/utils/__init__.py +++ b/packages/evo-objects/src/evo/objects/utils/__init__.py @@ -23,9 +23,8 @@ DataFrame = None # type: ignore from .data import ObjectDataClient -from .downhole import expand_hole_index, hole_chunks_from_ids from .table_formats import all_known_formats, get_known_format -from .tables import ArrowTableFormat, BaseTableFormat, KnownTableFormat +from .tables import ArrowTableFormat, BaseTableFormat, ColumnFormat, KnownTableFormat from .types import ArrayTableInfo, AttributeInfo, CategoryInfo, LookupTableInfo, TableInfo # We _used_ to export Table and DataFrame from this package as custom protocols, but we are using the actual @@ -39,12 +38,19 @@ "AttributeInfo", "BaseTableFormat", "CategoryInfo", + "ColumnFormat", "KnownTableFormat", "LookupTableInfo", "ObjectDataClient", "TableInfo", "all_known_formats", - "expand_hole_index", "get_known_format", - "hole_chunks_from_ids", ] + +if DataFrame is not None: + from .downhole import expand_hole_index, hole_chunks_from_ids + + __all__ += [ + "expand_hole_index", + "hole_chunks_from_ids", + ] diff --git a/packages/evo-objects/src/evo/objects/utils/data.py b/packages/evo-objects/src/evo/objects/utils/data.py index be6f22c50..af48a8fd4 100644 --- a/packages/evo-objects/src/evo/objects/utils/data.py +++ b/packages/evo-objects/src/evo/objects/utils/data.py @@ -26,7 +26,7 @@ from ..exceptions import TableFormatError from ..io import _CACHE_SCOPE, ObjectDataDownload, ObjectDataUpload from .table_formats import INTEGER_ARRAY_1_INT32, INTEGER_ARRAY_MD_INT32, LOOKUP_TABLE_INT32 -from .tables import KnownTableFormat +from .tables import ArrowTableFormat, KnownTableFormat, table_to_dataframe from .types import ArrayTableInfo, CategoryInfo, LookupTableInfo try: @@ -69,27 +69,19 @@ def _iter_refs(target: Any, _key: str | None = None) -> Iterator[str]: yield str(value) -def _get_schema_from_dataframe(dataframe: "pd.DataFrame") -> pa.Schema: - """Get a normalized pyarrow schema from a pandas dataframe. +def _table_from_dataframe(dataframe: "pd.DataFrame") -> pa.Table: + """Convert a pandas dataframe to a pyarrow table for upload. - This hook centralizes dataframe-to-Arrow schema adjustments before the - dataframe is converted to a table. It currently performs these conversions: - - large_string -> string - - dictionary<*, large_string> -> dictionary<*, string> + The dataframe index is not part of any Geoscience Object table format, so it is never stored. Pandas metadata is + removed, so readers rely only on the Geoscience Object schema. - :param dataframe: The pandas dataframe to get the schema from. - :return: A pyarrow schema with any configured type conversions applied. + :param dataframe: The pandas dataframe to convert. + :return: A pyarrow table with the normalized schema from `ArrowTableFormat.from_pandas`. """ - schema = pa.Schema.from_pandas(dataframe) - fields = [] - for field in schema: - if pa.types.is_large_string(field.type): - fields.append(field.with_type(pa.string())) - elif pa.types.is_dictionary(field.type) and pa.types.is_large_string(field.type.value_type): - fields.append(field.with_type(pa.dictionary(field.type.index_type, pa.string()))) - else: - fields.append(field) - return pa.schema(fields) + table_format = ArrowTableFormat.from_pandas(dataframe) + return pa.Table.from_pandas( + dataframe, schema=table_format.pa_schema, preserve_index=False + ).replace_schema_metadata() class ObjectDataClient: @@ -222,7 +214,8 @@ async def upload_category_table(self, table: pa.Table, fb: IFeedback = NoFeedbac both the value and lookup tables. The table columns must be string columns, but can be dictionary encoded. If they are dictionary encoded, this - method takes advantage of that, when building the value and lookup tables. + method takes advantage of that, when building the value and lookup tables. Large strings are converted to + strings. :param table: The table to be uploaded. :param fb: A feedback object for tracking upload progress. @@ -237,11 +230,13 @@ async def upload_category_table(self, table: pa.Table, fb: IFeedback = NoFeedbac columns_chunk_range = [] for column in table.itercolumns(): column = column.dictionary_encode() - if not pa.types.is_string(column.type.value_type): + value_type = column.type.value_type + if not (pa.types.is_string(value_type) or pa.types.is_large_string(value_type)): raise TableFormatError("Category columns must be of type string") - if not pa.types.is_int32(column.type.index_type): - # Currently, we only support int32 indices - column = pc.cast(column, pa.dictionary(pa.int32(), column.type.value_type)) + # Currently, we only support int32 indices and string values + category_type = pa.dictionary(pa.int32(), pa.string()) + if column.type != category_type: + column = pc.cast(column, category_type) start_chunk = len(all_chunks) if isinstance(column, pa.ChunkedArray): all_chunks.extend(column.chunks) @@ -422,8 +417,7 @@ def save_dataframe( no table formats are specified, raised when the table does not match any known format. :raises StorageFileNotFoundError: If the destination does not exist or is not a directory. """ - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema) + table = _table_from_dataframe(dataframe) return self.save_table(table, table_format=table_format) async def upload_dataframe( @@ -446,8 +440,7 @@ async def upload_dataframe( :raises TableFormatError: If the provided table does not match any of the specified formats. If no table formats are specified, raised when the table does not match any known format. """ - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema) + table = _table_from_dataframe(dataframe) table_info = await self.upload_table(table, table_format=table_format, fb=fb) return table_info @@ -468,8 +461,8 @@ async def upload_category_dataframe(self, dataframe: pd.DataFrame, fb: IFeedback :raises TableFormatError: If the table isn't a valid category table, or if the number of categories exceeds what int32 type can represent. """ - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema) + # Keep dictionary encoding until the categories are split into value and lookup tables. + table = pa.Table.from_pandas(dataframe, preserve_index=False).replace_schema_metadata() category_info = await self.upload_category_table(table, fb=fb) return category_info @@ -495,6 +488,6 @@ async def download_dataframe( """ table = await self.download_table(object_id, version_id, table_info, fb) try: - return table.to_pandas() + return table_to_dataframe(table) except ModuleNotFoundError: raise RuntimeError("Unable to download dataframe because the `pandas` package is not installed") diff --git a/packages/evo-objects/src/evo/objects/utils/tables.py b/packages/evo-objects/src/evo/objects/utils/tables.py index baae199a7..995a34894 100644 --- a/packages/evo-objects/src/evo/objects/utils/tables.py +++ b/packages/evo-objects/src/evo/objects/utils/tables.py @@ -13,6 +13,7 @@ import hashlib import os +from collections.abc import Callable from pathlib import Path from tempfile import NamedTemporaryFile from types import EllipsisType @@ -26,20 +27,50 @@ from ..exceptions import SchemaValidationError, TableFormatError +try: + import pandas as pd +except ImportError: + _PD_AVAILABLE = False +else: + _PD_AVAILABLE = True + logger = evo.logging.getLogger("object.tables") __all__ = [ "ArrowTableFormat", "BaseTableFormat", + "ColumnFormat", "KnownTableFormat", + "table_to_dataframe", ] # Block size to limit memory use while calculating the sha256 digest of a file. _DIGEST_BLOCK_SIZE = 4 * 1024 * 1024 - -class _ColumnFormat: - def __init__(self, format_spec: pa.DataType | str): +# Pandas dtype for every supported Geoscience Object column type. Integer and boolean columns use pandas nullable +# dtypes, so nulls do not change the dtype. `None` keeps the pyarrow default conversion: pyarrow `types_mapper` only +# accepts pandas extension dtypes, so numpy-backed `float64` and the pandas default string dtype cannot be named here. +_PANDAS_DTYPES: dict[str, str | None] = { + "float64": None, + "uint8": "UInt8", + "uint32": "UInt32", + "uint64": "UInt64", + "int32": "Int32", + "int64": "Int64", + "bool": "boolean", + "string": None, + "timestamp": "datetime64[us, UTC]", +} + + +class ColumnFormat: + """A Geoscience Object column type and its corresponding Arrow data type.""" + + def __init__(self, format_spec: pa.DataType | str) -> None: + """ + :param format_spec: A Geoscience Object column type identifier or a supported Arrow data type. + :raises TypeError: If the column type is not supported. + """ if isinstance(format_spec, str): self._type = self._get_data_type(format_spec) self._format_id = format_spec @@ -97,12 +128,29 @@ def _get_format_id(data_type: pa.DataType) -> str: @property def id(self) -> str: + """The Geoscience Object column type identifier.""" return self._format_id @property def type(self) -> pa.DataType: + """The corresponding Arrow data type.""" return self._type + @property + def pandas_dtype(self) -> pd.api.extensions.ExtensionDtype | None: + """The pandas dtype used when loading this column into a dataframe. + + Integer and boolean columns use pandas nullable dtypes, so nulls do not change the dtype. Returns `None` for + `float64` and `string` columns, which use the pyarrow default conversion. + + :raises ImportError: If pandas is not installed. + """ + if not _PD_AVAILABLE: + raise ModuleNotFoundError("No module named 'pandas'", name="pandas") + if (dtype_name := _PANDAS_DTYPES[self._format_id]) is None: + return None + return pd.api.types.pandas_dtype(dtype_name) + class BaseTableFormat: """Base type for comparing table formats""" @@ -114,7 +162,7 @@ def __init__(self, name: str, columns: list[pa.DataType | str | EllipsisType]) - ("...") indicates a multidimensional format. """ self._name = name - self._columns: list[_ColumnFormat] = [] + self._columns: list[ColumnFormat] = [] self._multi_dimensional = False for column_type in columns: if column_type is Ellipsis: @@ -122,7 +170,7 @@ def __init__(self, name: str, columns: list[pa.DataType | str | EllipsisType]) - elif self._multi_dimensional: raise AssertionError("Found new type specification after Ellipsis") else: - self._columns.append(_ColumnFormat(format_spec=column_type)) + self._columns.append(ColumnFormat(format_spec=column_type)) if self._multi_dimensional and len(self._columns) != 1: raise AssertionError("Multidimensional formats must only have one column type") @@ -147,6 +195,11 @@ def width(self) -> int: """The number of columns in this table format""" return len(self._columns) + @property + def columns(self) -> tuple[ColumnFormat, ...]: + """Column formats in order. A multidimensional format has one entry for its repeated column type.""" + return tuple(self._columns) + @property def data_type(self) -> str: """The column data type(s) in this table format""" @@ -156,15 +209,63 @@ def data_type(self) -> str: class ArrowTableFormat(BaseTableFormat): """Specialised table format type that can be generated from a pyarrow table""" + _pa_schema: pa.Schema | None = None + + @property + def pa_schema(self) -> pa.Schema: + """The Arrow schema supplied to `from_schema` or inferred by `from_pandas`, without any metadata. + + :raises ValueError: If the format was constructed directly without a schema. + """ + if self._pa_schema is None: + raise ValueError("Create the format with from_schema() or from_pandas() to access its Arrow schema") + return self._pa_schema + @classmethod def from_schema(cls, pa_schema: pa.Schema) -> ArrowTableFormat: """Generate an ArrowTableFormat instance that represents the structure of the provided table schema. - :param pa_schema: Table schema to generate a format representation for. + :param pa_schema: Table schema to generate a format representation for. Schema and field metadata are not kept. :return: A table format that can be used to compare against known formats. """ - return ArrowTableFormat(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types) + table_format = cls(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types) + table_format._pa_schema = pa.schema([field.remove_metadata() for field in pa_schema]) + return table_format + + @classmethod + def from_pandas(cls, dataframe: pd.DataFrame) -> ArrowTableFormat: + """Infer a table format from a pandas dataframe using the supported Geoscience Object column types. + + Large strings are normalized to strings. The inferred schema excludes the dataframe index and pandas metadata. + + :param dataframe: The dataframe to infer the format from. + :return: A format whose `pa_schema` can be passed to `pyarrow.Table.from_pandas`. + :raises TypeError: If a column type is not supported. Use `ObjectDataClient.upload_category_dataframe` + for categorical columns, which are split into value and lookup tables separately. + """ + schema = pa.Schema.from_pandas(dataframe, preserve_index=False) + fields = [field.with_type(pa.string()) if pa.types.is_large_string(field.type) else field for field in schema] + return cls.from_schema(pa.schema(fields)) + + def pandas_types_mapper(self) -> Callable[[pa.DataType], pd.api.extensions.ExtensionDtype | None]: + """Get a pyarrow `types_mapper` that converts this format's columns to their `ColumnFormat.pandas_dtype`. + + Pass the result to `table.replace_schema_metadata().to_pandas(types_mapper=...)` to ignore legacy metadata. + + :raises ImportError: If pandas is not installed. + """ + return {column.type: column.pandas_dtype for column in self._columns}.get + + +def table_to_dataframe(table: pa.Table) -> pd.DataFrame: + """Convert a table to a pandas dataframe using the Geoscience Object column types, ignoring pandas metadata. + + :param table: The table to convert. It is expected to have been validated against a known table format. + """ + types_mapper = ArrowTableFormat.from_schema(table.schema).pandas_types_mapper() + # PyArrow parses pandas metadata even with ignore_metadata=True. + return table.replace_schema_metadata().to_pandas(types_mapper=types_mapper) class KnownTableFormat(BaseTableFormat): diff --git a/packages/evo-objects/tests/helpers.py b/packages/evo-objects/tests/helpers.py index f3e604ef9..dd1b7b3c1 100644 --- a/packages/evo-objects/tests/helpers.py +++ b/packages/evo-objects/tests/helpers.py @@ -21,7 +21,7 @@ import pyarrow as pa import pyarrow.parquet as pq -from evo.objects.utils.tables import BaseTableFormat, _ColumnFormat +from evo.objects.utils.tables import BaseTableFormat, ColumnFormat class NoImport: @@ -123,16 +123,16 @@ def _generate_data(format_id: str, n_samples: int) -> Iterator: raise TypeError(f"Unsupported format '{unknown_format}'") -def _change_format(current_format: _ColumnFormat) -> _ColumnFormat: +def _change_format(current_format: ColumnFormat) -> ColumnFormat: match current_format.id: case "float64": - return _ColumnFormat("int64") + return ColumnFormat("int64") case "uint8" | "uint32" | "uint64" | "int32" | "int64": - return _ColumnFormat("float64") + return ColumnFormat("float64") case "bool" | "timestamp": - return _ColumnFormat("string") + return ColumnFormat("string") case "string": - return _ColumnFormat("bool") + return ColumnFormat("bool") case unknown_format: raise TypeError(f"Unsupported format '{unknown_format}'") @@ -140,10 +140,10 @@ def _change_format(current_format: _ColumnFormat) -> _ColumnFormat: def get_sample_table( table_format: BaseTableFormat, n_rows: int, add_column: bool = False, change_types: bool = False ) -> pa.Table: - column_formats = [column for column in table_format._columns] + column_formats = list(table_format.columns) if add_column: - column_formats.append(_ColumnFormat(column_formats[-1].type)) + column_formats.append(ColumnFormat(column_formats[-1].type)) if change_types: column_formats = [_change_format(column) for column in column_formats] diff --git a/packages/evo-objects/tests/test_data_client.py b/packages/evo-objects/tests/test_data_client.py index f3440bef6..a38ba976c 100644 --- a/packages/evo-objects/tests/test_data_client.py +++ b/packages/evo-objects/tests/test_data_client.py @@ -15,6 +15,7 @@ import pandas as pd import pyarrow as pa +import pyarrow.parquet as pq from pandas.testing import assert_frame_equal from parameterized import parameterized @@ -24,7 +25,7 @@ from evo.common.test_tools import TestWithConnector, TestWithStorage from evo.common.utils import NoFeedback, PartialFeedback, get_header_metadata from evo.objects.exceptions import TableFormatError -from evo.objects.utils import KnownTableFormat, ObjectDataClient, table_formats +from evo.objects.utils import ArrowTableFormat, KnownTableFormat, ObjectDataClient, table_formats from helpers import NoImport, UnloadModule, get_sample_table_and_bytes @@ -112,10 +113,9 @@ def test_save_dataframe(self, _name: str, pass_table_format: bool) -> None: with ( mock.patch("evo.objects.utils.table_formats.get_known_format") as mock_get_known_format, mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, - mock.patch("pyarrow.Table") as mock_pyarrow_table, - mock.patch("evo.objects.utils.data._get_schema_from_dataframe", return_value=None), + mock.patch("evo.objects.utils.data._table_from_dataframe") as mock_table_from_dataframe, ): - mock_pyarrow_table.from_pandas.return_value = mock_table = mock.Mock() + mock_table_from_dataframe.return_value = mock_table = mock.Mock() mock_get_known_format.return_value = mock_known_format = mock.Mock(spec=KnownTableFormat) mock_known_format.save_table.return_value = mock_table_info = {} @@ -125,6 +125,7 @@ def test_save_dataframe(self, _name: str, pass_table_format: bool) -> None: else: actual_table_info = self.data_client.save_dataframe(mock_dataframe) + mock_table_from_dataframe.assert_called_once_with(mock_dataframe) if pass_table_format: mock_get_known_format.assert_not_called() else: @@ -136,6 +137,21 @@ def test_save_dataframe(self, _name: str, pass_table_format: bool) -> None: self.transport.assert_no_requests() self.assertIs(mock_table_info, actual_table_info) + def test_save_dataframe_uses_format_schema_without_pandas_metadata(self) -> None: + dataframe = pd.DataFrame({"value": pd.array([2**63 - 1, None], dtype="Int64")}, index=[5, 3]) + with mock.patch( + "evo.objects.utils.data.ArrowTableFormat.from_pandas", wraps=ArrowTableFormat.from_pandas + ) as infer_format: + table_info = self.data_client.save_dataframe(dataframe) + + infer_format.assert_called_once_with(dataframe) + table = pq.ParquetFile(self.data_client.cache_location / table_info["data"]).read() + self.assertNotIn(b"pandas", table.schema.metadata or {}) + self.assertEqual(["value"], table.column_names) + self.assertEqual(pa.int64(), table.column("value").type) + self.assertEqual([2**63 - 1, None], table.column("value").to_pylist()) + self.transport.assert_no_requests() + async def test_upload_referenced_data(self) -> None: put_data_response = load_test_data("put_data_batch.json")[:5] test_pointset = { @@ -262,10 +278,9 @@ async def test_upload_dataframe(self, _name: str, pass_table_format: bool) -> No self.transport.set_http_response(status_code=200, content=json.dumps(put_data_response)), mock.patch("evo.objects.utils.table_formats.get_known_format") as mock_get_known_format, mock.patch("evo.common.io.upload.StorageDestination", autospec=True) as mock_destination, - mock.patch("pyarrow.Table") as mock_pyarrow_table, - mock.patch("evo.objects.utils.data._get_schema_from_dataframe", return_value=None), + mock.patch("evo.objects.utils.data._table_from_dataframe") as mock_table_from_dataframe, ): - mock_pyarrow_table.from_pandas.return_value = mock_table = mock.Mock() + mock_table_from_dataframe.return_value = mock_table = mock.Mock() mock_get_known_format.return_value = mock_known_format = mock.Mock(spec=KnownTableFormat) mock_known_format.save_table.return_value = mock_table_info = {} mock_table_info["data"] = mock_data_id = put_data_response[0]["name"] @@ -289,6 +304,7 @@ async def _mock_upload_file_side_effect(*args, **kwargs): else: actual_table_info = await self.data_client.upload_dataframe(mock_dataframe) + mock_table_from_dataframe.assert_called_once_with(mock_dataframe) if pass_table_format: mock_get_known_format.assert_not_called() else: @@ -351,10 +367,9 @@ async def test_upload_dataframe_exists(self) -> None: self.transport.set_http_response(status_code=200, content=json.dumps(put_data_response)), mock.patch("evo.objects.utils.table_formats.get_known_format") as mock_get_known_format, mock.patch("evo.common.io.upload.StorageDestination", autospec=True) as mock_destination, - mock.patch("pyarrow.Table") as mock_pyarrow_table, - mock.patch("evo.objects.utils.data._get_schema_from_dataframe", return_value=None), + mock.patch("evo.objects.utils.data._table_from_dataframe") as mock_table_from_dataframe, ): - mock_pyarrow_table.from_pandas.return_value = mock_table = mock.Mock() + mock_table_from_dataframe.return_value = mock_table = mock.Mock() mock_get_known_format.return_value = mock_known_format = mock.Mock(spec=KnownTableFormat) mock_known_format.save_table.return_value = mock_table_info = {} mock_table_info["data"] = mock_data_id = put_data_response[0]["name"] @@ -374,6 +389,7 @@ async def _mock_upload_file_side_effect(*args, **kwargs): mock_dataframe = mock.Mock() actual_table_info = await self.data_client.upload_dataframe(mock_dataframe) + mock_table_from_dataframe.assert_called_once_with(mock_dataframe) mock_get_known_format.assert_called_once_with(mock_table, table_formats=None) mock_known_format.save_table.assert_called_once_with( table=mock_table, destination=self.data_client.cache_location @@ -489,6 +505,41 @@ async def test_upload_category_table_error(self) -> None: await self.data_client.upload_category_table(table) assert "Category columns must be of type string" in str(cm.exception) + async def test_upload_category_table_normalizes_mixed_dictionary_types(self) -> None: + table = pa.table( + { + "ordered": pa.chunked_array( + [ + pa.DictionaryArray.from_arrays( + pa.array(indices, type=pa.int32()), + pa.array(["B", "A"], type=pa.large_string()), + ordered=True, + ) + for indices in ([0, None], [1, 0]) + ] + ), + "unordered": pa.array(["C", "A", None, "B"], type=pa.dictionary(pa.int8(), pa.string())), + } + ) + saved_tables: list[pa.Table] = [] + + def save_upload(table: pa.Table, *, table_format: KnownTableFormat, fb: IFeedback) -> dict: + saved_tables.append(table) + return self.data_client.save_table(table, table_format=table_format) + + with mock.patch.object(self.data_client, "upload_table", side_effect=save_upload): + category_info = await self.data_client.upload_category_table(table) + + values, lookup = saved_tables + self.assertEqual([pa.int32(), pa.int32()], values.schema.types) + self.assertEqual([pa.int32(), pa.string()], lookup.schema.types) + for name in table.column_names: + restored = pa.DictionaryArray.from_arrays(values[name].combine_chunks(), lookup["value"].combine_chunks()) + self.assertEqual(table[name].to_pylist(), restored.to_pylist()) + for table_info in (category_info["values"], category_info["table"]): + schema = pq.read_schema(self.data_client.cache_location / table_info["data"]) + self.assertNotIn(b"pandas", schema.metadata or {}) + @parameterized.expand( [ ( @@ -500,6 +551,31 @@ async def test_upload_category_table_error(self) -> None: pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), table_formats.INTEGER_ARRAY_1_INT32, ), + ( + "ordered_category", + pd.DataFrame({"Category": pd.Categorical(["A", "B", "A", "C"], ordered=True)}), + pa.table( + {"key": pa.array([0, 1, 2], type=pa.int32()), "value": pa.array(["A", "B", "C"], type=pa.string())} + ), + pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), + table_formats.INTEGER_ARRAY_1_INT32, + ), + ( + "large_string_category", + pd.DataFrame( + { + "Category": pd.Categorical( + ["A", "B", "A", "C"], + categories=pd.Index(pd.array(["A", "B", "C"], dtype="string[pyarrow]")), + ) + } + ), + pa.table( + {"key": pa.array([0, 1, 2], type=pa.int32()), "value": pa.array(["A", "B", "C"], type=pa.string())} + ), + pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), + table_formats.INTEGER_ARRAY_1_INT32, + ), ( "string", pd.DataFrame({"Category": ["A", "B", "A", "C"]}), @@ -552,6 +628,29 @@ def side_effect(table, table_format=None, fb=None): self.assertEqual(category_info["table"]["table_format"], table_formats.LOOKUP_TABLE_INT32) self.assertEqual(category_info["values"]["table"], values_table) self.assertEqual(category_info["values"]["table_format"], values_table_format) + for table_info in (category_info["values"], category_info["table"]): + table = table_info["table"] + self.assertNotIn(b"pandas", table.schema.metadata or {}) + saved_info = self.data_client.save_table(table, table_format=table_info["table_format"]) + schema = pq.read_schema(self.data_client.cache_location / saved_info["data"]) + self.assertNotIn(b"pandas", schema.metadata or {}) + + @parameterized.expand([("unordered", False), ("ordered", True)]) + async def test_category_dataframe_retains_dictionary_encoding(self, _name: str, ordered: bool) -> None: + categories = pd.Index(pd.array(["B", "A"], dtype="string[pyarrow]")) + dataframe = pd.DataFrame( + {"Category": pd.Categorical(["A", "B", None], categories=categories, ordered=ordered)}, + index=[5, 3, 1], + ) + with mock.patch.object(self.data_client, "upload_category_table", new_callable=mock.AsyncMock) as upload: + await self.data_client.upload_category_dataframe(dataframe) + + (table,), _ = upload.await_args + self.assertNotIn(b"pandas", table.schema.metadata or {}) + self.assertEqual(["Category"], table.column_names) + self.assertTrue(pa.types.is_dictionary(table.column("Category").type)) + self.assertEqual(ordered, table.column("Category").type.ordered) + self.assertEqual(["A", "B", None], table.column("Category").to_pylist()) async def test_upload_dataframe_converts_large_string_via_schema(self) -> None: """Test that upload_dataframe converts large_string columns (from pandas string[pyarrow]) to string via schema.""" @@ -561,6 +660,7 @@ async def test_upload_dataframe_converts_large_string_via_schema(self) -> None: await self.data_client.upload_dataframe(dataframe) (table,), _ = mock_upload.await_args + self.assertNotIn(b"pandas", table.schema.metadata or {}) self.assertTrue(pa.types.is_string(table.schema.field("col").type)) async def test_download_table(self) -> None: diff --git a/packages/evo-objects/tests/test_downloaded_object.py b/packages/evo-objects/tests/test_downloaded_object.py index af1b78cc3..e2001aec4 100644 --- a/packages/evo-objects/tests/test_downloaded_object.py +++ b/packages/evo-objects/tests/test_downloaded_object.py @@ -466,6 +466,20 @@ async def test_download_dataframe_nan_values(self): expected_dataframe = pd.DataFrame({"value": [np.nan, np.nan, 3.4, 5.3, np.nan, np.nan]}) assert_frame_equal(expected_dataframe, actual_dataframe) + async def test_download_dataframe_preserves_nullable_integers_and_sentinels(self) -> None: + values_path = "locations.attributes[1].values" + dataframe = pd.DataFrame( + {"value": pd.array([2**63 - 1, -999, None], dtype="Int64")}, + index=pd.RangeIndex(5, 8), + ) + self._set_property(f"{values_path}.data_type", "int64") + self._setup_table(values_path, pa.Table.from_pandas(dataframe)) + + actual_dataframe = await self.object.download_dataframe(values_path, nan_values=[-999], column_names=["count"]) + + expected_dataframe = pd.DataFrame({"count": pd.array([2**63 - 1, None, None], dtype="Int64")}) + assert_frame_equal(expected_dataframe, actual_dataframe) + async def test_download_dataframe_column_names(self): with self._patch_downloading_table_in_memory("locations.coordinates") as sample_table: actual_dataframe = await self.object.download_dataframe( diff --git a/packages/evo-objects/tests/test_pandas_metadata.py b/packages/evo-objects/tests/test_pandas_metadata.py new file mode 100644 index 000000000..0206d923f --- /dev/null +++ b/packages/evo-objects/tests/test_pandas_metadata.py @@ -0,0 +1,156 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DataFrame exports omit pandas metadata; dataframe readers derive dtypes from the Geoscience Object schema.""" + +import tempfile +import unittest +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from pandas.testing import assert_frame_equal + +from evo.objects.exceptions import SchemaValidationError, TableFormatError +from evo.objects.parquet import ParquetLoader +from evo.objects.utils import ArrowTableFormat, KnownTableFormat, get_known_format +from evo.objects.utils.data import _table_from_dataframe + + +def _load_dataframe(table_info: dict, source: Path) -> pd.DataFrame: + with pa.OSFile(str(source / str(table_info["data"])), mode="r") as parquet_file: + with ParquetLoader(parquet_file) as loader: + loader.validate_with_table_info(table_info) + return loader.load_as_dataframe() + + +class TestPandasMetadata(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.data_dir = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _save(self, table: pa.Table) -> dict: + return get_known_format(table).save_table(table, self.data_dir) + + def _schema_metadata(self, table_info: dict) -> dict: + return pq.read_schema(self.data_dir / table_info["data"]).metadata or {} + + def test_dataframe_table_has_pandas_metadata(self) -> None: + """Even an explicit metadata-free schema does not prevent PyArrow from adding pandas metadata.""" + dataframe = pd.DataFrame({"value": pd.array([1, None], dtype="Int64")}) + table_format = ArrowTableFormat.from_pandas(dataframe) + self.assertIsNone(table_format.pa_schema.metadata) + + table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + + self.assertIn(b"pandas", table.schema.metadata) + + def test_saved_dataframe_has_no_pandas_metadata(self) -> None: + table = _table_from_dataframe(pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")})) + + table_info = self._save(table) + + self.assertNotIn(b"pandas", self._schema_metadata(table_info)) + + def test_dataframe_index_is_not_stored(self) -> None: + dataframe = pd.DataFrame({"value": [1.0, 2.0, 3.0]}).iloc[[2, 0]] + + table = _table_from_dataframe(dataframe) + + self.assertEqual(["value"], table.column_names) + self.assertIsNone(table.schema.metadata) + + def test_load_as_dataframe_uses_nullable_dtypes_from_schema(self) -> None: + cases = [ + (pa.array([1, None, 3], type=pa.int64()), "Int64"), + (pa.array([1, 2, 3], type=pa.int64()), "Int64"), + (pa.array([1, None, 3], type=pa.int32()), "Int32"), + (pa.array([1, 2, 3], type=pa.int32()), "Int32"), + (pa.array([2**63 - 1, None, -(2**63)], type=pa.int64()), "Int64"), + (pa.array([255, None, 0], type=pa.uint8()), "UInt8"), + (pa.array([2**32 - 1, None, 0], type=pa.uint32()), "UInt32"), + (pa.array([2**64 - 1, None, 0], type=pa.uint64()), "UInt64"), + (pa.array([None, None], type=pa.int64()), "Int64"), + (pa.array([], type=pa.int64()), "Int64"), + (pa.array([True, None, False], type=pa.bool_()), "boolean"), + (pa.array([1.5, None, 3.5], type=pa.float64()), "float64"), + ( + pa.array([pd.Timestamp("2025-01-01", tz="UTC"), None], type=pa.timestamp("us", tz="UTC")), + "datetime64[us, UTC]", + ), + ] + for array, expected_dtype in cases: + with self.subTest(type=str(array.type), nulls=array.null_count): + table_info = self._save(pa.table({"value": array})) + + loaded = _load_dataframe(table_info, self.data_dir) + + self.assertEqual(pd.api.types.pandas_dtype(expected_dtype), loaded["value"].dtype) + assert_frame_equal(pd.DataFrame({"value": pd.array(array.to_pylist(), dtype=expected_dtype)}), loaded) + + def test_load_ignores_legacy_pandas_metadata(self) -> None: + """Files written before metadata stripping must load with the schema types, not the stored pandas dtypes.""" + dataframe = pd.DataFrame({"value": pd.array([1, 2, 3], dtype="Int64")}, index=pd.RangeIndex(10, 13)) + table = pa.Table.from_pandas(dataframe) + table_info = {"data": "legacy.parquet", "length": 3, "width": 1, "data_type": "int64"} + pq.write_table(table, self.data_dir / table_info["data"]) + self.assertIn(b"pandas", pq.read_schema(self.data_dir / table_info["data"]).metadata) + + loaded = _load_dataframe(table_info, self.data_dir) + + pd.testing.assert_index_equal(pd.RangeIndex(3), loaded.index) + self.assertEqual(pd.Int64Dtype(), loaded["value"].dtype) + + def test_load_ignores_malformed_pandas_metadata_without_changing_arrow_table(self) -> None: + schema = pa.schema( + [pa.field("value", pa.int64(), nullable=False, metadata={b"units": b"count"})], + metadata={b"pandas": b"not json", b"other": b"kept"}, + ) + table = pa.Table.from_arrays([pa.array([2**63 - 1])], schema=schema) + table_info = self._save(table) + + with ParquetLoader(pa.OSFile(str(self.data_dir / table_info["data"]), "r")) as loader: + loader.validate_with_table_info(table_info) + assert_frame_equal( + pd.DataFrame({"value": pd.array([2**63 - 1], dtype="Int64")}), loader.load_as_dataframe() + ) + self.assertTrue(table.equals(loader.load_as_table(), check_metadata=True)) + + def test_serialized_index_is_rejected_by_expected_schema(self) -> None: + dataframe = pd.DataFrame({"value": [1, 2]}, index=pd.Index([5, 3], name="row")) + table = pa.Table.from_pandas(dataframe, preserve_index=True) + table_info = {"data": "index.parquet", "length": 2, "width": 1, "data_type": "int64"} + table_format = KnownTableFormat.from_table_info(table_info) + + with self.assertRaisesRegex(TableFormatError, "Column count"): + table_format.save_table(table, self.data_dir) + self.assertEqual([], list(self.data_dir.iterdir())) + + pq.write_table(table, self.data_dir / table_info["data"]) + with ParquetLoader(pa.OSFile(str(self.data_dir / table_info["data"]), "r")) as loader: + with self.assertRaisesRegex(SchemaValidationError, "Column count"): + loader.validate_with_table_info(table_info) + + def test_string_and_float_columns_use_arrow_defaults_not_pandas_metadata(self) -> None: + for values, dtype in [(["A", None, "B"], "string[pyarrow]"), ([1.5, None, 3.5], "Float64")]: + with self.subTest(dtype=dtype): + table = pa.Table.from_pandas(pd.DataFrame({"value": pd.array(values, dtype=dtype)})) + if pa.types.is_large_string(table["value"].type): + table = table.cast(pa.schema([("value", pa.string())], metadata=table.schema.metadata)) + table_info = self._save(table) + + loaded = _load_dataframe(table_info, self.data_dir) + + assert_frame_equal(table.replace_schema_metadata().to_pandas(), loaded) diff --git a/packages/evo-objects/tests/test_tables.py b/packages/evo-objects/tests/test_tables.py index 416f5c22a..a5991156b 100644 --- a/packages/evo-objects/tests/test_tables.py +++ b/packages/evo-objects/tests/test_tables.py @@ -10,15 +10,19 @@ # limitations under the License. import hashlib +import subprocess +import sys import unittest import uuid from collections.abc import Iterator from io import BytesIO from pathlib import Path +from textwrap import dedent from typing import BinaryIO from unittest import mock from uuid import UUID +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from parameterized import parameterized, parameterized_class @@ -31,6 +35,7 @@ from evo.objects.utils import ( ArrowTableFormat, BaseTableFormat, + ColumnFormat, KnownTableFormat, all_known_formats, get_known_format, @@ -96,6 +101,264 @@ def _test_name_from_known_format(cls: type, num: int, params_dict: dict) -> str: return parameterized.to_safe_name(f"{cls.__name__}_{num}_{data_format.name}") +class TestOptionalPandas(unittest.TestCase): + @parameterized.expand([("with_pandas", False), ("without_pandas", True)]) + def test_cold_start(self, _name: str, missing_pandas: bool) -> None: + script = dedent( + """ + import importlib.abc + import sys + from pathlib import Path + from tempfile import TemporaryDirectory + + missing_pandas = sys.argv[1] == "True" + if missing_pandas: + assert not any(name == "pandas" or name.startswith("pandas.") for name in sys.modules) + + class NoPandasImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "pandas" or fullname.startswith("pandas."): + raise ModuleNotFoundError("No module named 'pandas'", name="pandas") + + sys.meta_path.insert(0, NoPandasImports()) + + import numpy as np + import pyarrow as pa + + from evo.objects import DownloadedObject, ObjectAPIClient, utils + from evo.objects.parquet import ParquetLoader + from evo.objects.utils import ArrowTableFormat, ObjectDataClient, get_known_format + + assert hasattr(ObjectAPIClient, "get_data_client") + assert all(hasattr(utils, name) for name in utils.__all__) + for name in ("expand_hole_index", "hole_chunks_from_ids"): + assert hasattr(utils, name) == (not missing_pandas), name + assert (name in utils.__all__) == (not missing_pandas), name + + arrow_methods = { + DownloadedObject: ("download_table", "download_category_table", "download_array"), + ObjectDataClient: ("save_table", "upload_table", "upload_category_table", "download_table"), + ParquetLoader: ("load_as_table", "load_as_array"), + } + for owner, methods in arrow_methods.items(): + for name in methods: + assert hasattr(owner, name), f"{owner.__name__}.{name}" + + dataframe_methods = { + DownloadedObject: ( + "download_dataframe", "download_category_dataframe", "download_attribute_dataframe" + ), + ObjectDataClient: ( + "save_dataframe", "upload_dataframe", "upload_category_dataframe", "download_dataframe" + ), + ParquetLoader: ("load_as_dataframe",), + } + for owner, methods in dataframe_methods.items(): + for name in methods: + assert hasattr(owner, name) == (not missing_pandas), f"{owner.__name__}.{name}" + + table = pa.table({"value": pa.array([1, 2, 3], type=pa.int64())}) + table_format = ArrowTableFormat.from_schema(table.schema) + assert table_format.pa_schema.equals(table.schema) + with TemporaryDirectory() as directory: + source = Path(directory) + table_info = get_known_format(table).save_table(table, source) + with ParquetLoader(pa.OSFile(str(source / table_info["data"]), "r")) as loader: + loader.validate_with_table_info(table_info) + assert loader.load_as_table().equals(table) + np.testing.assert_array_equal( + loader.load_as_array(), np.array([1, 2, 3], dtype=np.int64), strict=True + ) + + if missing_pandas: + try: + table_format.pandas_types_mapper() + except ModuleNotFoundError as error: + assert error.name == "pandas" + else: + raise AssertionError("Pandas dtype mapping must report the missing dependency") + assert not any(name == "pandas" or name.startswith("pandas.") for name in sys.modules) + """ + ) + # Cached pandas-dependent modules can hide import failures when only selected modules are reloaded. + result = subprocess.run( + [sys.executable, "-c", script, str(missing_pandas)], + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + +class TestColumnFormat(unittest.TestCase): + @parameterized.expand( + [ + ("float64", pa.float64()), + ("uint8", pa.uint8()), + ("uint32", pa.uint32()), + ("uint64", pa.uint64()), + ("int32", pa.int32()), + ("int64", pa.int64()), + ("bool", pa.bool_()), + ("string", pa.string()), + ("timestamp", pa.timestamp("us", tz="UTC")), + ] + ) + def test_column_types(self, format_id: str, data_type: pa.DataType) -> None: + for format_spec in (format_id, data_type): + with self.subTest(format_spec=format_spec): + column = ColumnFormat(format_spec) + self.assertEqual(format_id, column.id) + self.assertEqual(data_type, column.type) + + def test_unsupported_column_type(self) -> None: + for format_spec in ("float32", pa.float32()): + with self.subTest(format_spec=format_spec), self.assertRaises(TypeError): + ColumnFormat(format_spec) + + @parameterized.expand( + [ + ("float64", None), + ("uint8", "UInt8"), + ("uint32", "UInt32"), + ("uint64", "UInt64"), + ("int32", "Int32"), + ("int64", "Int64"), + ("bool", "boolean"), + ("string", None), + ("timestamp", "datetime64[us, UTC]"), + ] + ) + def test_pandas_dtype(self, format_id: str, expected_dtype: str | None) -> None: + column = ColumnFormat(format_id) + if expected_dtype is None: + self.assertIsNone(column.pandas_dtype) + return + + self.assertEqual(pd.api.types.pandas_dtype(expected_dtype), column.pandas_dtype) + # The pandas dtype must convert back to the same Arrow type, so import and export agree. + dataframe = pd.DataFrame({"value": pd.array([None], dtype=column.pandas_dtype)}) + self.assertEqual(column.type, ArrowTableFormat.from_pandas(dataframe).columns[0].type) + + +class TestBaseTableFormat(unittest.TestCase): + def test_columns(self) -> None: + table_format = BaseTableFormat("mixed", [pa.int32(), "string", pa.bool_()]) + + self.assertIsInstance(table_format.columns, tuple) + self.assertTrue(all(isinstance(column, ColumnFormat) for column in table_format.columns)) + self.assertEqual(["int32", "string", "bool"], [column.id for column in table_format.columns]) + self.assertEqual([pa.int32(), pa.string(), pa.bool_()], [column.type for column in table_format.columns]) + + def test_multidimensional_columns(self) -> None: + table_format = BaseTableFormat("multidimensional", [pa.float64(), ...]) + + self.assertEqual(1, len(table_format.columns)) + self.assertEqual(pa.float64(), table_format.columns[0].type) + + +class TestArrowTableFormat(unittest.TestCase): + def test_from_schema(self) -> None: + schema = pa.schema( + [ + pa.field("key", pa.int64(), nullable=False, metadata={b"field": b"kept"}), + pa.field("label", pa.string()), + ], + metadata={b"other": b"kept"}, + ) + + table_format = ArrowTableFormat.from_schema(schema) + + self.assertEqual( + pa.schema([pa.field("key", pa.int64(), nullable=False), pa.field("label", pa.string())]), + table_format.pa_schema, + ) + self.assertIsNone(table_format.pa_schema.metadata) + self.assertIsNone(table_format.pa_schema.field("key").metadata) + self.assertEqual("ArrowTableFormat", table_format.name) + self.assertEqual("int64/string", table_format.data_type) + self.assertEqual(2, table_format.width) + self.assertEqual(schema.types, [column.type for column in table_format.columns]) + + def test_pandas_types_mapper(self) -> None: + schema = pa.schema([("count", pa.int64()), ("flag", pa.bool_()), ("value", pa.float64())]) + + types_mapper = ArrowTableFormat.from_schema(schema).pandas_types_mapper() + + self.assertEqual(pd.Int64Dtype(), types_mapper(pa.int64())) + self.assertEqual(pd.BooleanDtype(), types_mapper(pa.bool_())) + self.assertIsNone(types_mapper(pa.float64())) + self.assertIsNone(types_mapper(pa.int32())) + + @parameterized.expand( + [ + ("Int32", [1, None], pa.int32()), + ("Int64", [2**63 - 1, None], pa.int64()), + ("UInt8", [255, None], pa.uint8()), + ("UInt32", [2**32 - 1, None], pa.uint32()), + ("UInt64", [2**64 - 1, None], pa.uint64()), + ("boolean", [True, None], pa.bool_()), + ("float64", [1.5, None], pa.float64()), + ("string[python]", ["A", None], pa.string()), + ("string[pyarrow]", ["A", None], pa.string()), + ("datetime64[us, UTC]", [pd.Timestamp("2025-01-01", tz="UTC"), None], pa.timestamp("us", tz="UTC")), + ] + ) + def test_from_pandas(self, dtype: str, values: list, expected_type: pa.DataType) -> None: + dataframe = pd.DataFrame({"value": pd.array(values, dtype=dtype)}) + + table_format = ArrowTableFormat.from_pandas(dataframe) + table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + + self.assertEqual(pa.schema([("value", expected_type)]), table_format.pa_schema) + self.assertIsNone(table_format.pa_schema.metadata) + self.assertEqual(expected_type, table_format.columns[0].type) + self.assertEqual(expected_type, table.column("value").type) + self.assertEqual(1, table.column("value").null_count) + self.assertEqual(values[0], table.column("value")[0].as_py()) + + def test_from_pandas_excludes_index(self) -> None: + indexes = [ + pd.RangeIndex(5, 7), + pd.Index([5, 3], name="row"), + pd.MultiIndex.from_tuples([("A", 5), ("B", 3)], names=["hole", "row"]), + ] + for index in indexes: + with self.subTest(index=index): + dataframe = pd.DataFrame({"value": [1.0, 2.0]}, index=index) + + table_format = ArrowTableFormat.from_pandas(dataframe) + table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + + self.assertEqual(["value"], table_format.pa_schema.names) + self.assertIsNone(table_format.pa_schema.metadata) + self.assertEqual({"value": [1.0, 2.0]}, table.to_pydict()) + + def test_from_pandas_rejects_unsupported_column_type(self) -> None: + dataframe = pd.DataFrame({"value": pd.array([1.0, 2.0], dtype="float32")}) + + with self.assertRaisesRegex(TypeError, "Unsupported column type"): + ArrowTableFormat.from_pandas(dataframe) + + def test_factory_preserves_subclass(self) -> None: + class CustomTableFormat(ArrowTableFormat): + pass + + table_format = CustomTableFormat.from_pandas(pd.DataFrame({"value": [1.0]})) + + self.assertIsInstance(table_format, CustomTableFormat) + self.assertEqual("CustomTableFormat", table_format.name) + + def test_direct_construction_remains_supported(self) -> None: + table_format = ArrowTableFormat("custom", [pa.float64()]) + + self.assertEqual("custom", table_format.name) + self.assertEqual("float64", table_format.data_type) + self.assertEqual(1, table_format.width) + with self.assertRaisesRegex(ValueError, "from_schema.*from_pandas"): + _ = table_format.pa_schema + + @parameterized_class(_all_known_formats_for_testing(), class_name_func=_test_name_from_known_format) class TestKnownFormat(unittest.TestCase): data_format: KnownTableFormat