From ae564c269904a2d1e740190acc2de2df533c88b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Peixoto?= Date: Wed, 4 Nov 2026 08:09:19 +0000 Subject: [PATCH 1/4] Strip pandas metadata from parquet tables and load dataframes using schema types Avoid pandas reconstruction metadata that can be incompatible across pandas versions. Load dataframes with nullable dtypes derived from the table schema. --- .../src/evo/objects/client/object_client.py | 9 +- .../src/evo/objects/parquet/loader.py | 18 +- .../evo-objects/src/evo/objects/utils/data.py | 27 ++- .../src/evo/objects/utils/tables.py | 93 +++++++++- .../evo-objects/tests/test_pandas_metadata.py | 161 ++++++++++++++++++ 5 files changed, 291 insertions(+), 17 deletions(-) create mode 100644 packages/evo-objects/tests/test_pandas_metadata.py diff --git a/packages/evo-objects/src/evo/objects/client/object_client.py b/packages/evo-objects/src/evo/objects/client/object_client.py index 7a5ce42a0..eac1b1715 100644 --- a/packages/evo-objects/src/evo/objects/client/object_client.py +++ b/packages/evo-objects/src/evo/objects/client/object_client.py @@ -40,6 +40,7 @@ ParquetLoader, TableInfo, ) + from ..utils.tables import KnownTableFormat, _table_to_dataframe except ImportError as e: print(e) _LOADER_AVAILABLE = False @@ -486,10 +487,12 @@ async def download_dataframe( nan_description structure. :param column_names: An optional list of column names for the table, instead of those from the Parquet file. - :returns: A Pandas DataFrame containing the downloaded data. + :returns: A Pandas DataFrame containing the downloaded data. Column dtypes are derived from the + Geoscience Object column types; integer and boolean columns use pandas nullable dtypes. """ + table_info = self._validate_typed_dict(table_info, _TABLE_INFO_VALIDATOR) table = await self.download_table(table_info, fb=fb, nan_values=nan_values, column_names=column_names) - return table.to_pandas() + return _table_to_dataframe(table, KnownTableFormat.from_table_info(table_info)) async def download_category_dataframe( self, @@ -512,7 +515,7 @@ async def download_category_dataframe( table = await self.download_category_table( category_info, fb=fb, nan_values=nan_values, column_names=column_names ) - return table.to_pandas() + return _table_to_dataframe(table) async def download_attribute_dataframe( self, diff --git a/packages/evo-objects/src/evo/objects/parquet/loader.py b/packages/evo-objects/src/evo/objects/parquet/loader.py index f4f529af6..37528fcc3 100644 --- a/packages/evo-objects/src/evo/objects/parquet/loader.py +++ b/packages/evo-objects/src/evo/objects/parquet/loader.py @@ -28,6 +28,7 @@ from ..exceptions import SchemaValidationError, TableFormatError from ..utils import ArrowTableFormat, KnownTableFormat, TableInfo +from ..utils.tables import _strip_pandas_metadata, _table_to_dataframe try: import pandas as pd @@ -67,6 +68,7 @@ def __init__(self, pa_file: pa.NativeFile) -> None: """ self._pa_file = pa_file self._parquet_file: pq.ParquetFile | None = None + self._expected_format: KnownTableFormat | None = None def __enter__(self) -> ParquetLoader: if self._parquet_file is not None: @@ -119,6 +121,7 @@ def validate_with_table_info(self, table_info: TableInfo) -> None: expected_format._check_format(actual_format) except TableFormatError as e: raise SchemaValidationError(str(e)) from None + self._expected_format = expected_format logger.debug("Checking parquet data length") actual_length = pa_file.metadata.num_rows @@ -130,19 +133,26 @@ def validate_with_table_info(self, table_info: TableInfo) -> None: logger.debug("Parquet metadata checks succeeded") def load_as_table(self) -> pa.Table: - """Load the Parquet data as a PyArrow Table.""" + """Load the Parquet data as a PyArrow Table. + + Any pandas reconstruction metadata in the Parquet file is discarded. + """ if self._parquet_file is None: raise RuntimeError("ParquetLoader context is not active") else: - return self._parquet_file.read() + return _strip_pandas_metadata(self._parquet_file.read()) if _PD_AVAILABLE: # Optional support for pandas dataframes def load_as_dataframe(self) -> pd.DataFrame: - """Load the Parquet data as a Pandas DataFrame.""" + """Load the Parquet data as a Pandas DataFrame. + + Column dtypes are derived from the Geoscience Object column types, never from pandas metadata in the file. + Integer and boolean columns use pandas nullable dtypes (e.g. `Int64`, `boolean`). + """ table = self.load_as_table() - return table.to_pandas() + return _table_to_dataframe(table, self._expected_format) if _NP_AVAILABLE: # Optional support for numpy arrays diff --git a/packages/evo-objects/src/evo/objects/utils/data.py b/packages/evo-objects/src/evo/objects/utils/data.py index be6f22c50..3583e2699 100644 --- a/packages/evo-objects/src/evo/objects/utils/data.py +++ b/packages/evo-objects/src/evo/objects/utils/data.py @@ -26,7 +26,7 @@ from ..exceptions import TableFormatError from ..io import _CACHE_SCOPE, ObjectDataDownload, ObjectDataUpload from .table_formats import INTEGER_ARRAY_1_INT32, INTEGER_ARRAY_MD_INT32, LOOKUP_TABLE_INT32 -from .tables import KnownTableFormat +from .tables import KnownTableFormat, _table_to_dataframe from .types import ArrayTableInfo, CategoryInfo, LookupTableInfo try: @@ -80,7 +80,7 @@ def _get_schema_from_dataframe(dataframe: "pd.DataFrame") -> pa.Schema: :param dataframe: The pandas dataframe to get the schema from. :return: A pyarrow schema with any configured type conversions applied. """ - schema = pa.Schema.from_pandas(dataframe) + schema = pa.Schema.from_pandas(dataframe, preserve_index=False) fields = [] for field in schema: if pa.types.is_large_string(field.type): @@ -92,6 +92,18 @@ def _get_schema_from_dataframe(dataframe: "pd.DataFrame") -> pa.Schema: return pa.schema(fields) +def _table_from_dataframe(dataframe: "pd.DataFrame") -> pa.Table: + """Convert a pandas dataframe to a pyarrow table for upload. + + The dataframe index is not part of any Geoscience Object table format, so it is never stored. + + :param dataframe: The pandas dataframe to convert. + :return: A pyarrow table with the normalized schema from `_get_schema_from_dataframe`. + """ + schema = _get_schema_from_dataframe(dataframe) + return pa.Table.from_pandas(dataframe, schema, preserve_index=False) + + class ObjectDataClient: """An optional wrapper around data upload and download functionality for geoscience objects. @@ -422,8 +434,7 @@ def save_dataframe( no table formats are specified, raised when the table does not match any known format. :raises StorageFileNotFoundError: If the destination does not exist or is not a directory. """ - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema) + table = _table_from_dataframe(dataframe) return self.save_table(table, table_format=table_format) async def upload_dataframe( @@ -446,8 +457,7 @@ async def upload_dataframe( :raises TableFormatError: If the provided table does not match any of the specified formats. If no table formats are specified, raised when the table does not match any known format. """ - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema) + table = _table_from_dataframe(dataframe) table_info = await self.upload_table(table, table_format=table_format, fb=fb) return table_info @@ -468,8 +478,7 @@ async def upload_category_dataframe(self, dataframe: pd.DataFrame, fb: IFeedback :raises TableFormatError: If the table isn't a valid category table, or if the number of categories exceeds what int32 type can represent. """ - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema) + table = _table_from_dataframe(dataframe) category_info = await self.upload_category_table(table, fb=fb) return category_info @@ -495,6 +504,6 @@ async def download_dataframe( """ table = await self.download_table(object_id, version_id, table_info, fb) try: - return table.to_pandas() + return _table_to_dataframe(table, KnownTableFormat.from_table_info(table_info)) except ModuleNotFoundError: raise RuntimeError("Unable to download dataframe because the `pandas` package is not installed") diff --git a/packages/evo-objects/src/evo/objects/utils/tables.py b/packages/evo-objects/src/evo/objects/utils/tables.py index baae199a7..e417bfefa 100644 --- a/packages/evo-objects/src/evo/objects/utils/tables.py +++ b/packages/evo-objects/src/evo/objects/utils/tables.py @@ -12,10 +12,13 @@ from __future__ import annotations import hashlib +import json import os +from collections.abc import Callable from pathlib import Path from tempfile import NamedTemporaryFile from types import EllipsisType +from typing import TYPE_CHECKING, Any import pyarrow as pa import pyarrow.parquet as pq @@ -26,6 +29,9 @@ from ..exceptions import SchemaValidationError, TableFormatError +if TYPE_CHECKING: + import pandas as pd + logger = evo.logging.getLogger("object.tables") __all__ = [ @@ -37,6 +43,85 @@ # Block size to limit memory use while calculating the sha256 digest of a file. _DIGEST_BLOCK_SIZE = 4 * 1024 * 1024 +# Arrow schema metadata key that pyarrow uses to reconstruct pandas dtypes and indexes. +_PANDAS_METADATA_KEY = b"pandas" + +# Pandas dtypes used for schema column types that numpy cannot represent with nulls. +_NULLABLE_PANDAS_DTYPES = { + "int32": "Int32", + "int64": "Int64", + "uint8": "UInt8", + "uint32": "UInt32", + "uint64": "UInt64", + "bool": "boolean", +} + + +def _strip_pandas_metadata(table: pa.Table, *, validate: bool = False) -> pa.Table: + """Remove pandas reconstruction metadata, so that readers rely only on the Arrow and Geoscience Object schemas. + + The pandas metadata is only a hint for rebuilding pandas dtypes and indexes; the Arrow data is not changed. + Other schema metadata is preserved. + + :param table: The table to remove pandas metadata from. + :param validate: Reject tables whose pandas metadata declares serialized index columns. Once the metadata is + removed, those columns would be indistinguishable from data columns. + + :return: The table without pandas metadata. + + :raises TableFormatError: If `validate` is true and the table contains serialized pandas index columns. + """ + metadata = table.schema.metadata + if not metadata or _PANDAS_METADATA_KEY not in metadata: + return table + + if validate: + try: + index_columns = json.loads(metadata[_PANDAS_METADATA_KEY]).get("index_columns", []) + except (UnicodeDecodeError, json.JSONDecodeError, AttributeError): + index_columns = [] # Invalid metadata is discarded below, it cannot describe stored columns. + stored_index_columns = [column for column in index_columns if isinstance(column, str)] + if stored_index_columns: + raise TableFormatError( + f"Table contains pandas index column(s) {stored_index_columns}, which are not part of any " + "Geoscience Object table format. Convert the dataframe with `preserve_index=False`." + ) + + remaining = {key: value for key, value in metadata.items() if key != _PANDAS_METADATA_KEY} + return table.replace_schema_metadata(remaining or None) + + +def _pandas_types_mapper(table_format: BaseTableFormat | None = None) -> Callable[[pa.DataType], Any]: + """Get a pyarrow `types_mapper` that maps Geoscience Object column types to pandas dtypes. + + Integer and boolean columns use pandas nullable dtypes, so they keep their type when they contain nulls, + rather than being converted to float or object columns depending on the data. Other types use pyarrow defaults. + + :param table_format: The expected table format. If omitted, all supported column types are mapped. + """ + import pandas as pd + + column_formats = ( + table_format._columns + if table_format is not None + else [_ColumnFormat(format_id) for format_id in _NULLABLE_PANDAS_DTYPES] + ) + mapping = { + column_format.type: pd.api.types.pandas_dtype(_NULLABLE_PANDAS_DTYPES[column_format.id]) + for column_format in column_formats + if column_format.id in _NULLABLE_PANDAS_DTYPES + } + return mapping.get + + +def _table_to_dataframe(table: pa.Table, table_format: BaseTableFormat | None = None) -> pd.DataFrame: + """Convert a table to a pandas dataframe using the Geoscience Object column types, ignoring pandas metadata. + + :param table: The table to convert. It is expected to have been validated against `table_format`. + :param table_format: The expected table format. If omitted, all supported column types are mapped. + """ + return _strip_pandas_metadata(table).to_pandas(types_mapper=_pandas_types_mapper(table_format)) + class _ColumnFormat: def __init__(self, format_spec: pa.DataType | str): @@ -234,16 +319,22 @@ def _get_file_digest(file_path: Path) -> str: def _save_table_as_parquet(cls, table: pa.Table, destination: Path) -> str: """Save a table in parquet format. + Pandas reconstruction metadata is not saved, so the parquet file can be read by any consumer. Readers must use + the Geoscience Object schema to interpret column types. + :param table: The table to save to parquet file. :param destination: A local directory to save the parquet file in. :return: The sha256 digest of the saved parquet file, which is also the filename. :raises StorageFileNotFoundError: If the destination does not exist or is not a directory. + :raises TableFormatError: If the table contains serialized pandas index columns. """ if not destination.is_dir(): raise StorageFileNotFoundError(f"'{destination}' is not a directory") + table = _strip_pandas_metadata(table, validate=True) + try: # Write temporary file. with NamedTemporaryFile(delete=False, dir=destination) as tmp_file: logger.debug(f"Writing temporary file {tmp_file.name}") @@ -383,7 +474,7 @@ def load_table(cls, table_info: dict, source: Path) -> pa.Table: ) logger.debug("Parquet metadata checks succeeded") - return parquet_data.read() + return _strip_pandas_metadata(parquet_data.read()) except FileNotFoundError: raise StorageFileNotFoundError(f"Could not find data for table with id '{parquet_file.name}'") from None diff --git a/packages/evo-objects/tests/test_pandas_metadata.py b/packages/evo-objects/tests/test_pandas_metadata.py new file mode 100644 index 000000000..bb4d66c31 --- /dev/null +++ b/packages/evo-objects/tests/test_pandas_metadata.py @@ -0,0 +1,161 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parquet files must not carry pandas reconstruction metadata; pandas dtypes come from the Geoscience Object schema.""" + +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from pandas.testing import assert_frame_equal + +from evo.objects.exceptions import TableFormatError +from evo.objects.parquet import ParquetLoader +from evo.objects.utils import get_known_format +from evo.objects.utils.data import _table_from_dataframe + + +def _load_dataframe(table_info: dict, source: Path) -> pd.DataFrame: + with pa.OSFile(str(source / str(table_info["data"])), mode="r") as parquet_file: + with ParquetLoader(parquet_file) as loader: + loader.validate_with_table_info(table_info) + return loader.load_as_dataframe() + + +class _BrokenPandasExtensionArrays: + """Simulate a consumer whose pandas cannot rebuild extension arrays from Arrow. + + pandas only calls `__from_arrow__` when told to by the pandas schema metadata, or by an explicit `types_mapper`. + """ + + def __enter__(self): + def broken(*_args, **_kwargs): + raise ModuleNotFoundError("No module named 'pandas.core.arrays.arrow._arrow_utils'") + + self._patches = [ + mock.patch.object(dtype, "__from_arrow__", broken) + for dtype in (pd.Int32Dtype, pd.Int64Dtype, pd.BooleanDtype, pd.StringDtype) + ] + for patch in self._patches: + patch.start() + return self + + def __exit__(self, *exc_info): + for patch in reversed(self._patches): + patch.stop() + + +class TestPandasMetadata(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.data_dir = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _save(self, table: pa.Table) -> dict: + return get_known_format(table).save_table(table, self.data_dir) + + def _schema_metadata(self, table_info: dict) -> dict: + return pq.read_schema(self.data_dir / table_info["data"]).metadata or {} + + def test_dataframe_table_has_pandas_metadata(self) -> None: + """Guard the premise of these tests: pyarrow always adds pandas metadata when converting a dataframe.""" + table = _table_from_dataframe(pd.DataFrame({"value": pd.array([1, None], dtype="Int64")})) + self.assertIn(b"pandas", table.schema.metadata) + + def test_save_table_strips_pandas_metadata(self) -> None: + table = _table_from_dataframe(pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")})) + table = table.replace_schema_metadata({**table.schema.metadata, b"other": b"kept"}) + + table_info = self._save(table) + + metadata = self._schema_metadata(table_info) + self.assertNotIn(b"pandas", metadata) + self.assertEqual(b"kept", metadata[b"other"]) + + def test_saved_table_can_be_read_by_pandas_without_extension_array_support(self) -> None: + dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) + table_info = self._save(_table_from_dataframe(dataframe)) + + with _BrokenPandasExtensionArrays(): + loaded = pq.read_table(self.data_dir / table_info["data"]).to_pandas() + + # Without schema information pandas falls back to float64, but no value is corrupted. + self.assertEqual([1.0, 3.0], loaded["value"].dropna().tolist()) + self.assertTrue(pd.isna(loaded["value"][1])) + + def test_legacy_pandas_metadata_breaks_pandas_without_extension_array_support(self) -> None: + """Document the failure mode that stripping the metadata prevents.""" + dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) + legacy_file = self.data_dir / "legacy.parquet" + pq.write_table(pa.Table.from_pandas(dataframe, preserve_index=False), legacy_file) + + with _BrokenPandasExtensionArrays(), self.assertRaises(ModuleNotFoundError): + pq.read_table(legacy_file).to_pandas() + + def test_save_table_rejects_pandas_index_columns(self) -> None: + dataframe = pd.DataFrame({"value": [1.0, 2.0]}, index=pd.Index([5, 3], name="row")) + table = pa.Table.from_pandas(dataframe, preserve_index=True).select(["value"]) + # Keep the metadata pointing to an index column, which is what a table saved with its index looks like. + index_metadata = json.loads(pa.Table.from_pandas(dataframe, preserve_index=True).schema.metadata[b"pandas"]) + table = table.replace_schema_metadata({b"pandas": json.dumps(index_metadata).encode()}) + + with self.assertRaises(TableFormatError): + self._save(table) + self.assertEqual([], list(self.data_dir.iterdir())) + + def test_dataframe_index_is_not_stored(self) -> None: + dataframe = pd.DataFrame({"value": [1.0, 2.0, 3.0]}).iloc[[2, 0]] + + table = _table_from_dataframe(dataframe) + + self.assertEqual(["value"], table.column_names) + self.assertEqual([], json.loads(table.schema.metadata[b"pandas"])["index_columns"]) + + def test_load_as_dataframe_uses_nullable_dtypes_from_schema(self) -> None: + cases = [ + (pa.array([1, None, 3], type=pa.int64()), "Int64"), + (pa.array([1, 2, 3], type=pa.int64()), "Int64"), + (pa.array([1, None, 3], type=pa.int32()), "Int32"), + (pa.array([1, 2, 3], type=pa.int32()), "Int32"), + (pa.array([True, None, False], type=pa.bool_()), "boolean"), + (pa.array([1.5, None, 3.5], type=pa.float64()), "float64"), + ] + for array, expected_dtype in cases: + with self.subTest(type=str(array.type), nulls=array.null_count): + table_info = self._save(pa.table({"value": array})) + + loaded = _load_dataframe(table_info, self.data_dir) + + self.assertEqual(pd.api.types.pandas_dtype(expected_dtype), loaded["value"].dtype) + assert_frame_equal(pd.DataFrame({"value": pd.array(array.to_pylist(), dtype=expected_dtype)}), loaded) + + def test_load_ignores_legacy_pandas_metadata(self) -> None: + """Files written before metadata stripping must load with the schema types, not the stored pandas dtypes.""" + dataframe = pd.DataFrame({"value": pd.array([1, 2, 3], dtype="Int64")}, index=pd.RangeIndex(10, 13)) + table = pa.Table.from_pandas(dataframe) + table_info = {"data": "legacy.parquet", "length": 3, "width": 1, "data_type": "int64"} + pq.write_table(table, self.data_dir / table_info["data"]) + self.assertIn(b"pandas", pq.read_schema(self.data_dir / table_info["data"]).metadata) + + loaded = _load_dataframe(table_info, self.data_dir) + + pd.testing.assert_index_equal(pd.RangeIndex(3), loaded.index) + self.assertEqual(pd.Int64Dtype(), loaded["value"].dtype) + with pa.OSFile(str(self.data_dir / table_info["data"]), mode="r") as parquet_file: + with ParquetLoader(parquet_file) as loader: + self.assertIsNone(loader.load_as_table().schema.metadata) From b5d5f507630f86de12ca92a2a4b3529d92b413c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Peixoto?= Date: Thu, 5 Nov 2026 08:55:59 +0000 Subject: [PATCH 2/4] Refactor ArrowTableFormat and ColumnFormat integration for schema handling --- packages/evo-objects/README.md | 26 +++ .../src/evo/objects/utils/__init__.py | 3 +- .../evo-objects/src/evo/objects/utils/data.py | 35 +--- .../src/evo/objects/utils/tables.py | 130 ++++++++++++--- packages/evo-objects/tests/helpers.py | 16 +- .../evo-objects/tests/test_data_client.py | 62 +++++-- .../tests/test_downloaded_object.py | 14 ++ .../evo-objects/tests/test_pandas_metadata.py | 77 ++++++++- packages/evo-objects/tests/test_tables.py | 156 ++++++++++++++++++ 9 files changed, 449 insertions(+), 70 deletions(-) diff --git a/packages/evo-objects/README.md b/packages/evo-objects/README.md index ab93e4d37..83567c654 100644 --- a/packages/evo-objects/README.md +++ b/packages/evo-objects/README.md @@ -69,6 +69,32 @@ You can also get a list of all objects. Internally, this recursively calls the ` Check out the other methods on the `ObjectAPIClient` for more details on how to upload and download objects, or get object versions. +### DataFrame and Parquet schemas + +`ArrowTableFormat.from_pandas()` infers the supported Geoscience Object column types, normalizes large strings to strings, +and excludes the DataFrame index from its Arrow schema. Use `pa_schema` for conversion and `columns` to inspect the ordered +`ColumnFormat` descriptors and their `id`, Arrow `type`, and `pandas_dtype` used when reading dataframes. + +```python +import pandas as pd +import pyarrow as pa + +from evo.objects.utils import ArrowTableFormat + +dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) +table_format = ArrowTableFormat.from_pandas(dataframe) +table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) +table_info = data_client.save_table(table) +``` + +PyArrow adds pandas reconstruction metadata even when an explicit schema is provided. The Geoscience Object helpers +strip that metadata when writing Parquet, while preserving other schema metadata. Readers ignore legacy pandas metadata +and derive dtypes from the Geoscience Object schema; integer and boolean columns use pandas nullable dtypes. +Saving rejects serialized pandas index columns and malformed index metadata with `TableFormatError`; recreate such +tables from the DataFrame with `preserve_index=False` rather than removing metadata from stored index columns. +For categorical DataFrames, use `upload_category_dataframe()` to retain dictionary encoding while building the value +and lookup tables. + ### Typed downhole collections `DownholeCollection` provides a DataFrame-based API for creating and reading downhole objects. A collection can contain diff --git a/packages/evo-objects/src/evo/objects/utils/__init__.py b/packages/evo-objects/src/evo/objects/utils/__init__.py index af0879537..1f3764990 100644 --- a/packages/evo-objects/src/evo/objects/utils/__init__.py +++ b/packages/evo-objects/src/evo/objects/utils/__init__.py @@ -25,7 +25,7 @@ from .data import ObjectDataClient from .downhole import expand_hole_index, hole_chunks_from_ids from .table_formats import all_known_formats, get_known_format -from .tables import ArrowTableFormat, BaseTableFormat, KnownTableFormat +from .tables import ArrowTableFormat, BaseTableFormat, ColumnFormat, KnownTableFormat from .types import ArrayTableInfo, AttributeInfo, CategoryInfo, LookupTableInfo, TableInfo # We _used_ to export Table and DataFrame from this package as custom protocols, but we are using the actual @@ -39,6 +39,7 @@ "AttributeInfo", "BaseTableFormat", "CategoryInfo", + "ColumnFormat", "KnownTableFormat", "LookupTableInfo", "ObjectDataClient", diff --git a/packages/evo-objects/src/evo/objects/utils/data.py b/packages/evo-objects/src/evo/objects/utils/data.py index 3583e2699..b6a7c7f63 100644 --- a/packages/evo-objects/src/evo/objects/utils/data.py +++ b/packages/evo-objects/src/evo/objects/utils/data.py @@ -26,7 +26,7 @@ from ..exceptions import TableFormatError from ..io import _CACHE_SCOPE, ObjectDataDownload, ObjectDataUpload from .table_formats import INTEGER_ARRAY_1_INT32, INTEGER_ARRAY_MD_INT32, LOOKUP_TABLE_INT32 -from .tables import KnownTableFormat, _table_to_dataframe +from .tables import ArrowTableFormat, KnownTableFormat, _get_schema_from_dataframe, _table_to_dataframe from .types import ArrayTableInfo, CategoryInfo, LookupTableInfo try: @@ -69,39 +69,16 @@ def _iter_refs(target: Any, _key: str | None = None) -> Iterator[str]: yield str(value) -def _get_schema_from_dataframe(dataframe: "pd.DataFrame") -> pa.Schema: - """Get a normalized pyarrow schema from a pandas dataframe. - - This hook centralizes dataframe-to-Arrow schema adjustments before the - dataframe is converted to a table. It currently performs these conversions: - - large_string -> string - - dictionary<*, large_string> -> dictionary<*, string> - - :param dataframe: The pandas dataframe to get the schema from. - :return: A pyarrow schema with any configured type conversions applied. - """ - schema = pa.Schema.from_pandas(dataframe, preserve_index=False) - fields = [] - for field in schema: - if pa.types.is_large_string(field.type): - fields.append(field.with_type(pa.string())) - elif pa.types.is_dictionary(field.type) and pa.types.is_large_string(field.type.value_type): - fields.append(field.with_type(pa.dictionary(field.type.index_type, pa.string()))) - else: - fields.append(field) - return pa.schema(fields) - - def _table_from_dataframe(dataframe: "pd.DataFrame") -> pa.Table: """Convert a pandas dataframe to a pyarrow table for upload. The dataframe index is not part of any Geoscience Object table format, so it is never stored. :param dataframe: The pandas dataframe to convert. - :return: A pyarrow table with the normalized schema from `_get_schema_from_dataframe`. + :return: A pyarrow table with the normalized schema from `ArrowTableFormat.from_pandas`. """ - schema = _get_schema_from_dataframe(dataframe) - return pa.Table.from_pandas(dataframe, schema, preserve_index=False) + table_format = ArrowTableFormat.from_pandas(dataframe) + return pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) class ObjectDataClient: @@ -478,7 +455,9 @@ async def upload_category_dataframe(self, dataframe: pd.DataFrame, fb: IFeedback :raises TableFormatError: If the table isn't a valid category table, or if the number of categories exceeds what int32 type can represent. """ - table = _table_from_dataframe(dataframe) + # Keep dictionary encoding until the categories are split into value and lookup tables. + schema = _get_schema_from_dataframe(dataframe) + table = pa.Table.from_pandas(dataframe, schema=schema, preserve_index=False) category_info = await self.upload_category_table(table, fb=fb) return category_info diff --git a/packages/evo-objects/src/evo/objects/utils/tables.py b/packages/evo-objects/src/evo/objects/utils/tables.py index e417bfefa..34635f370 100644 --- a/packages/evo-objects/src/evo/objects/utils/tables.py +++ b/packages/evo-objects/src/evo/objects/utils/tables.py @@ -18,7 +18,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile from types import EllipsisType -from typing import TYPE_CHECKING, Any +from typing import Any import pyarrow as pa import pyarrow.parquet as pq @@ -29,14 +29,19 @@ from ..exceptions import SchemaValidationError, TableFormatError -if TYPE_CHECKING: +try: import pandas as pd +except ImportError: + _PD_AVAILABLE = False +else: + _PD_AVAILABLE = True logger = evo.logging.getLogger("object.tables") __all__ = [ "ArrowTableFormat", "BaseTableFormat", + "ColumnFormat", "KnownTableFormat", ] @@ -57,6 +62,25 @@ } +def _get_schema_from_dataframe(dataframe: pd.DataFrame) -> pa.Schema: + """Infer an Arrow schema without the dataframe index or pandas metadata, normalizing large strings. + + Dictionary encoding is preserved for category uploads, including the ordering of categorical values. + """ + schema = pa.Schema.from_pandas(dataframe, preserve_index=False) + fields = [] + for field in schema: + if pa.types.is_large_string(field.type): + fields.append(field.with_type(pa.string())) + elif pa.types.is_dictionary(field.type) and pa.types.is_large_string(field.type.value_type): + fields.append( + field.with_type(pa.dictionary(field.type.index_type, pa.string(), ordered=field.type.ordered)) + ) + else: + fields.append(field) + return pa.schema(fields) + + def _strip_pandas_metadata(table: pa.Table, *, validate: bool = False) -> pa.Table: """Remove pandas reconstruction metadata, so that readers rely only on the Arrow and Geoscience Object schemas. @@ -64,12 +88,13 @@ def _strip_pandas_metadata(table: pa.Table, *, validate: bool = False) -> pa.Tab Other schema metadata is preserved. :param table: The table to remove pandas metadata from. - :param validate: Reject tables whose pandas metadata declares serialized index columns. Once the metadata is - removed, those columns would be indistinguishable from data columns. + :param validate: Reject tables whose pandas metadata declares serialized index columns or cannot be validated. + Once the metadata is removed, index columns would be indistinguishable from data columns. :return: The table without pandas metadata. - :raises TableFormatError: If `validate` is true and the table contains serialized pandas index columns. + :raises TableFormatError: If `validate` is true and the table contains serialized pandas index columns or malformed + pandas index metadata. """ metadata = table.schema.metadata if not metadata or _PANDAS_METADATA_KEY not in metadata: @@ -77,9 +102,20 @@ def _strip_pandas_metadata(table: pa.Table, *, validate: bool = False) -> pa.Tab if validate: try: - index_columns = json.loads(metadata[_PANDAS_METADATA_KEY]).get("index_columns", []) - except (UnicodeDecodeError, json.JSONDecodeError, AttributeError): - index_columns = [] # Invalid metadata is discarded below, it cannot describe stored columns. + pandas_metadata = json.loads(metadata[_PANDAS_METADATA_KEY]) + if not isinstance(pandas_metadata, dict): + raise ValueError("Expected a metadata object") + index_columns = pandas_metadata.get("index_columns", []) + if not isinstance(index_columns, list) or any( + not (isinstance(column, str) or isinstance(column, dict) and column.get("kind") == "range") + for column in index_columns + ): + raise ValueError("Expected index column names or RangeIndex descriptors") + except ValueError as error: + raise TableFormatError( + "Cannot validate pandas index metadata. Recreate the table from the dataframe with " + "`preserve_index=False` before saving." + ) from error stored_index_columns = [column for column in index_columns if isinstance(column, str)] if stored_index_columns: raise TableFormatError( @@ -99,17 +135,15 @@ def _pandas_types_mapper(table_format: BaseTableFormat | None = None) -> Callabl :param table_format: The expected table format. If omitted, all supported column types are mapped. """ - import pandas as pd - column_formats = ( - table_format._columns + table_format.columns if table_format is not None - else [_ColumnFormat(format_id) for format_id in _NULLABLE_PANDAS_DTYPES] + else [ColumnFormat(format_id) for format_id in _NULLABLE_PANDAS_DTYPES] ) mapping = { - column_format.type: pd.api.types.pandas_dtype(_NULLABLE_PANDAS_DTYPES[column_format.id]) + column_format.type: dtype for column_format in column_formats - if column_format.id in _NULLABLE_PANDAS_DTYPES + if (dtype := column_format.pandas_dtype) is not None } return mapping.get @@ -123,8 +157,14 @@ def _table_to_dataframe(table: pa.Table, table_format: BaseTableFormat | None = return _strip_pandas_metadata(table).to_pandas(types_mapper=_pandas_types_mapper(table_format)) -class _ColumnFormat: - def __init__(self, format_spec: pa.DataType | str): +class ColumnFormat: + """A Geoscience Object column type and its corresponding Arrow data type.""" + + def __init__(self, format_spec: pa.DataType | str) -> None: + """ + :param format_spec: A Geoscience Object column type identifier or a supported Arrow data type. + :raises TypeError: If the column type is not supported. + """ if isinstance(format_spec, str): self._type = self._get_data_type(format_spec) self._format_id = format_spec @@ -182,12 +222,29 @@ def _get_format_id(data_type: pa.DataType) -> str: @property def id(self) -> str: + """The Geoscience Object column type identifier.""" return self._format_id @property def type(self) -> pa.DataType: + """The corresponding Arrow data type.""" return self._type + @property + def pandas_dtype(self) -> pd.api.extensions.ExtensionDtype | None: + """The pandas dtype used when loading this column into a dataframe. + + Integer and boolean columns use pandas nullable dtypes, so nulls do not change the dtype. Returns `None` for + column types that use the pyarrow default conversion. + + :raises ImportError: If pandas is not installed. + """ + if (dtype_name := _NULLABLE_PANDAS_DTYPES.get(self._format_id)) is None: + return None + if not _PD_AVAILABLE: + raise ModuleNotFoundError("No module named 'pandas'", name="pandas") + return pd.api.types.pandas_dtype(dtype_name) + class BaseTableFormat: """Base type for comparing table formats""" @@ -199,7 +256,7 @@ def __init__(self, name: str, columns: list[pa.DataType | str | EllipsisType]) - ("...") indicates a multidimensional format. """ self._name = name - self._columns: list[_ColumnFormat] = [] + self._columns: list[ColumnFormat] = [] self._multi_dimensional = False for column_type in columns: if column_type is Ellipsis: @@ -207,7 +264,7 @@ def __init__(self, name: str, columns: list[pa.DataType | str | EllipsisType]) - elif self._multi_dimensional: raise AssertionError("Found new type specification after Ellipsis") else: - self._columns.append(_ColumnFormat(format_spec=column_type)) + self._columns.append(ColumnFormat(format_spec=column_type)) if self._multi_dimensional and len(self._columns) != 1: raise AssertionError("Multidimensional formats must only have one column type") @@ -232,6 +289,11 @@ def width(self) -> int: """The number of columns in this table format""" return len(self._columns) + @property + def columns(self) -> tuple[ColumnFormat, ...]: + """Column formats in order. A multidimensional format has one entry for its repeated column type.""" + return tuple(self._columns) + @property def data_type(self) -> str: """The column data type(s) in this table format""" @@ -241,6 +303,18 @@ def data_type(self) -> str: class ArrowTableFormat(BaseTableFormat): """Specialised table format type that can be generated from a pyarrow table""" + _pa_schema: pa.Schema | None = None + + @property + def pa_schema(self) -> pa.Schema: + """The Arrow schema supplied to `from_schema` or inferred by `from_pandas`. + + :raises ValueError: If the format was constructed directly without a schema. + """ + if self._pa_schema is None: + raise ValueError("Create the format with from_schema() or from_pandas() to access its Arrow schema") + return self._pa_schema + @classmethod def from_schema(cls, pa_schema: pa.Schema) -> ArrowTableFormat: """Generate an ArrowTableFormat instance that represents the structure of the provided table schema. @@ -249,7 +323,23 @@ def from_schema(cls, pa_schema: pa.Schema) -> ArrowTableFormat: :return: A table format that can be used to compare against known formats. """ - return ArrowTableFormat(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types) + table_format = cls(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types) + table_format._pa_schema = pa_schema + return table_format + + @classmethod + def from_pandas(cls, dataframe: pd.DataFrame) -> ArrowTableFormat: + """Infer a table format from a pandas dataframe using the supported Geoscience Object column types. + + Large strings are normalized to strings. The inferred schema excludes the dataframe index and pandas metadata. + PyArrow still adds pandas metadata when converting the dataframe; it is stripped when saving the Parquet file. + + :param dataframe: The dataframe to infer the format from. + :return: A format whose `pa_schema` can be passed to `pyarrow.Table.from_pandas`. + :raises TypeError: If a column type is not supported. Use `ObjectDataClient.upload_category_dataframe` + for categorical columns, which are split into value and lookup tables separately. + """ + return cls.from_schema(_get_schema_from_dataframe(dataframe)) class KnownTableFormat(BaseTableFormat): @@ -328,7 +418,7 @@ def _save_table_as_parquet(cls, table: pa.Table, destination: Path) -> str: :return: The sha256 digest of the saved parquet file, which is also the filename. :raises StorageFileNotFoundError: If the destination does not exist or is not a directory. - :raises TableFormatError: If the table contains serialized pandas index columns. + :raises TableFormatError: If the table contains serialized pandas index columns or malformed pandas index metadata. """ if not destination.is_dir(): raise StorageFileNotFoundError(f"'{destination}' is not a directory") diff --git a/packages/evo-objects/tests/helpers.py b/packages/evo-objects/tests/helpers.py index f3e604ef9..dd1b7b3c1 100644 --- a/packages/evo-objects/tests/helpers.py +++ b/packages/evo-objects/tests/helpers.py @@ -21,7 +21,7 @@ import pyarrow as pa import pyarrow.parquet as pq -from evo.objects.utils.tables import BaseTableFormat, _ColumnFormat +from evo.objects.utils.tables import BaseTableFormat, ColumnFormat class NoImport: @@ -123,16 +123,16 @@ def _generate_data(format_id: str, n_samples: int) -> Iterator: raise TypeError(f"Unsupported format '{unknown_format}'") -def _change_format(current_format: _ColumnFormat) -> _ColumnFormat: +def _change_format(current_format: ColumnFormat) -> ColumnFormat: match current_format.id: case "float64": - return _ColumnFormat("int64") + return ColumnFormat("int64") case "uint8" | "uint32" | "uint64" | "int32" | "int64": - return _ColumnFormat("float64") + return ColumnFormat("float64") case "bool" | "timestamp": - return _ColumnFormat("string") + return ColumnFormat("string") case "string": - return _ColumnFormat("bool") + return ColumnFormat("bool") case unknown_format: raise TypeError(f"Unsupported format '{unknown_format}'") @@ -140,10 +140,10 @@ def _change_format(current_format: _ColumnFormat) -> _ColumnFormat: def get_sample_table( table_format: BaseTableFormat, n_rows: int, add_column: bool = False, change_types: bool = False ) -> pa.Table: - column_formats = [column for column in table_format._columns] + column_formats = list(table_format.columns) if add_column: - column_formats.append(_ColumnFormat(column_formats[-1].type)) + column_formats.append(ColumnFormat(column_formats[-1].type)) if change_types: column_formats = [_change_format(column) for column in column_formats] diff --git a/packages/evo-objects/tests/test_data_client.py b/packages/evo-objects/tests/test_data_client.py index f3440bef6..5af33be8e 100644 --- a/packages/evo-objects/tests/test_data_client.py +++ b/packages/evo-objects/tests/test_data_client.py @@ -15,6 +15,7 @@ import pandas as pd import pyarrow as pa +import pyarrow.parquet as pq from pandas.testing import assert_frame_equal from parameterized import parameterized @@ -24,7 +25,7 @@ from evo.common.test_tools import TestWithConnector, TestWithStorage from evo.common.utils import NoFeedback, PartialFeedback, get_header_metadata from evo.objects.exceptions import TableFormatError -from evo.objects.utils import KnownTableFormat, ObjectDataClient, table_formats +from evo.objects.utils import ArrowTableFormat, KnownTableFormat, ObjectDataClient, table_formats from helpers import NoImport, UnloadModule, get_sample_table_and_bytes @@ -112,10 +113,9 @@ def test_save_dataframe(self, _name: str, pass_table_format: bool) -> None: with ( mock.patch("evo.objects.utils.table_formats.get_known_format") as mock_get_known_format, mock.patch("evo.common.io.upload.StorageDestination") as mock_destination, - mock.patch("pyarrow.Table") as mock_pyarrow_table, - mock.patch("evo.objects.utils.data._get_schema_from_dataframe", return_value=None), + mock.patch("evo.objects.utils.data._table_from_dataframe") as mock_table_from_dataframe, ): - mock_pyarrow_table.from_pandas.return_value = mock_table = mock.Mock() + mock_table_from_dataframe.return_value = mock_table = mock.Mock() mock_get_known_format.return_value = mock_known_format = mock.Mock(spec=KnownTableFormat) mock_known_format.save_table.return_value = mock_table_info = {} @@ -125,6 +125,7 @@ def test_save_dataframe(self, _name: str, pass_table_format: bool) -> None: else: actual_table_info = self.data_client.save_dataframe(mock_dataframe) + mock_table_from_dataframe.assert_called_once_with(mock_dataframe) if pass_table_format: mock_get_known_format.assert_not_called() else: @@ -136,6 +137,21 @@ def test_save_dataframe(self, _name: str, pass_table_format: bool) -> None: self.transport.assert_no_requests() self.assertIs(mock_table_info, actual_table_info) + def test_save_dataframe_uses_format_schema_without_pandas_metadata(self) -> None: + dataframe = pd.DataFrame({"value": pd.array([2**63 - 1, None], dtype="Int64")}, index=[5, 3]) + with mock.patch( + "evo.objects.utils.data.ArrowTableFormat.from_pandas", wraps=ArrowTableFormat.from_pandas + ) as infer_format: + table_info = self.data_client.save_dataframe(dataframe) + + infer_format.assert_called_once_with(dataframe) + table = pq.ParquetFile(self.data_client.cache_location / table_info["data"]).read() + self.assertNotIn(b"pandas", table.schema.metadata or {}) + self.assertEqual(["value"], table.column_names) + self.assertEqual(pa.int64(), table.column("value").type) + self.assertEqual([2**63 - 1, None], table.column("value").to_pylist()) + self.transport.assert_no_requests() + async def test_upload_referenced_data(self) -> None: put_data_response = load_test_data("put_data_batch.json")[:5] test_pointset = { @@ -262,10 +278,9 @@ async def test_upload_dataframe(self, _name: str, pass_table_format: bool) -> No self.transport.set_http_response(status_code=200, content=json.dumps(put_data_response)), mock.patch("evo.objects.utils.table_formats.get_known_format") as mock_get_known_format, mock.patch("evo.common.io.upload.StorageDestination", autospec=True) as mock_destination, - mock.patch("pyarrow.Table") as mock_pyarrow_table, - mock.patch("evo.objects.utils.data._get_schema_from_dataframe", return_value=None), + mock.patch("evo.objects.utils.data._table_from_dataframe") as mock_table_from_dataframe, ): - mock_pyarrow_table.from_pandas.return_value = mock_table = mock.Mock() + mock_table_from_dataframe.return_value = mock_table = mock.Mock() mock_get_known_format.return_value = mock_known_format = mock.Mock(spec=KnownTableFormat) mock_known_format.save_table.return_value = mock_table_info = {} mock_table_info["data"] = mock_data_id = put_data_response[0]["name"] @@ -289,6 +304,7 @@ async def _mock_upload_file_side_effect(*args, **kwargs): else: actual_table_info = await self.data_client.upload_dataframe(mock_dataframe) + mock_table_from_dataframe.assert_called_once_with(mock_dataframe) if pass_table_format: mock_get_known_format.assert_not_called() else: @@ -351,10 +367,9 @@ async def test_upload_dataframe_exists(self) -> None: self.transport.set_http_response(status_code=200, content=json.dumps(put_data_response)), mock.patch("evo.objects.utils.table_formats.get_known_format") as mock_get_known_format, mock.patch("evo.common.io.upload.StorageDestination", autospec=True) as mock_destination, - mock.patch("pyarrow.Table") as mock_pyarrow_table, - mock.patch("evo.objects.utils.data._get_schema_from_dataframe", return_value=None), + mock.patch("evo.objects.utils.data._table_from_dataframe") as mock_table_from_dataframe, ): - mock_pyarrow_table.from_pandas.return_value = mock_table = mock.Mock() + mock_table_from_dataframe.return_value = mock_table = mock.Mock() mock_get_known_format.return_value = mock_known_format = mock.Mock(spec=KnownTableFormat) mock_known_format.save_table.return_value = mock_table_info = {} mock_table_info["data"] = mock_data_id = put_data_response[0]["name"] @@ -374,6 +389,7 @@ async def _mock_upload_file_side_effect(*args, **kwargs): mock_dataframe = mock.Mock() actual_table_info = await self.data_client.upload_dataframe(mock_dataframe) + mock_table_from_dataframe.assert_called_once_with(mock_dataframe) mock_get_known_format.assert_called_once_with(mock_table, table_formats=None) mock_known_format.save_table.assert_called_once_with( table=mock_table, destination=self.data_client.cache_location @@ -500,6 +516,15 @@ async def test_upload_category_table_error(self) -> None: pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), table_formats.INTEGER_ARRAY_1_INT32, ), + ( + "ordered_category", + pd.DataFrame({"Category": pd.Categorical(["A", "B", "A", "C"], ordered=True)}), + pa.table( + {"key": pa.array([0, 1, 2], type=pa.int32()), "value": pa.array(["A", "B", "C"], type=pa.string())} + ), + pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), + table_formats.INTEGER_ARRAY_1_INT32, + ), ( "string", pd.DataFrame({"Category": ["A", "B", "A", "C"]}), @@ -553,6 +578,23 @@ def side_effect(table, table_format=None, fb=None): self.assertEqual(category_info["values"]["table"], values_table) self.assertEqual(category_info["values"]["table_format"], values_table_format) + @parameterized.expand([("unordered", False), ("ordered", True)]) + async def test_category_dataframe_retains_dictionary_encoding(self, _name: str, ordered: bool) -> None: + categories = pd.Index(pd.array(["B", "A"], dtype="string[pyarrow]")) + dataframe = pd.DataFrame( + {"Category": pd.Categorical(["A", "B", None], categories=categories, ordered=ordered)}, + index=[5, 3, 1], + ) + with mock.patch.object(self.data_client, "upload_category_table", new_callable=mock.AsyncMock) as upload: + await self.data_client.upload_category_dataframe(dataframe) + + (table,), _ = upload.await_args + self.assertEqual(["Category"], table.column_names) + self.assertTrue(pa.types.is_dictionary(table.column("Category").type)) + self.assertEqual(pa.string(), table.column("Category").type.value_type) + self.assertEqual(ordered, table.column("Category").type.ordered) + self.assertEqual(["A", "B", None], table.column("Category").to_pylist()) + async def test_upload_dataframe_converts_large_string_via_schema(self) -> None: """Test that upload_dataframe converts large_string columns (from pandas string[pyarrow]) to string via schema.""" dataframe = pd.DataFrame({"col": pd.Series(["A", "B", "C"], dtype="string[pyarrow]")}) diff --git a/packages/evo-objects/tests/test_downloaded_object.py b/packages/evo-objects/tests/test_downloaded_object.py index af1b78cc3..e2001aec4 100644 --- a/packages/evo-objects/tests/test_downloaded_object.py +++ b/packages/evo-objects/tests/test_downloaded_object.py @@ -466,6 +466,20 @@ async def test_download_dataframe_nan_values(self): expected_dataframe = pd.DataFrame({"value": [np.nan, np.nan, 3.4, 5.3, np.nan, np.nan]}) assert_frame_equal(expected_dataframe, actual_dataframe) + async def test_download_dataframe_preserves_nullable_integers_and_sentinels(self) -> None: + values_path = "locations.attributes[1].values" + dataframe = pd.DataFrame( + {"value": pd.array([2**63 - 1, -999, None], dtype="Int64")}, + index=pd.RangeIndex(5, 8), + ) + self._set_property(f"{values_path}.data_type", "int64") + self._setup_table(values_path, pa.Table.from_pandas(dataframe)) + + actual_dataframe = await self.object.download_dataframe(values_path, nan_values=[-999], column_names=["count"]) + + expected_dataframe = pd.DataFrame({"count": pd.array([2**63 - 1, None, None], dtype="Int64")}) + assert_frame_equal(expected_dataframe, actual_dataframe) + async def test_download_dataframe_column_names(self): with self._patch_downloading_table_in_memory("locations.coordinates") as sample_table: actual_dataframe = await self.object.download_dataframe( diff --git a/packages/evo-objects/tests/test_pandas_metadata.py b/packages/evo-objects/tests/test_pandas_metadata.py index bb4d66c31..96c66c427 100644 --- a/packages/evo-objects/tests/test_pandas_metadata.py +++ b/packages/evo-objects/tests/test_pandas_metadata.py @@ -24,7 +24,7 @@ from evo.objects.exceptions import TableFormatError from evo.objects.parquet import ParquetLoader -from evo.objects.utils import get_known_format +from evo.objects.utils import ArrowTableFormat, get_known_format from evo.objects.utils.data import _table_from_dataframe @@ -73,8 +73,13 @@ def _schema_metadata(self, table_info: dict) -> dict: return pq.read_schema(self.data_dir / table_info["data"]).metadata or {} def test_dataframe_table_has_pandas_metadata(self) -> None: - """Guard the premise of these tests: pyarrow always adds pandas metadata when converting a dataframe.""" - table = _table_from_dataframe(pd.DataFrame({"value": pd.array([1, None], dtype="Int64")})) + """Even an explicit metadata-free schema does not prevent PyArrow from adding pandas metadata.""" + dataframe = pd.DataFrame({"value": pd.array([1, None], dtype="Int64")}) + table_format = ArrowTableFormat.from_pandas(dataframe) + self.assertIsNone(table_format.pa_schema.metadata) + + table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + self.assertIn(b"pandas", table.schema.metadata) def test_save_table_strips_pandas_metadata(self) -> None: @@ -118,6 +123,62 @@ def test_save_table_rejects_pandas_index_columns(self) -> None: self._save(table) self.assertEqual([], list(self.data_dir.iterdir())) + def test_save_table_rejects_malformed_pandas_index_metadata(self) -> None: + for metadata in ( + b"not json", + b"\xff", + b"null", + b"[]", + b'{"index_columns": null}', + b'{"index_columns": 1}', + b'{"index_columns": "row"}', + b'{"index_columns": [1]}', + b'{"index_columns": [{}]}', + ): + with self.subTest(metadata=metadata): + table = pa.table({"value": [1.0]}).replace_schema_metadata({b"pandas": metadata}) + + with self.assertRaisesRegex(TableFormatError, "pandas index metadata"): + self._save(table) + self.assertEqual([], list(self.data_dir.iterdir())) + + def test_save_table_rejects_serialized_indexes(self) -> None: + indexes = [ + pd.Index([5, 3], name="row"), + pd.MultiIndex.from_tuples([(5, 1), (3, 2)], names=["hole", "row"]), + ] + for index in indexes: + with self.subTest(index=index): + dataframe = pd.DataFrame({"value": [1, 2]}, index=index) + table = pa.Table.from_pandas(dataframe, preserve_index=True) + + with self.assertRaisesRegex(TableFormatError, "pandas index column"): + self._save(table) + self.assertEqual([], list(self.data_dir.iterdir())) + + def test_metadata_only_range_index_does_not_change_saved_file(self) -> None: + dataframe = pd.DataFrame({"value": [1.0, 2.0]}, index=pd.RangeIndex(5, 7, name="row")) + table = pa.Table.from_pandas(dataframe) + + with_metadata = self._save(table) + without_metadata = self._save(table.replace_schema_metadata(None)) + + self.assertEqual(without_metadata, with_metadata) + self.assertNotIn(b"pandas", self._schema_metadata(with_metadata)) + + def test_save_preserves_field_metadata_and_nullability(self) -> None: + field = pa.field("value", pa.int64(), nullable=False, metadata={b"units": b"count"}) + schema = pa.schema([field], metadata={b"pandas": b'{"index_columns": []}', b"other": b"kept"}) + table = pa.Table.from_arrays([pa.array([2**63 - 1])], schema=schema) + + table_info = self._save(table) + saved = pq.ParquetFile(self.data_dir / table_info["data"]).read() + + self.assertTrue( + saved.schema.equals(schema.remove_metadata().with_metadata({b"other": b"kept"}), check_metadata=True) + ) + self.assertEqual(table.to_pydict(), saved.to_pydict()) + def test_dataframe_index_is_not_stored(self) -> None: dataframe = pd.DataFrame({"value": [1.0, 2.0, 3.0]}).iloc[[2, 0]] @@ -132,8 +193,18 @@ def test_load_as_dataframe_uses_nullable_dtypes_from_schema(self) -> None: (pa.array([1, 2, 3], type=pa.int64()), "Int64"), (pa.array([1, None, 3], type=pa.int32()), "Int32"), (pa.array([1, 2, 3], type=pa.int32()), "Int32"), + (pa.array([2**63 - 1, None, -(2**63)], type=pa.int64()), "Int64"), + (pa.array([255, None, 0], type=pa.uint8()), "UInt8"), + (pa.array([2**32 - 1, None, 0], type=pa.uint32()), "UInt32"), + (pa.array([2**64 - 1, None, 0], type=pa.uint64()), "UInt64"), + (pa.array([None, None], type=pa.int64()), "Int64"), + (pa.array([], type=pa.int64()), "Int64"), (pa.array([True, None, False], type=pa.bool_()), "boolean"), (pa.array([1.5, None, 3.5], type=pa.float64()), "float64"), + ( + pa.array([pd.Timestamp("2025-01-01", tz="UTC"), None], type=pa.timestamp("us", tz="UTC")), + "datetime64[us, UTC]", + ), ] for array, expected_dtype in cases: with self.subTest(type=str(array.type), nulls=array.null_count): diff --git a/packages/evo-objects/tests/test_tables.py b/packages/evo-objects/tests/test_tables.py index 416f5c22a..88c470dda 100644 --- a/packages/evo-objects/tests/test_tables.py +++ b/packages/evo-objects/tests/test_tables.py @@ -19,6 +19,7 @@ from unittest import mock from uuid import UUID +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from parameterized import parameterized, parameterized_class @@ -31,6 +32,7 @@ from evo.objects.utils import ( ArrowTableFormat, BaseTableFormat, + ColumnFormat, KnownTableFormat, all_known_formats, get_known_format, @@ -96,6 +98,160 @@ def _test_name_from_known_format(cls: type, num: int, params_dict: dict) -> str: return parameterized.to_safe_name(f"{cls.__name__}_{num}_{data_format.name}") +class TestColumnFormat(unittest.TestCase): + @parameterized.expand( + [ + ("float64", pa.float64()), + ("uint8", pa.uint8()), + ("uint32", pa.uint32()), + ("uint64", pa.uint64()), + ("int32", pa.int32()), + ("int64", pa.int64()), + ("bool", pa.bool_()), + ("string", pa.string()), + ("timestamp", pa.timestamp("us", tz="UTC")), + ] + ) + def test_column_types(self, format_id: str, data_type: pa.DataType) -> None: + for format_spec in (format_id, data_type): + with self.subTest(format_spec=format_spec): + column = ColumnFormat(format_spec) + self.assertEqual(format_id, column.id) + self.assertEqual(data_type, column.type) + + def test_unsupported_column_type(self) -> None: + for format_spec in ("float32", pa.float32()): + with self.subTest(format_spec=format_spec), self.assertRaises(TypeError): + ColumnFormat(format_spec) + + @parameterized.expand( + [ + ("float64", None), + ("uint8", "UInt8"), + ("uint32", "UInt32"), + ("uint64", "UInt64"), + ("int32", "Int32"), + ("int64", "Int64"), + ("bool", "boolean"), + ("string", None), + ("timestamp", None), + ] + ) + def test_pandas_dtype(self, format_id: str, expected_dtype: str | None) -> None: + column = ColumnFormat(format_id) + if expected_dtype is None: + self.assertIsNone(column.pandas_dtype) + return + + self.assertEqual(pd.api.types.pandas_dtype(expected_dtype), column.pandas_dtype) + # The pandas dtype must convert back to the same Arrow type, so import and export agree. + dataframe = pd.DataFrame({"value": pd.array([None], dtype=column.pandas_dtype)}) + self.assertEqual(column.type, ArrowTableFormat.from_pandas(dataframe).columns[0].type) + + +class TestBaseTableFormat(unittest.TestCase): + def test_columns(self) -> None: + table_format = BaseTableFormat("mixed", [pa.int32(), "string", pa.bool_()]) + + self.assertIsInstance(table_format.columns, tuple) + self.assertTrue(all(isinstance(column, ColumnFormat) for column in table_format.columns)) + self.assertEqual(["int32", "string", "bool"], [column.id for column in table_format.columns]) + self.assertEqual([pa.int32(), pa.string(), pa.bool_()], [column.type for column in table_format.columns]) + + def test_multidimensional_columns(self) -> None: + table_format = BaseTableFormat("multidimensional", [pa.float64(), ...]) + + self.assertEqual(1, len(table_format.columns)) + self.assertEqual(pa.float64(), table_format.columns[0].type) + + +class TestArrowTableFormat(unittest.TestCase): + def test_from_schema(self) -> None: + schema = pa.schema( + [ + pa.field("key", pa.int64(), nullable=False, metadata={b"field": b"kept"}), + pa.field("label", pa.string()), + ], + metadata={b"other": b"kept"}, + ) + + table_format = ArrowTableFormat.from_schema(schema) + + self.assertIs(schema, table_format.pa_schema) + self.assertEqual("ArrowTableFormat", table_format.name) + self.assertEqual("int64/string", table_format.data_type) + self.assertEqual(2, table_format.width) + self.assertEqual(schema.types, [column.type for column in table_format.columns]) + + @parameterized.expand( + [ + ("Int32", [1, None], pa.int32()), + ("Int64", [2**63 - 1, None], pa.int64()), + ("UInt8", [255, None], pa.uint8()), + ("UInt32", [2**32 - 1, None], pa.uint32()), + ("UInt64", [2**64 - 1, None], pa.uint64()), + ("boolean", [True, None], pa.bool_()), + ("float64", [1.5, None], pa.float64()), + ("string[python]", ["A", None], pa.string()), + ("string[pyarrow]", ["A", None], pa.string()), + ("datetime64[us, UTC]", [pd.Timestamp("2025-01-01", tz="UTC"), None], pa.timestamp("us", tz="UTC")), + ] + ) + def test_from_pandas(self, dtype: str, values: list, expected_type: pa.DataType) -> None: + dataframe = pd.DataFrame({"value": pd.array(values, dtype=dtype)}) + + table_format = ArrowTableFormat.from_pandas(dataframe) + table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + + self.assertEqual(pa.schema([("value", expected_type)]), table_format.pa_schema) + self.assertIsNone(table_format.pa_schema.metadata) + self.assertEqual(expected_type, table_format.columns[0].type) + self.assertEqual(expected_type, table.column("value").type) + self.assertEqual(1, table.column("value").null_count) + self.assertEqual(values[0], table.column("value")[0].as_py()) + + def test_from_pandas_excludes_index(self) -> None: + indexes = [ + pd.RangeIndex(5, 7), + pd.Index([5, 3], name="row"), + pd.MultiIndex.from_tuples([("A", 5), ("B", 3)], names=["hole", "row"]), + ] + for index in indexes: + with self.subTest(index=index): + dataframe = pd.DataFrame({"value": [1.0, 2.0]}, index=index) + + table_format = ArrowTableFormat.from_pandas(dataframe) + table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + + self.assertEqual(["value"], table_format.pa_schema.names) + self.assertIsNone(table_format.pa_schema.metadata) + self.assertEqual({"value": [1.0, 2.0]}, table.to_pydict()) + + def test_from_pandas_rejects_unsupported_column_type(self) -> None: + dataframe = pd.DataFrame({"value": pd.array([1.0, 2.0], dtype="float32")}) + + with self.assertRaisesRegex(TypeError, "Unsupported column type"): + ArrowTableFormat.from_pandas(dataframe) + + def test_factory_preserves_subclass(self) -> None: + class CustomTableFormat(ArrowTableFormat): + pass + + table_format = CustomTableFormat.from_pandas(pd.DataFrame({"value": [1.0]})) + + self.assertIsInstance(table_format, CustomTableFormat) + self.assertEqual("CustomTableFormat", table_format.name) + + def test_direct_construction_remains_supported(self) -> None: + table_format = ArrowTableFormat("custom", [pa.float64()]) + + self.assertEqual("custom", table_format.name) + self.assertEqual("float64", table_format.data_type) + self.assertEqual(1, table_format.width) + with self.assertRaisesRegex(ValueError, "from_schema.*from_pandas"): + _ = table_format.pa_schema + + @parameterized_class(_all_known_formats_for_testing(), class_name_func=_test_name_from_known_format) class TestKnownFormat(unittest.TestCase): data_format: KnownTableFormat From b6fb33359f4ce754b5751aeaaf37090789297f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Peixoto?= Date: Fri, 6 Nov 2026 12:03:51 +0000 Subject: [PATCH 3/4] Refine schema-based DataFrame conversion and pandas metadata handling - Centralize schema inference and dtype mapping in ArrowTableFormat. - Use public conversion helpers and remove redundant validation and state. - Strip pandas metadata during DataFrame export and before reconstruction. - Preserve Arrow-only operation without pandas. - Add regression coverage for legacy metadata, categories, and schema validation. --- packages/evo-objects/README.md | 18 +- .../src/evo/objects/client/object_client.py | 7 +- .../src/evo/objects/parquet/loader.py | 13 +- .../src/evo/objects/utils/__init__.py | 11 +- .../evo-objects/src/evo/objects/utils/data.py | 27 +-- .../src/evo/objects/utils/tables.py | 158 ++++------------ .../evo-objects/tests/test_data_client.py | 60 +++++- .../evo-objects/tests/test_pandas_metadata.py | 172 +++++------------- packages/evo-objects/tests/test_tables.py | 111 ++++++++++- 9 files changed, 298 insertions(+), 279 deletions(-) diff --git a/packages/evo-objects/README.md b/packages/evo-objects/README.md index 83567c654..4d2c7a6f2 100644 --- a/packages/evo-objects/README.md +++ b/packages/evo-objects/README.md @@ -71,6 +71,9 @@ Check out the other methods on the `ObjectAPIClient` for more details on how to ### DataFrame and Parquet schemas +Arrow table and NumPy array operations do not require pandas. Client and loader DataFrame methods, and the downhole +helper exports from `evo.objects.utils`, are available only when pandas is installed. + `ArrowTableFormat.from_pandas()` infers the supported Geoscience Object column types, normalizes large strings to strings, and excludes the DataFrame index from its Arrow schema. Use `pa_schema` for conversion and `columns` to inspect the ordered `ColumnFormat` descriptors and their `id`, Arrow `type`, and `pandas_dtype` used when reading dataframes. @@ -83,15 +86,18 @@ from evo.objects.utils import ArrowTableFormat dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) table_format = ArrowTableFormat.from_pandas(dataframe) -table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) +table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False).replace_schema_metadata() table_info = data_client.save_table(table) ``` -PyArrow adds pandas reconstruction metadata even when an explicit schema is provided. The Geoscience Object helpers -strip that metadata when writing Parquet, while preserving other schema metadata. Readers ignore legacy pandas metadata -and derive dtypes from the Geoscience Object schema; integer and boolean columns use pandas nullable dtypes. -Saving rejects serialized pandas index columns and malformed index metadata with `TableFormatError`; recreate such -tables from the DataFrame with `preserve_index=False` rather than removing metadata from stored index columns. +PyArrow adds pandas reconstruction metadata even when an explicit schema is provided, so remove it with +`replace_schema_metadata()` as shown above. The `save_dataframe()` and `upload_dataframe()` helpers do this for you. +Raw Arrow table save/load methods preserve caller-supplied metadata. +Dataframe readers ignore pandas metadata and derive dtypes from the Geoscience Object schema; integer and boolean +columns use pandas nullable dtypes. To apply the same dtypes yourself, pass +`ArrowTableFormat.from_schema(table.schema).pandas_types_mapper()` as the `types_mapper` of +`table.replace_schema_metadata().to_pandas()`. Removing metadata before conversion also avoids parsing malformed +legacy pandas metadata; `ignore_metadata=True` alone does not prevent that parsing. For categorical DataFrames, use `upload_category_dataframe()` to retain dictionary encoding while building the value and lookup tables. diff --git a/packages/evo-objects/src/evo/objects/client/object_client.py b/packages/evo-objects/src/evo/objects/client/object_client.py index eac1b1715..ac5112cc5 100644 --- a/packages/evo-objects/src/evo/objects/client/object_client.py +++ b/packages/evo-objects/src/evo/objects/client/object_client.py @@ -40,7 +40,7 @@ ParquetLoader, TableInfo, ) - from ..utils.tables import KnownTableFormat, _table_to_dataframe + from ..utils.tables import table_to_dataframe except ImportError as e: print(e) _LOADER_AVAILABLE = False @@ -490,9 +490,8 @@ async def download_dataframe( :returns: A Pandas DataFrame containing the downloaded data. Column dtypes are derived from the Geoscience Object column types; integer and boolean columns use pandas nullable dtypes. """ - table_info = self._validate_typed_dict(table_info, _TABLE_INFO_VALIDATOR) table = await self.download_table(table_info, fb=fb, nan_values=nan_values, column_names=column_names) - return _table_to_dataframe(table, KnownTableFormat.from_table_info(table_info)) + return table_to_dataframe(table) async def download_category_dataframe( self, @@ -515,7 +514,7 @@ async def download_category_dataframe( table = await self.download_category_table( category_info, fb=fb, nan_values=nan_values, column_names=column_names ) - return _table_to_dataframe(table) + return table.to_pandas() async def download_attribute_dataframe( self, diff --git a/packages/evo-objects/src/evo/objects/parquet/loader.py b/packages/evo-objects/src/evo/objects/parquet/loader.py index 37528fcc3..80921a00a 100644 --- a/packages/evo-objects/src/evo/objects/parquet/loader.py +++ b/packages/evo-objects/src/evo/objects/parquet/loader.py @@ -28,7 +28,7 @@ from ..exceptions import SchemaValidationError, TableFormatError from ..utils import ArrowTableFormat, KnownTableFormat, TableInfo -from ..utils.tables import _strip_pandas_metadata, _table_to_dataframe +from ..utils.tables import table_to_dataframe try: import pandas as pd @@ -68,7 +68,6 @@ def __init__(self, pa_file: pa.NativeFile) -> None: """ self._pa_file = pa_file self._parquet_file: pq.ParquetFile | None = None - self._expected_format: KnownTableFormat | None = None def __enter__(self) -> ParquetLoader: if self._parquet_file is not None: @@ -121,7 +120,6 @@ def validate_with_table_info(self, table_info: TableInfo) -> None: expected_format._check_format(actual_format) except TableFormatError as e: raise SchemaValidationError(str(e)) from None - self._expected_format = expected_format logger.debug("Checking parquet data length") actual_length = pa_file.metadata.num_rows @@ -133,14 +131,11 @@ def validate_with_table_info(self, table_info: TableInfo) -> None: logger.debug("Parquet metadata checks succeeded") def load_as_table(self) -> pa.Table: - """Load the Parquet data as a PyArrow Table. - - Any pandas reconstruction metadata in the Parquet file is discarded. - """ + """Load the Parquet data as a PyArrow Table.""" if self._parquet_file is None: raise RuntimeError("ParquetLoader context is not active") else: - return _strip_pandas_metadata(self._parquet_file.read()) + return self._parquet_file.read() if _PD_AVAILABLE: # Optional support for pandas dataframes @@ -152,7 +147,7 @@ def load_as_dataframe(self) -> pd.DataFrame: Integer and boolean columns use pandas nullable dtypes (e.g. `Int64`, `boolean`). """ table = self.load_as_table() - return _table_to_dataframe(table, self._expected_format) + return table_to_dataframe(table) if _NP_AVAILABLE: # Optional support for numpy arrays diff --git a/packages/evo-objects/src/evo/objects/utils/__init__.py b/packages/evo-objects/src/evo/objects/utils/__init__.py index 1f3764990..fc02bce8d 100644 --- a/packages/evo-objects/src/evo/objects/utils/__init__.py +++ b/packages/evo-objects/src/evo/objects/utils/__init__.py @@ -23,7 +23,6 @@ DataFrame = None # type: ignore from .data import ObjectDataClient -from .downhole import expand_hole_index, hole_chunks_from_ids from .table_formats import all_known_formats, get_known_format from .tables import ArrowTableFormat, BaseTableFormat, ColumnFormat, KnownTableFormat from .types import ArrayTableInfo, AttributeInfo, CategoryInfo, LookupTableInfo, TableInfo @@ -45,7 +44,13 @@ "ObjectDataClient", "TableInfo", "all_known_formats", - "expand_hole_index", "get_known_format", - "hole_chunks_from_ids", ] + +if DataFrame is not None: + from .downhole import expand_hole_index, hole_chunks_from_ids + + __all__ += [ + "expand_hole_index", + "hole_chunks_from_ids", + ] diff --git a/packages/evo-objects/src/evo/objects/utils/data.py b/packages/evo-objects/src/evo/objects/utils/data.py index b6a7c7f63..af48a8fd4 100644 --- a/packages/evo-objects/src/evo/objects/utils/data.py +++ b/packages/evo-objects/src/evo/objects/utils/data.py @@ -26,7 +26,7 @@ from ..exceptions import TableFormatError from ..io import _CACHE_SCOPE, ObjectDataDownload, ObjectDataUpload from .table_formats import INTEGER_ARRAY_1_INT32, INTEGER_ARRAY_MD_INT32, LOOKUP_TABLE_INT32 -from .tables import ArrowTableFormat, KnownTableFormat, _get_schema_from_dataframe, _table_to_dataframe +from .tables import ArrowTableFormat, KnownTableFormat, table_to_dataframe from .types import ArrayTableInfo, CategoryInfo, LookupTableInfo try: @@ -72,13 +72,16 @@ def _iter_refs(target: Any, _key: str | None = None) -> Iterator[str]: def _table_from_dataframe(dataframe: "pd.DataFrame") -> pa.Table: """Convert a pandas dataframe to a pyarrow table for upload. - The dataframe index is not part of any Geoscience Object table format, so it is never stored. + The dataframe index is not part of any Geoscience Object table format, so it is never stored. Pandas metadata is + removed, so readers rely only on the Geoscience Object schema. :param dataframe: The pandas dataframe to convert. :return: A pyarrow table with the normalized schema from `ArrowTableFormat.from_pandas`. """ table_format = ArrowTableFormat.from_pandas(dataframe) - return pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False) + return pa.Table.from_pandas( + dataframe, schema=table_format.pa_schema, preserve_index=False + ).replace_schema_metadata() class ObjectDataClient: @@ -211,7 +214,8 @@ async def upload_category_table(self, table: pa.Table, fb: IFeedback = NoFeedbac both the value and lookup tables. The table columns must be string columns, but can be dictionary encoded. If they are dictionary encoded, this - method takes advantage of that, when building the value and lookup tables. + method takes advantage of that, when building the value and lookup tables. Large strings are converted to + strings. :param table: The table to be uploaded. :param fb: A feedback object for tracking upload progress. @@ -226,11 +230,13 @@ async def upload_category_table(self, table: pa.Table, fb: IFeedback = NoFeedbac columns_chunk_range = [] for column in table.itercolumns(): column = column.dictionary_encode() - if not pa.types.is_string(column.type.value_type): + value_type = column.type.value_type + if not (pa.types.is_string(value_type) or pa.types.is_large_string(value_type)): raise TableFormatError("Category columns must be of type string") - if not pa.types.is_int32(column.type.index_type): - # Currently, we only support int32 indices - column = pc.cast(column, pa.dictionary(pa.int32(), column.type.value_type)) + # Currently, we only support int32 indices and string values + category_type = pa.dictionary(pa.int32(), pa.string()) + if column.type != category_type: + column = pc.cast(column, category_type) start_chunk = len(all_chunks) if isinstance(column, pa.ChunkedArray): all_chunks.extend(column.chunks) @@ -456,8 +462,7 @@ async def upload_category_dataframe(self, dataframe: pd.DataFrame, fb: IFeedback what int32 type can represent. """ # Keep dictionary encoding until the categories are split into value and lookup tables. - schema = _get_schema_from_dataframe(dataframe) - table = pa.Table.from_pandas(dataframe, schema=schema, preserve_index=False) + table = pa.Table.from_pandas(dataframe, preserve_index=False).replace_schema_metadata() category_info = await self.upload_category_table(table, fb=fb) return category_info @@ -483,6 +488,6 @@ async def download_dataframe( """ table = await self.download_table(object_id, version_id, table_info, fb) try: - return _table_to_dataframe(table, KnownTableFormat.from_table_info(table_info)) + return table_to_dataframe(table) except ModuleNotFoundError: raise RuntimeError("Unable to download dataframe because the `pandas` package is not installed") diff --git a/packages/evo-objects/src/evo/objects/utils/tables.py b/packages/evo-objects/src/evo/objects/utils/tables.py index 34635f370..995a34894 100644 --- a/packages/evo-objects/src/evo/objects/utils/tables.py +++ b/packages/evo-objects/src/evo/objects/utils/tables.py @@ -12,13 +12,11 @@ from __future__ import annotations import hashlib -import json import os from collections.abc import Callable from pathlib import Path from tempfile import NamedTemporaryFile from types import EllipsisType -from typing import Any import pyarrow as pa import pyarrow.parquet as pq @@ -43,120 +41,28 @@ "BaseTableFormat", "ColumnFormat", "KnownTableFormat", + "table_to_dataframe", ] # Block size to limit memory use while calculating the sha256 digest of a file. _DIGEST_BLOCK_SIZE = 4 * 1024 * 1024 -# Arrow schema metadata key that pyarrow uses to reconstruct pandas dtypes and indexes. -_PANDAS_METADATA_KEY = b"pandas" - -# Pandas dtypes used for schema column types that numpy cannot represent with nulls. -_NULLABLE_PANDAS_DTYPES = { - "int32": "Int32", - "int64": "Int64", +# Pandas dtype for every supported Geoscience Object column type. Integer and boolean columns use pandas nullable +# dtypes, so nulls do not change the dtype. `None` keeps the pyarrow default conversion: pyarrow `types_mapper` only +# accepts pandas extension dtypes, so numpy-backed `float64` and the pandas default string dtype cannot be named here. +_PANDAS_DTYPES: dict[str, str | None] = { + "float64": None, "uint8": "UInt8", "uint32": "UInt32", "uint64": "UInt64", + "int32": "Int32", + "int64": "Int64", "bool": "boolean", + "string": None, + "timestamp": "datetime64[us, UTC]", } -def _get_schema_from_dataframe(dataframe: pd.DataFrame) -> pa.Schema: - """Infer an Arrow schema without the dataframe index or pandas metadata, normalizing large strings. - - Dictionary encoding is preserved for category uploads, including the ordering of categorical values. - """ - schema = pa.Schema.from_pandas(dataframe, preserve_index=False) - fields = [] - for field in schema: - if pa.types.is_large_string(field.type): - fields.append(field.with_type(pa.string())) - elif pa.types.is_dictionary(field.type) and pa.types.is_large_string(field.type.value_type): - fields.append( - field.with_type(pa.dictionary(field.type.index_type, pa.string(), ordered=field.type.ordered)) - ) - else: - fields.append(field) - return pa.schema(fields) - - -def _strip_pandas_metadata(table: pa.Table, *, validate: bool = False) -> pa.Table: - """Remove pandas reconstruction metadata, so that readers rely only on the Arrow and Geoscience Object schemas. - - The pandas metadata is only a hint for rebuilding pandas dtypes and indexes; the Arrow data is not changed. - Other schema metadata is preserved. - - :param table: The table to remove pandas metadata from. - :param validate: Reject tables whose pandas metadata declares serialized index columns or cannot be validated. - Once the metadata is removed, index columns would be indistinguishable from data columns. - - :return: The table without pandas metadata. - - :raises TableFormatError: If `validate` is true and the table contains serialized pandas index columns or malformed - pandas index metadata. - """ - metadata = table.schema.metadata - if not metadata or _PANDAS_METADATA_KEY not in metadata: - return table - - if validate: - try: - pandas_metadata = json.loads(metadata[_PANDAS_METADATA_KEY]) - if not isinstance(pandas_metadata, dict): - raise ValueError("Expected a metadata object") - index_columns = pandas_metadata.get("index_columns", []) - if not isinstance(index_columns, list) or any( - not (isinstance(column, str) or isinstance(column, dict) and column.get("kind") == "range") - for column in index_columns - ): - raise ValueError("Expected index column names or RangeIndex descriptors") - except ValueError as error: - raise TableFormatError( - "Cannot validate pandas index metadata. Recreate the table from the dataframe with " - "`preserve_index=False` before saving." - ) from error - stored_index_columns = [column for column in index_columns if isinstance(column, str)] - if stored_index_columns: - raise TableFormatError( - f"Table contains pandas index column(s) {stored_index_columns}, which are not part of any " - "Geoscience Object table format. Convert the dataframe with `preserve_index=False`." - ) - - remaining = {key: value for key, value in metadata.items() if key != _PANDAS_METADATA_KEY} - return table.replace_schema_metadata(remaining or None) - - -def _pandas_types_mapper(table_format: BaseTableFormat | None = None) -> Callable[[pa.DataType], Any]: - """Get a pyarrow `types_mapper` that maps Geoscience Object column types to pandas dtypes. - - Integer and boolean columns use pandas nullable dtypes, so they keep their type when they contain nulls, - rather than being converted to float or object columns depending on the data. Other types use pyarrow defaults. - - :param table_format: The expected table format. If omitted, all supported column types are mapped. - """ - column_formats = ( - table_format.columns - if table_format is not None - else [ColumnFormat(format_id) for format_id in _NULLABLE_PANDAS_DTYPES] - ) - mapping = { - column_format.type: dtype - for column_format in column_formats - if (dtype := column_format.pandas_dtype) is not None - } - return mapping.get - - -def _table_to_dataframe(table: pa.Table, table_format: BaseTableFormat | None = None) -> pd.DataFrame: - """Convert a table to a pandas dataframe using the Geoscience Object column types, ignoring pandas metadata. - - :param table: The table to convert. It is expected to have been validated against `table_format`. - :param table_format: The expected table format. If omitted, all supported column types are mapped. - """ - return _strip_pandas_metadata(table).to_pandas(types_mapper=_pandas_types_mapper(table_format)) - - class ColumnFormat: """A Geoscience Object column type and its corresponding Arrow data type.""" @@ -235,14 +141,14 @@ def pandas_dtype(self) -> pd.api.extensions.ExtensionDtype | None: """The pandas dtype used when loading this column into a dataframe. Integer and boolean columns use pandas nullable dtypes, so nulls do not change the dtype. Returns `None` for - column types that use the pyarrow default conversion. + `float64` and `string` columns, which use the pyarrow default conversion. :raises ImportError: If pandas is not installed. """ - if (dtype_name := _NULLABLE_PANDAS_DTYPES.get(self._format_id)) is None: - return None if not _PD_AVAILABLE: raise ModuleNotFoundError("No module named 'pandas'", name="pandas") + if (dtype_name := _PANDAS_DTYPES[self._format_id]) is None: + return None return pd.api.types.pandas_dtype(dtype_name) @@ -307,7 +213,7 @@ class ArrowTableFormat(BaseTableFormat): @property def pa_schema(self) -> pa.Schema: - """The Arrow schema supplied to `from_schema` or inferred by `from_pandas`. + """The Arrow schema supplied to `from_schema` or inferred by `from_pandas`, without any metadata. :raises ValueError: If the format was constructed directly without a schema. """ @@ -319,12 +225,12 @@ def pa_schema(self) -> pa.Schema: def from_schema(cls, pa_schema: pa.Schema) -> ArrowTableFormat: """Generate an ArrowTableFormat instance that represents the structure of the provided table schema. - :param pa_schema: Table schema to generate a format representation for. + :param pa_schema: Table schema to generate a format representation for. Schema and field metadata are not kept. :return: A table format that can be used to compare against known formats. """ table_format = cls(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types) - table_format._pa_schema = pa_schema + table_format._pa_schema = pa.schema([field.remove_metadata() for field in pa_schema]) return table_format @classmethod @@ -332,14 +238,34 @@ def from_pandas(cls, dataframe: pd.DataFrame) -> ArrowTableFormat: """Infer a table format from a pandas dataframe using the supported Geoscience Object column types. Large strings are normalized to strings. The inferred schema excludes the dataframe index and pandas metadata. - PyArrow still adds pandas metadata when converting the dataframe; it is stripped when saving the Parquet file. :param dataframe: The dataframe to infer the format from. :return: A format whose `pa_schema` can be passed to `pyarrow.Table.from_pandas`. :raises TypeError: If a column type is not supported. Use `ObjectDataClient.upload_category_dataframe` for categorical columns, which are split into value and lookup tables separately. """ - return cls.from_schema(_get_schema_from_dataframe(dataframe)) + schema = pa.Schema.from_pandas(dataframe, preserve_index=False) + fields = [field.with_type(pa.string()) if pa.types.is_large_string(field.type) else field for field in schema] + return cls.from_schema(pa.schema(fields)) + + def pandas_types_mapper(self) -> Callable[[pa.DataType], pd.api.extensions.ExtensionDtype | None]: + """Get a pyarrow `types_mapper` that converts this format's columns to their `ColumnFormat.pandas_dtype`. + + Pass the result to `table.replace_schema_metadata().to_pandas(types_mapper=...)` to ignore legacy metadata. + + :raises ImportError: If pandas is not installed. + """ + return {column.type: column.pandas_dtype for column in self._columns}.get + + +def table_to_dataframe(table: pa.Table) -> pd.DataFrame: + """Convert a table to a pandas dataframe using the Geoscience Object column types, ignoring pandas metadata. + + :param table: The table to convert. It is expected to have been validated against a known table format. + """ + types_mapper = ArrowTableFormat.from_schema(table.schema).pandas_types_mapper() + # PyArrow parses pandas metadata even with ignore_metadata=True. + return table.replace_schema_metadata().to_pandas(types_mapper=types_mapper) class KnownTableFormat(BaseTableFormat): @@ -409,22 +335,16 @@ def _get_file_digest(file_path: Path) -> str: def _save_table_as_parquet(cls, table: pa.Table, destination: Path) -> str: """Save a table in parquet format. - Pandas reconstruction metadata is not saved, so the parquet file can be read by any consumer. Readers must use - the Geoscience Object schema to interpret column types. - :param table: The table to save to parquet file. :param destination: A local directory to save the parquet file in. :return: The sha256 digest of the saved parquet file, which is also the filename. :raises StorageFileNotFoundError: If the destination does not exist or is not a directory. - :raises TableFormatError: If the table contains serialized pandas index columns or malformed pandas index metadata. """ if not destination.is_dir(): raise StorageFileNotFoundError(f"'{destination}' is not a directory") - table = _strip_pandas_metadata(table, validate=True) - try: # Write temporary file. with NamedTemporaryFile(delete=False, dir=destination) as tmp_file: logger.debug(f"Writing temporary file {tmp_file.name}") @@ -564,7 +484,7 @@ def load_table(cls, table_info: dict, source: Path) -> pa.Table: ) logger.debug("Parquet metadata checks succeeded") - return _strip_pandas_metadata(parquet_data.read()) + return parquet_data.read() except FileNotFoundError: raise StorageFileNotFoundError(f"Could not find data for table with id '{parquet_file.name}'") from None diff --git a/packages/evo-objects/tests/test_data_client.py b/packages/evo-objects/tests/test_data_client.py index 5af33be8e..a38ba976c 100644 --- a/packages/evo-objects/tests/test_data_client.py +++ b/packages/evo-objects/tests/test_data_client.py @@ -505,6 +505,41 @@ async def test_upload_category_table_error(self) -> None: await self.data_client.upload_category_table(table) assert "Category columns must be of type string" in str(cm.exception) + async def test_upload_category_table_normalizes_mixed_dictionary_types(self) -> None: + table = pa.table( + { + "ordered": pa.chunked_array( + [ + pa.DictionaryArray.from_arrays( + pa.array(indices, type=pa.int32()), + pa.array(["B", "A"], type=pa.large_string()), + ordered=True, + ) + for indices in ([0, None], [1, 0]) + ] + ), + "unordered": pa.array(["C", "A", None, "B"], type=pa.dictionary(pa.int8(), pa.string())), + } + ) + saved_tables: list[pa.Table] = [] + + def save_upload(table: pa.Table, *, table_format: KnownTableFormat, fb: IFeedback) -> dict: + saved_tables.append(table) + return self.data_client.save_table(table, table_format=table_format) + + with mock.patch.object(self.data_client, "upload_table", side_effect=save_upload): + category_info = await self.data_client.upload_category_table(table) + + values, lookup = saved_tables + self.assertEqual([pa.int32(), pa.int32()], values.schema.types) + self.assertEqual([pa.int32(), pa.string()], lookup.schema.types) + for name in table.column_names: + restored = pa.DictionaryArray.from_arrays(values[name].combine_chunks(), lookup["value"].combine_chunks()) + self.assertEqual(table[name].to_pylist(), restored.to_pylist()) + for table_info in (category_info["values"], category_info["table"]): + schema = pq.read_schema(self.data_client.cache_location / table_info["data"]) + self.assertNotIn(b"pandas", schema.metadata or {}) + @parameterized.expand( [ ( @@ -525,6 +560,22 @@ async def test_upload_category_table_error(self) -> None: pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), table_formats.INTEGER_ARRAY_1_INT32, ), + ( + "large_string_category", + pd.DataFrame( + { + "Category": pd.Categorical( + ["A", "B", "A", "C"], + categories=pd.Index(pd.array(["A", "B", "C"], dtype="string[pyarrow]")), + ) + } + ), + pa.table( + {"key": pa.array([0, 1, 2], type=pa.int32()), "value": pa.array(["A", "B", "C"], type=pa.string())} + ), + pa.table({"Category": pa.array([0, 1, 0, 2], type=pa.int32())}), + table_formats.INTEGER_ARRAY_1_INT32, + ), ( "string", pd.DataFrame({"Category": ["A", "B", "A", "C"]}), @@ -577,6 +628,12 @@ def side_effect(table, table_format=None, fb=None): self.assertEqual(category_info["table"]["table_format"], table_formats.LOOKUP_TABLE_INT32) self.assertEqual(category_info["values"]["table"], values_table) self.assertEqual(category_info["values"]["table_format"], values_table_format) + for table_info in (category_info["values"], category_info["table"]): + table = table_info["table"] + self.assertNotIn(b"pandas", table.schema.metadata or {}) + saved_info = self.data_client.save_table(table, table_format=table_info["table_format"]) + schema = pq.read_schema(self.data_client.cache_location / saved_info["data"]) + self.assertNotIn(b"pandas", schema.metadata or {}) @parameterized.expand([("unordered", False), ("ordered", True)]) async def test_category_dataframe_retains_dictionary_encoding(self, _name: str, ordered: bool) -> None: @@ -589,9 +646,9 @@ async def test_category_dataframe_retains_dictionary_encoding(self, _name: str, await self.data_client.upload_category_dataframe(dataframe) (table,), _ = upload.await_args + self.assertNotIn(b"pandas", table.schema.metadata or {}) self.assertEqual(["Category"], table.column_names) self.assertTrue(pa.types.is_dictionary(table.column("Category").type)) - self.assertEqual(pa.string(), table.column("Category").type.value_type) self.assertEqual(ordered, table.column("Category").type.ordered) self.assertEqual(["A", "B", None], table.column("Category").to_pylist()) @@ -603,6 +660,7 @@ async def test_upload_dataframe_converts_large_string_via_schema(self) -> None: await self.data_client.upload_dataframe(dataframe) (table,), _ = mock_upload.await_args + self.assertNotIn(b"pandas", table.schema.metadata or {}) self.assertTrue(pa.types.is_string(table.schema.field("col").type)) async def test_download_table(self) -> None: diff --git a/packages/evo-objects/tests/test_pandas_metadata.py b/packages/evo-objects/tests/test_pandas_metadata.py index 96c66c427..0206d923f 100644 --- a/packages/evo-objects/tests/test_pandas_metadata.py +++ b/packages/evo-objects/tests/test_pandas_metadata.py @@ -9,22 +9,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Parquet files must not carry pandas reconstruction metadata; pandas dtypes come from the Geoscience Object schema.""" +"""DataFrame exports omit pandas metadata; dataframe readers derive dtypes from the Geoscience Object schema.""" -import json import tempfile import unittest from pathlib import Path -from unittest import mock import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from pandas.testing import assert_frame_equal -from evo.objects.exceptions import TableFormatError +from evo.objects.exceptions import SchemaValidationError, TableFormatError from evo.objects.parquet import ParquetLoader -from evo.objects.utils import ArrowTableFormat, get_known_format +from evo.objects.utils import ArrowTableFormat, KnownTableFormat, get_known_format from evo.objects.utils.data import _table_from_dataframe @@ -35,29 +33,6 @@ def _load_dataframe(table_info: dict, source: Path) -> pd.DataFrame: return loader.load_as_dataframe() -class _BrokenPandasExtensionArrays: - """Simulate a consumer whose pandas cannot rebuild extension arrays from Arrow. - - pandas only calls `__from_arrow__` when told to by the pandas schema metadata, or by an explicit `types_mapper`. - """ - - def __enter__(self): - def broken(*_args, **_kwargs): - raise ModuleNotFoundError("No module named 'pandas.core.arrays.arrow._arrow_utils'") - - self._patches = [ - mock.patch.object(dtype, "__from_arrow__", broken) - for dtype in (pd.Int32Dtype, pd.Int64Dtype, pd.BooleanDtype, pd.StringDtype) - ] - for patch in self._patches: - patch.start() - return self - - def __exit__(self, *exc_info): - for patch in reversed(self._patches): - patch.stop() - - class TestPandasMetadata(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() @@ -82,102 +57,12 @@ def test_dataframe_table_has_pandas_metadata(self) -> None: self.assertIn(b"pandas", table.schema.metadata) - def test_save_table_strips_pandas_metadata(self) -> None: + def test_saved_dataframe_has_no_pandas_metadata(self) -> None: table = _table_from_dataframe(pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")})) - table = table.replace_schema_metadata({**table.schema.metadata, b"other": b"kept"}) - - table_info = self._save(table) - - metadata = self._schema_metadata(table_info) - self.assertNotIn(b"pandas", metadata) - self.assertEqual(b"kept", metadata[b"other"]) - - def test_saved_table_can_be_read_by_pandas_without_extension_array_support(self) -> None: - dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) - table_info = self._save(_table_from_dataframe(dataframe)) - - with _BrokenPandasExtensionArrays(): - loaded = pq.read_table(self.data_dir / table_info["data"]).to_pandas() - - # Without schema information pandas falls back to float64, but no value is corrupted. - self.assertEqual([1.0, 3.0], loaded["value"].dropna().tolist()) - self.assertTrue(pd.isna(loaded["value"][1])) - - def test_legacy_pandas_metadata_breaks_pandas_without_extension_array_support(self) -> None: - """Document the failure mode that stripping the metadata prevents.""" - dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) - legacy_file = self.data_dir / "legacy.parquet" - pq.write_table(pa.Table.from_pandas(dataframe, preserve_index=False), legacy_file) - - with _BrokenPandasExtensionArrays(), self.assertRaises(ModuleNotFoundError): - pq.read_table(legacy_file).to_pandas() - - def test_save_table_rejects_pandas_index_columns(self) -> None: - dataframe = pd.DataFrame({"value": [1.0, 2.0]}, index=pd.Index([5, 3], name="row")) - table = pa.Table.from_pandas(dataframe, preserve_index=True).select(["value"]) - # Keep the metadata pointing to an index column, which is what a table saved with its index looks like. - index_metadata = json.loads(pa.Table.from_pandas(dataframe, preserve_index=True).schema.metadata[b"pandas"]) - table = table.replace_schema_metadata({b"pandas": json.dumps(index_metadata).encode()}) - - with self.assertRaises(TableFormatError): - self._save(table) - self.assertEqual([], list(self.data_dir.iterdir())) - - def test_save_table_rejects_malformed_pandas_index_metadata(self) -> None: - for metadata in ( - b"not json", - b"\xff", - b"null", - b"[]", - b'{"index_columns": null}', - b'{"index_columns": 1}', - b'{"index_columns": "row"}', - b'{"index_columns": [1]}', - b'{"index_columns": [{}]}', - ): - with self.subTest(metadata=metadata): - table = pa.table({"value": [1.0]}).replace_schema_metadata({b"pandas": metadata}) - - with self.assertRaisesRegex(TableFormatError, "pandas index metadata"): - self._save(table) - self.assertEqual([], list(self.data_dir.iterdir())) - - def test_save_table_rejects_serialized_indexes(self) -> None: - indexes = [ - pd.Index([5, 3], name="row"), - pd.MultiIndex.from_tuples([(5, 1), (3, 2)], names=["hole", "row"]), - ] - for index in indexes: - with self.subTest(index=index): - dataframe = pd.DataFrame({"value": [1, 2]}, index=index) - table = pa.Table.from_pandas(dataframe, preserve_index=True) - - with self.assertRaisesRegex(TableFormatError, "pandas index column"): - self._save(table) - self.assertEqual([], list(self.data_dir.iterdir())) - - def test_metadata_only_range_index_does_not_change_saved_file(self) -> None: - dataframe = pd.DataFrame({"value": [1.0, 2.0]}, index=pd.RangeIndex(5, 7, name="row")) - table = pa.Table.from_pandas(dataframe) - - with_metadata = self._save(table) - without_metadata = self._save(table.replace_schema_metadata(None)) - - self.assertEqual(without_metadata, with_metadata) - self.assertNotIn(b"pandas", self._schema_metadata(with_metadata)) - - def test_save_preserves_field_metadata_and_nullability(self) -> None: - field = pa.field("value", pa.int64(), nullable=False, metadata={b"units": b"count"}) - schema = pa.schema([field], metadata={b"pandas": b'{"index_columns": []}', b"other": b"kept"}) - table = pa.Table.from_arrays([pa.array([2**63 - 1])], schema=schema) table_info = self._save(table) - saved = pq.ParquetFile(self.data_dir / table_info["data"]).read() - self.assertTrue( - saved.schema.equals(schema.remove_metadata().with_metadata({b"other": b"kept"}), check_metadata=True) - ) - self.assertEqual(table.to_pydict(), saved.to_pydict()) + self.assertNotIn(b"pandas", self._schema_metadata(table_info)) def test_dataframe_index_is_not_stored(self) -> None: dataframe = pd.DataFrame({"value": [1.0, 2.0, 3.0]}).iloc[[2, 0]] @@ -185,7 +70,7 @@ def test_dataframe_index_is_not_stored(self) -> None: table = _table_from_dataframe(dataframe) self.assertEqual(["value"], table.column_names) - self.assertEqual([], json.loads(table.schema.metadata[b"pandas"])["index_columns"]) + self.assertIsNone(table.schema.metadata) def test_load_as_dataframe_uses_nullable_dtypes_from_schema(self) -> None: cases = [ @@ -227,6 +112,45 @@ def test_load_ignores_legacy_pandas_metadata(self) -> None: pd.testing.assert_index_equal(pd.RangeIndex(3), loaded.index) self.assertEqual(pd.Int64Dtype(), loaded["value"].dtype) - with pa.OSFile(str(self.data_dir / table_info["data"]), mode="r") as parquet_file: - with ParquetLoader(parquet_file) as loader: - self.assertIsNone(loader.load_as_table().schema.metadata) + + def test_load_ignores_malformed_pandas_metadata_without_changing_arrow_table(self) -> None: + schema = pa.schema( + [pa.field("value", pa.int64(), nullable=False, metadata={b"units": b"count"})], + metadata={b"pandas": b"not json", b"other": b"kept"}, + ) + table = pa.Table.from_arrays([pa.array([2**63 - 1])], schema=schema) + table_info = self._save(table) + + with ParquetLoader(pa.OSFile(str(self.data_dir / table_info["data"]), "r")) as loader: + loader.validate_with_table_info(table_info) + assert_frame_equal( + pd.DataFrame({"value": pd.array([2**63 - 1], dtype="Int64")}), loader.load_as_dataframe() + ) + self.assertTrue(table.equals(loader.load_as_table(), check_metadata=True)) + + def test_serialized_index_is_rejected_by_expected_schema(self) -> None: + dataframe = pd.DataFrame({"value": [1, 2]}, index=pd.Index([5, 3], name="row")) + table = pa.Table.from_pandas(dataframe, preserve_index=True) + table_info = {"data": "index.parquet", "length": 2, "width": 1, "data_type": "int64"} + table_format = KnownTableFormat.from_table_info(table_info) + + with self.assertRaisesRegex(TableFormatError, "Column count"): + table_format.save_table(table, self.data_dir) + self.assertEqual([], list(self.data_dir.iterdir())) + + pq.write_table(table, self.data_dir / table_info["data"]) + with ParquetLoader(pa.OSFile(str(self.data_dir / table_info["data"]), "r")) as loader: + with self.assertRaisesRegex(SchemaValidationError, "Column count"): + loader.validate_with_table_info(table_info) + + def test_string_and_float_columns_use_arrow_defaults_not_pandas_metadata(self) -> None: + for values, dtype in [(["A", None, "B"], "string[pyarrow]"), ([1.5, None, 3.5], "Float64")]: + with self.subTest(dtype=dtype): + table = pa.Table.from_pandas(pd.DataFrame({"value": pd.array(values, dtype=dtype)})) + if pa.types.is_large_string(table["value"].type): + table = table.cast(pa.schema([("value", pa.string())], metadata=table.schema.metadata)) + table_info = self._save(table) + + loaded = _load_dataframe(table_info, self.data_dir) + + assert_frame_equal(table.replace_schema_metadata().to_pandas(), loaded) diff --git a/packages/evo-objects/tests/test_tables.py b/packages/evo-objects/tests/test_tables.py index 88c470dda..a5991156b 100644 --- a/packages/evo-objects/tests/test_tables.py +++ b/packages/evo-objects/tests/test_tables.py @@ -10,11 +10,14 @@ # limitations under the License. import hashlib +import subprocess +import sys import unittest import uuid from collections.abc import Iterator from io import BytesIO from pathlib import Path +from textwrap import dedent from typing import BinaryIO from unittest import mock from uuid import UUID @@ -98,6 +101,95 @@ def _test_name_from_known_format(cls: type, num: int, params_dict: dict) -> str: return parameterized.to_safe_name(f"{cls.__name__}_{num}_{data_format.name}") +class TestOptionalPandas(unittest.TestCase): + @parameterized.expand([("with_pandas", False), ("without_pandas", True)]) + def test_cold_start(self, _name: str, missing_pandas: bool) -> None: + script = dedent( + """ + import importlib.abc + import sys + from pathlib import Path + from tempfile import TemporaryDirectory + + missing_pandas = sys.argv[1] == "True" + if missing_pandas: + assert not any(name == "pandas" or name.startswith("pandas.") for name in sys.modules) + + class NoPandasImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "pandas" or fullname.startswith("pandas."): + raise ModuleNotFoundError("No module named 'pandas'", name="pandas") + + sys.meta_path.insert(0, NoPandasImports()) + + import numpy as np + import pyarrow as pa + + from evo.objects import DownloadedObject, ObjectAPIClient, utils + from evo.objects.parquet import ParquetLoader + from evo.objects.utils import ArrowTableFormat, ObjectDataClient, get_known_format + + assert hasattr(ObjectAPIClient, "get_data_client") + assert all(hasattr(utils, name) for name in utils.__all__) + for name in ("expand_hole_index", "hole_chunks_from_ids"): + assert hasattr(utils, name) == (not missing_pandas), name + assert (name in utils.__all__) == (not missing_pandas), name + + arrow_methods = { + DownloadedObject: ("download_table", "download_category_table", "download_array"), + ObjectDataClient: ("save_table", "upload_table", "upload_category_table", "download_table"), + ParquetLoader: ("load_as_table", "load_as_array"), + } + for owner, methods in arrow_methods.items(): + for name in methods: + assert hasattr(owner, name), f"{owner.__name__}.{name}" + + dataframe_methods = { + DownloadedObject: ( + "download_dataframe", "download_category_dataframe", "download_attribute_dataframe" + ), + ObjectDataClient: ( + "save_dataframe", "upload_dataframe", "upload_category_dataframe", "download_dataframe" + ), + ParquetLoader: ("load_as_dataframe",), + } + for owner, methods in dataframe_methods.items(): + for name in methods: + assert hasattr(owner, name) == (not missing_pandas), f"{owner.__name__}.{name}" + + table = pa.table({"value": pa.array([1, 2, 3], type=pa.int64())}) + table_format = ArrowTableFormat.from_schema(table.schema) + assert table_format.pa_schema.equals(table.schema) + with TemporaryDirectory() as directory: + source = Path(directory) + table_info = get_known_format(table).save_table(table, source) + with ParquetLoader(pa.OSFile(str(source / table_info["data"]), "r")) as loader: + loader.validate_with_table_info(table_info) + assert loader.load_as_table().equals(table) + np.testing.assert_array_equal( + loader.load_as_array(), np.array([1, 2, 3], dtype=np.int64), strict=True + ) + + if missing_pandas: + try: + table_format.pandas_types_mapper() + except ModuleNotFoundError as error: + assert error.name == "pandas" + else: + raise AssertionError("Pandas dtype mapping must report the missing dependency") + assert not any(name == "pandas" or name.startswith("pandas.") for name in sys.modules) + """ + ) + # Cached pandas-dependent modules can hide import failures when only selected modules are reloaded. + result = subprocess.run( + [sys.executable, "-c", script, str(missing_pandas)], + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + class TestColumnFormat(unittest.TestCase): @parameterized.expand( [ @@ -134,7 +226,7 @@ def test_unsupported_column_type(self) -> None: ("int64", "Int64"), ("bool", "boolean"), ("string", None), - ("timestamp", None), + ("timestamp", "datetime64[us, UTC]"), ] ) def test_pandas_dtype(self, format_id: str, expected_dtype: str | None) -> None: @@ -177,12 +269,27 @@ def test_from_schema(self) -> None: table_format = ArrowTableFormat.from_schema(schema) - self.assertIs(schema, table_format.pa_schema) + self.assertEqual( + pa.schema([pa.field("key", pa.int64(), nullable=False), pa.field("label", pa.string())]), + table_format.pa_schema, + ) + self.assertIsNone(table_format.pa_schema.metadata) + self.assertIsNone(table_format.pa_schema.field("key").metadata) self.assertEqual("ArrowTableFormat", table_format.name) self.assertEqual("int64/string", table_format.data_type) self.assertEqual(2, table_format.width) self.assertEqual(schema.types, [column.type for column in table_format.columns]) + def test_pandas_types_mapper(self) -> None: + schema = pa.schema([("count", pa.int64()), ("flag", pa.bool_()), ("value", pa.float64())]) + + types_mapper = ArrowTableFormat.from_schema(schema).pandas_types_mapper() + + self.assertEqual(pd.Int64Dtype(), types_mapper(pa.int64())) + self.assertEqual(pd.BooleanDtype(), types_mapper(pa.bool_())) + self.assertIsNone(types_mapper(pa.float64())) + self.assertIsNone(types_mapper(pa.int32())) + @parameterized.expand( [ ("Int32", [1, None], pa.int32()), From f4c692a6b895b6444c2b26a879debc77428283b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Peixoto?= Date: Tue, 17 Nov 2026 08:01:44 +0000 Subject: [PATCH 4/4] Update README to clarify ObjectAPIClient and ObjectDataClient usage without pandas --- packages/evo-objects/README.md | 43 +++++++++------------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/packages/evo-objects/README.md b/packages/evo-objects/README.md index 4d2c7a6f2..c2b70de5e 100644 --- a/packages/evo-objects/README.md +++ b/packages/evo-objects/README.md @@ -25,6 +25,17 @@ data, empowering our users to build responsive modern workflows. pip install evo-objects ``` +The base installation supports object operations and low-level data transfers without PyArrow. +`ObjectAPIClient.prepare_data_upload()` and `prepare_data_download()` return contexts for transferring existing Parquet files. You must create or read the files yourself and provide matching table descriptions when creating objects. + +The optional `ObjectDataClient` provides higher-level operations on PyArrow tables without requiring pandas. Its +DataFrame methods and other pandas-based helpers additionally require pandas. Install the `utils` extra for these +dependencies: + +```shell +pip install "evo-objects[utils]" +``` + ## Usage See [the evo-sdk-common documentation](https://github.com/SeequentEvo/evo-python-sdk/blob/main/packages/evo-sdk-common/README.md) @@ -69,38 +80,6 @@ You can also get a list of all objects. Internally, this recursively calls the ` Check out the other methods on the `ObjectAPIClient` for more details on how to upload and download objects, or get object versions. -### DataFrame and Parquet schemas - -Arrow table and NumPy array operations do not require pandas. Client and loader DataFrame methods, and the downhole -helper exports from `evo.objects.utils`, are available only when pandas is installed. - -`ArrowTableFormat.from_pandas()` infers the supported Geoscience Object column types, normalizes large strings to strings, -and excludes the DataFrame index from its Arrow schema. Use `pa_schema` for conversion and `columns` to inspect the ordered -`ColumnFormat` descriptors and their `id`, Arrow `type`, and `pandas_dtype` used when reading dataframes. - -```python -import pandas as pd -import pyarrow as pa - -from evo.objects.utils import ArrowTableFormat - -dataframe = pd.DataFrame({"value": pd.array([1, None, 3], dtype="Int64")}) -table_format = ArrowTableFormat.from_pandas(dataframe) -table = pa.Table.from_pandas(dataframe, schema=table_format.pa_schema, preserve_index=False).replace_schema_metadata() -table_info = data_client.save_table(table) -``` - -PyArrow adds pandas reconstruction metadata even when an explicit schema is provided, so remove it with -`replace_schema_metadata()` as shown above. The `save_dataframe()` and `upload_dataframe()` helpers do this for you. -Raw Arrow table save/load methods preserve caller-supplied metadata. -Dataframe readers ignore pandas metadata and derive dtypes from the Geoscience Object schema; integer and boolean -columns use pandas nullable dtypes. To apply the same dtypes yourself, pass -`ArrowTableFormat.from_schema(table.schema).pandas_types_mapper()` as the `types_mapper` of -`table.replace_schema_metadata().to_pandas()`. Removing metadata before conversion also avoids parsing malformed -legacy pandas metadata; `ignore_metadata=True` alone does not prevent that parsing. -For categorical DataFrames, use `upload_category_dataframe()` to retain dictionary encoding while building the value -and lookup tables. - ### Typed downhole collections `DownholeCollection` provides a DataFrame-based API for creating and reading downhole objects. A collection can contain