Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions packages/evo-objects/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,17 @@ data, empowering our users to build responsive modern workflows.
pip install evo-objects
```

The base installation supports object operations and low-level data transfers without PyArrow.
`ObjectAPIClient.prepare_data_upload()` and `prepare_data_download()` return contexts for transferring existing Parquet files. You must create or read the files yourself and provide matching table descriptions when creating objects.

The optional `ObjectDataClient` provides higher-level operations on PyArrow tables without requiring pandas. Its
DataFrame methods and other pandas-based helpers additionally require pandas. Install the `utils` extra for these
dependencies:

```shell
pip install "evo-objects[utils]"
```

## Usage

See [the evo-sdk-common documentation](https://github.com/SeequentEvo/evo-python-sdk/blob/main/packages/evo-sdk-common/README.md)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
ParquetLoader,
TableInfo,
)
from ..utils.tables import table_to_dataframe
except ImportError as e:
print(e)
_LOADER_AVAILABLE = False
Expand Down Expand Up @@ -486,10 +487,11 @@ async def download_dataframe(
nan_description structure.
:param column_names: An optional list of column names for the table, instead of those from the Parquet file.

:returns: A Pandas DataFrame containing the downloaded data.
:returns: A Pandas DataFrame containing the downloaded data. Column dtypes are derived from the
Geoscience Object column types; integer and boolean columns use pandas nullable dtypes.
"""
table = await self.download_table(table_info, fb=fb, nan_values=nan_values, column_names=column_names)
return table.to_pandas()
return table_to_dataframe(table)

async def download_category_dataframe(
self,
Expand Down
9 changes: 7 additions & 2 deletions packages/evo-objects/src/evo/objects/parquet/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

from ..exceptions import SchemaValidationError, TableFormatError
from ..utils import ArrowTableFormat, KnownTableFormat, TableInfo
from ..utils.tables import table_to_dataframe

try:
import pandas as pd
Expand Down Expand Up @@ -140,9 +141,13 @@ def load_as_table(self) -> pa.Table:
# Optional support for pandas dataframes

def load_as_dataframe(self) -> pd.DataFrame:
"""Load the Parquet data as a Pandas DataFrame."""
"""Load the Parquet data as a Pandas DataFrame.

Column dtypes are derived from the Geoscience Object column types, never from pandas metadata in the file.
Integer and boolean columns use pandas nullable dtypes (e.g. `Int64`, `boolean`).
"""
table = self.load_as_table()
return table.to_pandas()
return table_to_dataframe(table)

if _NP_AVAILABLE:
# Optional support for numpy arrays
Expand Down
14 changes: 10 additions & 4 deletions packages/evo-objects/src/evo/objects/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@
DataFrame = None # type: ignore

from .data import ObjectDataClient
from .downhole import expand_hole_index, hole_chunks_from_ids
from .table_formats import all_known_formats, get_known_format
from .tables import ArrowTableFormat, BaseTableFormat, KnownTableFormat
from .tables import ArrowTableFormat, BaseTableFormat, ColumnFormat, KnownTableFormat
from .types import ArrayTableInfo, AttributeInfo, CategoryInfo, LookupTableInfo, TableInfo

# We _used_ to export Table and DataFrame from this package as custom protocols, but we are using the actual
Expand All @@ -39,12 +38,19 @@
"AttributeInfo",
"BaseTableFormat",
"CategoryInfo",
"ColumnFormat",
"KnownTableFormat",
"LookupTableInfo",
"ObjectDataClient",
"TableInfo",
"all_known_formats",
"expand_hole_index",
"get_known_format",
"hole_chunks_from_ids",
]

if DataFrame is not None:
from .downhole import expand_hole_index, hole_chunks_from_ids

__all__ += [
"expand_hole_index",
"hole_chunks_from_ids",
]
55 changes: 24 additions & 31 deletions packages/evo-objects/src/evo/objects/utils/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from ..exceptions import TableFormatError
from ..io import _CACHE_SCOPE, ObjectDataDownload, ObjectDataUpload
from .table_formats import INTEGER_ARRAY_1_INT32, INTEGER_ARRAY_MD_INT32, LOOKUP_TABLE_INT32
from .tables import KnownTableFormat
from .tables import ArrowTableFormat, KnownTableFormat, table_to_dataframe
from .types import ArrayTableInfo, CategoryInfo, LookupTableInfo

try:
Expand Down Expand Up @@ -69,27 +69,19 @@ def _iter_refs(target: Any, _key: str | None = None) -> Iterator[str]:
yield str(value)


def _get_schema_from_dataframe(dataframe: "pd.DataFrame") -> pa.Schema:
"""Get a normalized pyarrow schema from a pandas dataframe.
def _table_from_dataframe(dataframe: "pd.DataFrame") -> pa.Table:
"""Convert a pandas dataframe to a pyarrow table for upload.

This hook centralizes dataframe-to-Arrow schema adjustments before the
dataframe is converted to a table. It currently performs these conversions:
- large_string -> string
- dictionary<*, large_string> -> dictionary<*, string>
The dataframe index is not part of any Geoscience Object table format, so it is never stored. Pandas metadata is
removed, so readers rely only on the Geoscience Object schema.

:param dataframe: The pandas dataframe to get the schema from.
:return: A pyarrow schema with any configured type conversions applied.
:param dataframe: The pandas dataframe to convert.
:return: A pyarrow table with the normalized schema from `ArrowTableFormat.from_pandas`.
"""
schema = pa.Schema.from_pandas(dataframe)
fields = []
for field in schema:
if pa.types.is_large_string(field.type):
fields.append(field.with_type(pa.string()))
elif pa.types.is_dictionary(field.type) and pa.types.is_large_string(field.type.value_type):
fields.append(field.with_type(pa.dictionary(field.type.index_type, pa.string())))
else:
fields.append(field)
return pa.schema(fields)
table_format = ArrowTableFormat.from_pandas(dataframe)
return pa.Table.from_pandas(
dataframe, schema=table_format.pa_schema, preserve_index=False
).replace_schema_metadata()


class ObjectDataClient:
Expand Down Expand Up @@ -222,7 +214,8 @@ async def upload_category_table(self, table: pa.Table, fb: IFeedback = NoFeedbac
both the value and lookup tables.

The table columns must be string columns, but can be dictionary encoded. If they are dictionary encoded, this
method takes advantage of that, when building the value and lookup tables.
method takes advantage of that, when building the value and lookup tables. Large strings are converted to
strings.

:param table: The table to be uploaded.
:param fb: A feedback object for tracking upload progress.
Expand All @@ -237,11 +230,13 @@ async def upload_category_table(self, table: pa.Table, fb: IFeedback = NoFeedbac
columns_chunk_range = []
for column in table.itercolumns():
column = column.dictionary_encode()
if not pa.types.is_string(column.type.value_type):
value_type = column.type.value_type
if not (pa.types.is_string(value_type) or pa.types.is_large_string(value_type)):
raise TableFormatError("Category columns must be of type string")
if not pa.types.is_int32(column.type.index_type):
# Currently, we only support int32 indices
column = pc.cast(column, pa.dictionary(pa.int32(), column.type.value_type))
# Currently, we only support int32 indices and string values
category_type = pa.dictionary(pa.int32(), pa.string())
if column.type != category_type:
column = pc.cast(column, category_type)
start_chunk = len(all_chunks)
if isinstance(column, pa.ChunkedArray):
all_chunks.extend(column.chunks)
Expand Down Expand Up @@ -422,8 +417,7 @@ def save_dataframe(
no table formats are specified, raised when the table does not match any known format.
:raises StorageFileNotFoundError: If the destination does not exist or is not a directory.
"""
schema = _get_schema_from_dataframe(dataframe)
table = pa.Table.from_pandas(dataframe, schema)
table = _table_from_dataframe(dataframe)
return self.save_table(table, table_format=table_format)

async def upload_dataframe(
Expand All @@ -446,8 +440,7 @@ async def upload_dataframe(
:raises TableFormatError: If the provided table does not match any of the specified formats. If
no table formats are specified, raised when the table does not match any known format.
"""
schema = _get_schema_from_dataframe(dataframe)
table = pa.Table.from_pandas(dataframe, schema)
table = _table_from_dataframe(dataframe)
table_info = await self.upload_table(table, table_format=table_format, fb=fb)
return table_info

Expand All @@ -468,8 +461,8 @@ async def upload_category_dataframe(self, dataframe: pd.DataFrame, fb: IFeedback
:raises TableFormatError: If the table isn't a valid category table, or if the number of categories exceeds
what int32 type can represent.
"""
schema = _get_schema_from_dataframe(dataframe)
table = pa.Table.from_pandas(dataframe, schema)
# Keep dictionary encoding until the categories are split into value and lookup tables.
table = pa.Table.from_pandas(dataframe, preserve_index=False).replace_schema_metadata()
category_info = await self.upload_category_table(table, fb=fb)
return category_info

Expand All @@ -495,6 +488,6 @@ async def download_dataframe(
"""
table = await self.download_table(object_id, version_id, table_info, fb)
try:
return table.to_pandas()
return table_to_dataframe(table)
except ModuleNotFoundError:
raise RuntimeError("Unable to download dataframe because the `pandas` package is not installed")
115 changes: 108 additions & 7 deletions packages/evo-objects/src/evo/objects/utils/tables.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import hashlib
import os
from collections.abc import Callable
from pathlib import Path
from tempfile import NamedTemporaryFile
from types import EllipsisType
Expand All @@ -26,20 +27,50 @@

from ..exceptions import SchemaValidationError, TableFormatError

try:
import pandas as pd
except ImportError:
_PD_AVAILABLE = False
else:
_PD_AVAILABLE = True

logger = evo.logging.getLogger("object.tables")

__all__ = [
"ArrowTableFormat",
"BaseTableFormat",
"ColumnFormat",
"KnownTableFormat",
"table_to_dataframe",
]

# Block size to limit memory use while calculating the sha256 digest of a file.
_DIGEST_BLOCK_SIZE = 4 * 1024 * 1024


class _ColumnFormat:
def __init__(self, format_spec: pa.DataType | str):
# Pandas dtype for every supported Geoscience Object column type. Integer and boolean columns use pandas nullable
# dtypes, so nulls do not change the dtype. `None` keeps the pyarrow default conversion: pyarrow `types_mapper` only
# accepts pandas extension dtypes, so numpy-backed `float64` and the pandas default string dtype cannot be named here.
_PANDAS_DTYPES: dict[str, str | None] = {
"float64": None,
"uint8": "UInt8",
"uint32": "UInt32",
"uint64": "UInt64",
"int32": "Int32",
"int64": "Int64",
"bool": "boolean",
"string": None,
"timestamp": "datetime64[us, UTC]",
}


class ColumnFormat:
"""A Geoscience Object column type and its corresponding Arrow data type."""

def __init__(self, format_spec: pa.DataType | str) -> None:
"""
:param format_spec: A Geoscience Object column type identifier or a supported Arrow data type.
:raises TypeError: If the column type is not supported.
"""
if isinstance(format_spec, str):
self._type = self._get_data_type(format_spec)
self._format_id = format_spec
Expand Down Expand Up @@ -97,12 +128,29 @@ def _get_format_id(data_type: pa.DataType) -> str:

@property
def id(self) -> str:
"""The Geoscience Object column type identifier."""
return self._format_id

@property
def type(self) -> pa.DataType:
"""The corresponding Arrow data type."""
return self._type

@property
def pandas_dtype(self) -> pd.api.extensions.ExtensionDtype | None:
"""The pandas dtype used when loading this column into a dataframe.

Integer and boolean columns use pandas nullable dtypes, so nulls do not change the dtype. Returns `None` for
`float64` and `string` columns, which use the pyarrow default conversion.

:raises ImportError: If pandas is not installed.
"""
if not _PD_AVAILABLE:
raise ModuleNotFoundError("No module named 'pandas'", name="pandas")
if (dtype_name := _PANDAS_DTYPES[self._format_id]) is None:
return None
return pd.api.types.pandas_dtype(dtype_name)


class BaseTableFormat:
"""Base type for comparing table formats"""
Expand All @@ -114,15 +162,15 @@ def __init__(self, name: str, columns: list[pa.DataType | str | EllipsisType]) -
("...") indicates a multidimensional format.
"""
self._name = name
self._columns: list[_ColumnFormat] = []
self._columns: list[ColumnFormat] = []
self._multi_dimensional = False
for column_type in columns:
if column_type is Ellipsis:
self._multi_dimensional = True
elif self._multi_dimensional:
raise AssertionError("Found new type specification after Ellipsis")
else:
self._columns.append(_ColumnFormat(format_spec=column_type))
self._columns.append(ColumnFormat(format_spec=column_type))

if self._multi_dimensional and len(self._columns) != 1:
raise AssertionError("Multidimensional formats must only have one column type")
Expand All @@ -147,6 +195,11 @@ def width(self) -> int:
"""The number of columns in this table format"""
return len(self._columns)

@property
def columns(self) -> tuple[ColumnFormat, ...]:
"""Column formats in order. A multidimensional format has one entry for its repeated column type."""
return tuple(self._columns)

@property
def data_type(self) -> str:
"""The column data type(s) in this table format"""
Expand All @@ -156,15 +209,63 @@ def data_type(self) -> str:
class ArrowTableFormat(BaseTableFormat):
"""Specialised table format type that can be generated from a pyarrow table"""

_pa_schema: pa.Schema | None = None

@property
def pa_schema(self) -> pa.Schema:
"""The Arrow schema supplied to `from_schema` or inferred by `from_pandas`, without any metadata.

:raises ValueError: If the format was constructed directly without a schema.
"""
if self._pa_schema is None:
raise ValueError("Create the format with from_schema() or from_pandas() to access its Arrow schema")
return self._pa_schema
Comment thread
wordsworthc marked this conversation as resolved.
Resolved

@classmethod
def from_schema(cls, pa_schema: pa.Schema) -> ArrowTableFormat:
"""Generate an ArrowTableFormat instance that represents the structure of the provided table schema.

:param pa_schema: Table schema to generate a format representation for.
:param pa_schema: Table schema to generate a format representation for. Schema and field metadata are not kept.

:return: A table format that can be used to compare against known formats.
"""
return ArrowTableFormat(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types)
table_format = cls(name=f"{cls.__name__}<{'/'.join(pa_schema.names)}>", columns=pa_schema.types)
table_format._pa_schema = pa.schema([field.remove_metadata() for field in pa_schema])
return table_format

@classmethod
def from_pandas(cls, dataframe: pd.DataFrame) -> ArrowTableFormat:
"""Infer a table format from a pandas dataframe using the supported Geoscience Object column types.

Large strings are normalized to strings. The inferred schema excludes the dataframe index and pandas metadata.

:param dataframe: The dataframe to infer the format from.
:return: A format whose `pa_schema` can be passed to `pyarrow.Table.from_pandas`.
:raises TypeError: If a column type is not supported. Use `ObjectDataClient.upload_category_dataframe`
for categorical columns, which are split into value and lookup tables separately.
"""
schema = pa.Schema.from_pandas(dataframe, preserve_index=False)
fields = [field.with_type(pa.string()) if pa.types.is_large_string(field.type) else field for field in schema]
return cls.from_schema(pa.schema(fields))

def pandas_types_mapper(self) -> Callable[[pa.DataType], pd.api.extensions.ExtensionDtype | None]:
"""Get a pyarrow `types_mapper` that converts this format's columns to their `ColumnFormat.pandas_dtype`.

Pass the result to `table.replace_schema_metadata().to_pandas(types_mapper=...)` to ignore legacy metadata.

:raises ImportError: If pandas is not installed.
"""
return {column.type: column.pandas_dtype for column in self._columns}.get


def table_to_dataframe(table: pa.Table) -> pd.DataFrame:
"""Convert a table to a pandas dataframe using the Geoscience Object column types, ignoring pandas metadata.

:param table: The table to convert. It is expected to have been validated against a known table format.
"""
types_mapper = ArrowTableFormat.from_schema(table.schema).pandas_types_mapper()
# PyArrow parses pandas metadata even with ignore_metadata=True.
return table.replace_schema_metadata().to_pandas(types_mapper=types_mapper)


class KnownTableFormat(BaseTableFormat):
Expand Down
Loading