diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index f62e53a911..1e90fd0de3 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -41,6 +41,7 @@ from collections.abc import Callable, Iterable, Iterator from copy import copy from dataclasses import dataclass +from decimal import Decimal from enum import Enum from functools import lru_cache, singledispatch from typing import ( @@ -886,10 +887,22 @@ def visit_geography(self, geography_type: GeographyType) -> pa.DataType: return pa.large_binary() +def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa.DataType: + """Return the Arrow type for literals compared with a column of the given type. + + A decimal literal can have more digits than the column, e.g. 1000.00 for decimal(5, 2), + so the precision is widened to hold every literal. The scale always stays the column's. + """ + if isinstance(iceberg_type, DecimalType): + digits = [len(value.as_tuple().digits) for value in values if isinstance(value, Decimal)] + return pa.decimal128(max([iceberg_type.precision, *digits]), iceberg_type.scale) + return visit(iceberg_type, _ConvertToArrowSchema()) + + def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value=value, type=schema_to_pyarrow(iceberg_type)) + return pa.scalar(value=value, type=_literal_arrow_type(iceberg_type, [value])) class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): @@ -926,11 +939,11 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]: return term.ref().field.name def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) return pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm) -> pc.Expression: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 419d130965..122e2f8cff 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -27,6 +27,7 @@ from collections.abc import Iterator from dataclasses import replace from datetime import UTC, date, datetime +from decimal import Decimal from pathlib import Path from typing import Any, Literal from unittest.mock import MagicMock, patch @@ -940,6 +941,49 @@ def test_always_false_to_pyarrow(bound_reference: BoundReference) -> None: assert repr(expression_to_pyarrow(AlwaysFalse())) == "" +def test_decimal_literal_wider_than_column_to_pyarrow() -> None: + schema = Schema(NestedField(1, "amt", DecimalType(5, 2), required=False)) + ref = BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + wide = literal(Decimal("1000.00")).to(DecimalType(5, 2)) + narrow = literal(Decimal("1.25")).to(DecimalType(5, 2)) + arrow_table = pa.table({"amt": pa.array([Decimal("1.25"), Decimal("99.99")], pa.decimal128(5, 2))}) + for bound_expr, expected in [ + (BoundLessThan(term=ref, literal=wide), [Decimal("1.25"), Decimal("99.99")]), + (BoundNotEqualTo(term=ref, literal=wide), [Decimal("1.25"), Decimal("99.99")]), + (BoundIn(term=ref, literals={wide, narrow}), [Decimal("1.25")]), + (BoundNotIn(term=ref, literals={wide, narrow}), [Decimal("99.99")]), + ]: + assert arrow_table.filter(expression_to_pyarrow(bound_expr))["amt"].to_pylist() == expected, bound_expr + + +def test_row_filter_decimal_literal_wider_than_column(catalog: InMemoryCatalog) -> None: + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_decimal_literal_wider_than_column", + schema=Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "amt", DecimalType(5, 2), required=False), + ), + ) + table.append( + pa.table( + { + "id": pa.array([1, 2, 3], pa.int32()), + "amt": pa.array([Decimal("1.25"), Decimal("99.99"), None], pa.decimal128(5, 2)), + } + ) + ) + + # 1000.00 needs one more digit than decimal(5, 2) holds, but every value is below it + for row_filter, expected in [ + ("amt < 1000.00", [1, 2]), + ("amt > -1000.00", [1, 2]), + ("amt < 50.00", [1]), + ("amt IN (1.25, 1000.00)", [1]), + ]: + assert sorted(table.scan(row_filter=row_filter).to_arrow()["id"].to_pylist()) == expected, row_filter + + @pytest.fixture def schema_int() -> Schema: return Schema(NestedField(1, "id", IntegerType(), required=False))