Skip to content

Commit bccbed0

Browse files
rahulsmahadevIsaac
andcommitted
Parse exponent literals as doubles and convert whole doubles to integer types
A number with an exponent is an approximate numeric literal in ANSI SQL, and Spark, Trino and DuckDB parse 1e-3 as a double. As a DecimalLiteral, 1e3 has scale -3 and could not bind to decimal columns. Parse the exponent forms as DoubleLiteral instead, and convert a whole DoubleLiteral to int and long columns (a fractional value raises, as for decimals) so that long_col = 1e3 binds. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent f81b827 commit bccbed0

4 files changed

Lines changed: 84 additions & 13 deletions

File tree

‎pyiceberg/expressions/literals.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,16 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]:
445445
raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite")
446446
return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP))
447447

448+
@to.register(IntegerType)
449+
@to.register(LongType)
450+
def _(self, type_var: IntegerType | LongType) -> Literal[int]:
451+
if not isfinite(self.value):
452+
raise ValueError(f"Could not convert {self.value} into a {type_var}, value is not finite")
453+
if not self.value.is_integer():
454+
raise ValueError(f"Could not convert {self.value} into a {type_var}, value has a fractional part")
455+
# The decimal conversion checks the range of the column type
456+
return DecimalLiteral(Decimal(self.value)).to(type_var)
457+
448458

449459
class DateLiteral(Literal[int]):
450460
def __init__(self, value: int) -> None:

‎pyiceberg/expressions/parser.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
ParserElement,
2727
ParseResults,
2828
QuotedString,
29+
Regex,
2930
Suppress,
3031
Word,
3132
alphanums,
@@ -62,6 +63,7 @@
6263
from pyiceberg.expressions.literals import (
6364
BooleanLiteral,
6465
DecimalLiteral,
66+
DoubleLiteral,
6567
Literal,
6668
LongLiteral,
6769
StringLiteral,
@@ -105,10 +107,12 @@ def _(result: ParseResults) -> Reference:
105107

106108
boolean = one_of(["true", "false"], caseless=True)
107109
string = sgl_quoted_string.set_results_name("raw_quoted_string")
108-
decimal = common.sci_real().set_results_name("decimal")
110+
# A number with an exponent is an approximate numeric literal (a double), as in ANSI SQL
111+
double = Regex(r"[+-]?(?:\d+\.?\d*|\.\d+)[eE][+-]?\d+").set_results_name("double")
112+
decimal = common.real().set_results_name("decimal")
109113
integer = common.signed_integer().set_results_name("integer")
110-
literal = Group(string | decimal | integer | boolean).set_results_name("literal")
111-
literal_set = Group(DelimitedList(string) | DelimitedList(decimal | integer) | DelimitedList(boolean)).set_results_name(
114+
literal = Group(string | double | decimal | integer | boolean).set_results_name("literal")
115+
literal_set = Group(DelimitedList(string) | DelimitedList(double | decimal | integer) | DelimitedList(boolean)).set_results_name(
112116
"literal_set"
113117
)
114118

@@ -126,6 +130,11 @@ def _(result: ParseResults) -> Literal[str]:
126130
return StringLiteral(result.raw_quoted_string[1:-1].replace("''", "'"))
127131

128132

133+
@double.set_parse_action
134+
def _(result: ParseResults) -> Literal[float]:
135+
return DoubleLiteral(float(result.double))
136+
137+
129138
@decimal.set_parse_action
130139
def _(result: ParseResults) -> Literal[Decimal]:
131140
return DecimalLiteral(Decimal(result.decimal))

‎tests/expressions/test_literals.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,21 @@ def test_double_to_float_outside_bound() -> None:
268268
assert below_min_lit == FloatBelowMin()
269269

270270

271+
def test_double_to_integer_and_long_conversion() -> None:
272+
for type_var in (IntegerType(), LongType()):
273+
assert DoubleLiteral(1e3).to(type_var) == LongLiteral(1000)
274+
assert DoubleLiteral(-2.0).to(type_var) == LongLiteral(-2)
275+
with pytest.raises(ValueError, match="value has a fractional part"):
276+
DoubleLiteral(1.5).to(type_var)
277+
with pytest.raises(ValueError, match="value is not finite"):
278+
DoubleLiteral(float("inf")).to(type_var)
279+
# A float literal still does not convert to an integer type
280+
with pytest.raises(TypeError):
281+
FloatLiteral(1e3).to(type_var)
282+
assert DoubleLiteral(1e10).to(IntegerType()) == IntAboveMax()
283+
assert DoubleLiteral(-1e19).to(LongType()) == LongBelowMin()
284+
285+
271286
@pytest.mark.parametrize(
272287
"decimal_type, decimal_value", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")]
273288
)
@@ -703,8 +718,6 @@ def test_invalid_long_conversions() -> None:
703718
"test_type",
704719
[
705720
BooleanType(),
706-
IntegerType(),
707-
LongType(),
708721
DateType(),
709722
TimeType(),
710723
TimestampType(),

‎tests/expressions/test_parser.py‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@
4343
Reference,
4444
StartsWith,
4545
)
46-
from pyiceberg.expressions.literals import DecimalLiteral, LongLiteral, literal
46+
from pyiceberg.expressions.literals import DecimalLiteral, DoubleLiteral, LongLiteral, literal
47+
from pyiceberg.expressions.visitors import bind
48+
from pyiceberg.schema import Schema
49+
from pyiceberg.types import DecimalType, DoubleType, LongType, NestedField
4750

4851

4952
def test_always_true() -> None:
@@ -152,18 +155,54 @@ def test_in_mixed_numbers() -> None:
152155
assert result == In("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5"))}), values
153156
assert isinstance(result, In)
154157
assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral}, values
155-
assert NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("1e3"))}) == parser.parse("foo not in (5, 1e3)")
158+
159+
result = parser.parse("foo not in (5, 6.5, 1e3)")
160+
assert result == NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5")), DoubleLiteral(1000.0)})
161+
assert isinstance(result, NotIn)
162+
assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral, DoubleLiteral}
156163

157164

158165
def test_scientific_notation() -> None:
166+
# A number with an exponent is a double, as in ANSI SQL
167+
for expression, expected in [
168+
("foo > 1e-3", GreaterThan("foo", 0.001)),
169+
("1e-3 < foo", GreaterThan("foo", 0.001)),
170+
("foo < -1.5E3", LessThan("foo", -1500.0)),
171+
("foo = 2e2", EqualTo("foo", 200.0)),
172+
("foo = .5e1", EqualTo("foo", 5.0)),
173+
]:
174+
result = parser.parse(expression)
175+
assert result == expected, expression
176+
assert isinstance(result.literal, DoubleLiteral), expression
177+
178+
assert And(GreaterThanOrEqual("foo", 10.0), LessThanOrEqual("foo", 25.0)) == parser.parse("foo between 1e1 and 2.5e1")
179+
180+
181+
def test_bind_numeric_literals() -> None:
182+
schema = Schema(
183+
NestedField(1, "dec", DecimalType(9, 2), required=False),
184+
NestedField(2, "lng", LongType(), required=False),
185+
NestedField(3, "dbl", DoubleType(), required=False),
186+
)
159187
for expression, expected in [
160-
("foo > 1e-3", GreaterThan("foo", Decimal("0.001"))),
161-
("1e-3 < foo", GreaterThan("foo", Decimal("0.001"))),
162-
("foo < -1.5E3", LessThan("foo", Decimal("-1500"))),
163-
("foo = 2e2", EqualTo("foo", Decimal("200"))),
164-
("foo between 1e1 and 2.5e1", And(GreaterThanOrEqual("foo", Decimal("10")), LessThanOrEqual("foo", Decimal("25")))),
188+
("dec = 1e3", DecimalLiteral(Decimal("1000.00"))),
189+
("dec > 2.5e-1", DecimalLiteral(Decimal("0.25"))),
190+
("dec = 12.34", DecimalLiteral(Decimal("12.34"))),
191+
("lng = 1e3", LongLiteral(1000)),
192+
("lng = 7", LongLiteral(7)),
193+
("dbl > 1e-3", DoubleLiteral(0.001)),
194+
("dbl > 2.5", DoubleLiteral(2.5)),
165195
]:
166-
assert parser.parse(expression) == expected, expression
196+
bound = bind(schema, parser.parse(expression), case_sensitive=True)
197+
assert bound.literal == expected, expression
198+
assert type(bound.literal) is type(expected), expression
199+
if isinstance(expected, DecimalLiteral):
200+
assert bound.literal.value.as_tuple() == expected.value.as_tuple(), expression
201+
202+
bound = bind(schema, parser.parse("lng in (1, 2e1, 3)"), case_sensitive=True)
203+
assert bound.literals == {LongLiteral(1), LongLiteral(20), LongLiteral(3)}
204+
with pytest.raises(ValueError, match="value has a fractional part"):
205+
bind(schema, parser.parse("lng = 1.5e0"), case_sensitive=True)
167206

168207

169208
def test_simple_and() -> None:

0 commit comments

Comments
 (0)