Skip to content

Commit f81b827

Browse files
rahulsmahadevIsaac
andcommitted
Parse scientific notation and mixed int/decimal IN lists in filters
The row filter parser only accepted decimals with a fractional part and literal sets of a single numeric kind, so `x > 1e-3` and `x in (1, 2.5)` raised a ParseException. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 9eb8395 commit f81b827

2 files changed

Lines changed: 24 additions & 4 deletions

File tree

‎pyiceberg/expressions/parser.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,12 @@ def _(result: ParseResults) -> Reference:
105105

106106
boolean = one_of(["true", "false"], caseless=True)
107107
string = sgl_quoted_string.set_results_name("raw_quoted_string")
108-
decimal = common.real().set_results_name("decimal")
108+
decimal = common.sci_real().set_results_name("decimal")
109109
integer = common.signed_integer().set_results_name("integer")
110110
literal = Group(string | decimal | integer | boolean).set_results_name("literal")
111-
literal_set = Group(
112-
DelimitedList(string) | DelimitedList(decimal) | DelimitedList(integer) | DelimitedList(boolean)
113-
).set_results_name("literal_set")
111+
literal_set = Group(DelimitedList(string) | DelimitedList(decimal | integer) | DelimitedList(boolean)).set_results_name(
112+
"literal_set"
113+
)
114114

115115

116116
@boolean.set_parse_action

‎tests/expressions/test_parser.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,26 @@ def test_not_in_different_types() -> None:
146146
parser.parse("foo not in (5, 'a')")
147147

148148

149+
def test_in_mixed_numbers() -> None:
150+
for values in ("5, 6.5", "6.5, 5"):
151+
result = parser.parse(f"foo in ({values})")
152+
assert result == In("foo", {LongLiteral(5), DecimalLiteral(Decimal("6.5"))}), values
153+
assert isinstance(result, In)
154+
assert {type(value) for value in result.literals} == {LongLiteral, DecimalLiteral}, values
155+
assert NotIn("foo", {LongLiteral(5), DecimalLiteral(Decimal("1e3"))}) == parser.parse("foo not in (5, 1e3)")
156+
157+
158+
def test_scientific_notation() -> None:
159+
for expression, expected in [
160+
("foo > 1e-3", GreaterThan("foo", Decimal("0.001"))),
161+
("1e-3 < foo", GreaterThan("foo", Decimal("0.001"))),
162+
("foo < -1.5E3", LessThan("foo", Decimal("-1500"))),
163+
("foo = 2e2", EqualTo("foo", Decimal("200"))),
164+
("foo between 1e1 and 2.5e1", And(GreaterThanOrEqual("foo", Decimal("10")), LessThanOrEqual("foo", Decimal("25")))),
165+
]:
166+
assert parser.parse(expression) == expected, expression
167+
168+
149169
def test_simple_and() -> None:
150170
assert And(GreaterThanOrEqual("foo", 5), LessThan("foo", 10)) == parser.parse("5 <= foo and foo < 10")
151171

0 commit comments

Comments
 (0)