Skip to content

Commit aeaddb1

Browse files
Order -0.0 below 0.0 in data file metrics bounds (#4331)
* Order -0.0 below 0.0 in data file metrics bounds StatsAggregator.update_min/update_max feed DataFile lower/upper bounds, but min(val, current) and max(val, current) keep the first argument on a tie, so whichever zero arrives last wins: a late -0.0 lowers an existing +0.0 upper bound, and a late +0.0 raises an existing -0.0 lower bound. The serialized bounds then order above a value present in the file. Mirror the fix that #4324 applied to partition field summaries: restore the stored bound when a zero arrives that would invert the -0.0 < 0.0 ordering. The argument order here (val first) flips which direction needs the guard compared to manifest.py. Verified failing-first: the new tests fail on main for both arrival orders and both float types, and pass with the guard. Signed-off-by: Ayush Kumar Saha <225407166+AP24110011695@users.noreply.github.com> * Simplify the negative-zero guards per review Flip the min/max argument order so a tie keeps the existing value (the same shape as PartitionFieldStats.update in manifest.py), which lets the guard only handle an incoming zero of the wrong sign and drops the previous-value capture. Behavior verified unchanged: the regression tests and all zero-arrival interleavings still produce the canonical -0.0-below-0.0 bounds. Signed-off-by: Ayush Kumar Saha <225407166+AP24110011695@users.noreply.github.com> --------- Signed-off-by: Ayush Kumar Saha <225407166+AP24110011695@users.noreply.github.com> Co-authored-by: Ayush Kumar Saha <225407166+AP24110011695@users.noreply.github.com>
1 parent 0008d65 commit aeaddb1

2 files changed

Lines changed: 32 additions & 2 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2402,13 +2402,19 @@ def update_min(self, val: Any | None) -> None:
24022402
if self.current_min is None:
24032403
self.current_min = val
24042404
elif val is not None:
2405-
self.current_min = min(val, self.current_min)
2405+
self.current_min = min(self.current_min, val)
2406+
if isinstance(val, float) and val == 0.0 and math.copysign(1.0, val) < 0 and self.current_min == 0.0:
2407+
# min keeps the existing zero on a tie, but bounds must order -0.0 below 0.0
2408+
self.current_min = val
24062409

24072410
def update_max(self, val: Any | None) -> None:
24082411
if self.current_max is None:
24092412
self.current_max = val
24102413
elif val is not None:
2411-
self.current_max = max(val, self.current_max)
2414+
self.current_max = max(self.current_max, val)
2415+
if isinstance(val, float) and val == 0.0 and math.copysign(1.0, val) > 0 and self.current_max == 0.0:
2416+
# max keeps the existing zero on a tie, but bounds must order -0.0 below 0.0
2417+
self.current_max = val
24122418

24132419
def min_as_bytes(self) -> bytes | None:
24142420
if self.current_min is None:

‎tests/io/test_pyarrow.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
from pyroaring import BitMap
4545

4646
from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTDeleteFile, RESTFileScanTask, RESTPositionDeleteFile
47+
from pyiceberg.conversions import to_bytes
4748
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
4849
from pyiceberg.encryption.stream import _encrypt_ags1
4950
from pyiceberg.exceptions import ResolveError
@@ -2954,6 +2955,29 @@ def test_stats_aggregator_update_max(vals: list[Any], primitive_type: PrimitiveT
29542955
assert stats.current_max == expected_result
29552956

29562957

2958+
@pytest.mark.parametrize(
2959+
"vals, primitive_type",
2960+
[
2961+
([0.0, -0.0], DoubleType()),
2962+
([-0.0, 0.0], DoubleType()),
2963+
([0.0, -0.0], FloatType()),
2964+
([-0.0, 0.0], FloatType()),
2965+
],
2966+
)
2967+
def test_stats_aggregator_orders_negative_zero_below_zero(vals: list[float], primitive_type: PrimitiveType) -> None:
2968+
# min/max keep whichever zero came first, but the serialized bounds must
2969+
# order -0.0 below 0.0, whichever order the values arrive in
2970+
stats_min = StatsAggregator(primitive_type, _primitive_to_physical(primitive_type))
2971+
stats_max = StatsAggregator(primitive_type, _primitive_to_physical(primitive_type))
2972+
2973+
for val in vals:
2974+
stats_min.update_min(val)
2975+
stats_max.update_max(val)
2976+
2977+
assert stats_min.min_as_bytes() == to_bytes(primitive_type, -0.0)
2978+
assert stats_max.max_as_bytes() == to_bytes(primitive_type, 0.0)
2979+
2980+
29572981
@pytest.mark.parametrize(
29582982
"iceberg_type, physical_type_string",
29592983
[

0 commit comments

Comments
 (0)