From 22dc94f6c753a81d9362a937823a541fa037f67b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Fri, 30 Oct 2026 15:05:23 +0100 Subject: [PATCH 1/2] GH-3913: Avoid Comparing Equal Binary Statistics Values BinaryStatistics.updateStats(Binary) compares each value with the current minimum and maximum. When the value is the same instance as one of these bounds, the result of the comparison is always equal. This occurs when a writer sends the same constant Binary instance again, for example values decoded from a dictionary. Update BinaryStatistics in the parquet-column module to skip the comparisons when the value is the same instance as the minimum or the maximum. The resulting statistics do not change. Add tests in TestStatistics that make sure the behavior does not change: a single constant instance that repeats, bound instances that repeat, and other instances with equal, lower, or higher values. Unit tests cannot detect the skipped comparisons, because the result is the same with or without them. --- .../column/statistics/BinaryStatistics.java | 17 ++++--- .../column/statistics/TestStatistics.java | 46 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java index b171106804..e31c170ecc 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java @@ -57,12 +57,17 @@ public void updateStats(Binary value) { min = value.copy(); max = value.copy(); this.markAsNotEmpty(); - } else { - if (comparator().compare(min, value) > 0) { - min = value.copy(); - } else if (comparator().compare(max, value) < 0) { - max = value.copy(); - } + return; + } + // copy() keeps the same instance when the backing bytes are not reused. + // Thus, a value can be the same instance as min or max. That value is equal to the bound, so do not compare it. + if (value == min || value == max) { + return; + } + if (comparator().compare(min, value) > 0) { + min = value.copy(); + } else if (comparator().compare(max, value) < 0) { + max = value.copy(); } } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java index e4783b700e..83d59512db 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java @@ -521,6 +521,52 @@ public void testBinaryMinMaxForReusedBackingByteArray() { assertThat(stats.getMinBytes()).isEqualTo(new byte[] {10}); } + @Test + public void testBinaryMinMaxForRepeatedSingleConstantInstance() { + BinaryStatistics stats = new BinaryStatistics(); + + Binary value = Binary.fromConstantByteArray(new byte[] {7}); + stats.updateStats(value); + stats.updateStats(value); + stats.updateStats(value); + + assertThat(stats.genericGetMin()).isSameAs(value); + assertThat(stats.genericGetMax()).isSameAs(value); + } + + @Test + public void testBinaryMinMaxKeepsBoundInstancesForConstantValues() { + BinaryStatistics stats = new BinaryStatistics(); + + Binary low = Binary.fromConstantByteArray(new byte[] {10}); + Binary high = Binary.fromConstantByteArray(new byte[] {30}); + stats.updateStats(low); + stats.updateStats(high); + // The bounds keep the constant instances. An update with a bound instance does not change the bounds. + assertThat(stats.genericGetMin()).isSameAs(low); + assertThat(stats.genericGetMax()).isSameAs(high); + stats.updateStats(low); + assertThat(stats.genericGetMin()).isSameAs(low); + assertThat(stats.genericGetMax()).isSameAs(high); + stats.updateStats(high); + assertThat(stats.genericGetMin()).isSameAs(low); + assertThat(stats.genericGetMax()).isSameAs(high); + stats.updateStats(Binary.fromConstantByteArray(new byte[] {20})); + assertThat(stats.genericGetMin()).isSameAs(low); + assertThat(stats.genericGetMax()).isSameAs(high); + + // A different instance with equal content does not change the bounds. Lower or higher values change them. + stats.updateStats(Binary.fromConstantByteArray(new byte[] {10})); + assertThat(stats.genericGetMin()).isSameAs(low); + assertThat(stats.genericGetMax()).isSameAs(high); + stats.updateStats(Binary.fromConstantByteArray(new byte[] {5})); + assertThat(stats.getMinBytes()).isEqualTo(new byte[] {5}); + assertThat(stats.genericGetMax()).isSameAs(high); + stats.updateStats(Binary.fromConstantByteArray(new byte[] {40})); + assertThat(stats.getMinBytes()).isEqualTo(new byte[] {5}); + assertThat(stats.getMaxBytes()).isEqualTo(new byte[] {40}); + } + @Test public void testMergingStatistics() { testMergingIntStats(); From 723d973d57f658518dc4ad55724cc3231eaec2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rou=C3=A9l?= Date: Mon, 2 Nov 2026 21:48:43 +0100 Subject: [PATCH 2/2] GH-3913: Verify That Bound Instances Do Not Call the Comparator The tests for repeated bound instances passed also without the fast path in BinaryStatistics.updateStats(Binary). Thus, the tests did not protect the optimization. Use Mockito spies for the repeated instances in TestStatistics. After the first updates, clear the recorded calls, repeat the bound instances, and make sure that no method is called on them. A comparison always calls methods on the value. If the fast path is removed, both tests fail. --- .../column/statistics/TestStatistics.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java index 83d59512db..ea9b9f7ea4 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatistics.java @@ -36,6 +36,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.data.Offset.offset; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verifyNoInteractions; import java.nio.ByteBuffer; import java.util.Locale; @@ -525,11 +528,15 @@ public void testBinaryMinMaxForReusedBackingByteArray() { public void testBinaryMinMaxForRepeatedSingleConstantInstance() { BinaryStatistics stats = new BinaryStatistics(); - Binary value = Binary.fromConstantByteArray(new byte[] {7}); + // A spy records each call. A comparison calls methods on the value, so no calls means no comparison. + Binary value = spy(Binary.fromConstantByteArray(new byte[] {7})); stats.updateStats(value); + clearInvocations(value); + stats.updateStats(value); stats.updateStats(value); + verifyNoInteractions(value); assertThat(stats.genericGetMin()).isSameAs(value); assertThat(stats.genericGetMax()).isSameAs(value); } @@ -538,17 +545,17 @@ public void testBinaryMinMaxForRepeatedSingleConstantInstance() { public void testBinaryMinMaxKeepsBoundInstancesForConstantValues() { BinaryStatistics stats = new BinaryStatistics(); - Binary low = Binary.fromConstantByteArray(new byte[] {10}); - Binary high = Binary.fromConstantByteArray(new byte[] {30}); + Binary low = spy(Binary.fromConstantByteArray(new byte[] {10})); + Binary high = spy(Binary.fromConstantByteArray(new byte[] {30})); stats.updateStats(low); stats.updateStats(high); - // The bounds keep the constant instances. An update with a bound instance does not change the bounds. + // The bounds keep the constant instances. An update with a bound instance does not compare the values. assertThat(stats.genericGetMin()).isSameAs(low); assertThat(stats.genericGetMax()).isSameAs(high); + clearInvocations(low, high); stats.updateStats(low); - assertThat(stats.genericGetMin()).isSameAs(low); - assertThat(stats.genericGetMax()).isSameAs(high); stats.updateStats(high); + verifyNoInteractions(low, high); assertThat(stats.genericGetMin()).isSameAs(low); assertThat(stats.genericGetMax()).isSameAs(high); stats.updateStats(Binary.fromConstantByteArray(new byte[] {20}));