diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index d29738a6d8..baf9a5586c 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -1135,13 +1135,15 @@ public PageReadStore readRowGroup(int blockIndex) throws IOException { * @throws IOException if an error occurs while reading */ public PageReadStore readNextRowGroup() throws IOException { - ColumnChunkPageReadStore rowGroup = null; - try { - rowGroup = internalReadRowGroup(currentBlock); - } catch (ParquetEmptyBlockException e) { - LOG.warn("Read empty block at index {} from {}", currentBlock, getFile()); - advanceToNextBlock(); - return readNextRowGroup(); + ColumnChunkPageReadStore rowGroup; + while (true) { + try { + rowGroup = internalReadRowGroup(currentBlock); + break; + } catch (ParquetEmptyBlockException e) { + LOG.warn("Read empty block at index {} from {}", currentBlock, getFile()); + advanceToNextBlock(); + } } closeCurrentRowGroup(); @@ -1531,36 +1533,39 @@ public PageReadStore readNextFilteredRowGroup() throws IOException { if (!options.useColumnIndexFilter() || !FilterCompat.isFilteringRequired(options.getRecordFilter())) { return readNextRowGroup(); } - BlockMetaData block = blocks.get(currentBlock); - if (block.getRowCount() == 0L) { - LOG.warn("Read empty block at index {} from {}", currentBlock, getFile()); - // Skip the empty block - advanceToNextBlock(); - return readNextFilteredRowGroup(); - } - RowRanges rowRanges = getRowRanges(currentBlock); - long rowCount = rowRanges.rowCount(); - if (rowCount == 0) { - // There are no matching rows -> skipping this row-group - advanceToNextBlock(); - return readNextFilteredRowGroup(); - } - if (rowCount == block.getRowCount()) { - // All rows are matching -> fall back to the non-filtering path - return readNextRowGroup(); - } + while (currentBlock < blocks.size()) { + BlockMetaData block = blocks.get(currentBlock); + if (block.getRowCount() == 0L) { + LOG.warn("Read empty block at index {} from {}", currentBlock, getFile()); + // Skip the empty block + advanceToNextBlock(); + continue; + } + RowRanges rowRanges = getRowRanges(currentBlock); + long rowCount = rowRanges.rowCount(); + if (rowCount == 0) { + // There are no matching rows -> skipping this row-group + advanceToNextBlock(); + continue; + } + if (rowCount == block.getRowCount()) { + // All rows are matching -> fall back to the non-filtering path + return readNextRowGroup(); + } - closeCurrentRowGroup(); - this.currentRowGroup = internalReadFilteredRowGroup(block, rowRanges, getColumnIndexStore(currentBlock)); + closeCurrentRowGroup(); + this.currentRowGroup = internalReadFilteredRowGroup(block, rowRanges, getColumnIndexStore(currentBlock)); - // avoid re-reading bytes the dictionary reader is used after this call - if (nextDictionaryReader != null) { - nextDictionaryReader.setRowGroup(currentRowGroup); - } + // avoid re-reading bytes the dictionary reader is used after this call + if (nextDictionaryReader != null) { + nextDictionaryReader.setRowGroup(currentRowGroup); + } - advanceToNextBlock(); + advanceToNextBlock(); - return this.currentRowGroup; + return this.currentRowGroup; + } + return null; } private ColumnChunkPageReadStore internalReadFilteredRowGroup( diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetReaderEmptyBlock.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetReaderEmptyBlock.java index 1901f8160f..74837f7c92 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetReaderEmptyBlock.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetReaderEmptyBlock.java @@ -24,17 +24,33 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.filter2.compat.FilterCompat; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.parquet.io.InputFile; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class TestParquetReaderEmptyBlock { + private static final int SKIPPED_GROUPS = 50_000; + + @TempDir + private java.nio.file.Path tempDir; // The parquet file contains only one empty row group private static final Path EMPTY_BLOCK_FILE_1 = createPathFromCP("/test-empty-row-group_1.parquet"); @@ -165,4 +181,110 @@ public void testSkipEmptyBlocksNextToEachOther() throws IOException { assertThat(pages).isNull(); } } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testSkipManyEmptyBlocks(boolean filtered) throws IOException { + InputFile inputFile = HadoopInputFile.fromPath(EMPTY_BLOCK_FILE_3, new Configuration()); + ParquetReadOptions options = readOptions(filtered); + ParquetMetadata footer; + try (ParquetFileReader reader = new ParquetFileReader(inputFile, options)) { + footer = reader.getFooter(); + } + List source = footer.getBlocks(); + // Repeat existing footer entries to exercise long runs without adding a large binary fixture. + List blocks = new ArrayList<>(); + blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, source.get(1))); + blocks.add(source.get(0)); + blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, source.get(1))); + blocks.add(source.get(3)); + blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, source.get(1))); + ParquetMetadata repeatedFooter = new ParquetMetadata(footer.getFileMetaData(), blocks); + try (ParquetFileReader reader = + new ParquetFileReader(inputFile, repeatedFooter, options, inputFile.newStream())) { + assertThat(reader.getRowGroups()).hasSize(3 * SKIPPED_GROUPS + 2); + PageReadStore first = filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup(); + assertThat(first).isNotNull(); + assertThat(first.getRowCount()).isEqualTo(1); + assertThat(reader.getCurrentRowGroupIndex()).isEqualTo(SKIPPED_GROUPS); + PageReadStore last = filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup(); + assertThat(last).isNotNull(); + assertThat(last.getRowCount()).isEqualTo(4); + assertThat(reader.getCurrentRowGroupIndex()).isEqualTo(2 * SKIPPED_GROUPS + 1); + assertThat(filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup()) + .isNull(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testReadManyOnlyEmptyBlocks(boolean filtered) throws IOException { + InputFile inputFile = HadoopInputFile.fromPath(EMPTY_BLOCK_FILE_1, new Configuration()); + ParquetReadOptions options = readOptions(filtered); + ParquetMetadata footer; + try (ParquetFileReader reader = new ParquetFileReader(inputFile, options)) { + footer = reader.getFooter(); + } + ParquetMetadata repeatedFooter = new ParquetMetadata( + footer.getFileMetaData(), + Collections.nCopies(SKIPPED_GROUPS, footer.getBlocks().get(0))); + try (ParquetFileReader reader = + new ParquetFileReader(inputFile, repeatedFooter, options, inputFile.newStream())) { + assertThat(reader.getRowGroups()).hasSize(SKIPPED_GROUPS); + assertThat(filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup()) + .isNull(); + } + } + + @Test + public void testSkipManyNonMatchingBlocks() throws IOException { + Path path = new Path(tempDir.resolve("column-index.parquet").toUri()); + MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; }"); + SimpleGroupFactory groups = new SimpleGroupFactory(schema); + try (ParquetWriter writer = ExampleParquetWriter.builder(path) + .withType(schema) + .withRowGroupSize(1) + .withMinRowCountForPageSizeCheck(1) + .withMaxRowCountForPageSizeCheck(1) + .build()) { + writer.write(groups.newGroup().append("a", 0)); + writer.write(groups.newGroup().append("a", 2)); + } + InputFile inputFile = HadoopInputFile.fromPath(path, new Configuration()); + ParquetReadOptions options = ParquetReadOptions.builder() + .copy(readOptions(true)) + .withRecordFilter(FilterCompat.get(gt(intColumn("a"), 1))) + .build(); + ParquetMetadata footer; + try (ParquetFileReader reader = new ParquetFileReader(inputFile, options)) { + footer = reader.getFooter(); + } + assertThat(footer.getBlocks()).hasSize(2); + List blocks = new ArrayList<>(); + blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, footer.getBlocks().get(0))); + blocks.add(footer.getBlocks().get(1)); + blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, footer.getBlocks().get(0))); + ParquetMetadata repeatedFooter = new ParquetMetadata(footer.getFileMetaData(), blocks); + try (ParquetFileReader reader = + new ParquetFileReader(inputFile, repeatedFooter, options, inputFile.newStream())) { + assertThat(reader.getRowGroups()).hasSize(2 * SKIPPED_GROUPS + 1); + PageReadStore pages = reader.readNextFilteredRowGroup(); + assertThat(pages).isNotNull(); + assertThat(pages.getRowCount()).isEqualTo(1); + assertThat(reader.getCurrentRowGroupIndex()).isEqualTo(SKIPPED_GROUPS); + assertThat(reader.readNextFilteredRowGroup()).isNull(); + } + } + + private static ParquetReadOptions readOptions(boolean filtered) { + ParquetReadOptions.Builder builder = ParquetReadOptions.builder() + .useStatsFilter(false) + .useDictionaryFilter(false) + .useBloomFilter(false) + .useColumnIndexFilter(true); + if (filtered) { + builder.withRecordFilter(FilterCompat.get(gt(intColumn("a"), -1))); + } + return builder.build(); + } }