Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1135,13 +1135,15 @@ public PageReadStore readRowGroup(int blockIndex) throws IOException {
* @throws IOException if an error occurs while reading
*/
public PageReadStore readNextRowGroup() throws IOException {
ColumnChunkPageReadStore rowGroup = null;
try {
rowGroup = internalReadRowGroup(currentBlock);
} catch (ParquetEmptyBlockException e) {
LOG.warn("Read empty block at index {} from {}", currentBlock, getFile());
advanceToNextBlock();
return readNextRowGroup();
ColumnChunkPageReadStore rowGroup;
while (true) {
try {
rowGroup = internalReadRowGroup(currentBlock);
break;
} catch (ParquetEmptyBlockException e) {
LOG.warn("Read empty block at index {} from {}", currentBlock, getFile());
advanceToNextBlock();
}
}

closeCurrentRowGroup();
Expand Down Expand Up @@ -1531,36 +1533,39 @@ public PageReadStore readNextFilteredRowGroup() throws IOException {
if (!options.useColumnIndexFilter() || !FilterCompat.isFilteringRequired(options.getRecordFilter())) {
return readNextRowGroup();
}
BlockMetaData block = blocks.get(currentBlock);
if (block.getRowCount() == 0L) {
LOG.warn("Read empty block at index {} from {}", currentBlock, getFile());
// Skip the empty block
advanceToNextBlock();
return readNextFilteredRowGroup();
}
RowRanges rowRanges = getRowRanges(currentBlock);
long rowCount = rowRanges.rowCount();
if (rowCount == 0) {
// There are no matching rows -> skipping this row-group
advanceToNextBlock();
return readNextFilteredRowGroup();
}
if (rowCount == block.getRowCount()) {
// All rows are matching -> fall back to the non-filtering path
return readNextRowGroup();
}
while (currentBlock < blocks.size()) {
BlockMetaData block = blocks.get(currentBlock);
if (block.getRowCount() == 0L) {
LOG.warn("Read empty block at index {} from {}", currentBlock, getFile());
// Skip the empty block
advanceToNextBlock();
continue;
}
RowRanges rowRanges = getRowRanges(currentBlock);
long rowCount = rowRanges.rowCount();
if (rowCount == 0) {
// There are no matching rows -> skipping this row-group
advanceToNextBlock();
continue;
}
if (rowCount == block.getRowCount()) {
// All rows are matching -> fall back to the non-filtering path
return readNextRowGroup();
}

closeCurrentRowGroup();
this.currentRowGroup = internalReadFilteredRowGroup(block, rowRanges, getColumnIndexStore(currentBlock));
closeCurrentRowGroup();
this.currentRowGroup = internalReadFilteredRowGroup(block, rowRanges, getColumnIndexStore(currentBlock));

// avoid re-reading bytes the dictionary reader is used after this call
if (nextDictionaryReader != null) {
nextDictionaryReader.setRowGroup(currentRowGroup);
}
// avoid re-reading bytes the dictionary reader is used after this call
if (nextDictionaryReader != null) {
nextDictionaryReader.setRowGroup(currentRowGroup);
}

advanceToNextBlock();
advanceToNextBlock();

return this.currentRowGroup;
return this.currentRowGroup;
}
return null;
}

private ColumnChunkPageReadStore internalReadFilteredRowGroup(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,33 @@

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.ParquetReadOptions;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.filter2.compat.FilterCompat;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.parquet.io.InputFile;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

public class TestParquetReaderEmptyBlock {
private static final int SKIPPED_GROUPS = 50_000;

@TempDir
private java.nio.file.Path tempDir;

// The parquet file contains only one empty row group
private static final Path EMPTY_BLOCK_FILE_1 = createPathFromCP("/test-empty-row-group_1.parquet");
Expand Down Expand Up @@ -165,4 +181,110 @@ public void testSkipEmptyBlocksNextToEachOther() throws IOException {
assertThat(pages).isNull();
}
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testSkipManyEmptyBlocks(boolean filtered) throws IOException {
InputFile inputFile = HadoopInputFile.fromPath(EMPTY_BLOCK_FILE_3, new Configuration());
ParquetReadOptions options = readOptions(filtered);
ParquetMetadata footer;
try (ParquetFileReader reader = new ParquetFileReader(inputFile, options)) {
footer = reader.getFooter();
}
List<BlockMetaData> source = footer.getBlocks();
// Repeat existing footer entries to exercise long runs without adding a large binary fixture.
List<BlockMetaData> blocks = new ArrayList<>();
blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, source.get(1)));
blocks.add(source.get(0));
blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, source.get(1)));
blocks.add(source.get(3));
blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, source.get(1)));
ParquetMetadata repeatedFooter = new ParquetMetadata(footer.getFileMetaData(), blocks);
try (ParquetFileReader reader =
new ParquetFileReader(inputFile, repeatedFooter, options, inputFile.newStream())) {
assertThat(reader.getRowGroups()).hasSize(3 * SKIPPED_GROUPS + 2);
PageReadStore first = filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup();
assertThat(first).isNotNull();
assertThat(first.getRowCount()).isEqualTo(1);
assertThat(reader.getCurrentRowGroupIndex()).isEqualTo(SKIPPED_GROUPS);
PageReadStore last = filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup();
assertThat(last).isNotNull();
assertThat(last.getRowCount()).isEqualTo(4);
assertThat(reader.getCurrentRowGroupIndex()).isEqualTo(2 * SKIPPED_GROUPS + 1);
assertThat(filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup())
.isNull();
}
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadManyOnlyEmptyBlocks(boolean filtered) throws IOException {
InputFile inputFile = HadoopInputFile.fromPath(EMPTY_BLOCK_FILE_1, new Configuration());
ParquetReadOptions options = readOptions(filtered);
ParquetMetadata footer;
try (ParquetFileReader reader = new ParquetFileReader(inputFile, options)) {
footer = reader.getFooter();
}
ParquetMetadata repeatedFooter = new ParquetMetadata(
footer.getFileMetaData(),
Collections.nCopies(SKIPPED_GROUPS, footer.getBlocks().get(0)));
try (ParquetFileReader reader =
new ParquetFileReader(inputFile, repeatedFooter, options, inputFile.newStream())) {
assertThat(reader.getRowGroups()).hasSize(SKIPPED_GROUPS);
assertThat(filtered ? reader.readNextFilteredRowGroup() : reader.readNextRowGroup())
.isNull();
}
}

@Test
public void testSkipManyNonMatchingBlocks() throws IOException {
Path path = new Path(tempDir.resolve("column-index.parquet").toUri());
MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; }");
SimpleGroupFactory groups = new SimpleGroupFactory(schema);
try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
.withType(schema)
.withRowGroupSize(1)
.withMinRowCountForPageSizeCheck(1)
.withMaxRowCountForPageSizeCheck(1)
.build()) {
writer.write(groups.newGroup().append("a", 0));
writer.write(groups.newGroup().append("a", 2));
}
InputFile inputFile = HadoopInputFile.fromPath(path, new Configuration());
ParquetReadOptions options = ParquetReadOptions.builder()
.copy(readOptions(true))
.withRecordFilter(FilterCompat.get(gt(intColumn("a"), 1)))
.build();
ParquetMetadata footer;
try (ParquetFileReader reader = new ParquetFileReader(inputFile, options)) {
footer = reader.getFooter();
}
assertThat(footer.getBlocks()).hasSize(2);
List<BlockMetaData> blocks = new ArrayList<>();
blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, footer.getBlocks().get(0)));
blocks.add(footer.getBlocks().get(1));
blocks.addAll(Collections.nCopies(SKIPPED_GROUPS, footer.getBlocks().get(0)));
ParquetMetadata repeatedFooter = new ParquetMetadata(footer.getFileMetaData(), blocks);
try (ParquetFileReader reader =
new ParquetFileReader(inputFile, repeatedFooter, options, inputFile.newStream())) {
assertThat(reader.getRowGroups()).hasSize(2 * SKIPPED_GROUPS + 1);
PageReadStore pages = reader.readNextFilteredRowGroup();
assertThat(pages).isNotNull();
assertThat(pages.getRowCount()).isEqualTo(1);
assertThat(reader.getCurrentRowGroupIndex()).isEqualTo(SKIPPED_GROUPS);
assertThat(reader.readNextFilteredRowGroup()).isNull();
}
}

private static ParquetReadOptions readOptions(boolean filtered) {
ParquetReadOptions.Builder builder = ParquetReadOptions.builder()
.useStatsFilter(false)
.useDictionaryFilter(false)
.useBloomFilter(false)
.useColumnIndexFilter(true);
if (filtered) {
builder.withRecordFilter(FilterCompat.get(gt(intColumn("a"), -1)));
}
return builder.build();
}
}
Loading