Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -171,13 +171,13 @@ public void writeBatch(VectorSchemaRoot batch) throws IOException {
return;
}

// Estimate rows per page based on schema width
int estimatedRowBytes = 0;
// Every column is cut at the same row boundaries, and each column chunk gets its own pages,
// so size the pages by the widest column rather than by the whole row.
int widestValueBytes = 1;
for (ColumnDescriptor col : schema.getColumns()) {
int typeLen = col.getPrimitiveType().getTypeLength();
estimatedRowBytes += typeLen > 0 ? typeLen : VAR_WIDTH_ESTIMATE_BYTES;
widestValueBytes = Math.max(widestValueBytes, estimatedValueBytes(col));
}
int rowsPerPage = Math.max(1, TARGET_PAGE_SIZE_BYTES / Math.max(estimatedRowBytes, 1));
int rowsPerPage = Math.max(1, TARGET_PAGE_SIZE_BYTES / widestValueBytes);

// Write in page-sized chunks
int offset = 0;
Expand Down Expand Up @@ -244,6 +244,26 @@ private void initRowGroup() {
rowGroupRowCount = 0;
}

/** Estimated encoded bytes of one PLAIN value of the column. */
static int estimatedValueBytes(ColumnDescriptor col) {
switch (col.getPrimitiveType().getPrimitiveTypeName()) {
case BOOLEAN:
return 1;
case INT32:
case FLOAT:
return 4;
case INT64:
case DOUBLE:
return 8;
case INT96:
return 12;
case FIXED_LEN_BYTE_ARRAY:
return col.getPrimitiveType().getTypeLength();
default:
return VAR_WIDTH_ESTIMATE_BYTES;
}
}

private boolean shouldFlushRowGroup() {
// Query actual buffered size from page writers
long totalBuffered = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
Expand All @@ -41,6 +43,7 @@
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.LocalInputFile;
import org.apache.parquet.io.LocalOutputFile;
Expand Down Expand Up @@ -278,10 +281,9 @@ void testLargeBatchProducesMultiplePages() throws IOException {
.sum();
assertThat(totalRows).isEqualTo(numRows);

// Verify the column chunk has multiple pages (data_page_offset implies pages were written)
// With 500K x 4 bytes = 2MB and 1MB target page size, expect at least 2 pages
long totalSize = footer.getBlocks().get(0).getColumns().get(0).getTotalSize();
assertThat(totalSize).isGreaterThan(0);
// 500K x 4 bytes = 2MB at a 1MB target page size: 262144 rows a page, so 2 pages
assertThat(fileReader.readOffsetIndex(footer.getBlocks().get(0).getColumns().get(0)).getPageCount())
.isEqualTo(2);
}

// Verify round-trip of first and last values
Expand All @@ -299,6 +301,45 @@ void testLargeBatchProducesMultiplePages() throws IOException {
}
}

@Test
void testPageSizeDoesNotShrinkWithColumnCount() throws IOException {
// 20 INT32 columns x 100K rows: 400KB a column, under the 1MB target, so one page each.
// Sizing pages by the whole row would cut every column into many small pages.
int numColumns = 20;
int numRows = 100_000;
List<Field> fields = new ArrayList<>();
for (int c = 0; c < numColumns; c++) {
fields.add(new Field("c" + c, FieldType.notNullable(new ArrowType.Int(32, true)), null));
}
Schema arrowSchema = new Schema(fields);
MessageType parquetSchema = new SchemaConverter().fromArrow(arrowSchema).getParquetSchema();
Path path = tempDir.toPath().resolve("wide_schema.parquet");

try (VectorSchemaRoot batch = VectorSchemaRoot.create(arrowSchema, allocator)) {
for (int c = 0; c < numColumns; c++) {
IntVector v = (IntVector) batch.getVector(c);
v.allocateNew(numRows);
for (int i = 0; i < numRows; i++) {
v.set(i, i * c);
}
}
batch.setRowCount(numRows);
try (ArrowParquetWriter writer = new ArrowParquetWriter(new LocalOutputFile(path), parquetSchema)) {
writer.writeBatch(batch);
}
}

try (ParquetFileReader fileReader = ParquetFileReader.open(new LocalInputFile(path))) {
ParquetMetadata footer = fileReader.getFooter();
assertThat(footer.getBlocks()).hasSize(1);
for (ColumnChunkMetaData column : footer.getBlocks().get(0).getColumns()) {
assertThat(fileReader.readOffsetIndex(column).getPageCount())
.as(column.getPath().toDotString())
.isEqualTo(1);
}
}
}

@Test
void testFooterRowCount() throws IOException {
Schema arrowSchema = new Schema(asList(
Expand Down