From 3c516939012467636d3f8b3fd34b42e2e76f5638 Mon Sep 17 00:00:00 2001 From: mikepapadim Date: Fri, 13 Nov 2026 21:59:03 +0000 Subject: [PATCH] Size ArrowParquetWriter pages by the widest column rowsPerPage divided the 1 MB page target by the sum of all column widths, and getTypeLength() is 0 for every primitive type except FIXED_LEN_BYTE_ARRAY, so each column was estimated at 32 bytes. On a 19-column schema that gave 1724 rows a page (~7 KB INT32 pages), and ZSTD, which compresses each page on its own, compressed them poorly. Size pages by the widest column, using each primitive type's PLAIN width. On a 19-column TPC-H lineitem-like table (512 MB of Arrow, ZSTD, 16384-row batches) this cuts the write from 2.57 s to 1.73 s and the file from 107.5 MB to 87.5 MB, the same size ExampleParquetWriter produces with dictionary encoding off. --- .../arrow/writer/ArrowParquetWriter.java | 30 ++++++++++-- .../arrow/writer/TestArrowParquetWriter.java | 49 +++++++++++++++++-- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/parquet-arrow/src/main/java/org/apache/parquet/arrow/writer/ArrowParquetWriter.java b/parquet-arrow/src/main/java/org/apache/parquet/arrow/writer/ArrowParquetWriter.java index b14e39c539..5d18705d59 100644 --- a/parquet-arrow/src/main/java/org/apache/parquet/arrow/writer/ArrowParquetWriter.java +++ b/parquet-arrow/src/main/java/org/apache/parquet/arrow/writer/ArrowParquetWriter.java @@ -171,13 +171,13 @@ public void writeBatch(VectorSchemaRoot batch) throws IOException { return; } - // Estimate rows per page based on schema width - int estimatedRowBytes = 0; + // Every column is cut at the same row boundaries, and each column chunk gets its own pages, + // so size the pages by the widest column rather than by the whole row. + int widestValueBytes = 1; for (ColumnDescriptor col : schema.getColumns()) { - int typeLen = col.getPrimitiveType().getTypeLength(); - estimatedRowBytes += typeLen > 0 ? typeLen : VAR_WIDTH_ESTIMATE_BYTES; + widestValueBytes = Math.max(widestValueBytes, estimatedValueBytes(col)); } - int rowsPerPage = Math.max(1, TARGET_PAGE_SIZE_BYTES / Math.max(estimatedRowBytes, 1)); + int rowsPerPage = Math.max(1, TARGET_PAGE_SIZE_BYTES / widestValueBytes); // Write in page-sized chunks int offset = 0; @@ -244,6 +244,26 @@ private void initRowGroup() { rowGroupRowCount = 0; } + /** Estimated encoded bytes of one PLAIN value of the column. */ + static int estimatedValueBytes(ColumnDescriptor col) { + switch (col.getPrimitiveType().getPrimitiveTypeName()) { + case BOOLEAN: + return 1; + case INT32: + case FLOAT: + return 4; + case INT64: + case DOUBLE: + return 8; + case INT96: + return 12; + case FIXED_LEN_BYTE_ARRAY: + return col.getPrimitiveType().getTypeLength(); + default: + return VAR_WIDTH_ESTIMATE_BYTES; + } + } + private boolean shouldFlushRowGroup() { // Query actual buffered size from page writers long totalBuffered = 0; diff --git a/parquet-arrow/src/test/java/org/apache/parquet/arrow/writer/TestArrowParquetWriter.java b/parquet-arrow/src/test/java/org/apache/parquet/arrow/writer/TestArrowParquetWriter.java index 5fced4a9d1..ff5dc6276a 100644 --- a/parquet-arrow/src/test/java/org/apache/parquet/arrow/writer/TestArrowParquetWriter.java +++ b/parquet-arrow/src/test/java/org/apache/parquet/arrow/writer/TestArrowParquetWriter.java @@ -24,6 +24,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -41,6 +43,7 @@ import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.LocalInputFile; import org.apache.parquet.io.LocalOutputFile; @@ -278,10 +281,9 @@ void testLargeBatchProducesMultiplePages() throws IOException { .sum(); assertThat(totalRows).isEqualTo(numRows); - // Verify the column chunk has multiple pages (data_page_offset implies pages were written) - // With 500K x 4 bytes = 2MB and 1MB target page size, expect at least 2 pages - long totalSize = footer.getBlocks().get(0).getColumns().get(0).getTotalSize(); - assertThat(totalSize).isGreaterThan(0); + // 500K x 4 bytes = 2MB at a 1MB target page size: 262144 rows a page, so 2 pages + assertThat(fileReader.readOffsetIndex(footer.getBlocks().get(0).getColumns().get(0)).getPageCount()) + .isEqualTo(2); } // Verify round-trip of first and last values @@ -299,6 +301,45 @@ void testLargeBatchProducesMultiplePages() throws IOException { } } + @Test + void testPageSizeDoesNotShrinkWithColumnCount() throws IOException { + // 20 INT32 columns x 100K rows: 400KB a column, under the 1MB target, so one page each. + // Sizing pages by the whole row would cut every column into many small pages. + int numColumns = 20; + int numRows = 100_000; + List fields = new ArrayList<>(); + for (int c = 0; c < numColumns; c++) { + fields.add(new Field("c" + c, FieldType.notNullable(new ArrowType.Int(32, true)), null)); + } + Schema arrowSchema = new Schema(fields); + MessageType parquetSchema = new SchemaConverter().fromArrow(arrowSchema).getParquetSchema(); + Path path = tempDir.toPath().resolve("wide_schema.parquet"); + + try (VectorSchemaRoot batch = VectorSchemaRoot.create(arrowSchema, allocator)) { + for (int c = 0; c < numColumns; c++) { + IntVector v = (IntVector) batch.getVector(c); + v.allocateNew(numRows); + for (int i = 0; i < numRows; i++) { + v.set(i, i * c); + } + } + batch.setRowCount(numRows); + try (ArrowParquetWriter writer = new ArrowParquetWriter(new LocalOutputFile(path), parquetSchema)) { + writer.writeBatch(batch); + } + } + + try (ParquetFileReader fileReader = ParquetFileReader.open(new LocalInputFile(path))) { + ParquetMetadata footer = fileReader.getFooter(); + assertThat(footer.getBlocks()).hasSize(1); + for (ColumnChunkMetaData column : footer.getBlocks().get(0).getColumns()) { + assertThat(fileReader.readOffsetIndex(column).getPageCount()) + .as(column.getPath().toDotString()) + .isEqualTo(1); + } + } + } + @Test void testFooterRowCount() throws IOException { Schema arrowSchema = new Schema(asList(