|
78 | 78 | import org.apache.parquet.column.values.bloomfilter.BloomFilter; |
79 | 79 | import org.apache.parquet.example.data.Group; |
80 | 80 | import org.apache.parquet.example.data.simple.SimpleGroup; |
| 81 | +import org.apache.parquet.example.data.simple.SimpleGroupFactory; |
81 | 82 | import org.apache.parquet.format.Statistics; |
82 | 83 | import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel; |
83 | 84 | import org.apache.parquet.hadoop.example.GroupReadSupport; |
84 | 85 | import org.apache.parquet.hadoop.example.GroupWriteSupport; |
| 86 | +import org.apache.parquet.hadoop.api.WriteSupport; |
| 87 | +import org.apache.parquet.hadoop.api.WriteSupport.FinalizedWriteContext; |
85 | 88 | import org.apache.parquet.hadoop.metadata.BlockMetaData; |
86 | 89 | import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; |
87 | 90 | import org.apache.parquet.hadoop.metadata.CompressionCodecName; |
|
98 | 101 | import org.apache.parquet.internal.column.columnindex.BoundaryOrder; |
99 | 102 | import org.apache.parquet.internal.column.columnindex.ColumnIndex; |
100 | 103 | import org.apache.parquet.internal.column.columnindex.OffsetIndex; |
| 104 | +import org.apache.parquet.io.LocalOutputFile; |
101 | 105 | import org.apache.parquet.io.OutputFile; |
102 | 106 | import org.apache.parquet.io.ParquetEncodingException; |
103 | 107 | import org.apache.parquet.io.PositionOutputStream; |
@@ -1548,6 +1552,55 @@ private void assertEndFailureDoesNotFlush(Throwable toThrow) throws Exception { |
1548 | 1552 | .isEqualTo(0); |
1549 | 1553 | } |
1550 | 1554 |
|
| 1555 | + @Test |
| 1556 | + public void testNoFlushWhenRecordWriterCloseFailsWithError() throws Exception { |
| 1557 | + // An Error thrown from InternalParquetRecordWriter.close() outside any ParquetFileWriter |
| 1558 | + // method (here, WriteSupport.finalizeWrite()) must abort the writer before the finally |
| 1559 | + // flushes, so an incomplete, footerless file is never committed on storage that commits |
| 1560 | + // on stream close. See the review thread on GH-3706: catch (Exception) missed Errors. |
| 1561 | + java.nio.file.Path path = tempDir.resolve(java.util.UUID.randomUUID() + ".tmp"); |
| 1562 | + Configuration conf = getTestConfiguration(false); |
| 1563 | + |
| 1564 | + FaultInjectingOutputFile out = new FaultInjectingOutputFile(new LocalOutputFile(path)); |
| 1565 | + MessageType schema = MessageTypeParser.parseMessageType("message m { required int64 id; }"); |
| 1566 | + WriteSupport<Group> writeSupport = new GroupWriteSupport() { |
| 1567 | + @Override |
| 1568 | + public FinalizedWriteContext finalizeWrite() { |
| 1569 | + throw new AssertionError("injected finalizeWrite() failure"); |
| 1570 | + } |
| 1571 | + }; |
| 1572 | + GroupWriteSupport.setSchema(schema, conf); |
| 1573 | + |
| 1574 | + ParquetWriter<Group> writer = new ParquetWriter<Group>( |
| 1575 | + out, |
| 1576 | + ParquetFileWriter.Mode.CREATE, |
| 1577 | + writeSupport, |
| 1578 | + CompressionCodecName.UNCOMPRESSED, |
| 1579 | + DEFAULT_BLOCK_SIZE, |
| 1580 | + false, |
| 1581 | + conf, |
| 1582 | + MAX_PADDING_SIZE_DEFAULT, |
| 1583 | + ParquetProperties.builder().withAllocator(allocator).build(), |
| 1584 | + null); |
| 1585 | + |
| 1586 | + // Write enough rows to form a row group, so close() flushes pages before finalizeWrite(). |
| 1587 | + SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema); |
| 1588 | + for (int i = 0; i < 100; i++) { |
| 1589 | + writer.write(groupFactory.newGroup().append("id", (long) i)); |
| 1590 | + } |
| 1591 | + |
| 1592 | + out.flushCount = 0; |
| 1593 | + assertThatThrownBy(writer::close) |
| 1594 | + .isInstanceOf(AssertionError.class) |
| 1595 | + .hasMessageContaining("injected finalizeWrite() failure"); |
| 1596 | + |
| 1597 | + // The writer must be aborted before close(), so ParquetFileWriter.close() skips the flush |
| 1598 | + // and the incomplete stream is never flushed to storage. |
| 1599 | + assertThat(out.flushCount) |
| 1600 | + .as("close() must not flush the output stream when finalizeWrite() fails with an Error") |
| 1601 | + .isEqualTo(0); |
| 1602 | + } |
| 1603 | + |
1551 | 1604 | /** |
1552 | 1605 | * OutputFile whose stream throws a preset Throwable on write (simulating a mid-write failure) and |
1553 | 1606 | * which counts flush() calls, so a test can assert nothing was flushed on the failure path. |
|
0 commit comments