Skip to content

Commit 6454716

Browse files
author
Yigit Colakoglu
committed
GH-3706: Catch Throwable in InternalParquetRecordWriter.close() to abort on Error
Address review feedback on #3707: close() caught only Exception, so an Error thrown outside ParquetFileWriter (e.g. OOM during flushRowGroupToStore, or finalizeWrite) left the writer un-aborted and the finally flushed an incomplete, footerless file. Match write()'s catch (Throwable t) so abort runs before close().
1 parent 201ee57 commit 6454716

2 files changed

Lines changed: 55 additions & 2 deletions

File tree

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,9 @@ public void close() throws IOException, InterruptedException {
143143
}
144144
finalMetadata.putAll(finalWriteContext.getExtraMetaData());
145145
parquetFileWriter.end(finalMetadata);
146-
} catch (Exception e) {
146+
} catch (Throwable t) {
147147
parquetFileWriter.abort();
148-
throw e;
148+
throw t;
149149
} finally {
150150
AutoCloseables.uncheckedClose(columnStore, pageStore, bloomFilterWriteStore, parquetFileWriter);
151151
closed = true;

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,13 @@
7878
import org.apache.parquet.column.values.bloomfilter.BloomFilter;
7979
import org.apache.parquet.example.data.Group;
8080
import org.apache.parquet.example.data.simple.SimpleGroup;
81+
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
8182
import org.apache.parquet.format.Statistics;
8283
import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel;
8384
import org.apache.parquet.hadoop.example.GroupReadSupport;
8485
import org.apache.parquet.hadoop.example.GroupWriteSupport;
86+
import org.apache.parquet.hadoop.api.WriteSupport;
87+
import org.apache.parquet.hadoop.api.WriteSupport.FinalizedWriteContext;
8588
import org.apache.parquet.hadoop.metadata.BlockMetaData;
8689
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
8790
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
@@ -98,6 +101,7 @@
98101
import org.apache.parquet.internal.column.columnindex.BoundaryOrder;
99102
import org.apache.parquet.internal.column.columnindex.ColumnIndex;
100103
import org.apache.parquet.internal.column.columnindex.OffsetIndex;
104+
import org.apache.parquet.io.LocalOutputFile;
101105
import org.apache.parquet.io.OutputFile;
102106
import org.apache.parquet.io.ParquetEncodingException;
103107
import org.apache.parquet.io.PositionOutputStream;
@@ -1548,6 +1552,55 @@ private void assertEndFailureDoesNotFlush(Throwable toThrow) throws Exception {
15481552
.isEqualTo(0);
15491553
}
15501554

1555+
@Test
1556+
public void testNoFlushWhenRecordWriterCloseFailsWithError() throws Exception {
1557+
// An Error thrown from InternalParquetRecordWriter.close() outside any ParquetFileWriter
1558+
// method (here, WriteSupport.finalizeWrite()) must abort the writer before the finally
1559+
// flushes, so an incomplete, footerless file is never committed on storage that commits
1560+
// on stream close. See the review thread on GH-3706: catch (Exception) missed Errors.
1561+
java.nio.file.Path path = tempDir.resolve(java.util.UUID.randomUUID() + ".tmp");
1562+
Configuration conf = getTestConfiguration(false);
1563+
1564+
FaultInjectingOutputFile out = new FaultInjectingOutputFile(new LocalOutputFile(path));
1565+
MessageType schema = MessageTypeParser.parseMessageType("message m { required int64 id; }");
1566+
WriteSupport<Group> writeSupport = new GroupWriteSupport() {
1567+
@Override
1568+
public FinalizedWriteContext finalizeWrite() {
1569+
throw new AssertionError("injected finalizeWrite() failure");
1570+
}
1571+
};
1572+
GroupWriteSupport.setSchema(schema, conf);
1573+
1574+
ParquetWriter<Group> writer = new ParquetWriter<Group>(
1575+
out,
1576+
ParquetFileWriter.Mode.CREATE,
1577+
writeSupport,
1578+
CompressionCodecName.UNCOMPRESSED,
1579+
DEFAULT_BLOCK_SIZE,
1580+
false,
1581+
conf,
1582+
MAX_PADDING_SIZE_DEFAULT,
1583+
ParquetProperties.builder().withAllocator(allocator).build(),
1584+
null);
1585+
1586+
// Write enough rows to form a row group, so close() flushes pages before finalizeWrite().
1587+
SimpleGroupFactory groupFactory = new SimpleGroupFactory(schema);
1588+
for (int i = 0; i < 100; i++) {
1589+
writer.write(groupFactory.newGroup().append("id", (long) i));
1590+
}
1591+
1592+
out.flushCount = 0;
1593+
assertThatThrownBy(writer::close)
1594+
.isInstanceOf(AssertionError.class)
1595+
.hasMessageContaining("injected finalizeWrite() failure");
1596+
1597+
// The writer must be aborted before close(), so ParquetFileWriter.close() skips the flush
1598+
// and the incomplete stream is never flushed to storage.
1599+
assertThat(out.flushCount)
1600+
.as("close() must not flush the output stream when finalizeWrite() fails with an Error")
1601+
.isEqualTo(0);
1602+
}
1603+
15511604
/**
15521605
* OutputFile whose stream throws a preset Throwable on write (simulating a mid-write failure) and
15531606
* which counts flush() calls, so a test can assert nothing was flushed on the failure path.

0 commit comments

Comments
 (0)