Skip to content

Commit 7d2fc0f

Browse files
authored
MINOR: Retire H1SeekableInputStream (#3584)
Hadoop 1 has long been unsupported, and from Hadoop 3.3 onward the stream capability probes make a separate Hadoop 1 wrapper unnecessary. HadoopStreams.wrap() now returns H3ByteBufferInputStream for streams advertising in:preadbytebuffer and H2SeekableInputStream otherwise. H2SeekableInputStream detects whether the wrapped stream supports read(ByteBuffer) and, if not, falls back to the byte-array based reads inherited from DelegatingSeekableInputStream instead of hitting FSDataInputStream's UnsupportedOperationException.
1 parent f1cfacf commit 7d2fc0f

5 files changed

Lines changed: 105 additions & 128 deletions

File tree

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java‎

Lines changed: 0 additions & 77 deletions
This file was deleted.

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,21 @@
2121

2222
import java.io.EOFException;
2323
import java.io.IOException;
24+
import java.io.InputStream;
2425
import java.nio.ByteBuffer;
2526
import java.util.List;
27+
import org.apache.hadoop.fs.ByteBufferReadable;
2628
import org.apache.hadoop.fs.FSDataInputStream;
2729
import org.apache.parquet.bytes.ByteBufferAllocator;
2830
import org.apache.parquet.hadoop.util.wrapped.io.VectorIoBridge;
2931
import org.apache.parquet.io.DelegatingSeekableInputStream;
3032
import org.apache.parquet.io.ParquetFileRange;
3133

3234
/**
33-
* SeekableInputStream implementation for FSDataInputStream that implements
34-
* ByteBufferReadable in Hadoop 2.
35+
* SeekableInputStream implementation for FSDataInputStream.
36+
* Uses the underlying stream's native {@code read(ByteBuffer)} when the
37+
* stream advertises {@link ByteBufferReadable} support; otherwise falls back
38+
* to the byte-array based reads inherited from {@link DelegatingSeekableInputStream}.
3539
* It implements {@link #readVectored(List, ByteBufferAllocator)}) by
3640
* handing off to VectorIoBridge which uses reflection to offer the API if it is found.
3741
* The return value of {@link #readVectoredAvailable(ByteBufferAllocator)}
@@ -46,10 +50,12 @@ interface Reader {
4650

4751
private final FSDataInputStream stream;
4852
private final Reader reader;
53+
private final boolean byteBufferReadable;
4954

5055
public H2SeekableInputStream(FSDataInputStream stream) {
5156
super(stream);
5257
this.stream = stream;
58+
this.byteBufferReadable = isByteBufferReadable(stream);
5359
this.reader = new H2Reader();
5460
}
5561

@@ -75,12 +81,19 @@ public void readFully(byte[] bytes, int start, int len) throws IOException {
7581

7682
@Override
7783
public int read(ByteBuffer buf) throws IOException {
78-
return stream.read(buf);
84+
if (byteBufferReadable) {
85+
return stream.read(buf);
86+
}
87+
return super.read(buf);
7988
}
8089

8190
@Override
8291
public void readFully(ByteBuffer buf) throws IOException {
83-
readFully(reader, buf);
92+
if (byteBufferReadable) {
93+
readFully(reader, buf);
94+
} else {
95+
super.readFully(buf);
96+
}
8497
}
8598

8699
private class H2Reader implements Reader {
@@ -90,6 +103,29 @@ public int read(ByteBuffer buf) throws IOException {
90103
}
91104
}
92105

106+
/**
107+
* Whether the given FSDataInputStream supports {@code read(ByteBuffer)}.
108+
* <p>
109+
* In Hadoop 3.3.0+, only streams which provide the read(ByteBuffer)
110+
* semantics MAY return true for the probe "in:readbytebuffer" (HDFS-14111);
111+
* FSDataInputStream passes the probe down to the underlying stream.
112+
* Otherwise, walk the wrapped-stream chain and check whether the innermost
113+
* non-FSDataInputStream implements {@link ByteBufferReadable}.
114+
*
115+
* @param stream stream to probe
116+
* @return true if {@code stream.read(ByteBuffer)} is supported
117+
*/
118+
private static boolean isByteBufferReadable(FSDataInputStream stream) {
119+
if (stream.hasCapability("in:readbytebuffer")) {
120+
return true;
121+
}
122+
InputStream wrapped = stream.getWrappedStream();
123+
if (wrapped instanceof FSDataInputStream) {
124+
return isByteBufferReadable((FSDataInputStream) wrapped);
125+
}
126+
return wrapped instanceof ByteBufferReadable;
127+
}
128+
93129
@Override
94130
public boolean readVectoredAvailable(final ByteBufferAllocator allocator) {
95131
return VectorIoBridge.instance().readVectoredAvailable(stream, allocator);

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java‎

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@
1919

2020
package org.apache.parquet.hadoop.util;
2121

22-
import java.io.InputStream;
2322
import java.util.Objects;
24-
import org.apache.hadoop.fs.ByteBufferReadable;
2523
import org.apache.hadoop.fs.FSDataInputStream;
2624
import org.apache.hadoop.fs.FSDataOutputStream;
2725
import org.apache.parquet.io.PositionOutputStream;
@@ -50,46 +48,8 @@ public static SeekableInputStream wrap(FSDataInputStream stream) {
5048
if (stream.hasCapability("in:preadbytebuffer")) {
5149
LOG.debug("Using ByteBufferPositionedReadable to read {}", stream);
5250
return new H3ByteBufferInputStream(stream);
53-
} else if (isWrappedStreamByteBufferReadable(stream)) {
54-
return new H2SeekableInputStream(stream);
55-
} else {
56-
return new H1SeekableInputStream(stream);
5751
}
58-
}
59-
60-
/**
61-
* Is the inner stream byte buffer readable?
62-
* The test is 'the stream is not FSDataInputStream
63-
* and implements ByteBufferReadable'
64-
* <p>
65-
* That is: all streams which implement ByteBufferReadable
66-
* other than FSDataInputStream successfully support read(ByteBuffer).
67-
* This is true for all filesystem clients the hadoop codebase.
68-
* <p>
69-
* In hadoop 3.3.0+, the StreamCapabilities probe can be used to
70-
* check this: only those streams which provide the read(ByteBuffer)
71-
* semantics MAY return true for the probe "in:readbytebuffer";
72-
* FSDataInputStream will pass the probe down to the underlying stream.
73-
*
74-
* @param stream stream to probe
75-
* @return true if it is safe to a H2SeekableInputStream to access
76-
* the data, null when it cannot be determined because of missing hasCapabilities
77-
*/
78-
private static Boolean isWrappedStreamByteBufferReadable(FSDataInputStream stream) {
79-
boolean isByteBufferReadable = stream.hasCapability("in:readbytebuffer");
80-
81-
if (isByteBufferReadable) {
82-
// stream is issuing the guarantee that it implements the
83-
// API. Holds for all implementations in hadoop-*
84-
// since Hadoop 3.3.0 (HDFS-14111).
85-
return true;
86-
}
87-
InputStream wrapped = stream.getWrappedStream();
88-
if (wrapped instanceof FSDataInputStream) {
89-
LOG.debug("Checking on wrapped stream {} of {} whether is ByteBufferReadable", wrapped, stream);
90-
return isWrappedStreamByteBufferReadable(((FSDataInputStream) wrapped));
91-
}
92-
return wrapped instanceof ByteBufferReadable;
52+
return new H2SeekableInputStream(stream);
9353
}
9454

9555
/**

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop2ByteBufferReads.java‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -382,13 +382,71 @@ public void testDirectReadFullyPositionAndLimit() throws Exception {
382382
@Test
383383
public void testCreateStreamNoByteBufferReadable() {
384384
final SeekableInputStream s = wrap(new FSDataInputStream(new MockHadoopInputStream()));
385-
assertThat(s).as("Wrong wrapper: " + s).isInstanceOf(H1SeekableInputStream.class);
385+
assertThat(s).as("Wrong wrapper: " + s).isInstanceOf(H2SeekableInputStream.class);
386386
}
387387

388388
@Test
389389
public void testDoubleWrapNoByteBufferReadable() {
390390
final SeekableInputStream s = wrap(new FSDataInputStream(new FSDataInputStream(new MockHadoopInputStream())));
391-
assertThat(s).as("Wrong wrapper: " + s).isInstanceOf(H1SeekableInputStream.class);
391+
assertThat(s).as("Wrong wrapper: " + s).isInstanceOf(H2SeekableInputStream.class);
392+
}
393+
394+
/**
395+
* A stream which is not ByteBufferReadable must fall back to byte-array reads
396+
* rather than calling {@code FSDataInputStream.read(ByteBuffer)}, which would
397+
* throw UnsupportedOperationException.
398+
*/
399+
@Test
400+
public void testHeapReadNoByteBufferReadable() throws Exception {
401+
final SeekableInputStream s = wrap(new FSDataInputStream(new MockHadoopInputStream()));
402+
final ByteBuffer readBuffer = ByteBuffer.allocate(20);
403+
404+
assertThat(s.read(readBuffer)).isEqualTo(10);
405+
assertThat(readBuffer.position()).isEqualTo(10);
406+
readBuffer.flip();
407+
assertThat(readBuffer).as("Buffer contents should match").isEqualTo(ByteBuffer.wrap(TEST_ARRAY));
408+
}
409+
410+
@Test
411+
public void testDirectReadNoByteBufferReadable() throws Exception {
412+
final SeekableInputStream s = wrap(new FSDataInputStream(new MockHadoopInputStream()));
413+
final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20);
414+
415+
assertThat(s.read(readBuffer)).isEqualTo(10);
416+
assertThat(readBuffer.position()).isEqualTo(10);
417+
readBuffer.flip();
418+
assertThat(readBuffer).as("Buffer contents should match").isEqualTo(ByteBuffer.wrap(TEST_ARRAY));
419+
}
420+
421+
@Test
422+
public void testHeapReadFullyNoByteBufferReadable() throws Exception {
423+
final SeekableInputStream s = wrap(new FSDataInputStream(new MockHadoopInputStream(2, 3)));
424+
final ByteBuffer readBuffer = ByteBuffer.allocate(10);
425+
426+
s.readFully(readBuffer);
427+
assertThat(readBuffer.position()).isEqualTo(10);
428+
readBuffer.flip();
429+
assertThat(readBuffer).as("Buffer contents should match").isEqualTo(ByteBuffer.wrap(TEST_ARRAY));
430+
}
431+
432+
@Test
433+
public void testDirectReadFullyNoByteBufferReadable() throws Exception {
434+
final SeekableInputStream s =
435+
wrap(new FSDataInputStream(new FSDataInputStream(new MockHadoopInputStream(2, 3))));
436+
final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10);
437+
438+
s.readFully(readBuffer);
439+
assertThat(readBuffer.position()).isEqualTo(10);
440+
readBuffer.flip();
441+
assertThat(readBuffer).as("Buffer contents should match").isEqualTo(ByteBuffer.wrap(TEST_ARRAY));
442+
}
443+
444+
@Test
445+
public void testReadFullyPastEOFNoByteBufferReadable() {
446+
final SeekableInputStream s = wrap(new FSDataInputStream(new MockHadoopInputStream()));
447+
final ByteBuffer readBuffer = ByteBuffer.allocate(20);
448+
449+
assertThatThrownBy(() -> s.readFully(readBuffer)).isInstanceOf(EOFException.class);
392450
}
393451

394452
@Test

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop3ByteBufferReadFully.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -246,12 +246,12 @@ private static void assertThrowsEOFException(final FSDataInputStream hadoopStrea
246246
}
247247

248248
/**
249-
* Regression test: verify that creating a stream for {@link MockHadoopInputStream}
250-
* still generates an {@link H1SeekableInputStream}.
249+
* Regression test: verify that creating a stream for {@link MockHadoopInputStream},
250+
* which is not ByteBufferReadable, generates an {@link H2SeekableInputStream}.
251251
*/
252252
@Test
253-
public void testCreateH1Stream() {
254-
assertStreamClass(H1SeekableInputStream.class, wrap(new FSDataInputStream(new MockHadoopInputStream())));
253+
public void testCreateStreamNoByteBufferReadable() {
254+
assertStreamClass(H2SeekableInputStream.class, wrap(new FSDataInputStream(new MockHadoopInputStream())));
255255
}
256256

257257
/**

0 commit comments

Comments
 (0)