Skip to content

Commit ce4fd45

Browse files
committed
feat(parquet): expose ParquetReadOptions via registerParquet and readParquet
1 parent e46f393 commit ce4fd45

2 files changed

Lines changed: 200 additions & 12 deletions

File tree

‎native/src/lib.rs‎

Lines changed: 107 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@ mod errors;
1919

2020
use std::sync::{Arc, OnceLock};
2121

22-
use datafusion::arrow::datatypes::SchemaRef;
22+
use datafusion::arrow::datatypes::{Schema, SchemaRef};
2323
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
24+
use datafusion::arrow::ipc::reader::StreamReader;
2425
use datafusion::arrow::record_batch::RecordBatchIterator;
2526
use datafusion::dataframe::DataFrame;
2627
use datafusion::error::DataFusionError;
2728
use datafusion::prelude::{ParquetReadOptions, SessionContext};
28-
use jni::objects::{JClass, JString};
29-
use jni::sys::jlong;
29+
use jni::objects::{JByteArray, JClass, JString};
30+
use jni::sys::{jboolean, jlong};
3031
use jni::JNIEnv;
3132
use tokio::runtime::Runtime;
3233

@@ -129,13 +130,61 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_closeSessionCon
129130
})
130131
}
131132

133+
#[allow(clippy::too_many_arguments)]
134+
fn with_parquet_options<R>(
135+
env: &mut JNIEnv,
136+
file_extension: JString,
137+
parquet_pruning_set: jboolean,
138+
parquet_pruning_value: jboolean,
139+
skip_metadata_set: jboolean,
140+
skip_metadata_value: jboolean,
141+
metadata_size_hint: jlong,
142+
schema_ipc_bytes: JByteArray,
143+
f: impl FnOnce(ParquetReadOptions) -> JniResult<R>,
144+
) -> JniResult<R> {
145+
let file_ext: String = env.get_string(&file_extension)?.into();
146+
147+
let schema: Option<Schema> = if !schema_ipc_bytes.is_null() {
148+
let bytes: Vec<u8> = env.convert_byte_array(&schema_ipc_bytes)?;
149+
let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
150+
Some((*reader.schema()).clone())
151+
} else {
152+
None
153+
};
154+
155+
let mut opts = ParquetReadOptions::default().file_extension(&file_ext);
156+
if parquet_pruning_set != 0 {
157+
opts = opts.parquet_pruning(parquet_pruning_value != 0);
158+
}
159+
if skip_metadata_set != 0 {
160+
opts = opts.skip_metadata(skip_metadata_value != 0);
161+
}
162+
if metadata_size_hint >= 0 {
163+
opts = opts.metadata_size_hint(Some(metadata_size_hint as usize));
164+
}
165+
if let Some(ref s) = schema {
166+
opts = opts.schema(s);
167+
}
168+
169+
f(opts)
170+
}
171+
132172
#[no_mangle]
133-
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquet<'local>(
173+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquetWithOptions<
174+
'local,
175+
>(
134176
mut env: JNIEnv<'local>,
135177
_class: JClass<'local>,
136178
handle: jlong,
137179
name: JString<'local>,
138180
path: JString<'local>,
181+
file_extension: JString<'local>,
182+
parquet_pruning_set: jboolean,
183+
parquet_pruning_value: jboolean,
184+
skip_metadata_set: jboolean,
185+
skip_metadata_value: jboolean,
186+
metadata_size_hint: jlong,
187+
schema_ipc_bytes: JByteArray<'local>,
139188
) {
140189
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
141190
if handle == 0 {
@@ -144,11 +193,59 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquet
144193
let ctx = unsafe { &*(handle as *const SessionContext) };
145194
let name: String = env.get_string(&name)?.into();
146195
let path: String = env.get_string(&path)?.into();
147-
runtime().block_on(async {
148-
ctx.register_parquet(&name, &path, ParquetReadOptions::default())
149-
.await?;
150-
Ok::<(), DataFusionError>(())
151-
})?;
152-
Ok(())
196+
with_parquet_options(
197+
env,
198+
file_extension,
199+
parquet_pruning_set,
200+
parquet_pruning_value,
201+
skip_metadata_set,
202+
skip_metadata_value,
203+
metadata_size_hint,
204+
schema_ipc_bytes,
205+
|opts| {
206+
runtime().block_on(async {
207+
ctx.register_parquet(&name, &path, opts).await?;
208+
Ok::<(), DataFusionError>(())
209+
})?;
210+
Ok(())
211+
},
212+
)
213+
})
214+
}
215+
216+
#[no_mangle]
217+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readParquetWithOptions<'local>(
218+
mut env: JNIEnv<'local>,
219+
_class: JClass<'local>,
220+
handle: jlong,
221+
path: JString<'local>,
222+
file_extension: JString<'local>,
223+
parquet_pruning_set: jboolean,
224+
parquet_pruning_value: jboolean,
225+
skip_metadata_set: jboolean,
226+
skip_metadata_value: jboolean,
227+
metadata_size_hint: jlong,
228+
schema_ipc_bytes: JByteArray<'local>,
229+
) -> jlong {
230+
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
231+
if handle == 0 {
232+
return Err("SessionContext handle is null".into());
233+
}
234+
let ctx = unsafe { &*(handle as *const SessionContext) };
235+
let path: String = env.get_string(&path)?.into();
236+
with_parquet_options(
237+
env,
238+
file_extension,
239+
parquet_pruning_set,
240+
parquet_pruning_value,
241+
skip_metadata_set,
242+
skip_metadata_value,
243+
metadata_size_hint,
244+
schema_ipc_bytes,
245+
|opts| {
246+
let df = runtime().block_on(ctx.read_parquet(path, opts))?;
247+
Ok(Box::into_raw(Box::new(df)) as jlong)
248+
},
249+
)
153250
})
154251
}

‎src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,16 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.ByteArrayOutputStream;
23+
import java.io.IOException;
24+
import java.nio.channels.Channels;
25+
26+
import org.apache.arrow.memory.BufferAllocator;
27+
import org.apache.arrow.memory.RootAllocator;
28+
import org.apache.arrow.vector.VectorSchemaRoot;
29+
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
30+
import org.apache.arrow.vector.types.pojo.Schema;
31+
2232
/**
2333
* A DataFusion session context.
2434
*
@@ -54,10 +64,70 @@ public DataFrame sql(String query) {
5464
}
5565

5666
public void registerParquet(String name, String path) {
67+
registerParquet(name, path, new ParquetReadOptions());
68+
}
69+
70+
/**
71+
* Register a parquet file as a table with the supplied {@link ParquetReadOptions}.
72+
*
73+
* @throws RuntimeException if registration fails (path not found, schema mismatch, etc.).
74+
*/
75+
public void registerParquet(String name, String path, ParquetReadOptions options) {
76+
if (nativeHandle == 0) {
77+
throw new IllegalStateException("SessionContext is closed");
78+
}
79+
registerParquetWithOptions(
80+
nativeHandle,
81+
name,
82+
path,
83+
options.fileExtension(),
84+
options.parquetPruning() != null,
85+
options.parquetPruning() != null && options.parquetPruning(),
86+
options.skipMetadata() != null,
87+
options.skipMetadata() != null && options.skipMetadata(),
88+
options.metadataSizeHint() != null ? options.metadataSizeHint() : -1L,
89+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
90+
}
91+
92+
/** Read a parquet file as a {@link DataFrame} without registering it. */
93+
public DataFrame readParquet(String path) {
94+
return readParquet(path, new ParquetReadOptions());
95+
}
96+
97+
/**
98+
* Read a parquet file as a {@link DataFrame} with the supplied {@link ParquetReadOptions}.
99+
*
100+
* @throws RuntimeException if the read fails.
101+
*/
102+
public DataFrame readParquet(String path, ParquetReadOptions options) {
57103
if (nativeHandle == 0) {
58104
throw new IllegalStateException("SessionContext is closed");
59105
}
60-
registerParquet(nativeHandle, name, path);
106+
long dfHandle =
107+
readParquetWithOptions(
108+
nativeHandle,
109+
path,
110+
options.fileExtension(),
111+
options.parquetPruning() != null,
112+
options.parquetPruning() != null && options.parquetPruning(),
113+
options.skipMetadata() != null,
114+
options.skipMetadata() != null && options.skipMetadata(),
115+
options.metadataSizeHint() != null ? options.metadataSizeHint() : -1L,
116+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
117+
return new DataFrame(dfHandle);
118+
}
119+
120+
private static byte[] serializeSchemaIpc(Schema schema) {
121+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
122+
try (BufferAllocator allocator = new RootAllocator();
123+
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
124+
ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(baos))) {
125+
writer.start();
126+
writer.end();
127+
} catch (IOException e) {
128+
throw new RuntimeException("Failed to serialize Arrow schema for JNI", e);
129+
}
130+
return baos.toByteArray();
61131
}
62132

63133
@Override
@@ -72,7 +142,28 @@ public void close() {
72142

73143
private static native long createDataFrame(long handle, String sql);
74144

75-
private static native void registerParquet(long handle, String name, String path);
145+
private static native void registerParquetWithOptions(
146+
long handle,
147+
String name,
148+
String path,
149+
String fileExtension,
150+
boolean parquetPruningSet,
151+
boolean parquetPruningValue,
152+
boolean skipMetadataSet,
153+
boolean skipMetadataValue,
154+
long metadataSizeHint,
155+
byte[] schemaIpcBytes);
156+
157+
private static native long readParquetWithOptions(
158+
long handle,
159+
String path,
160+
String fileExtension,
161+
boolean parquetPruningSet,
162+
boolean parquetPruningValue,
163+
boolean skipMetadataSet,
164+
boolean skipMetadataValue,
165+
long metadataSizeHint,
166+
byte[] schemaIpcBytes);
76167

77168
private static native void closeSessionContext(long handle);
78169
}

0 commit comments

Comments
 (0)