Skip to content

Commit 111e70e

Browse files
committed
refactor: pass csv and parquet read options via protobuf
1 parent 4e182d7 commit 111e70e

7 files changed

Lines changed: 177 additions & 333 deletions

File tree

‎native/src/csv.rs‎

Lines changed: 47 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -15,73 +15,65 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::str::FromStr;
19-
20-
use datafusion::arrow::datatypes::Schema;
21-
use datafusion::arrow::ipc::reader::StreamReader;
2218
use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
2319
use datafusion::error::DataFusionError;
2420
use datafusion::prelude::{CsvReadOptions, SessionContext};
2521
use jni::objects::{JByteArray, JClass, JString};
26-
use jni::sys::{jboolean, jbyte, jlong};
22+
use jni::sys::jlong;
2723
use jni::JNIEnv;
24+
use prost::Message;
2825

2926
use crate::errors::{try_unwrap_or_throw, JniResult};
3027
use crate::runtime;
28+
use crate::schema::decode_optional_schema;
29+
use crate::session_options::{
30+
CsvReadOptionsProto, FileCompressionType as ProtoFileCompressionType,
31+
};
3132

32-
#[allow(clippy::too_many_arguments)]
3333
fn with_csv_options<R>(
3434
env: &mut JNIEnv,
35-
has_header: jboolean,
36-
delimiter: jbyte,
37-
quote: jbyte,
38-
terminator_set: jboolean,
39-
terminator_value: jbyte,
40-
escape_set: jboolean,
41-
escape_value: jbyte,
42-
comment_set: jboolean,
43-
comment_value: jbyte,
44-
newlines_in_values_set: jboolean,
45-
newlines_in_values_value: jboolean,
46-
schema_infer_max_records: jlong,
47-
file_extension: JString,
48-
file_compression_type: JString,
35+
options_bytes: JByteArray,
4936
schema_ipc_bytes: JByteArray,
5037
f: impl FnOnce(CsvReadOptions) -> JniResult<R>,
5138
) -> JniResult<R> {
52-
let file_ext: String = env.get_string(&file_extension)?.into();
53-
let compression: String = env.get_string(&file_compression_type)?.into();
54-
let compression = FileCompressionType::from_str(&compression)?;
39+
let bytes: Vec<u8> = env.convert_byte_array(&options_bytes)?;
40+
let p = CsvReadOptionsProto::decode(bytes.as_slice())?;
41+
42+
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
5543

56-
let schema: Option<Schema> = if !schema_ipc_bytes.is_null() {
57-
let bytes: Vec<u8> = env.convert_byte_array(&schema_ipc_bytes)?;
58-
let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
59-
Some((*reader.schema()).clone())
60-
} else {
61-
None
44+
let compression = match p.file_compression_type() {
45+
ProtoFileCompressionType::Unspecified => {
46+
return Err("CsvReadOptionsProto.file_compression_type is UNSPECIFIED".into());
47+
}
48+
ProtoFileCompressionType::Uncompressed => FileCompressionType::UNCOMPRESSED,
49+
ProtoFileCompressionType::Gzip => FileCompressionType::GZIP,
50+
ProtoFileCompressionType::Bzip2 => FileCompressionType::BZIP2,
51+
ProtoFileCompressionType::Xz => FileCompressionType::XZ,
52+
ProtoFileCompressionType::Zstd => FileCompressionType::ZSTD,
6253
};
6354

55+
let file_ext = p.file_extension;
6456
let mut opts = CsvReadOptions::new()
65-
.has_header(has_header != 0)
66-
.delimiter(delimiter as u8)
67-
.quote(quote as u8)
57+
.has_header(p.has_header)
58+
.delimiter(p.delimiter as u8)
59+
.quote(p.quote as u8)
6860
.file_extension(&file_ext)
6961
.file_compression_type(compression);
7062

71-
if terminator_set != 0 {
72-
opts = opts.terminator(Some(terminator_value as u8));
63+
if let Some(t) = p.terminator {
64+
opts = opts.terminator(Some(t as u8));
7365
}
74-
if escape_set != 0 {
75-
opts = opts.escape(escape_value as u8);
66+
if let Some(e) = p.escape {
67+
opts = opts.escape(e as u8);
7668
}
77-
if comment_set != 0 {
78-
opts = opts.comment(comment_value as u8);
69+
if let Some(c) = p.comment {
70+
opts = opts.comment(c as u8);
7971
}
80-
if newlines_in_values_set != 0 {
81-
opts = opts.newlines_in_values(newlines_in_values_value != 0);
72+
if let Some(n) = p.newlines_in_values {
73+
opts = opts.newlines_in_values(n);
8274
}
83-
if schema_infer_max_records >= 0 {
84-
opts = opts.schema_infer_max_records(schema_infer_max_records as usize);
75+
if let Some(n) = p.schema_infer_max_records {
76+
opts = opts.schema_infer_max_records(n as usize);
8577
}
8678
if let Some(ref s) = schema {
8779
opts = opts.schema(s);
@@ -90,28 +82,14 @@ fn with_csv_options<R>(
9082
f(opts)
9183
}
9284

93-
#[allow(clippy::too_many_arguments)]
9485
#[no_mangle]
9586
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWithOptions<'local>(
9687
mut env: JNIEnv<'local>,
9788
_class: JClass<'local>,
9889
handle: jlong,
9990
name: JString<'local>,
10091
path: JString<'local>,
101-
has_header: jboolean,
102-
delimiter: jbyte,
103-
quote: jbyte,
104-
terminator_set: jboolean,
105-
terminator_value: jbyte,
106-
escape_set: jboolean,
107-
escape_value: jbyte,
108-
comment_set: jboolean,
109-
comment_value: jbyte,
110-
newlines_in_values_set: jboolean,
111-
newlines_in_values_value: jboolean,
112-
schema_infer_max_records: jlong,
113-
file_extension: JString<'local>,
114-
file_compression_type: JString<'local>,
92+
options_bytes: JByteArray<'local>,
11593
schema_ipc_bytes: JByteArray<'local>,
11694
) {
11795
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
@@ -121,55 +99,23 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWith
12199
let ctx = unsafe { &*(handle as *const SessionContext) };
122100
let name: String = env.get_string(&name)?.into();
123101
let path: String = env.get_string(&path)?.into();
124-
with_csv_options(
125-
env,
126-
has_header,
127-
delimiter,
128-
quote,
129-
terminator_set,
130-
terminator_value,
131-
escape_set,
132-
escape_value,
133-
comment_set,
134-
comment_value,
135-
newlines_in_values_set,
136-
newlines_in_values_value,
137-
schema_infer_max_records,
138-
file_extension,
139-
file_compression_type,
140-
schema_ipc_bytes,
141-
|opts| {
142-
runtime().block_on(async {
143-
ctx.register_csv(&name, &path, opts).await?;
144-
Ok::<(), DataFusionError>(())
145-
})?;
146-
Ok(())
147-
},
148-
)
102+
with_csv_options(env, options_bytes, schema_ipc_bytes, |opts| {
103+
runtime().block_on(async {
104+
ctx.register_csv(&name, &path, opts).await?;
105+
Ok::<(), DataFusionError>(())
106+
})?;
107+
Ok(())
108+
})
149109
})
150110
}
151111

152-
#[allow(clippy::too_many_arguments)]
153112
#[no_mangle]
154113
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOptions<'local>(
155114
mut env: JNIEnv<'local>,
156115
_class: JClass<'local>,
157116
handle: jlong,
158117
path: JString<'local>,
159-
has_header: jboolean,
160-
delimiter: jbyte,
161-
quote: jbyte,
162-
terminator_set: jboolean,
163-
terminator_value: jbyte,
164-
escape_set: jboolean,
165-
escape_value: jbyte,
166-
comment_set: jboolean,
167-
comment_value: jbyte,
168-
newlines_in_values_set: jboolean,
169-
newlines_in_values_value: jboolean,
170-
schema_infer_max_records: jlong,
171-
file_extension: JString<'local>,
172-
file_compression_type: JString<'local>,
118+
options_bytes: JByteArray<'local>,
173119
schema_ipc_bytes: JByteArray<'local>,
174120
) -> jlong {
175121
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
@@ -178,27 +124,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOpti
178124
}
179125
let ctx = unsafe { &*(handle as *const SessionContext) };
180126
let path: String = env.get_string(&path)?.into();
181-
with_csv_options(
182-
env,
183-
has_header,
184-
delimiter,
185-
quote,
186-
terminator_set,
187-
terminator_value,
188-
escape_set,
189-
escape_value,
190-
comment_set,
191-
comment_value,
192-
newlines_in_values_set,
193-
newlines_in_values_value,
194-
schema_infer_max_records,
195-
file_extension,
196-
file_compression_type,
197-
schema_ipc_bytes,
198-
|opts| {
199-
let df = runtime().block_on(ctx.read_csv(path, opts))?;
200-
Ok(Box::into_raw(Box::new(df)) as jlong)
201-
},
202-
)
127+
with_csv_options(env, options_bytes, schema_ipc_bytes, |opts| {
128+
let df = runtime().block_on(ctx.read_csv(path, opts))?;
129+
Ok(Box::into_raw(Box::new(df)) as jlong)
130+
})
203131
})
204132
}

‎native/src/lib.rs‎

Lines changed: 28 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
mod csv;
1919
mod errors;
2020
mod proto;
21+
mod schema;
2122

2223
pub(crate) mod session_options {
2324
include!(concat!(env!("OUT_DIR"), "/datafusion_java.rs"));
@@ -26,9 +27,8 @@ pub(crate) mod session_options {
2627
use std::path::PathBuf;
2728
use std::sync::{Arc, OnceLock};
2829

29-
use datafusion::arrow::datatypes::{Schema, SchemaRef};
30+
use datafusion::arrow::datatypes::SchemaRef;
3031
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
31-
use datafusion::arrow::ipc::reader::StreamReader;
3232
use datafusion::arrow::record_batch::RecordBatchIterator;
3333
use datafusion::config::TableParquetOptions;
3434
use datafusion::dataframe::DataFrame;
@@ -43,6 +43,8 @@ use prost::Message;
4343
use tokio::runtime::Runtime;
4444

4545
use crate::errors::{try_unwrap_or_throw, JniResult};
46+
use crate::schema::decode_optional_schema;
47+
use crate::session_options::ParquetReadOptionsProto;
4648
use crate::session_options::SessionOptions;
4749

4850
pub(crate) fn runtime() -> &'static Runtime {
@@ -313,37 +315,27 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_closeSessionCon
313315
})
314316
}
315317

316-
#[allow(clippy::too_many_arguments)]
317318
fn with_parquet_options<R>(
318319
env: &mut JNIEnv,
319-
file_extension: JString,
320-
parquet_pruning_set: jboolean,
321-
parquet_pruning_value: jboolean,
322-
skip_metadata_set: jboolean,
323-
skip_metadata_value: jboolean,
324-
metadata_size_hint: jlong,
320+
options_bytes: JByteArray,
325321
schema_ipc_bytes: JByteArray,
326322
f: impl FnOnce(ParquetReadOptions) -> JniResult<R>,
327323
) -> JniResult<R> {
328-
let file_ext: String = env.get_string(&file_extension)?.into();
324+
let bytes: Vec<u8> = env.convert_byte_array(&options_bytes)?;
325+
let p = ParquetReadOptionsProto::decode(bytes.as_slice())?;
329326

330-
let schema: Option<Schema> = if !schema_ipc_bytes.is_null() {
331-
let bytes: Vec<u8> = env.convert_byte_array(&schema_ipc_bytes)?;
332-
let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
333-
Some((*reader.schema()).clone())
334-
} else {
335-
None
336-
};
327+
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
337328

329+
let file_ext = p.file_extension;
338330
let mut opts = ParquetReadOptions::default().file_extension(&file_ext);
339-
if parquet_pruning_set != 0 {
340-
opts = opts.parquet_pruning(parquet_pruning_value != 0);
331+
if let Some(v) = p.parquet_pruning {
332+
opts = opts.parquet_pruning(v);
341333
}
342-
if skip_metadata_set != 0 {
343-
opts = opts.skip_metadata(skip_metadata_value != 0);
334+
if let Some(v) = p.skip_metadata {
335+
opts = opts.skip_metadata(v);
344336
}
345-
if metadata_size_hint >= 0 {
346-
opts = opts.metadata_size_hint(Some(metadata_size_hint as usize));
337+
if let Some(v) = p.metadata_size_hint {
338+
opts = opts.metadata_size_hint(Some(v as usize));
347339
}
348340
if let Some(ref s) = schema {
349341
opts = opts.schema(s);
@@ -361,12 +353,7 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquet
361353
handle: jlong,
362354
name: JString<'local>,
363355
path: JString<'local>,
364-
file_extension: JString<'local>,
365-
parquet_pruning_set: jboolean,
366-
parquet_pruning_value: jboolean,
367-
skip_metadata_set: jboolean,
368-
skip_metadata_value: jboolean,
369-
metadata_size_hint: jlong,
356+
options_bytes: JByteArray<'local>,
370357
schema_ipc_bytes: JByteArray<'local>,
371358
) {
372359
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
@@ -376,23 +363,13 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquet
376363
let ctx = unsafe { &*(handle as *const SessionContext) };
377364
let name: String = env.get_string(&name)?.into();
378365
let path: String = env.get_string(&path)?.into();
379-
with_parquet_options(
380-
env,
381-
file_extension,
382-
parquet_pruning_set,
383-
parquet_pruning_value,
384-
skip_metadata_set,
385-
skip_metadata_value,
386-
metadata_size_hint,
387-
schema_ipc_bytes,
388-
|opts| {
389-
runtime().block_on(async {
390-
ctx.register_parquet(&name, &path, opts).await?;
391-
Ok::<(), DataFusionError>(())
392-
})?;
393-
Ok(())
394-
},
395-
)
366+
with_parquet_options(env, options_bytes, schema_ipc_bytes, |opts| {
367+
runtime().block_on(async {
368+
ctx.register_parquet(&name, &path, opts).await?;
369+
Ok::<(), DataFusionError>(())
370+
})?;
371+
Ok(())
372+
})
396373
})
397374
}
398375

@@ -402,12 +379,7 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_readParquetWith
402379
_class: JClass<'local>,
403380
handle: jlong,
404381
path: JString<'local>,
405-
file_extension: JString<'local>,
406-
parquet_pruning_set: jboolean,
407-
parquet_pruning_value: jboolean,
408-
skip_metadata_set: jboolean,
409-
skip_metadata_value: jboolean,
410-
metadata_size_hint: jlong,
382+
options_bytes: JByteArray<'local>,
411383
schema_ipc_bytes: JByteArray<'local>,
412384
) -> jlong {
413385
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
@@ -416,19 +388,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_readParquetWith
416388
}
417389
let ctx = unsafe { &*(handle as *const SessionContext) };
418390
let path: String = env.get_string(&path)?.into();
419-
with_parquet_options(
420-
env,
421-
file_extension,
422-
parquet_pruning_set,
423-
parquet_pruning_value,
424-
skip_metadata_set,
425-
skip_metadata_value,
426-
metadata_size_hint,
427-
schema_ipc_bytes,
428-
|opts| {
429-
let df = runtime().block_on(ctx.read_parquet(path, opts))?;
430-
Ok(Box::into_raw(Box::new(df)) as jlong)
431-
},
432-
)
391+
with_parquet_options(env, options_bytes, schema_ipc_bytes, |opts| {
392+
let df = runtime().block_on(ctx.read_parquet(path, opts))?;
393+
Ok(Box::into_raw(Box::new(df)) as jlong)
394+
})
433395
})
434396
}

0 commit comments

Comments
 (0)