Skip to content

Commit 9dacdc8

Browse files
authored
refactor: pass csv and parquet read options via protobuf (#29)
1 parent 3698b9a commit 9dacdc8

13 files changed

Lines changed: 382 additions & 390 deletions

File tree

‎docs/source/contributor-guide/development.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,40 @@ disk space.
7474
- `mvnw`, `mvnw.cmd` — bundled Maven wrapper.
7575
- `src/` — Java sources and tests.
7676
- `native/` — Rust crate (JNI + Arrow C Data Interface).
77+
- `proto/` — Protobuf definitions shared between Java and Rust.
7778
- `docs/` — Sphinx documentation source and build scripts.
79+
80+
## Passing structured options across the JNI boundary
81+
82+
When a JNI call needs to carry more than a handful of scalar arguments —
83+
for example, a struct of nullable knobs like `CsvReadOptions` or
84+
`SessionOptions` — encode the call's configuration as a protobuf message
85+
rather than expanding the JNI signature with one parameter per field.
86+
87+
Add a `.proto` file under `proto/`, declare `package datafusion_java;`,
88+
and follow the conventions already in use:
89+
90+
- One message per logical bundle of options.
91+
- Use `optional` for fields whose unset-ness must survive the boundary
92+
(so the Rust side can leave a DataFusion default in place).
93+
- Use proto enums for closed sets of choices instead of strings.
94+
Prefix every value with the enum's name (e.g.
95+
`FILE_COMPRESSION_TYPE_GZIP`, not bare `GZIP`) because proto3 enum
96+
values are scoped at the package level, and the zero value must be a
97+
`_UNSPECIFIED` sentinel — the Rust side should reject `UNSPECIFIED`
98+
rather than silently default it.
99+
- Keep large opaque payloads (Arrow IPC schemas, plan nodes) as separate
100+
`byte[]` JNI arguments next to the options proto, not inside it.
101+
- Suffix the message name with `Proto` if a sibling Java class would
102+
otherwise shadow it (e.g. `CsvReadOptionsProto` vs the public
103+
`CsvReadOptions`).
104+
105+
The proto is compiled by both `prost-build` (Rust, via `native/build.rs`)
106+
and the Maven `protobuf-maven-plugin` (Java). The Java side builds the
107+
message, serializes to bytes, and passes the byte array through JNI; the
108+
Rust side decodes once and folds the fields into the corresponding
109+
DataFusion struct.
110+
111+
This pattern keeps JNI signatures short, makes nullable and enum fields
112+
explicit in a single typed schema, and lets new fields be added without
113+
touching the signature on either side.

‎native/build.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,15 @@
1616
// under the License.
1717

1818
fn main() {
19-
println!("cargo:rerun-if-changed=../proto/session_options.proto");
19+
const PROTOS: &[&str] = &[
20+
"../proto/session_options.proto",
21+
"../proto/csv_read_options.proto",
22+
"../proto/parquet_read_options.proto",
23+
];
24+
for p in PROTOS {
25+
println!("cargo:rerun-if-changed={p}");
26+
}
2027
let protoc = protoc_bin_vendored::protoc_bin_path().expect("vendored protoc not available");
2128
std::env::set_var("PROTOC", protoc);
22-
prost_build::compile_protos(&["../proto/session_options.proto"], &["../proto"])
23-
.expect("failed to compile session_options.proto");
29+
prost_build::compile_protos(PROTOS, &["../proto"]).expect("failed to compile protos");
2430
}

‎native/src/csv.rs‎

Lines changed: 45 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -15,73 +15,63 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::str::FromStr;
19-
20-
use datafusion::arrow::datatypes::Schema;
21-
use datafusion::arrow::ipc::reader::StreamReader;
2218
use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
2319
use datafusion::error::DataFusionError;
2420
use datafusion::prelude::{CsvReadOptions, SessionContext};
2521
use jni::objects::{JByteArray, JClass, JString};
26-
use jni::sys::{jboolean, jbyte, jlong};
22+
use jni::sys::jlong;
2723
use jni::JNIEnv;
24+
use prost::Message;
2825

2926
use crate::errors::{try_unwrap_or_throw, JniResult};
27+
use crate::proto_gen::{CsvReadOptionsProto, FileCompressionType as ProtoFileCompressionType};
3028
use crate::runtime;
29+
use crate::schema::decode_optional_schema;
3130

32-
#[allow(clippy::too_many_arguments)]
3331
fn with_csv_options<R>(
3432
env: &mut JNIEnv,
35-
has_header: jboolean,
36-
delimiter: jbyte,
37-
quote: jbyte,
38-
terminator_set: jboolean,
39-
terminator_value: jbyte,
40-
escape_set: jboolean,
41-
escape_value: jbyte,
42-
comment_set: jboolean,
43-
comment_value: jbyte,
44-
newlines_in_values_set: jboolean,
45-
newlines_in_values_value: jboolean,
46-
schema_infer_max_records: jlong,
47-
file_extension: JString,
48-
file_compression_type: JString,
33+
options_bytes: JByteArray,
4934
schema_ipc_bytes: JByteArray,
5035
f: impl FnOnce(CsvReadOptions) -> JniResult<R>,
5136
) -> JniResult<R> {
52-
let file_ext: String = env.get_string(&file_extension)?.into();
53-
let compression: String = env.get_string(&file_compression_type)?.into();
54-
let compression = FileCompressionType::from_str(&compression)?;
37+
let bytes: Vec<u8> = env.convert_byte_array(&options_bytes)?;
38+
let p = CsvReadOptionsProto::decode(bytes.as_slice())?;
39+
40+
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
5541

56-
let schema: Option<Schema> = if !schema_ipc_bytes.is_null() {
57-
let bytes: Vec<u8> = env.convert_byte_array(&schema_ipc_bytes)?;
58-
let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
59-
Some((*reader.schema()).clone())
60-
} else {
61-
None
42+
let compression = match p.file_compression_type() {
43+
ProtoFileCompressionType::Unspecified => {
44+
return Err("CsvReadOptionsProto.file_compression_type is UNSPECIFIED".into());
45+
}
46+
ProtoFileCompressionType::Uncompressed => FileCompressionType::UNCOMPRESSED,
47+
ProtoFileCompressionType::Gzip => FileCompressionType::GZIP,
48+
ProtoFileCompressionType::Bzip2 => FileCompressionType::BZIP2,
49+
ProtoFileCompressionType::Xz => FileCompressionType::XZ,
50+
ProtoFileCompressionType::Zstd => FileCompressionType::ZSTD,
6251
};
6352

53+
let file_ext = p.file_extension;
6454
let mut opts = CsvReadOptions::new()
65-
.has_header(has_header != 0)
66-
.delimiter(delimiter as u8)
67-
.quote(quote as u8)
55+
.has_header(p.has_header)
56+
.delimiter(p.delimiter as u8)
57+
.quote(p.quote as u8)
6858
.file_extension(&file_ext)
6959
.file_compression_type(compression);
7060

71-
if terminator_set != 0 {
72-
opts = opts.terminator(Some(terminator_value as u8));
61+
if let Some(t) = p.terminator {
62+
opts = opts.terminator(Some(t as u8));
7363
}
74-
if escape_set != 0 {
75-
opts = opts.escape(escape_value as u8);
64+
if let Some(e) = p.escape {
65+
opts = opts.escape(e as u8);
7666
}
77-
if comment_set != 0 {
78-
opts = opts.comment(comment_value as u8);
67+
if let Some(c) = p.comment {
68+
opts = opts.comment(c as u8);
7969
}
80-
if newlines_in_values_set != 0 {
81-
opts = opts.newlines_in_values(newlines_in_values_value != 0);
70+
if let Some(n) = p.newlines_in_values {
71+
opts = opts.newlines_in_values(n);
8272
}
83-
if schema_infer_max_records >= 0 {
84-
opts = opts.schema_infer_max_records(schema_infer_max_records as usize);
73+
if let Some(n) = p.schema_infer_max_records {
74+
opts = opts.schema_infer_max_records(n as usize);
8575
}
8676
if let Some(ref s) = schema {
8777
opts = opts.schema(s);
@@ -90,28 +80,14 @@ fn with_csv_options<R>(
9080
f(opts)
9181
}
9282

93-
#[allow(clippy::too_many_arguments)]
9483
#[no_mangle]
9584
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWithOptions<'local>(
9685
mut env: JNIEnv<'local>,
9786
_class: JClass<'local>,
9887
handle: jlong,
9988
name: JString<'local>,
10089
path: JString<'local>,
101-
has_header: jboolean,
102-
delimiter: jbyte,
103-
quote: jbyte,
104-
terminator_set: jboolean,
105-
terminator_value: jbyte,
106-
escape_set: jboolean,
107-
escape_value: jbyte,
108-
comment_set: jboolean,
109-
comment_value: jbyte,
110-
newlines_in_values_set: jboolean,
111-
newlines_in_values_value: jboolean,
112-
schema_infer_max_records: jlong,
113-
file_extension: JString<'local>,
114-
file_compression_type: JString<'local>,
90+
options_bytes: JByteArray<'local>,
11591
schema_ipc_bytes: JByteArray<'local>,
11692
) {
11793
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
@@ -121,55 +97,23 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWith
12197
let ctx = unsafe { &*(handle as *const SessionContext) };
12298
let name: String = env.get_string(&name)?.into();
12399
let path: String = env.get_string(&path)?.into();
124-
with_csv_options(
125-
env,
126-
has_header,
127-
delimiter,
128-
quote,
129-
terminator_set,
130-
terminator_value,
131-
escape_set,
132-
escape_value,
133-
comment_set,
134-
comment_value,
135-
newlines_in_values_set,
136-
newlines_in_values_value,
137-
schema_infer_max_records,
138-
file_extension,
139-
file_compression_type,
140-
schema_ipc_bytes,
141-
|opts| {
142-
runtime().block_on(async {
143-
ctx.register_csv(&name, &path, opts).await?;
144-
Ok::<(), DataFusionError>(())
145-
})?;
146-
Ok(())
147-
},
148-
)
100+
with_csv_options(env, options_bytes, schema_ipc_bytes, |opts| {
101+
runtime().block_on(async {
102+
ctx.register_csv(&name, &path, opts).await?;
103+
Ok::<(), DataFusionError>(())
104+
})?;
105+
Ok(())
106+
})
149107
})
150108
}
151109

152-
#[allow(clippy::too_many_arguments)]
153110
#[no_mangle]
154111
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOptions<'local>(
155112
mut env: JNIEnv<'local>,
156113
_class: JClass<'local>,
157114
handle: jlong,
158115
path: JString<'local>,
159-
has_header: jboolean,
160-
delimiter: jbyte,
161-
quote: jbyte,
162-
terminator_set: jboolean,
163-
terminator_value: jbyte,
164-
escape_set: jboolean,
165-
escape_value: jbyte,
166-
comment_set: jboolean,
167-
comment_value: jbyte,
168-
newlines_in_values_set: jboolean,
169-
newlines_in_values_value: jboolean,
170-
schema_infer_max_records: jlong,
171-
file_extension: JString<'local>,
172-
file_compression_type: JString<'local>,
116+
options_bytes: JByteArray<'local>,
173117
schema_ipc_bytes: JByteArray<'local>,
174118
) -> jlong {
175119
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
@@ -178,27 +122,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOpti
178122
}
179123
let ctx = unsafe { &*(handle as *const SessionContext) };
180124
let path: String = env.get_string(&path)?.into();
181-
with_csv_options(
182-
env,
183-
has_header,
184-
delimiter,
185-
quote,
186-
terminator_set,
187-
terminator_value,
188-
escape_set,
189-
escape_value,
190-
comment_set,
191-
comment_value,
192-
newlines_in_values_set,
193-
newlines_in_values_value,
194-
schema_infer_max_records,
195-
file_extension,
196-
file_compression_type,
197-
schema_ipc_bytes,
198-
|opts| {
199-
let df = runtime().block_on(ctx.read_csv(path, opts))?;
200-
Ok(Box::into_raw(Box::new(df)) as jlong)
201-
},
202-
)
125+
with_csv_options(env, options_bytes, schema_ipc_bytes, |opts| {
126+
let df = runtime().block_on(ctx.read_csv(path, opts))?;
127+
Ok(Box::into_raw(Box::new(df)) as jlong)
128+
})
203129
})
204130
}

0 commit comments

Comments
 (0)