Skip to content

Commit 6edb29f

Browse files
committed
feat(csv): expose CsvReadOptions via registerCsv and readCsv
Mirror the parquet pattern (#18, #19): add CsvReadOptions builder, four SessionContext methods (registerCsv x2, readCsv x2), and JNI plumbing in native/src/csv.rs. Builder exposes the subset of DataFusion's Rust CsvReadOptions that has a parquet analog already on the Java side -- header, delimiter, quote, escape, terminator, comment, newlinesInValues, schemaInferMaxRecords, fileExtension, fileCompressionType, and explicit Arrow schema. Tests cover: option-builder fluent API, header-inferred schema with SQL round-trip, explicit-schema header-less file with custom delimiter, and a custom file extension with a tab delimiter. table_partition_cols, file_sort_order, null_regex and truncated_rows are intentionally deferred -- they have no parquet-side counterpart yet on the Java side.
1 parent ca7ebef commit 6edb29f

7 files changed

Lines changed: 677 additions & 1 deletion

File tree

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ Query interfaces:
4848
Data sources:
4949

5050
- [x] Parquet via `registerParquet` / `readParquet`, with `ParquetReadOptions`
51-
- [ ] CSV, JSON, Avro
51+
- [x] CSV via `registerCsv` / `readCsv`, with `CsvReadOptions`
52+
- [ ] JSON, Avro
5253
- [ ] Custom catalog and table providers
5354

5455
Results:

‎native/src/csv.rs‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use std::str::FromStr;
19+
20+
use datafusion::arrow::datatypes::Schema;
21+
use datafusion::arrow::ipc::reader::StreamReader;
22+
use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
23+
use datafusion::error::DataFusionError;
24+
use datafusion::prelude::{CsvReadOptions, SessionContext};
25+
use jni::objects::{JByteArray, JClass, JString};
26+
use jni::sys::{jboolean, jbyte, jlong};
27+
use jni::JNIEnv;
28+
29+
use crate::errors::{try_unwrap_or_throw, JniResult};
30+
use crate::runtime;
31+
32+
#[allow(clippy::too_many_arguments)]
33+
fn with_csv_options<R>(
34+
env: &mut JNIEnv,
35+
has_header: jboolean,
36+
delimiter: jbyte,
37+
quote: jbyte,
38+
terminator_set: jboolean,
39+
terminator_value: jbyte,
40+
escape_set: jboolean,
41+
escape_value: jbyte,
42+
comment_set: jboolean,
43+
comment_value: jbyte,
44+
newlines_in_values_set: jboolean,
45+
newlines_in_values_value: jboolean,
46+
schema_infer_max_records: jlong,
47+
file_extension: JString,
48+
file_compression_type: JString,
49+
schema_ipc_bytes: JByteArray,
50+
f: impl FnOnce(CsvReadOptions) -> JniResult<R>,
51+
) -> JniResult<R> {
52+
let file_ext: String = env.get_string(&file_extension)?.into();
53+
let compression: String = env.get_string(&file_compression_type)?.into();
54+
let compression = FileCompressionType::from_str(&compression)?;
55+
56+
let schema: Option<Schema> = if !schema_ipc_bytes.is_null() {
57+
let bytes: Vec<u8> = env.convert_byte_array(&schema_ipc_bytes)?;
58+
let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
59+
Some((*reader.schema()).clone())
60+
} else {
61+
None
62+
};
63+
64+
let mut opts = CsvReadOptions::new()
65+
.has_header(has_header != 0)
66+
.delimiter(delimiter as u8)
67+
.quote(quote as u8)
68+
.file_extension(&file_ext)
69+
.file_compression_type(compression);
70+
71+
if terminator_set != 0 {
72+
opts = opts.terminator(Some(terminator_value as u8));
73+
}
74+
if escape_set != 0 {
75+
opts = opts.escape(escape_value as u8);
76+
}
77+
if comment_set != 0 {
78+
opts = opts.comment(comment_value as u8);
79+
}
80+
if newlines_in_values_set != 0 {
81+
opts = opts.newlines_in_values(newlines_in_values_value != 0);
82+
}
83+
if schema_infer_max_records >= 0 {
84+
opts = opts.schema_infer_max_records(schema_infer_max_records as usize);
85+
}
86+
if let Some(ref s) = schema {
87+
opts = opts.schema(s);
88+
}
89+
90+
f(opts)
91+
}
92+
93+
#[allow(clippy::too_many_arguments)]
94+
#[no_mangle]
95+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWithOptions<'local>(
96+
mut env: JNIEnv<'local>,
97+
_class: JClass<'local>,
98+
handle: jlong,
99+
name: JString<'local>,
100+
path: JString<'local>,
101+
has_header: jboolean,
102+
delimiter: jbyte,
103+
quote: jbyte,
104+
terminator_set: jboolean,
105+
terminator_value: jbyte,
106+
escape_set: jboolean,
107+
escape_value: jbyte,
108+
comment_set: jboolean,
109+
comment_value: jbyte,
110+
newlines_in_values_set: jboolean,
111+
newlines_in_values_value: jboolean,
112+
schema_infer_max_records: jlong,
113+
file_extension: JString<'local>,
114+
file_compression_type: JString<'local>,
115+
schema_ipc_bytes: JByteArray<'local>,
116+
) {
117+
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
118+
if handle == 0 {
119+
return Err("SessionContext handle is null".into());
120+
}
121+
let ctx = unsafe { &*(handle as *const SessionContext) };
122+
let name: String = env.get_string(&name)?.into();
123+
let path: String = env.get_string(&path)?.into();
124+
with_csv_options(
125+
env,
126+
has_header,
127+
delimiter,
128+
quote,
129+
terminator_set,
130+
terminator_value,
131+
escape_set,
132+
escape_value,
133+
comment_set,
134+
comment_value,
135+
newlines_in_values_set,
136+
newlines_in_values_value,
137+
schema_infer_max_records,
138+
file_extension,
139+
file_compression_type,
140+
schema_ipc_bytes,
141+
|opts| {
142+
runtime().block_on(async {
143+
ctx.register_csv(&name, &path, opts).await?;
144+
Ok::<(), DataFusionError>(())
145+
})?;
146+
Ok(())
147+
},
148+
)
149+
})
150+
}
151+
152+
#[allow(clippy::too_many_arguments)]
153+
#[no_mangle]
154+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOptions<'local>(
155+
mut env: JNIEnv<'local>,
156+
_class: JClass<'local>,
157+
handle: jlong,
158+
path: JString<'local>,
159+
has_header: jboolean,
160+
delimiter: jbyte,
161+
quote: jbyte,
162+
terminator_set: jboolean,
163+
terminator_value: jbyte,
164+
escape_set: jboolean,
165+
escape_value: jbyte,
166+
comment_set: jboolean,
167+
comment_value: jbyte,
168+
newlines_in_values_set: jboolean,
169+
newlines_in_values_value: jboolean,
170+
schema_infer_max_records: jlong,
171+
file_extension: JString<'local>,
172+
file_compression_type: JString<'local>,
173+
schema_ipc_bytes: JByteArray<'local>,
174+
) -> jlong {
175+
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
176+
if handle == 0 {
177+
return Err("SessionContext handle is null".into());
178+
}
179+
let ctx = unsafe { &*(handle as *const SessionContext) };
180+
let path: String = env.get_string(&path)?.into();
181+
with_csv_options(
182+
env,
183+
has_header,
184+
delimiter,
185+
quote,
186+
terminator_set,
187+
terminator_value,
188+
escape_set,
189+
escape_value,
190+
comment_set,
191+
comment_value,
192+
newlines_in_values_set,
193+
newlines_in_values_value,
194+
schema_infer_max_records,
195+
file_extension,
196+
file_compression_type,
197+
schema_ipc_bytes,
198+
|opts| {
199+
let df = runtime().block_on(ctx.read_csv(path, opts))?;
200+
Ok(Box::into_raw(Box::new(df)) as jlong)
201+
},
202+
)
203+
})
204+
}

‎native/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
mod csv;
1819
mod errors;
1920
mod proto;
2021

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
24+
/**
25+
* Configuration knobs for CSV sources passed to {@link SessionContext#registerCsv(String, String,
26+
* CsvReadOptions)} and {@link SessionContext#readCsv(String, CsvReadOptions)}.
27+
*
28+
* <p>Mirrors a subset of DataFusion's {@code CsvReadOptions}. All setters return {@code this} for
29+
* fluent chaining. Defaults match the Rust struct: {@code hasHeader = true}, {@code delimiter =
30+
* ','}, {@code quote = '"'}, {@code fileExtension = ".csv"}, {@code fileCompressionType =
31+
* UNCOMPRESSED}, all other fields {@code null} (meaning the DataFusion default is used, or the
32+
* schema is inferred from the file).
33+
*/
34+
public final class CsvReadOptions {
35+
36+
/** Compression of the file. Names match DataFusion's {@code FileCompressionType} variants. */
37+
public enum FileCompressionType {
38+
UNCOMPRESSED,
39+
GZIP,
40+
BZIP2,
41+
XZ,
42+
ZSTD
43+
}
44+
45+
private boolean hasHeader = true;
46+
private byte delimiter = (byte) ',';
47+
private byte quote = (byte) '"';
48+
private Byte terminator;
49+
private Byte escape;
50+
private Byte comment;
51+
private Boolean newlinesInValues;
52+
private Long schemaInferMaxRecords;
53+
private String fileExtension = ".csv";
54+
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
55+
private Schema schema;
56+
57+
public CsvReadOptions hasHeader(boolean v) {
58+
this.hasHeader = v;
59+
return this;
60+
}
61+
62+
public CsvReadOptions delimiter(byte b) {
63+
this.delimiter = b;
64+
return this;
65+
}
66+
67+
public CsvReadOptions quote(byte b) {
68+
this.quote = b;
69+
return this;
70+
}
71+
72+
public CsvReadOptions terminator(byte b) {
73+
this.terminator = b;
74+
return this;
75+
}
76+
77+
public CsvReadOptions escape(byte b) {
78+
this.escape = b;
79+
return this;
80+
}
81+
82+
public CsvReadOptions comment(byte b) {
83+
this.comment = b;
84+
return this;
85+
}
86+
87+
public CsvReadOptions newlinesInValues(boolean v) {
88+
this.newlinesInValues = v;
89+
return this;
90+
}
91+
92+
public CsvReadOptions schemaInferMaxRecords(long n) {
93+
this.schemaInferMaxRecords = n;
94+
return this;
95+
}
96+
97+
public CsvReadOptions fileExtension(String ext) {
98+
this.fileExtension = ext;
99+
return this;
100+
}
101+
102+
public CsvReadOptions fileCompressionType(FileCompressionType t) {
103+
this.fileCompressionType = t;
104+
return this;
105+
}
106+
107+
public CsvReadOptions schema(Schema schema) {
108+
this.schema = schema;
109+
return this;
110+
}
111+
112+
boolean hasHeader() {
113+
return hasHeader;
114+
}
115+
116+
byte delimiter() {
117+
return delimiter;
118+
}
119+
120+
byte quote() {
121+
return quote;
122+
}
123+
124+
Byte terminator() {
125+
return terminator;
126+
}
127+
128+
Byte escape() {
129+
return escape;
130+
}
131+
132+
Byte comment() {
133+
return comment;
134+
}
135+
136+
Boolean newlinesInValues() {
137+
return newlinesInValues;
138+
}
139+
140+
Long schemaInferMaxRecords() {
141+
return schemaInferMaxRecords;
142+
}
143+
144+
String fileExtension() {
145+
return fileExtension;
146+
}
147+
148+
FileCompressionType fileCompressionType() {
149+
return fileCompressionType;
150+
}
151+
152+
Schema schema() {
153+
return schema;
154+
}
155+
}

0 commit comments

Comments
 (0)