1515// specific language governing permissions and limitations
1616// under the License.
1717
18- use std:: str:: FromStr ;
19-
20- use datafusion:: arrow:: datatypes:: Schema ;
21- use datafusion:: arrow:: ipc:: reader:: StreamReader ;
2218use datafusion:: datasource:: file_format:: file_compression_type:: FileCompressionType ;
2319use datafusion:: error:: DataFusionError ;
2420use datafusion:: prelude:: { CsvReadOptions , SessionContext } ;
2521use jni:: objects:: { JByteArray , JClass , JString } ;
26- use jni:: sys:: { jboolean , jbyte , jlong} ;
22+ use jni:: sys:: jlong;
2723use jni:: JNIEnv ;
24+ use prost:: Message ;
2825
2926use crate :: errors:: { try_unwrap_or_throw, JniResult } ;
27+ use crate :: proto_gen:: { CsvReadOptionsProto , FileCompressionType as ProtoFileCompressionType } ;
3028use crate :: runtime;
29+ use crate :: schema:: decode_optional_schema;
3130
32- #[ allow( clippy:: too_many_arguments) ]
3331fn with_csv_options < R > (
3432 env : & mut JNIEnv ,
35- has_header : jboolean ,
36- delimiter : jbyte ,
37- quote : jbyte ,
38- terminator_set : jboolean ,
39- terminator_value : jbyte ,
40- escape_set : jboolean ,
41- escape_value : jbyte ,
42- comment_set : jboolean ,
43- comment_value : jbyte ,
44- newlines_in_values_set : jboolean ,
45- newlines_in_values_value : jboolean ,
46- schema_infer_max_records : jlong ,
47- file_extension : JString ,
48- file_compression_type : JString ,
33+ options_bytes : JByteArray ,
4934 schema_ipc_bytes : JByteArray ,
5035 f : impl FnOnce ( CsvReadOptions ) -> JniResult < R > ,
5136) -> JniResult < R > {
52- let file_ext: String = env. get_string ( & file_extension) ?. into ( ) ;
53- let compression: String = env. get_string ( & file_compression_type) ?. into ( ) ;
54- let compression = FileCompressionType :: from_str ( & compression) ?;
37+ let bytes: Vec < u8 > = env. convert_byte_array ( & options_bytes) ?;
38+ let p = CsvReadOptionsProto :: decode ( bytes. as_slice ( ) ) ?;
39+
40+ let schema = decode_optional_schema ( env, schema_ipc_bytes) ?;
5541
56- let schema: Option < Schema > = if !schema_ipc_bytes. is_null ( ) {
57- let bytes: Vec < u8 > = env. convert_byte_array ( & schema_ipc_bytes) ?;
58- let reader = StreamReader :: try_new ( std:: io:: Cursor :: new ( bytes) , None ) ?;
59- Some ( ( * reader. schema ( ) ) . clone ( ) )
60- } else {
61- None
42+ let compression = match p. file_compression_type ( ) {
43+ ProtoFileCompressionType :: Unspecified => {
44+ return Err ( "CsvReadOptionsProto.file_compression_type is UNSPECIFIED" . into ( ) ) ;
45+ }
46+ ProtoFileCompressionType :: Uncompressed => FileCompressionType :: UNCOMPRESSED ,
47+ ProtoFileCompressionType :: Gzip => FileCompressionType :: GZIP ,
48+ ProtoFileCompressionType :: Bzip2 => FileCompressionType :: BZIP2 ,
49+ ProtoFileCompressionType :: Xz => FileCompressionType :: XZ ,
50+ ProtoFileCompressionType :: Zstd => FileCompressionType :: ZSTD ,
6251 } ;
6352
53+ let file_ext = p. file_extension ;
6454 let mut opts = CsvReadOptions :: new ( )
65- . has_header ( has_header != 0 )
66- . delimiter ( delimiter as u8 )
67- . quote ( quote as u8 )
55+ . has_header ( p . has_header )
56+ . delimiter ( p . delimiter as u8 )
57+ . quote ( p . quote as u8 )
6858 . file_extension ( & file_ext)
6959 . file_compression_type ( compression) ;
7060
71- if terminator_set != 0 {
72- opts = opts. terminator ( Some ( terminator_value as u8 ) ) ;
61+ if let Some ( t ) = p . terminator {
62+ opts = opts. terminator ( Some ( t as u8 ) ) ;
7363 }
74- if escape_set != 0 {
75- opts = opts. escape ( escape_value as u8 ) ;
64+ if let Some ( e ) = p . escape {
65+ opts = opts. escape ( e as u8 ) ;
7666 }
77- if comment_set != 0 {
78- opts = opts. comment ( comment_value as u8 ) ;
67+ if let Some ( c ) = p . comment {
68+ opts = opts. comment ( c as u8 ) ;
7969 }
80- if newlines_in_values_set != 0 {
81- opts = opts. newlines_in_values ( newlines_in_values_value != 0 ) ;
70+ if let Some ( n ) = p . newlines_in_values {
71+ opts = opts. newlines_in_values ( n ) ;
8272 }
83- if schema_infer_max_records >= 0 {
84- opts = opts. schema_infer_max_records ( schema_infer_max_records as usize ) ;
73+ if let Some ( n ) = p . schema_infer_max_records {
74+ opts = opts. schema_infer_max_records ( n as usize ) ;
8575 }
8676 if let Some ( ref s) = schema {
8777 opts = opts. schema ( s) ;
@@ -90,28 +80,14 @@ fn with_csv_options<R>(
9080 f ( opts)
9181}
9282
93- #[ allow( clippy:: too_many_arguments) ]
9483#[ no_mangle]
9584pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWithOptions < ' local > (
9685 mut env : JNIEnv < ' local > ,
9786 _class : JClass < ' local > ,
9887 handle : jlong ,
9988 name : JString < ' local > ,
10089 path : JString < ' local > ,
101- has_header : jboolean ,
102- delimiter : jbyte ,
103- quote : jbyte ,
104- terminator_set : jboolean ,
105- terminator_value : jbyte ,
106- escape_set : jboolean ,
107- escape_value : jbyte ,
108- comment_set : jboolean ,
109- comment_value : jbyte ,
110- newlines_in_values_set : jboolean ,
111- newlines_in_values_value : jboolean ,
112- schema_infer_max_records : jlong ,
113- file_extension : JString < ' local > ,
114- file_compression_type : JString < ' local > ,
90+ options_bytes : JByteArray < ' local > ,
11591 schema_ipc_bytes : JByteArray < ' local > ,
11692) {
11793 try_unwrap_or_throw ( & mut env, ( ) , |env| -> JniResult < ( ) > {
@@ -121,55 +97,23 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWith
12197 let ctx = unsafe { & * ( handle as * const SessionContext ) } ;
12298 let name: String = env. get_string ( & name) ?. into ( ) ;
12399 let path: String = env. get_string ( & path) ?. into ( ) ;
124- with_csv_options (
125- env,
126- has_header,
127- delimiter,
128- quote,
129- terminator_set,
130- terminator_value,
131- escape_set,
132- escape_value,
133- comment_set,
134- comment_value,
135- newlines_in_values_set,
136- newlines_in_values_value,
137- schema_infer_max_records,
138- file_extension,
139- file_compression_type,
140- schema_ipc_bytes,
141- |opts| {
142- runtime ( ) . block_on ( async {
143- ctx. register_csv ( & name, & path, opts) . await ?;
144- Ok :: < ( ) , DataFusionError > ( ( ) )
145- } ) ?;
146- Ok ( ( ) )
147- } ,
148- )
100+ with_csv_options ( env, options_bytes, schema_ipc_bytes, |opts| {
101+ runtime ( ) . block_on ( async {
102+ ctx. register_csv ( & name, & path, opts) . await ?;
103+ Ok :: < ( ) , DataFusionError > ( ( ) )
104+ } ) ?;
105+ Ok ( ( ) )
106+ } )
149107 } )
150108}
151109
152- #[ allow( clippy:: too_many_arguments) ]
153110#[ no_mangle]
154111pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOptions < ' local > (
155112 mut env : JNIEnv < ' local > ,
156113 _class : JClass < ' local > ,
157114 handle : jlong ,
158115 path : JString < ' local > ,
159- has_header : jboolean ,
160- delimiter : jbyte ,
161- quote : jbyte ,
162- terminator_set : jboolean ,
163- terminator_value : jbyte ,
164- escape_set : jboolean ,
165- escape_value : jbyte ,
166- comment_set : jboolean ,
167- comment_value : jbyte ,
168- newlines_in_values_set : jboolean ,
169- newlines_in_values_value : jboolean ,
170- schema_infer_max_records : jlong ,
171- file_extension : JString < ' local > ,
172- file_compression_type : JString < ' local > ,
116+ options_bytes : JByteArray < ' local > ,
173117 schema_ipc_bytes : JByteArray < ' local > ,
174118) -> jlong {
175119 try_unwrap_or_throw ( & mut env, 0 , |env| -> JniResult < jlong > {
@@ -178,27 +122,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOpti
178122 }
179123 let ctx = unsafe { & * ( handle as * const SessionContext ) } ;
180124 let path: String = env. get_string ( & path) ?. into ( ) ;
181- with_csv_options (
182- env,
183- has_header,
184- delimiter,
185- quote,
186- terminator_set,
187- terminator_value,
188- escape_set,
189- escape_value,
190- comment_set,
191- comment_value,
192- newlines_in_values_set,
193- newlines_in_values_value,
194- schema_infer_max_records,
195- file_extension,
196- file_compression_type,
197- schema_ipc_bytes,
198- |opts| {
199- let df = runtime ( ) . block_on ( ctx. read_csv ( path, opts) ) ?;
200- Ok ( Box :: into_raw ( Box :: new ( df) ) as jlong )
201- } ,
202- )
125+ with_csv_options ( env, options_bytes, schema_ipc_bytes, |opts| {
126+ let df = runtime ( ) . block_on ( ctx. read_csv ( path, opts) ) ?;
127+ Ok ( Box :: into_raw ( Box :: new ( df) ) as jlong )
128+ } )
203129 } )
204130}
0 commit comments