1515// specific language governing permissions and limitations
1616// under the License.
1717
18- use std:: str:: FromStr ;
19-
20- use datafusion:: arrow:: datatypes:: Schema ;
21- use datafusion:: arrow:: ipc:: reader:: StreamReader ;
2218use datafusion:: datasource:: file_format:: file_compression_type:: FileCompressionType ;
2319use datafusion:: error:: DataFusionError ;
2420use datafusion:: prelude:: { CsvReadOptions , SessionContext } ;
2521use jni:: objects:: { JByteArray , JClass , JString } ;
26- use jni:: sys:: { jboolean , jbyte , jlong} ;
22+ use jni:: sys:: jlong;
2723use jni:: JNIEnv ;
24+ use prost:: Message ;
2825
2926use crate :: errors:: { try_unwrap_or_throw, JniResult } ;
3027use crate :: runtime;
28+ use crate :: schema:: decode_optional_schema;
29+ use crate :: session_options:: {
30+ CsvReadOptionsProto , FileCompressionType as ProtoFileCompressionType ,
31+ } ;
3132
32- #[ allow( clippy:: too_many_arguments) ]
3333fn with_csv_options < R > (
3434 env : & mut JNIEnv ,
35- has_header : jboolean ,
36- delimiter : jbyte ,
37- quote : jbyte ,
38- terminator_set : jboolean ,
39- terminator_value : jbyte ,
40- escape_set : jboolean ,
41- escape_value : jbyte ,
42- comment_set : jboolean ,
43- comment_value : jbyte ,
44- newlines_in_values_set : jboolean ,
45- newlines_in_values_value : jboolean ,
46- schema_infer_max_records : jlong ,
47- file_extension : JString ,
48- file_compression_type : JString ,
35+ options_bytes : JByteArray ,
4936 schema_ipc_bytes : JByteArray ,
5037 f : impl FnOnce ( CsvReadOptions ) -> JniResult < R > ,
5138) -> JniResult < R > {
52- let file_ext: String = env. get_string ( & file_extension) ?. into ( ) ;
53- let compression: String = env. get_string ( & file_compression_type) ?. into ( ) ;
54- let compression = FileCompressionType :: from_str ( & compression) ?;
39+ let bytes: Vec < u8 > = env. convert_byte_array ( & options_bytes) ?;
40+ let p = CsvReadOptionsProto :: decode ( bytes. as_slice ( ) ) ?;
41+
42+ let schema = decode_optional_schema ( env, schema_ipc_bytes) ?;
5543
56- let schema: Option < Schema > = if !schema_ipc_bytes. is_null ( ) {
57- let bytes: Vec < u8 > = env. convert_byte_array ( & schema_ipc_bytes) ?;
58- let reader = StreamReader :: try_new ( std:: io:: Cursor :: new ( bytes) , None ) ?;
59- Some ( ( * reader. schema ( ) ) . clone ( ) )
60- } else {
61- None
44+ let compression = match p. file_compression_type ( ) {
45+ ProtoFileCompressionType :: Unspecified => {
46+ return Err ( "CsvReadOptionsProto.file_compression_type is UNSPECIFIED" . into ( ) ) ;
47+ }
48+ ProtoFileCompressionType :: Uncompressed => FileCompressionType :: UNCOMPRESSED ,
49+ ProtoFileCompressionType :: Gzip => FileCompressionType :: GZIP ,
50+ ProtoFileCompressionType :: Bzip2 => FileCompressionType :: BZIP2 ,
51+ ProtoFileCompressionType :: Xz => FileCompressionType :: XZ ,
52+ ProtoFileCompressionType :: Zstd => FileCompressionType :: ZSTD ,
6253 } ;
6354
55+ let file_ext = p. file_extension ;
6456 let mut opts = CsvReadOptions :: new ( )
65- . has_header ( has_header != 0 )
66- . delimiter ( delimiter as u8 )
67- . quote ( quote as u8 )
57+ . has_header ( p . has_header )
58+ . delimiter ( p . delimiter as u8 )
59+ . quote ( p . quote as u8 )
6860 . file_extension ( & file_ext)
6961 . file_compression_type ( compression) ;
7062
71- if terminator_set != 0 {
72- opts = opts. terminator ( Some ( terminator_value as u8 ) ) ;
63+ if let Some ( t ) = p . terminator {
64+ opts = opts. terminator ( Some ( t as u8 ) ) ;
7365 }
74- if escape_set != 0 {
75- opts = opts. escape ( escape_value as u8 ) ;
66+ if let Some ( e ) = p . escape {
67+ opts = opts. escape ( e as u8 ) ;
7668 }
77- if comment_set != 0 {
78- opts = opts. comment ( comment_value as u8 ) ;
69+ if let Some ( c ) = p . comment {
70+ opts = opts. comment ( c as u8 ) ;
7971 }
80- if newlines_in_values_set != 0 {
81- opts = opts. newlines_in_values ( newlines_in_values_value != 0 ) ;
72+ if let Some ( n ) = p . newlines_in_values {
73+ opts = opts. newlines_in_values ( n ) ;
8274 }
83- if schema_infer_max_records >= 0 {
84- opts = opts. schema_infer_max_records ( schema_infer_max_records as usize ) ;
75+ if let Some ( n ) = p . schema_infer_max_records {
76+ opts = opts. schema_infer_max_records ( n as usize ) ;
8577 }
8678 if let Some ( ref s) = schema {
8779 opts = opts. schema ( s) ;
@@ -90,28 +82,14 @@ fn with_csv_options<R>(
9082 f ( opts)
9183}
9284
93- #[ allow( clippy:: too_many_arguments) ]
9485#[ no_mangle]
9586pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWithOptions < ' local > (
9687 mut env : JNIEnv < ' local > ,
9788 _class : JClass < ' local > ,
9889 handle : jlong ,
9990 name : JString < ' local > ,
10091 path : JString < ' local > ,
101- has_header : jboolean ,
102- delimiter : jbyte ,
103- quote : jbyte ,
104- terminator_set : jboolean ,
105- terminator_value : jbyte ,
106- escape_set : jboolean ,
107- escape_value : jbyte ,
108- comment_set : jboolean ,
109- comment_value : jbyte ,
110- newlines_in_values_set : jboolean ,
111- newlines_in_values_value : jboolean ,
112- schema_infer_max_records : jlong ,
113- file_extension : JString < ' local > ,
114- file_compression_type : JString < ' local > ,
92+ options_bytes : JByteArray < ' local > ,
11593 schema_ipc_bytes : JByteArray < ' local > ,
11694) {
11795 try_unwrap_or_throw ( & mut env, ( ) , |env| -> JniResult < ( ) > {
@@ -121,55 +99,23 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWith
12199 let ctx = unsafe { & * ( handle as * const SessionContext ) } ;
122100 let name: String = env. get_string ( & name) ?. into ( ) ;
123101 let path: String = env. get_string ( & path) ?. into ( ) ;
124- with_csv_options (
125- env,
126- has_header,
127- delimiter,
128- quote,
129- terminator_set,
130- terminator_value,
131- escape_set,
132- escape_value,
133- comment_set,
134- comment_value,
135- newlines_in_values_set,
136- newlines_in_values_value,
137- schema_infer_max_records,
138- file_extension,
139- file_compression_type,
140- schema_ipc_bytes,
141- |opts| {
142- runtime ( ) . block_on ( async {
143- ctx. register_csv ( & name, & path, opts) . await ?;
144- Ok :: < ( ) , DataFusionError > ( ( ) )
145- } ) ?;
146- Ok ( ( ) )
147- } ,
148- )
102+ with_csv_options ( env, options_bytes, schema_ipc_bytes, |opts| {
103+ runtime ( ) . block_on ( async {
104+ ctx. register_csv ( & name, & path, opts) . await ?;
105+ Ok :: < ( ) , DataFusionError > ( ( ) )
106+ } ) ?;
107+ Ok ( ( ) )
108+ } )
149109 } )
150110}
151111
152- #[ allow( clippy:: too_many_arguments) ]
153112#[ no_mangle]
154113pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOptions < ' local > (
155114 mut env : JNIEnv < ' local > ,
156115 _class : JClass < ' local > ,
157116 handle : jlong ,
158117 path : JString < ' local > ,
159- has_header : jboolean ,
160- delimiter : jbyte ,
161- quote : jbyte ,
162- terminator_set : jboolean ,
163- terminator_value : jbyte ,
164- escape_set : jboolean ,
165- escape_value : jbyte ,
166- comment_set : jboolean ,
167- comment_value : jbyte ,
168- newlines_in_values_set : jboolean ,
169- newlines_in_values_value : jboolean ,
170- schema_infer_max_records : jlong ,
171- file_extension : JString < ' local > ,
172- file_compression_type : JString < ' local > ,
118+ options_bytes : JByteArray < ' local > ,
173119 schema_ipc_bytes : JByteArray < ' local > ,
174120) -> jlong {
175121 try_unwrap_or_throw ( & mut env, 0 , |env| -> JniResult < jlong > {
@@ -178,27 +124,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOpti
178124 }
179125 let ctx = unsafe { & * ( handle as * const SessionContext ) } ;
180126 let path: String = env. get_string ( & path) ?. into ( ) ;
181- with_csv_options (
182- env,
183- has_header,
184- delimiter,
185- quote,
186- terminator_set,
187- terminator_value,
188- escape_set,
189- escape_value,
190- comment_set,
191- comment_value,
192- newlines_in_values_set,
193- newlines_in_values_value,
194- schema_infer_max_records,
195- file_extension,
196- file_compression_type,
197- schema_ipc_bytes,
198- |opts| {
199- let df = runtime ( ) . block_on ( ctx. read_csv ( path, opts) ) ?;
200- Ok ( Box :: into_raw ( Box :: new ( df) ) as jlong )
201- } ,
202- )
127+ with_csv_options ( env, options_bytes, schema_ipc_bytes, |opts| {
128+ let df = runtime ( ) . block_on ( ctx. read_csv ( path, opts) ) ?;
129+ Ok ( Box :: into_raw ( Box :: new ( df) ) as jlong )
130+ } )
203131 } )
204132}
0 commit comments