Skip to content

Commit 11131a9

Browse files
committed
Merge remote-tracking branch 'apache/main' into execute-proto-plan
# Conflicts: # src/main/java/org/apache/datafusion/SessionContext.java
2 parents cbf4739 + 233243c commit 11131a9

6 files changed

Lines changed: 499 additions & 17 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,19 @@ jobs:
4141
cache: maven
4242

4343
- name: Set up Rust toolchain
44-
uses: dtolnay/rust-toolchain@stable
44+
run: |
45+
rustup update stable
46+
rustup default stable
4547
46-
- name: Cache cargo build
47-
uses: Swatinem/rust-cache@v2
48+
- name: Cache cargo
49+
uses: actions/cache@v4
4850
with:
49-
workspaces: native
51+
path: |
52+
~/.cargo/registry
53+
~/.cargo/git
54+
native/target
55+
key: ${{ runner.os }}-cargo-${{ hashFiles('native/Cargo.lock') }}
56+
restore-keys: ${{ runner.os }}-cargo-
5057

5158
- name: Build native and run JVM tests
5259
run: make test

‎native/src/lib.rs‎

Lines changed: 107 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@ mod proto;
2020

2121
use std::sync::{Arc, OnceLock};
2222

23-
use datafusion::arrow::datatypes::SchemaRef;
23+
use datafusion::arrow::datatypes::{Schema, SchemaRef};
2424
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
25+
use datafusion::arrow::ipc::reader::StreamReader;
2526
use datafusion::arrow::record_batch::RecordBatchIterator;
2627
use datafusion::dataframe::DataFrame;
2728
use datafusion::error::DataFusionError;
2829
use datafusion::prelude::{ParquetReadOptions, SessionContext};
29-
use jni::objects::{JClass, JString};
30-
use jni::sys::jlong;
30+
use jni::objects::{JByteArray, JClass, JString};
31+
use jni::sys::{jboolean, jlong};
3132
use jni::JNIEnv;
3233
use tokio::runtime::Runtime;
3334

@@ -130,13 +131,61 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_closeSessionCon
130131
})
131132
}
132133

134+
#[allow(clippy::too_many_arguments)]
135+
fn with_parquet_options<R>(
136+
env: &mut JNIEnv,
137+
file_extension: JString,
138+
parquet_pruning_set: jboolean,
139+
parquet_pruning_value: jboolean,
140+
skip_metadata_set: jboolean,
141+
skip_metadata_value: jboolean,
142+
metadata_size_hint: jlong,
143+
schema_ipc_bytes: JByteArray,
144+
f: impl FnOnce(ParquetReadOptions) -> JniResult<R>,
145+
) -> JniResult<R> {
146+
let file_ext: String = env.get_string(&file_extension)?.into();
147+
148+
let schema: Option<Schema> = if !schema_ipc_bytes.is_null() {
149+
let bytes: Vec<u8> = env.convert_byte_array(&schema_ipc_bytes)?;
150+
let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
151+
Some((*reader.schema()).clone())
152+
} else {
153+
None
154+
};
155+
156+
let mut opts = ParquetReadOptions::default().file_extension(&file_ext);
157+
if parquet_pruning_set != 0 {
158+
opts = opts.parquet_pruning(parquet_pruning_value != 0);
159+
}
160+
if skip_metadata_set != 0 {
161+
opts = opts.skip_metadata(skip_metadata_value != 0);
162+
}
163+
if metadata_size_hint >= 0 {
164+
opts = opts.metadata_size_hint(Some(metadata_size_hint as usize));
165+
}
166+
if let Some(ref s) = schema {
167+
opts = opts.schema(s);
168+
}
169+
170+
f(opts)
171+
}
172+
133173
#[no_mangle]
134-
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquet<'local>(
174+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquetWithOptions<
175+
'local,
176+
>(
135177
mut env: JNIEnv<'local>,
136178
_class: JClass<'local>,
137179
handle: jlong,
138180
name: JString<'local>,
139181
path: JString<'local>,
182+
file_extension: JString<'local>,
183+
parquet_pruning_set: jboolean,
184+
parquet_pruning_value: jboolean,
185+
skip_metadata_set: jboolean,
186+
skip_metadata_value: jboolean,
187+
metadata_size_hint: jlong,
188+
schema_ipc_bytes: JByteArray<'local>,
140189
) {
141190
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
142191
if handle == 0 {
@@ -145,11 +194,59 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquet
145194
let ctx = unsafe { &*(handle as *const SessionContext) };
146195
let name: String = env.get_string(&name)?.into();
147196
let path: String = env.get_string(&path)?.into();
148-
runtime().block_on(async {
149-
ctx.register_parquet(&name, &path, ParquetReadOptions::default())
150-
.await?;
151-
Ok::<(), DataFusionError>(())
152-
})?;
153-
Ok(())
197+
with_parquet_options(
198+
env,
199+
file_extension,
200+
parquet_pruning_set,
201+
parquet_pruning_value,
202+
skip_metadata_set,
203+
skip_metadata_value,
204+
metadata_size_hint,
205+
schema_ipc_bytes,
206+
|opts| {
207+
runtime().block_on(async {
208+
ctx.register_parquet(&name, &path, opts).await?;
209+
Ok::<(), DataFusionError>(())
210+
})?;
211+
Ok(())
212+
},
213+
)
214+
})
215+
}
216+
217+
#[no_mangle]
218+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readParquetWithOptions<'local>(
219+
mut env: JNIEnv<'local>,
220+
_class: JClass<'local>,
221+
handle: jlong,
222+
path: JString<'local>,
223+
file_extension: JString<'local>,
224+
parquet_pruning_set: jboolean,
225+
parquet_pruning_value: jboolean,
226+
skip_metadata_set: jboolean,
227+
skip_metadata_value: jboolean,
228+
metadata_size_hint: jlong,
229+
schema_ipc_bytes: JByteArray<'local>,
230+
) -> jlong {
231+
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
232+
if handle == 0 {
233+
return Err("SessionContext handle is null".into());
234+
}
235+
let ctx = unsafe { &*(handle as *const SessionContext) };
236+
let path: String = env.get_string(&path)?.into();
237+
with_parquet_options(
238+
env,
239+
file_extension,
240+
parquet_pruning_set,
241+
parquet_pruning_value,
242+
skip_metadata_set,
243+
skip_metadata_value,
244+
metadata_size_hint,
245+
schema_ipc_bytes,
246+
|opts| {
247+
let df = runtime().block_on(ctx.read_parquet(path, opts))?;
248+
Ok(Box::into_raw(Box::new(df)) as jlong)
249+
},
250+
)
154251
})
155252
}
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
24+
/**
25+
* Configuration knobs for parquet sources passed to {@link SessionContext#registerParquet(String,
26+
* String, ParquetReadOptions)} and {@link SessionContext#readParquet(String, ParquetReadOptions)}.
27+
*
28+
* <p>Mirrors a subset of DataFusion's {@code ParquetReadOptions}. All setters return {@code this}
29+
* for fluent chaining. Defaults: {@code fileExtension = ".parquet"}; all other fields {@code null}
30+
* (meaning the SessionConfig default is used, or the schema is inferred from the file).
31+
*/
32+
public final class ParquetReadOptions {
33+
34+
private String fileExtension = ".parquet";
35+
private Boolean parquetPruning;
36+
private Boolean skipMetadata;
37+
private Long metadataSizeHint;
38+
private Schema schema;
39+
40+
public ParquetReadOptions fileExtension(String ext) {
41+
this.fileExtension = ext;
42+
return this;
43+
}
44+
45+
public ParquetReadOptions parquetPruning(boolean v) {
46+
this.parquetPruning = v;
47+
return this;
48+
}
49+
50+
public ParquetReadOptions skipMetadata(boolean v) {
51+
this.skipMetadata = v;
52+
return this;
53+
}
54+
55+
public ParquetReadOptions metadataSizeHint(long bytes) {
56+
this.metadataSizeHint = bytes;
57+
return this;
58+
}
59+
60+
public ParquetReadOptions schema(Schema schema) {
61+
this.schema = schema;
62+
return this;
63+
}
64+
65+
String fileExtension() {
66+
return fileExtension;
67+
}
68+
69+
Boolean parquetPruning() {
70+
return parquetPruning;
71+
}
72+
73+
Boolean skipMetadata() {
74+
return skipMetadata;
75+
}
76+
77+
Long metadataSizeHint() {
78+
return metadataSizeHint;
79+
}
80+
81+
Schema schema() {
82+
return schema;
83+
}
84+
}

‎src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,14 @@
2020
package org.apache.datafusion;
2121

2222
import java.io.ByteArrayInputStream;
23+
import java.io.ByteArrayOutputStream;
24+
import java.io.IOException;
2325
import java.nio.channels.Channels;
2426

27+
import org.apache.arrow.memory.BufferAllocator;
28+
import org.apache.arrow.memory.RootAllocator;
29+
import org.apache.arrow.vector.VectorSchemaRoot;
30+
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
2531
import org.apache.arrow.vector.ipc.ReadChannel;
2632
import org.apache.arrow.vector.ipc.message.MessageSerializer;
2733
import org.apache.arrow.vector.types.pojo.Schema;
@@ -92,16 +98,76 @@ public Schema tableSchema(String tableName) {
9298
try {
9399
return MessageSerializer.deserializeSchema(
94100
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
95-
} catch (java.io.IOException e) {
101+
} catch (IOException e) {
96102
throw new RuntimeException("Failed to deserialize IPC schema", e);
97103
}
98104
}
99105

100106
public void registerParquet(String name, String path) {
107+
registerParquet(name, path, new ParquetReadOptions());
108+
}
109+
110+
/**
111+
* Register a parquet file as a table with the supplied {@link ParquetReadOptions}.
112+
*
113+
* @throws RuntimeException if registration fails (path not found, schema mismatch, etc.).
114+
*/
115+
public void registerParquet(String name, String path, ParquetReadOptions options) {
101116
if (nativeHandle == 0) {
102117
throw new IllegalStateException("SessionContext is closed");
103118
}
104-
registerParquet(nativeHandle, name, path);
119+
registerParquetWithOptions(
120+
nativeHandle,
121+
name,
122+
path,
123+
options.fileExtension(),
124+
options.parquetPruning() != null,
125+
options.parquetPruning() != null && options.parquetPruning(),
126+
options.skipMetadata() != null,
127+
options.skipMetadata() != null && options.skipMetadata(),
128+
options.metadataSizeHint() != null ? options.metadataSizeHint() : -1L,
129+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
130+
}
131+
132+
/** Read a parquet file as a {@link DataFrame} without registering it. */
133+
public DataFrame readParquet(String path) {
134+
return readParquet(path, new ParquetReadOptions());
135+
}
136+
137+
/**
138+
* Read a parquet file as a {@link DataFrame} with the supplied {@link ParquetReadOptions}.
139+
*
140+
* @throws RuntimeException if the read fails.
141+
*/
142+
public DataFrame readParquet(String path, ParquetReadOptions options) {
143+
if (nativeHandle == 0) {
144+
throw new IllegalStateException("SessionContext is closed");
145+
}
146+
long dfHandle =
147+
readParquetWithOptions(
148+
nativeHandle,
149+
path,
150+
options.fileExtension(),
151+
options.parquetPruning() != null,
152+
options.parquetPruning() != null && options.parquetPruning(),
153+
options.skipMetadata() != null,
154+
options.skipMetadata() != null && options.skipMetadata(),
155+
options.metadataSizeHint() != null ? options.metadataSizeHint() : -1L,
156+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
157+
return new DataFrame(dfHandle);
158+
}
159+
160+
private static byte[] serializeSchemaIpc(Schema schema) {
161+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
162+
try (BufferAllocator allocator = new RootAllocator();
163+
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
164+
ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(baos))) {
165+
writer.start();
166+
writer.end();
167+
} catch (IOException e) {
168+
throw new RuntimeException("Failed to serialize Arrow schema for JNI", e);
169+
}
170+
return baos.toByteArray();
105171
}
106172

107173
@Override
@@ -120,7 +186,28 @@ public void close() {
120186

121187
private static native byte[] tableSchemaIpc(long handle, String tableName);
122188

123-
private static native void registerParquet(long handle, String name, String path);
189+
private static native void registerParquetWithOptions(
190+
long handle,
191+
String name,
192+
String path,
193+
String fileExtension,
194+
boolean parquetPruningSet,
195+
boolean parquetPruningValue,
196+
boolean skipMetadataSet,
197+
boolean skipMetadataValue,
198+
long metadataSizeHint,
199+
byte[] schemaIpcBytes);
200+
201+
private static native long readParquetWithOptions(
202+
long handle,
203+
String path,
204+
String fileExtension,
205+
boolean parquetPruningSet,
206+
boolean parquetPruningValue,
207+
boolean skipMetadataSet,
208+
boolean skipMetadataValue,
209+
long metadataSizeHint,
210+
byte[] schemaIpcBytes);
124211

125212
private static native void closeSessionContext(long handle);
126213
}

0 commit comments

Comments
 (0)