Skip to content

Commit 8db633a

Browse files
committed
feat(proto): add SessionContext.fromProto for executing protobuf plans
1 parent 08cc530 commit 8db633a

5 files changed

Lines changed: 138 additions & 1 deletion

File tree

‎native/Cargo.lock‎

Lines changed: 64 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎native/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,7 @@ crate-type = ["cdylib"]
2727
[dependencies]
2828
arrow = { version = "58", features = ["ffi"] }
2929
datafusion = "53.1.0"
30+
datafusion-proto = "53.1.0"
3031
jni = "0.21"
32+
prost = "0.14"
3133
tokio = { version = "1", features = ["rt-multi-thread"] }

‎native/src/lib.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717

1818
mod errors;
19+
mod proto;
1920

2021
use std::sync::{Arc, OnceLock};
2122

@@ -32,7 +33,7 @@ use tokio::runtime::Runtime;
3233

3334
use crate::errors::{try_unwrap_or_throw, JniResult};
3435

35-
fn runtime() -> &'static Runtime {
36+
pub(crate) fn runtime() -> &'static Runtime {
3637
static RT: OnceLock<Runtime> = OnceLock::new();
3738
RT.get_or_init(|| Runtime::new().expect("failed to create Tokio runtime"))
3839
}

‎native/src/proto.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use datafusion::dataframe::DataFrame;
19+
use datafusion::prelude::SessionContext;
20+
use datafusion_proto::logical_plan::{AsLogicalPlan, DefaultLogicalExtensionCodec};
21+
use datafusion_proto::protobuf::LogicalPlanNode;
22+
use jni::objects::{JByteArray, JClass};
23+
use jni::sys::jlong;
24+
use jni::JNIEnv;
25+
use prost::Message;
26+
27+
use crate::errors::{try_unwrap_or_throw, JniResult};
28+
use crate::runtime;
29+
30+
#[no_mangle]
31+
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createDataFrameFromProto<'local>(
32+
mut env: JNIEnv<'local>,
33+
_class: JClass<'local>,
34+
handle: jlong,
35+
plan_bytes: JByteArray<'local>,
36+
) -> jlong {
37+
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
38+
if handle == 0 {
39+
return Err("SessionContext handle is null".into());
40+
}
41+
let ctx = unsafe { &*(handle as *const SessionContext) };
42+
let bytes: Vec<u8> = env.convert_byte_array(&plan_bytes)?;
43+
let node = LogicalPlanNode::decode(bytes.as_slice())?;
44+
let codec = DefaultLogicalExtensionCodec {};
45+
let task_ctx = ctx.task_ctx();
46+
let plan = node.try_into_logical_plan(task_ctx.as_ref(), &codec)?;
47+
let df: DataFrame = runtime().block_on(ctx.execute_logical_plan(plan))?;
48+
Ok(Box::into_raw(Box::new(df)) as jlong)
49+
})
50+
}

‎src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,24 @@ public DataFrame sql(String query) {
5353
return new DataFrame(dfHandle);
5454
}
5555

56+
/**
57+
* Decode a DataFusion-Proto {@code LogicalPlanNode} and return a lazy {@link DataFrame}. The
58+
* plan is not executed until {@link DataFrame#collect} is called.
59+
*
60+
* <p>The bytes must be a serialized {@code datafusion.LogicalPlanNode} (see {@code
61+
* org.apache.datafusion.protobuf.LogicalPlanNode}).
62+
*
63+
* @throws RuntimeException if the bytes are not a valid {@code LogicalPlanNode} or if logical
64+
* planning fails.
65+
*/
66+
public DataFrame fromProto(byte[] planBytes) {
67+
if (nativeHandle == 0) {
68+
throw new IllegalStateException("SessionContext is closed");
69+
}
70+
long dfHandle = createDataFrameFromProto(nativeHandle, planBytes);
71+
return new DataFrame(dfHandle);
72+
}
73+
5674
public void registerParquet(String name, String path) {
5775
if (nativeHandle == 0) {
5876
throw new IllegalStateException("SessionContext is closed");
@@ -72,6 +90,8 @@ public void close() {
7290

7391
private static native long createDataFrame(long handle, String sql);
7492

93+
private static native long createDataFrameFromProto(long handle, byte[] planBytes);
94+
7595
private static native void registerParquet(long handle, String name, String path);
7696

7797
private static native void closeSessionContext(long handle);

0 commit comments

Comments
 (0)