Skip to content

Commit dd5d6fc

Browse files
authored
feat: add Java scalar UDF support (#46)
1 parent 678eee1 commit dd5d6fc

12 files changed

Lines changed: 1384 additions & 3 deletions

File tree

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.List;
23+
24+
import org.apache.arrow.memory.BufferAllocator;
25+
import org.apache.arrow.vector.FieldVector;
26+
import org.apache.arrow.vector.types.pojo.ArrowType;
27+
28+
/**
29+
* A Java-implemented scalar SQL function. Implementations declare their own name, signature, and
30+
* volatility, and evaluate one input batch at a time.
31+
*
32+
* <p>Mirrors DataFusion's {@code ScalarUDFImpl} trait. Wrap an instance in a {@link ScalarUdf} and
33+
* register it via {@link SessionContext#registerUdf(ScalarUdf)} to make it callable from SQL or
34+
* DataFrame plans.
35+
*
36+
* <p>Implementations may be invoked concurrently by DataFusion on multiple worker threads. If the
37+
* implementation carries mutable state, the implementation must synchronize it.
38+
*/
39+
public interface ScalarFunction {
40+
/** SQL name under which this function is invoked (e.g., {@code "add_one"}). */
41+
String name();
42+
43+
/**
44+
* Declared argument types, in positional order. The function is registered with an exact
45+
* signature; calls whose argument types do not match exactly are rejected.
46+
*/
47+
List<ArrowType> argTypes();
48+
49+
/** Declared return type. The returned {@link FieldVector} must have this exact type. */
50+
ArrowType returnType();
51+
52+
/**
53+
* Volatility classification. Use {@link Volatility#IMMUTABLE} for pure functions, {@link
54+
* Volatility#STABLE} for functions deterministic within a query, and {@link Volatility#VOLATILE}
55+
* for non-deterministic functions.
56+
*/
57+
Volatility volatility();
58+
59+
/**
60+
* Compute the function result for one input batch.
61+
*
62+
* @param allocator the {@link BufferAllocator} that MUST be used for any new {@link FieldVector}
63+
* allocation, including the result. Buffers allocated from other allocators will not survive
64+
* the JNI handoff.
65+
* @param args one {@link FieldVector} per declared argument, all of the same length. These are
66+
* read-only views; the implementation must NOT close them.
67+
* @return a {@link FieldVector} of the declared return type and the same length as the inputs.
68+
* Ownership transfers to the framework on return; the implementation must NOT close the
69+
* returned vector.
70+
*/
71+
FieldVector evaluate(BufferAllocator allocator, List<FieldVector> args);
72+
}
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.List;
23+
import java.util.Objects;
24+
25+
import org.apache.arrow.vector.types.pojo.ArrowType;
26+
27+
/**
28+
* A scalar UDF registration handle: pairs a {@link ScalarFunction} implementation with the metadata
29+
* DataFusion needs to dispatch SQL calls to it.
30+
*
31+
* <p>Mirrors DataFusion's {@code ScalarUDF} struct. Construct one from a {@link ScalarFunction} via
32+
* {@link #fromImpl(ScalarFunction)} (or the public constructor) and pass it to {@link
33+
* SessionContext#registerUdf(ScalarUdf)}.
34+
*/
35+
public final class ScalarUdf {
36+
private final ScalarFunction impl;
37+
private final String name;
38+
private final List<ArrowType> argTypes;
39+
private final ArrowType returnType;
40+
private final Volatility volatility;
41+
42+
/**
43+
* Wrap a {@link ScalarFunction} for registration. Each metadata getter is invoked once here and
44+
* cached, so a user impl that allocates per call won't be double-evaluated at registration.
45+
*
46+
* @throws NullPointerException if {@code impl} or any of its declared metadata is null
47+
*/
48+
public ScalarUdf(ScalarFunction impl) {
49+
this.impl = Objects.requireNonNull(impl, "impl");
50+
this.name = Objects.requireNonNull(impl.name(), "impl.name()");
51+
this.argTypes = Objects.requireNonNull(impl.argTypes(), "impl.argTypes()");
52+
this.returnType = Objects.requireNonNull(impl.returnType(), "impl.returnType()");
53+
this.volatility = Objects.requireNonNull(impl.volatility(), "impl.volatility()");
54+
}
55+
56+
/** Equivalent to {@code new ScalarUdf(impl)}; mirrors Rust's {@code ScalarUDF::new_from_impl}. */
57+
public static ScalarUdf fromImpl(ScalarFunction impl) {
58+
return new ScalarUdf(impl);
59+
}
60+
61+
/** The wrapped implementation. */
62+
public ScalarFunction impl() {
63+
return impl;
64+
}
65+
66+
/** SQL name; cached from {@link ScalarFunction#name()}. */
67+
public String name() {
68+
return name;
69+
}
70+
71+
/** Declared argument types; cached from {@link ScalarFunction#argTypes()}. */
72+
public List<ArrowType> argTypes() {
73+
return argTypes;
74+
}
75+
76+
/** Declared return type; cached from {@link ScalarFunction#returnType()}. */
77+
public ArrowType returnType() {
78+
return returnType;
79+
}
80+
81+
/** Volatility classification; cached from {@link ScalarFunction#volatility()}. */
82+
public Volatility volatility() {
83+
return volatility;
84+
}
85+
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,18 @@
2323
import java.io.ByteArrayOutputStream;
2424
import java.io.IOException;
2525
import java.nio.channels.Channels;
26+
import java.util.ArrayList;
27+
import java.util.List;
2628

2729
import org.apache.arrow.memory.BufferAllocator;
2830
import org.apache.arrow.memory.RootAllocator;
2931
import org.apache.arrow.vector.VectorSchemaRoot;
3032
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
3133
import org.apache.arrow.vector.ipc.ReadChannel;
3234
import org.apache.arrow.vector.ipc.message.MessageSerializer;
35+
import org.apache.arrow.vector.types.pojo.ArrowType;
36+
import org.apache.arrow.vector.types.pojo.Field;
37+
import org.apache.arrow.vector.types.pojo.FieldType;
3338
import org.apache.arrow.vector.types.pojo.Schema;
3439

3540
/**
@@ -293,6 +298,36 @@ public DataFrame readParquet(String path, ParquetReadOptions options) {
293298
return new DataFrame(dfHandle);
294299
}
295300

301+
/**
302+
* Register a Java-implemented scalar UDF. After registration, the function can be invoked by SQL
303+
* via the UDF's name or referenced in DataFusion plans deserialised with {@link #fromProto}.
304+
*
305+
* <p>The UDF is registered with an exact signature: the runtime will reject calls whose argument
306+
* types do not match the declared {@link ScalarFunction#argTypes()} exactly.
307+
*
308+
* @throws RuntimeException if registration fails (e.g., name already registered with an
309+
* incompatible signature, schema serialisation failure).
310+
*/
311+
public void registerUdf(ScalarUdf udf) {
312+
if (nativeHandle == 0) {
313+
throw new IllegalStateException("SessionContext is closed");
314+
}
315+
java.util.Objects.requireNonNull(udf, "udf");
316+
ScalarFunction impl = udf.impl();
317+
String name = udf.name();
318+
ArrowType returnType = udf.returnType();
319+
List<ArrowType> argTypes = udf.argTypes();
320+
Volatility volatility = udf.volatility();
321+
List<Field> fields = new ArrayList<>(argTypes.size() + 1);
322+
fields.add(new Field("return", FieldType.nullable(returnType), null));
323+
for (int i = 0; i < argTypes.size(); i++) {
324+
fields.add(new Field("arg" + i, FieldType.nullable(argTypes.get(i)), null));
325+
}
326+
Schema signatureSchema = new Schema(fields);
327+
byte[] signatureBytes = serializeSchemaIpc(signatureSchema);
328+
registerScalarUdf(nativeHandle, name, signatureBytes, volatility.code(), impl);
329+
}
330+
296331
private static byte[] serializeSchemaIpc(Schema schema) {
297332
ByteArrayOutputStream baos = new ByteArrayOutputStream();
298333
try (BufferAllocator allocator = new RootAllocator();
@@ -345,4 +380,7 @@ private static native long readJsonWithOptions(
345380
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
346381

347382
private static native void closeSessionContext(long handle);
383+
384+
private static native void registerScalarUdf(
385+
long handle, String name, byte[] signatureSchemaBytes, byte volatility, ScalarFunction impl);
348386
}
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Volatility classification for a UDF. Mirrors DataFusion's {@code Volatility} enum.
24+
*
25+
* <ul>
26+
* <li>{@link #IMMUTABLE} — pure function: same inputs always produce the same output; safe to
27+
* constant-fold and common-subexpression-eliminate.
28+
* <li>{@link #STABLE} — deterministic within a single query but not across queries (e.g., {@code
29+
* now()}).
30+
* <li>{@link #VOLATILE} — may return a different value on every call (e.g., {@code random()}).
31+
* </ul>
32+
*/
33+
public enum Volatility {
34+
IMMUTABLE((byte) 0),
35+
STABLE((byte) 1),
36+
VOLATILE((byte) 2);
37+
38+
private final byte code;
39+
40+
Volatility(byte code) {
41+
this.code = code;
42+
}
43+
44+
/** Stable byte code for FFI. */
45+
public byte code() {
46+
return code;
47+
}
48+
}
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion.internal;
21+
22+
import java.util.List;
23+
24+
import org.apache.arrow.c.ArrowArray;
25+
import org.apache.arrow.c.ArrowSchema;
26+
import org.apache.arrow.c.Data;
27+
import org.apache.arrow.memory.RootAllocator;
28+
import org.apache.arrow.vector.FieldVector;
29+
import org.apache.arrow.vector.VectorSchemaRoot;
30+
import org.apache.datafusion.ScalarFunction;
31+
32+
/** Internal trampoline invoked from native code on every UDF call. Not part of the public API. */
33+
public final class JniBridge {
34+
/**
35+
* Shared allocator for UDF inputs/outputs. Created once at class-load time; never closed.
36+
* Outstanding allocations are released by the FFI structs' release callbacks when the native side
37+
* drops them after {@code from_ffi}.
38+
*/
39+
private static final RootAllocator ALLOCATOR = new RootAllocator();
40+
41+
private JniBridge() {}
42+
43+
/**
44+
* Invoke a scalar UDF for one batch. Called from native code; not for application use.
45+
*
46+
* @param impl the registered {@link ScalarFunction} implementation
47+
* @param argsArrayAddr address of a populated {@code FFI_ArrowArray} struct holding the input
48+
* batch as a struct array (one field per UDF argument)
49+
* @param argsSchemaAddr address of the matching {@code FFI_ArrowSchema}
50+
* @param resultArrayAddr address of an empty {@code FFI_ArrowArray} the bridge writes into
51+
* @param resultSchemaAddr address of an empty {@code FFI_ArrowSchema} the bridge writes into
52+
* @param expectedRowCount the row count the result vector must have
53+
*/
54+
public static void invokeScalarUdf(
55+
ScalarFunction impl,
56+
long argsArrayAddr,
57+
long argsSchemaAddr,
58+
long resultArrayAddr,
59+
long resultSchemaAddr,
60+
int expectedRowCount) {
61+
ArrowArray argsArr = ArrowArray.wrap(argsArrayAddr);
62+
ArrowSchema argsSch = ArrowSchema.wrap(argsSchemaAddr);
63+
ArrowArray resultArr = ArrowArray.wrap(resultArrayAddr);
64+
ArrowSchema resultSch = ArrowSchema.wrap(resultSchemaAddr);
65+
66+
try (VectorSchemaRoot root = Data.importVectorSchemaRoot(ALLOCATOR, argsArr, argsSch, null)) {
67+
List<FieldVector> argVectors = root.getFieldVectors();
68+
69+
FieldVector result = impl.evaluate(ALLOCATOR, argVectors);
70+
71+
if (result == null) {
72+
throw new IllegalStateException("ScalarFunction.evaluate returned null");
73+
}
74+
if (result.getValueCount() != expectedRowCount) {
75+
try {
76+
throw new IllegalStateException(
77+
"ScalarFunction.evaluate returned vector with "
78+
+ result.getValueCount()
79+
+ " rows; expected "
80+
+ expectedRowCount);
81+
} finally {
82+
result.close();
83+
}
84+
}
85+
86+
try {
87+
Data.exportVector(ALLOCATOR, result, null, resultArr, resultSch);
88+
} finally {
89+
result.close();
90+
}
91+
}
92+
}
93+
}

‎core/src/test/java/org/apache/datafusion/DataFrameTransformationsTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,7 @@ void limitRejectsNegativeArgs() {
250250
@Test
251251
void distinctRemovesDuplicates() {
252252
try (SessionContext ctx = new SessionContext();
253-
DataFrame source =
254-
ctx.sql("SELECT * FROM (VALUES (1), (1), (2), (2), (3)) AS t(x)");
253+
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (1), (2), (2), (3)) AS t(x)");
255254
DataFrame deduped = source.distinct()) {
256255
assertEquals(3L, deduped.count());
257256
}

0 commit comments

Comments
 (0)