Skip to content

Commit d1c9803

Browse files
committed
refactor(udf): split ScalarUdf into ScalarFunction impl iface and holder class
Mirror DataFusion's Rust shape: ScalarFunction (formerly the eval-only ScalarUdf interface) now carries name/argTypes/returnType/volatility and the per-batch evaluate body, mirroring ScalarUDFImpl. ScalarUdf becomes a holder that wraps a ScalarFunction and is what SessionContext.registerUdf takes — mirroring the ScalarUDF struct. The holder caches the four metadata values in its constructor so impls that allocate per call aren't double-evaluated at registration. The JNI bridge takes a ScalarFunction directly; the wrapper is bypassed on the hot path.
1 parent 296b3be commit d1c9803

9 files changed

Lines changed: 302 additions & 166 deletions

File tree

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.List;
23+
24+
import org.apache.arrow.memory.BufferAllocator;
25+
import org.apache.arrow.vector.FieldVector;
26+
import org.apache.arrow.vector.types.pojo.ArrowType;
27+
28+
/**
29+
* A Java-implemented scalar SQL function. Implementations declare their own name, signature, and
30+
* volatility, and evaluate one input batch at a time.
31+
*
32+
* <p>Mirrors DataFusion's {@code ScalarUDFImpl} trait. Wrap an instance in a {@link ScalarUdf} and
33+
* register it via {@link SessionContext#registerUdf(ScalarUdf)} to make it callable from SQL or
34+
* DataFrame plans.
35+
*
36+
* <p>Implementations may be invoked concurrently by DataFusion on multiple worker threads. If the
37+
* implementation carries mutable state, the implementation must synchronize it.
38+
*/
39+
public interface ScalarFunction {
40+
/** SQL name under which this function is invoked (e.g., {@code "add_one"}). */
41+
String name();
42+
43+
/**
44+
* Declared argument types, in positional order. The function is registered with an exact
45+
* signature; calls whose argument types do not match exactly are rejected.
46+
*/
47+
List<ArrowType> argTypes();
48+
49+
/** Declared return type. The returned {@link FieldVector} must have this exact type. */
50+
ArrowType returnType();
51+
52+
/**
53+
* Volatility classification. Use {@link Volatility#IMMUTABLE} for pure functions, {@link
54+
* Volatility#STABLE} for functions deterministic within a query, and {@link Volatility#VOLATILE}
55+
* for non-deterministic functions.
56+
*/
57+
Volatility volatility();
58+
59+
/**
60+
* Compute the function result for one input batch.
61+
*
62+
* @param allocator the {@link BufferAllocator} that MUST be used for any new {@link FieldVector}
63+
* allocation, including the result. Buffers allocated from other allocators will not survive
64+
* the JNI handoff.
65+
* @param args one {@link FieldVector} per declared argument, all of the same length. These are
66+
* read-only views; the implementation must NOT close them.
67+
* @return a {@link FieldVector} of the declared return type and the same length as the inputs.
68+
* Ownership transfers to the framework on return; the implementation must NOT close the
69+
* returned vector.
70+
*/
71+
FieldVector evaluate(BufferAllocator allocator, List<FieldVector> args);
72+
}

‎core/src/main/java/org/apache/datafusion/ScalarUdf.java‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,30 +20,66 @@
2020
package org.apache.datafusion;
2121

2222
import java.util.List;
23+
import java.util.Objects;
2324

24-
import org.apache.arrow.memory.BufferAllocator;
25-
import org.apache.arrow.vector.FieldVector;
25+
import org.apache.arrow.vector.types.pojo.ArrowType;
2626

2727
/**
28-
* A Java-implemented scalar SQL function. Register an instance with {@link
29-
* SessionContext#registerUdf} to make it callable from SQL or DataFrame plans.
28+
* A scalar UDF registration handle: pairs a {@link ScalarFunction} implementation with the metadata
29+
* DataFusion needs to dispatch SQL calls to it.
3030
*
31-
* <p>Implementations may be invoked concurrently by DataFusion on multiple worker threads. If the
32-
* implementation carries mutable state, the implementation must synchronize it.
31+
* <p>Mirrors DataFusion's {@code ScalarUDF} struct. Construct one from a {@link ScalarFunction} via
32+
* {@link #fromImpl(ScalarFunction)} (or the public constructor) and pass it to {@link
33+
* SessionContext#registerUdf(ScalarUdf)}.
3334
*/
34-
@FunctionalInterface
35-
public interface ScalarUdf {
35+
public final class ScalarUdf {
36+
private final ScalarFunction impl;
37+
private final String name;
38+
private final List<ArrowType> argTypes;
39+
private final ArrowType returnType;
40+
private final Volatility volatility;
41+
3642
/**
37-
* Compute the function result for one input batch.
43+
* Wrap a {@link ScalarFunction} for registration. Each metadata getter is invoked once here and
44+
* cached, so a user impl that allocates per call won't be double-evaluated at registration.
3845
*
39-
* @param allocator the {@link BufferAllocator} that MUST be used for any new {@link FieldVector}
40-
* allocation, including the result. Buffers allocated from other allocators will not survive
41-
* the JNI handoff.
42-
* @param args one {@link FieldVector} per declared argument, all of the same length. These are
43-
* read-only views; the implementation must NOT close them.
44-
* @return a {@link FieldVector} of the declared return type and the same length as the inputs.
45-
* Ownership transfers to the framework on return; the implementation must NOT close the
46-
* returned vector.
46+
* @throws NullPointerException if {@code impl} or any of its declared metadata is null
4747
*/
48-
FieldVector evaluate(BufferAllocator allocator, List<FieldVector> args);
48+
public ScalarUdf(ScalarFunction impl) {
49+
this.impl = Objects.requireNonNull(impl, "impl");
50+
this.name = Objects.requireNonNull(impl.name(), "impl.name()");
51+
this.argTypes = Objects.requireNonNull(impl.argTypes(), "impl.argTypes()");
52+
this.returnType = Objects.requireNonNull(impl.returnType(), "impl.returnType()");
53+
this.volatility = Objects.requireNonNull(impl.volatility(), "impl.volatility()");
54+
}
55+
56+
/** Equivalent to {@code new ScalarUdf(impl)}; mirrors Rust's {@code ScalarUDF::new_from_impl}. */
57+
public static ScalarUdf fromImpl(ScalarFunction impl) {
58+
return new ScalarUdf(impl);
59+
}
60+
61+
/** The wrapped implementation. */
62+
public ScalarFunction impl() {
63+
return impl;
64+
}
65+
66+
/** SQL name; cached from {@link ScalarFunction#name()}. */
67+
public String name() {
68+
return name;
69+
}
70+
71+
/** Declared argument types; cached from {@link ScalarFunction#argTypes()}. */
72+
public List<ArrowType> argTypes() {
73+
return argTypes;
74+
}
75+
76+
/** Declared return type; cached from {@link ScalarFunction#returnType()}. */
77+
public ArrowType returnType() {
78+
return returnType;
79+
}
80+
81+
/** Volatility classification; cached from {@link ScalarFunction#volatility()}. */
82+
public Volatility volatility() {
83+
return volatility;
84+
}
4985
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -211,37 +211,32 @@ public DataFrame readParquet(String path, ParquetReadOptions options) {
211211

212212
/**
213213
* Register a Java-implemented scalar UDF. After registration, the function can be invoked by SQL
214-
* via its {@code name} or referenced in DataFusion plans deserialised with {@link #fromProto}.
214+
* via the UDF's name or referenced in DataFusion plans deserialised with {@link #fromProto}.
215215
*
216-
* <p>Argument and return types are declared at registration time. The UDF is registered with an
217-
* exact signature: the runtime will reject calls whose argument types do not match {@code
218-
* argTypes} exactly.
216+
* <p>The UDF is registered with an exact signature: the runtime will reject calls whose argument
217+
* types do not match the declared {@link ScalarFunction#argTypes()} exactly.
219218
*
220219
* @throws RuntimeException if registration fails (e.g., name already registered with an
221220
* incompatible signature, schema serialisation failure).
222221
*/
223-
public void registerUdf(
224-
String name,
225-
ScalarUdf udf,
226-
ArrowType returnType,
227-
List<ArrowType> argTypes,
228-
Volatility volatility) {
222+
public void registerUdf(ScalarUdf udf) {
229223
if (nativeHandle == 0) {
230224
throw new IllegalStateException("SessionContext is closed");
231225
}
232-
java.util.Objects.requireNonNull(name, "name");
233226
java.util.Objects.requireNonNull(udf, "udf");
234-
java.util.Objects.requireNonNull(returnType, "returnType");
235-
java.util.Objects.requireNonNull(argTypes, "argTypes");
236-
java.util.Objects.requireNonNull(volatility, "volatility");
227+
ScalarFunction impl = udf.impl();
228+
String name = udf.name();
229+
ArrowType returnType = udf.returnType();
230+
List<ArrowType> argTypes = udf.argTypes();
231+
Volatility volatility = udf.volatility();
237232
List<Field> fields = new ArrayList<>(argTypes.size() + 1);
238233
fields.add(new Field("return", FieldType.nullable(returnType), null));
239234
for (int i = 0; i < argTypes.size(); i++) {
240235
fields.add(new Field("arg" + i, FieldType.nullable(argTypes.get(i)), null));
241236
}
242237
Schema signatureSchema = new Schema(fields);
243238
byte[] signatureBytes = serializeSchemaIpc(signatureSchema);
244-
registerScalarUdf(nativeHandle, name, signatureBytes, volatility.code(), udf);
239+
registerScalarUdf(nativeHandle, name, signatureBytes, volatility.code(), impl);
245240
}
246241

247242
private static byte[] serializeSchemaIpc(Schema schema) {
@@ -290,5 +285,5 @@ private static native long readCsvWithOptions(
290285
private static native void closeSessionContext(long handle);
291286

292287
private static native void registerScalarUdf(
293-
long handle, String name, byte[] signatureSchemaBytes, byte volatility, ScalarUdf udf);
288+
long handle, String name, byte[] signatureSchemaBytes, byte volatility, ScalarFunction impl);
294289
}

‎core/src/main/java/org/apache/datafusion/internal/JniBridge.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import org.apache.arrow.memory.RootAllocator;
2828
import org.apache.arrow.vector.FieldVector;
2929
import org.apache.arrow.vector.VectorSchemaRoot;
30-
import org.apache.datafusion.ScalarUdf;
30+
import org.apache.datafusion.ScalarFunction;
3131

3232
/** Internal trampoline invoked from native code on every UDF call. Not part of the public API. */
3333
public final class JniBridge {
@@ -43,7 +43,7 @@ private JniBridge() {}
4343
/**
4444
* Invoke a scalar UDF for one batch. Called from native code; not for application use.
4545
*
46-
* @param udf the registered {@link ScalarUdf} instance
46+
* @param impl the registered {@link ScalarFunction} implementation
4747
* @param argsArrayAddr address of a populated {@code FFI_ArrowArray} struct holding the input
4848
* batch as a struct array (one field per UDF argument)
4949
* @param argsSchemaAddr address of the matching {@code FFI_ArrowSchema}
@@ -52,7 +52,7 @@ private JniBridge() {}
5252
* @param expectedRowCount the row count the result vector must have
5353
*/
5454
public static void invokeScalarUdf(
55-
ScalarUdf udf,
55+
ScalarFunction impl,
5656
long argsArrayAddr,
5757
long argsSchemaAddr,
5858
long resultArrayAddr,
@@ -66,15 +66,15 @@ public static void invokeScalarUdf(
6666
try (VectorSchemaRoot root = Data.importVectorSchemaRoot(ALLOCATOR, argsArr, argsSch, null)) {
6767
List<FieldVector> argVectors = root.getFieldVectors();
6868

69-
FieldVector result = udf.evaluate(ALLOCATOR, argVectors);
69+
FieldVector result = impl.evaluate(ALLOCATOR, argVectors);
7070

7171
if (result == null) {
72-
throw new IllegalStateException("ScalarUdf.evaluate returned null");
72+
throw new IllegalStateException("ScalarFunction.evaluate returned null");
7373
}
7474
if (result.getValueCount() != expectedRowCount) {
7575
try {
7676
throw new IllegalStateException(
77-
"ScalarUdf.evaluate returned vector with "
77+
"ScalarFunction.evaluate returned vector with "
7878
+ result.getValueCount()
7979
+ " rows; expected "
8080
+ expectedRowCount);

0 commit comments

Comments
 (0)