From d0120e06475d885b0628a519b144219e6e4b8382 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Wed, 11 Nov 2026 10:00:20 -0800 Subject: [PATCH 1/3] Add flat CASE serialization and JNI support for Gandiva --- gandiva/proto/gandiva/types.proto | 8 +++ gandiva/src/main/cpp/jni_common.cc | 17 ++++++ .../arrow/gandiva/expression/CaseNode.java | 54 +++++++++++++++++++ .../arrow/gandiva/expression/TreeBuilder.java | 6 +++ .../gandiva/evaluator/ProjectorTest.java | 32 +++++++++++ .../gandiva/expression/TreeBuilderTest.java | 45 ++++++++++++++++ 6 files changed, 162 insertions(+) create mode 100644 gandiva/src/main/java/org/apache/arrow/gandiva/expression/CaseNode.java diff --git a/gandiva/proto/gandiva/types.proto b/gandiva/proto/gandiva/types.proto index 4ce342681d..a7b39546c7 100644 --- a/gandiva/proto/gandiva/types.proto +++ b/gandiva/proto/gandiva/types.proto @@ -117,6 +117,13 @@ message AndNode { repeated TreeNode args = 1; } +message CaseNode { + repeated TreeNode conditions = 1; + repeated TreeNode results = 2; + optional TreeNode elseNode = 3; + optional ExtGandivaType returnType = 4; +} + message OrNode { repeated TreeNode args = 1; } @@ -168,6 +175,7 @@ message TreeNode { optional IfNode ifNode = 6; optional AndNode andNode = 7; optional OrNode orNode = 8; + optional CaseNode caseNode = 9; // literals optional NullNode nullNode = 11; diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index 2851250072..50cd8a75af 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -319,6 +319,19 @@ NodePtr ProtoTypeToIfNode(const gandiva::types::IfNode& node) { return TreeExprBuilder::MakeIf(cond, then_node, else_node, return_type); } +NodePtr ProtoTypeToCaseNode(const gandiva::types::CaseNode& node) { + NodeVector conditions; + NodeVector results; + for (const auto&condition : node.conditions()) { + conditions.push_back(ProtoTypeToNode(condition)); + } + for (const auto&result : node.results()) { + results.push_back(ProtoTypeToNode(result)); + } + return TreeExprBuilder::MakeCase(conditions, results, ProtoTypeToNode(node.elsenode()), + ProtoTypeToDataType(node.returntype())); +} + NodePtr ProtoTypeToAndNode(const gandiva::types::AndNode& node) { NodeVector children; @@ -436,6 +449,10 @@ NodePtr ProtoTypeToNode(const gandiva::types::TreeNode& node) { return ProtoTypeToFnNode(node.fnnode()); } + if (node.has_casenode()) { + return ProtoTypeToCaseNode(node.casenode()); + } + if (node.has_ifnode()) { return ProtoTypeToIfNode(node.ifnode()); } diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/expression/CaseNode.java b/gandiva/src/main/java/org/apache/arrow/gandiva/expression/CaseNode.java new file mode 100644 index 0000000000..7747943199 --- /dev/null +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/expression/CaseNode.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.gandiva.expression; + +import java.util.List; +import java.util.Objects; +import org.apache.arrow.gandiva.exceptions.GandivaException; +import org.apache.arrow.gandiva.ipc.GandivaTypes; +import org.apache.arrow.vector.types.pojo.ArrowType; + +/** An ordered CASE expression without a nested ELSE chain. */ +class CaseNode implements TreeNode { + private final List conditions; + private final List results; + private final TreeNode elseNode; + private final ArrowType retType; + + CaseNode( + List conditions, List results, TreeNode elseNode, ArrowType retType) { + if (conditions.isEmpty() || conditions.size() != results.size()) { + throw new IllegalArgumentException("CASE requires matching conditions and results"); + } + this.conditions = List.copyOf(conditions); + this.results = List.copyOf(results); + this.elseNode = Objects.requireNonNull(elseNode); + this.retType = Objects.requireNonNull(retType); + } + + @Override + public GandivaTypes.TreeNode toProtobuf() throws GandivaException { + GandivaTypes.CaseNode.Builder caseNode = GandivaTypes.CaseNode.newBuilder(); + for (int i = 0; i < conditions.size(); i++) { + caseNode.addConditions(conditions.get(i).toProtobuf()); + caseNode.addResults(results.get(i).toProtobuf()); + } + caseNode.setElseNode(elseNode.toProtobuf()); + caseNode.setReturnType(ArrowTypeHelper.arrowTypeToProtobuf(retType)); + return GandivaTypes.TreeNode.newBuilder().setCaseNode(caseNode).build(); + } +} diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/expression/TreeBuilder.java b/gandiva/src/main/java/org/apache/arrow/gandiva/expression/TreeBuilder.java index 96838e5ad1..ce7ddc9772 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/expression/TreeBuilder.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/expression/TreeBuilder.java @@ -101,6 +101,12 @@ public static TreeNode makeIf( return new IfNode(condition, thenNode, elseNode, retType); } + /** Creates an ordered CASE expression with first-match semantics. */ + public static TreeNode makeCase( + List conditions, List results, TreeNode elseNode, ArrowType retType) { + return new CaseNode(conditions, results, elseNode, retType); + } + /** * Invoke this function to create a node representing an and-clause. * diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java index f2d626f7ab..1d07a4bb5e 100644 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java @@ -380,6 +380,38 @@ public void testMakeProjectorValidationError() throws InterruptedException { assertTrue(exceptionThrown); } + @Test + public void testFlatCaseBeyondProtobufRecursionLimit() throws Exception { + Field input = Field.nullable("a", int32); + TreeNode a = TreeBuilder.makeField(input); + List conditions = new ArrayList<>(); + List results = new ArrayList<>(); + int branches = 5000; + for (int i = 0; i < branches; i++) { + conditions.add( + TreeBuilder.makeFunction( + "equal", List.of(a, TreeBuilder.makeLiteral(i)), new ArrowType.Bool())); + results.add(TreeBuilder.makeLiteral(i + 10)); + } + TreeNode node = TreeBuilder.makeCase(conditions, results, TreeBuilder.makeNull(int32), int32); + ExpressionTree expression = TreeBuilder.makeExpression(node, Field.nullable("out", int32)); + Schema schema = new Schema(List.of(input)); + Projector projector = Projector.make(schema, List.of(expression)); + try (ArrowBuf validity = buf(new byte[] {(byte) 15}); + ArrowBuf values = intBuf(new int[] {0, 2500, 4999, 5000, 0}); + ArrowRecordBatch batch = + new ArrowRecordBatch(5, List.of(new ArrowFieldNode(5, 1)), List.of(validity, values)); + IntVector output = new IntVector("out", allocator)) { + output.allocateNew(5); + projector.evaluate(batch, List.of(output)); + assertEquals(List.of(10, 2510, 5009), List.of(output.get(0), output.get(1), output.get(2))); + assertTrue(output.isNull(3)); + assertTrue(output.isNull(4)); + } finally { + projector.close(); + } + } + @Test public void testEvaluate() throws GandivaException, Exception { Field a = Field.nullable("a", int32); diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java index 63b31f1fb3..5aa70ca951 100644 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java @@ -18,8 +18,10 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.protobuf.CodedInputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -32,6 +34,49 @@ public class TreeBuilderTest { + @Test + public void testFlatCaseSerialization() throws Exception { + ArrowType type = new ArrowType.Int(32, true); + List conditions = new ArrayList<>(); + List results = new ArrayList<>(); + GandivaTypes.CaseNode.Builder expected = GandivaTypes.CaseNode.newBuilder(); + for (int i = 0; i < 5000; i++) { + TreeNode condition = TreeBuilder.makeLiteral(i % 2 == 0); + TreeNode result = TreeBuilder.makeLiteral(i); + conditions.add(condition); + results.add(result); + expected.addConditions(condition.toProtobuf()); + expected.addResults(result.toProtobuf()); + } + TreeNode fallback = TreeBuilder.makeNull(type); + TreeNode node = TreeBuilder.makeCase(conditions, results, fallback, type); + conditions.clear(); + results.clear(); + expected.setElseNode(fallback.toProtobuf()); + expected.setReturnType(ArrowTypeHelper.arrowTypeToProtobuf(type)); + GandivaTypes.TreeNode expectedTree = + GandivaTypes.TreeNode.newBuilder().setCaseNode(expected).build(); + CodedInputStream input = CodedInputStream.newInstance(node.toProtobuf().toByteArray()); + input.setRecursionLimit(8); + assertEquals(expectedTree, GandivaTypes.TreeNode.parseFrom(input)); + } + + @Test + public void testFlatCaseRejectsMismatchedArms() { + TreeNode fallback = TreeBuilder.makeLiteral(0); + assertThrows( + IllegalArgumentException.class, + () -> TreeBuilder.makeCase(List.of(), List.of(), fallback, new ArrowType.Int(32, true))); + assertThrows( + IllegalArgumentException.class, + () -> + TreeBuilder.makeCase( + List.of(TreeBuilder.makeLiteral(true)), + List.of(), + fallback, + new ArrowType.Int(32, true))); + } + @Test public void testMakeLiteral() throws GandivaException { TreeNode n = TreeBuilder.makeLiteral(Boolean.TRUE); From 039f4ba1870468bed12f12803e770c749ed30d79 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Thu, 12 Nov 2026 15:33:18 -0800 Subject: [PATCH 2/3] Expose cache-only Gandiva construction through Java and JNI --- gandiva/src/main/cpp/jni_common.cc | 58 +++++++++-- .../arrow/gandiva/evaluator/Filter.java | 23 ++++- .../arrow/gandiva/evaluator/JniWrapper.java | 9 ++ .../arrow/gandiva/evaluator/Projector.java | 40 +++++++- .../evaluator/CacheOnlyEvaluatorTest.java | 99 +++++++++++++++++++ 5 files changed, 218 insertions(+), 11 deletions(-) create mode 100644 gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index 50cd8a75af..eb1a3b3fa6 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -618,9 +618,9 @@ void releaseProjectorInput(jbyteArray schema_arr, jbyte* schema_bytes, env->ReleaseByteArrayElements(exprs_arr, exprs_bytes, JNI_ABORT); } -JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildProjector( - JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray exprs_arr, - jint selection_vector_type, jlong configuration_id) { +static jlong BuildProjector(JNIEnv* env, jbyteArray schema_arr, jbyteArray exprs_arr, + jint selection_vector_type, jlong configuration_id, + bool cache_only) { jlong module_id = 0LL; std::shared_ptr projector; std::shared_ptr holder; @@ -694,7 +694,10 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_build break; } // good to invoke the evaluator now - status = Projector::Make(schema_ptr, expr_vector, mode, config, &projector); + status = + cache_only + ? Projector::TryMakeFromCache(schema_ptr, expr_vector, mode, config, &projector) + : Projector::Make(schema_ptr, expr_vector, mode, config, &projector); if (!status.ok()) { ss << "Failed to make LLVM module due to " << status.message() << "\n"; @@ -702,6 +705,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_build goto err_out; } + if (projector == nullptr) { + releaseProjectorInput(schema_arr, schema_bytes, exprs_arr, exprs_bytes, env); + return 0; + } + // store the result in a map holder = std::shared_ptr( new ProjectorHolder(schema_ptr, ret_types, std::move(projector))); @@ -714,6 +722,21 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_build return module_id; } +JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildProjector( + JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray exprs_arr, + jint selection_vector_type, jlong configuration_id) { + return BuildProjector(env, schema_arr, exprs_arr, selection_vector_type, + configuration_id, false); +} + +JNIEXPORT jlong JNICALL +Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildProjectorFromCache( + JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray exprs_arr, + jint selection_vector_type, jlong configuration_id) { + return BuildProjector(env, schema_arr, exprs_arr, selection_vector_type, + configuration_id, true); +} + /// /// \brief Resizable buffer which resizes by doing a callback into java. /// @@ -919,9 +942,8 @@ void releaseFilterInput(jbyteArray schema_arr, jbyte* schema_bytes, env->ReleaseByteArrayElements(condition_arr, condition_bytes, JNI_ABORT); } -JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildFilter( - JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray condition_arr, - jlong configuration_id) { +static jlong BuildFilter(JNIEnv* env, jbyteArray schema_arr, jbyteArray condition_arr, + jlong configuration_id, bool cache_only) { jlong module_id = 0LL; std::shared_ptr filter; std::shared_ptr holder; @@ -976,13 +998,20 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_build } // good to invoke the filter builder now - status = Filter::Make(schema_ptr, condition_ptr, config, &filter); + status = cache_only + ? Filter::TryMakeFromCache(schema_ptr, condition_ptr, config, &filter) + : Filter::Make(schema_ptr, condition_ptr, config, &filter); if (!status.ok()) { ss << "Failed to make LLVM module due to " << status.message() << "\n"; releaseFilterInput(schema_arr, schema_bytes, condition_arr, condition_bytes, env); goto err_out; } + if (filter == nullptr) { + releaseFilterInput(schema_arr, schema_bytes, condition_arr, condition_bytes, env); + return 0; + } + // store the result in a map holder = std::shared_ptr(new FilterHolder(schema_ptr, std::move(filter))); module_id = filter_modules_.Insert(holder); @@ -994,6 +1023,19 @@ JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_build return module_id; } +JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildFilter( + JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray condition_arr, + jlong configuration_id) { + return BuildFilter(env, schema_arr, condition_arr, configuration_id, false); +} + +JNIEXPORT jlong JNICALL +Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildFilterFromCache( + JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray condition_arr, + jlong configuration_id) { + return BuildFilter(env, schema_arr, condition_arr, configuration_id, true); +} + JNIEXPORT jint JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_evaluateFilter( JNIEnv* env, jobject cls, jlong module_id, jint num_rows, jlongArray buf_addrs, jlongArray buf_sizes, jint jselection_vector_type, jlong out_buf_addr, diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java index 06577a516e..03203369a4 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.apache.arrow.gandiva.exceptions.EvaluatorClosedException; import org.apache.arrow.gandiva.exceptions.GandivaException; import org.apache.arrow.gandiva.expression.ArrowTypeHelper; @@ -113,6 +114,20 @@ public static Filter make(Schema schema, Condition condition, boolean optimize) */ public static Filter make(Schema schema, Condition condition, long configurationId) throws GandivaException { + return makeInternal(schema, condition, configurationId, false); + } + + /** Constructs an evaluator from cached object code, without compiling on a miss. */ + public static Optional tryMakeFromCache( + Schema schema, Condition condition, ConfigurationBuilder.ConfigOptions options) + throws GandivaException { + return Optional.ofNullable( + makeInternal(schema, condition, JniLoader.getConfiguration(options), true)); + } + + private static Filter makeInternal( + Schema schema, Condition condition, long configurationId, boolean cacheOnly) + throws GandivaException { // Invoke the JNI layer to create the LLVM module representing the filter. GandivaTypes.Condition conditionBuf = condition.toProtobuf(); GandivaTypes.Schema schemaBuf = ArrowTypeHelper.arrowSchemaToProtobuf(schema); @@ -122,7 +137,13 @@ public static Filter make(Schema schema, Condition condition, long configuration // No lock here, deliberately -- see the equivalent comment in Projector.make(). The // duplicate-LLVM-symbol race from GH-601 is fixed in Gandiva's native Filter::Make(), so this // no longer needs to be serialized. - long moduleId = wrapper.buildFilter(schemaBytes, conditionBytes, configurationId); + long moduleId = + cacheOnly + ? wrapper.buildFilterFromCache(schemaBytes, conditionBytes, configurationId) + : wrapper.buildFilter(schemaBytes, conditionBytes, configurationId); + if (cacheOnly && moduleId == 0) { + return null; + } logger.debug("Created module for the filter with id {}", moduleId); return new Filter(wrapper, moduleId, schema); } diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java index acb25ebae2..8b05148147 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java @@ -36,6 +36,11 @@ public class JniWrapper { * @param configId Configuration to gandiva. * @return A moduleId that is passed to the evaluateProjector() and closeProjector() methods */ + /** Returns zero on an object-cache miss without compiling. */ + native long buildProjectorFromCache( + byte[] schemaBuf, byte[] exprListBuf, int selectionVectorType, long configId) + throws GandivaException; + native long buildProjector( byte[] schemaBuf, byte[] exprListBuf, int selectionVectorType, long configId) throws GandivaException; @@ -86,6 +91,10 @@ native void evaluateProjector( * @param configId Configuration to gandiva. * @return A moduleId that is passed to the evaluateFilter() and closeFilter() methods */ + /** Returns zero on an object-cache miss without compiling. */ + native long buildFilterFromCache(byte[] schemaBuf, byte[] conditionBuf, long configId) + throws GandivaException; + native long buildFilter(byte[] schemaBuf, byte[] conditionBuf, long configId) throws GandivaException; diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java index 13fc040f34..b0e01234e5 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.apache.arrow.gandiva.exceptions.EvaluatorClosedException; import org.apache.arrow.gandiva.exceptions.GandivaException; import org.apache.arrow.gandiva.exceptions.UnsupportedTypeException; @@ -194,6 +195,35 @@ public static Projector make( SelectionVectorType selectionVectorType, long configurationId) throws GandivaException { + return makeInternal(schema, exprs, selectionVectorType, configurationId, false); + } + + /** Constructs an evaluator from cached object code, without compiling on a miss. */ + public static Optional tryMakeFromCache( + Schema schema, List exprs, ConfigurationBuilder.ConfigOptions options) + throws GandivaException { + return tryMakeFromCache(schema, exprs, SelectionVectorType.SV_NONE, options); + } + + /** Constructs a cached evaluator for the specified selection-vector mode. */ + public static Optional tryMakeFromCache( + Schema schema, + List exprs, + SelectionVectorType selectionVectorType, + ConfigurationBuilder.ConfigOptions options) + throws GandivaException { + return Optional.ofNullable( + makeInternal( + schema, exprs, selectionVectorType, JniLoader.getConfiguration(options), true)); + } + + private static Projector makeInternal( + Schema schema, + List exprs, + SelectionVectorType selectionVectorType, + long configurationId, + boolean cacheOnly) + throws GandivaException { // serialize the schema and the list of expressions as a protobuf GandivaTypes.ExpressionList.Builder builder = GandivaTypes.ExpressionList.newBuilder(); for (ExpressionTree expr : exprs) { @@ -213,8 +243,14 @@ public static Projector make( // in the arrow C++ tree. Concurrent buildProjector() calls are safe against a Gandiva that // contains that fix; do not re-add a lock here without first checking the native side. long moduleId = - wrapper.buildProjector( - schemaBytes, exprBytes, selectionVectorType.getNumber(), configurationId); + cacheOnly + ? wrapper.buildProjectorFromCache( + schemaBytes, exprBytes, selectionVectorType.getNumber(), configurationId) + : wrapper.buildProjector( + schemaBytes, exprBytes, selectionVectorType.getNumber(), configurationId); + if (cacheOnly && moduleId == 0) { + return null; + } logger.debug("Created module for the projector with id {}", moduleId); return new Projector(wrapper, moduleId, schema, exprs.size()); } diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java new file mode 100644 index 0000000000..69c1babe25 --- /dev/null +++ b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.gandiva.evaluator; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.arrow.gandiva.expression.TreeBuilder; +import org.apache.arrow.gandiva.ipc.GandivaTypes.SelectionVectorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; + +class CacheOnlyEvaluatorTest extends BaseEvaluatorTest { + @Test + void projectorMissDoesNotCompileAndOptimizationKeysAreDistinct() throws Exception { + Field input = Field.nullable("java_cache_only_project", new ArrowType.Int(32, true)); + Schema schema = new Schema(List.of(input)); + var expr = + TreeBuilder.makeExpression( + TreeBuilder.makeFunction( + "add", + List.of(TreeBuilder.makeField(input), TreeBuilder.makeField(input)), + input.getType()), + Field.nullable("result", input.getType())); + var o3 = new ConfigurationBuilder.ConfigOptions().withOptimize(true); + var o0 = new ConfigurationBuilder.ConfigOptions().withOptimize(false); + assertTrue(Projector.tryMakeFromCache(schema, List.of(expr), o3).isEmpty()); + assertTrue(Projector.tryMakeFromCache(schema, List.of(expr), o3).isEmpty()); + var o0Evaluator = Projector.make(schema, List.of(expr), o0); + try { + assertTrue(Projector.tryMakeFromCache(schema, List.of(expr), o3).isEmpty()); + } finally { + o0Evaluator.close(); + } + var o3Evaluator = Projector.make(schema, List.of(expr), o3); + try { + var hit = Projector.tryMakeFromCache(schema, List.of(expr), o3); + assertTrue(hit.isPresent()); + hit.orElseThrow().close(); + assertTrue( + Projector.tryMakeFromCache(schema, List.of(expr), SelectionVectorType.SV_INT16, o3) + .isEmpty()); + assertTrue( + Projector.tryMakeFromCache( + schema, + List.of(expr), + new ConfigurationBuilder.ConfigOptions().withOptimize(true).withTargetCPU(false)) + .isEmpty()); + } finally { + o3Evaluator.close(); + } + } + + @Test + void filterMissDoesNotCompileAndOptimizationKeysAreDistinct() throws Exception { + Field input = Field.nullable("java_cache_only_filter", new ArrowType.Int(32, true)); + Schema schema = new Schema(List.of(input)); + var condition = + TreeBuilder.makeCondition( + TreeBuilder.makeFunction( + "less_than", + List.of(TreeBuilder.makeField(input), TreeBuilder.makeLiteral(7)), + new ArrowType.Bool())); + var o3 = new ConfigurationBuilder.ConfigOptions().withOptimize(true); + var o0 = new ConfigurationBuilder.ConfigOptions().withOptimize(false); + assertTrue(Filter.tryMakeFromCache(schema, condition, o3).isEmpty()); + assertTrue(Filter.tryMakeFromCache(schema, condition, o3).isEmpty()); + var o0Evaluator = Filter.make(schema, condition, o0); + try { + assertTrue(Filter.tryMakeFromCache(schema, condition, o3).isEmpty()); + } finally { + o0Evaluator.close(); + } + var o3Evaluator = Filter.make(schema, condition, o3); + try { + var hit = Filter.tryMakeFromCache(schema, condition, o3); + assertTrue(hit.isPresent()); + hit.orElseThrow().close(); + } finally { + o3Evaluator.close(); + } + } +} From 2d9c59e91949a372a654db76d58177bb3fdbccf7 Mon Sep 17 00:00:00 2001 From: Steven Phillips Date: Fri, 13 Nov 2026 10:58:47 -0800 Subject: [PATCH 3/3] Revert "Expose cache-only Gandiva construction through Java and JNI" This reverts commit 039f4ba1870468bed12f12803e770c749ed30d79. --- gandiva/src/main/cpp/jni_common.cc | 58 ++--------- .../arrow/gandiva/evaluator/Filter.java | 23 +---- .../arrow/gandiva/evaluator/JniWrapper.java | 9 -- .../arrow/gandiva/evaluator/Projector.java | 40 +------- .../evaluator/CacheOnlyEvaluatorTest.java | 99 ------------------- 5 files changed, 11 insertions(+), 218 deletions(-) delete mode 100644 gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java diff --git a/gandiva/src/main/cpp/jni_common.cc b/gandiva/src/main/cpp/jni_common.cc index eb1a3b3fa6..50cd8a75af 100644 --- a/gandiva/src/main/cpp/jni_common.cc +++ b/gandiva/src/main/cpp/jni_common.cc @@ -618,9 +618,9 @@ void releaseProjectorInput(jbyteArray schema_arr, jbyte* schema_bytes, env->ReleaseByteArrayElements(exprs_arr, exprs_bytes, JNI_ABORT); } -static jlong BuildProjector(JNIEnv* env, jbyteArray schema_arr, jbyteArray exprs_arr, - jint selection_vector_type, jlong configuration_id, - bool cache_only) { +JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildProjector( + JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray exprs_arr, + jint selection_vector_type, jlong configuration_id) { jlong module_id = 0LL; std::shared_ptr projector; std::shared_ptr holder; @@ -694,10 +694,7 @@ static jlong BuildProjector(JNIEnv* env, jbyteArray schema_arr, jbyteArray exprs break; } // good to invoke the evaluator now - status = - cache_only - ? Projector::TryMakeFromCache(schema_ptr, expr_vector, mode, config, &projector) - : Projector::Make(schema_ptr, expr_vector, mode, config, &projector); + status = Projector::Make(schema_ptr, expr_vector, mode, config, &projector); if (!status.ok()) { ss << "Failed to make LLVM module due to " << status.message() << "\n"; @@ -705,11 +702,6 @@ static jlong BuildProjector(JNIEnv* env, jbyteArray schema_arr, jbyteArray exprs goto err_out; } - if (projector == nullptr) { - releaseProjectorInput(schema_arr, schema_bytes, exprs_arr, exprs_bytes, env); - return 0; - } - // store the result in a map holder = std::shared_ptr( new ProjectorHolder(schema_ptr, ret_types, std::move(projector))); @@ -722,21 +714,6 @@ static jlong BuildProjector(JNIEnv* env, jbyteArray schema_arr, jbyteArray exprs return module_id; } -JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildProjector( - JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray exprs_arr, - jint selection_vector_type, jlong configuration_id) { - return BuildProjector(env, schema_arr, exprs_arr, selection_vector_type, - configuration_id, false); -} - -JNIEXPORT jlong JNICALL -Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildProjectorFromCache( - JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray exprs_arr, - jint selection_vector_type, jlong configuration_id) { - return BuildProjector(env, schema_arr, exprs_arr, selection_vector_type, - configuration_id, true); -} - /// /// \brief Resizable buffer which resizes by doing a callback into java. /// @@ -942,8 +919,9 @@ void releaseFilterInput(jbyteArray schema_arr, jbyte* schema_bytes, env->ReleaseByteArrayElements(condition_arr, condition_bytes, JNI_ABORT); } -static jlong BuildFilter(JNIEnv* env, jbyteArray schema_arr, jbyteArray condition_arr, - jlong configuration_id, bool cache_only) { +JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildFilter( + JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray condition_arr, + jlong configuration_id) { jlong module_id = 0LL; std::shared_ptr filter; std::shared_ptr holder; @@ -998,20 +976,13 @@ static jlong BuildFilter(JNIEnv* env, jbyteArray schema_arr, jbyteArray conditio } // good to invoke the filter builder now - status = cache_only - ? Filter::TryMakeFromCache(schema_ptr, condition_ptr, config, &filter) - : Filter::Make(schema_ptr, condition_ptr, config, &filter); + status = Filter::Make(schema_ptr, condition_ptr, config, &filter); if (!status.ok()) { ss << "Failed to make LLVM module due to " << status.message() << "\n"; releaseFilterInput(schema_arr, schema_bytes, condition_arr, condition_bytes, env); goto err_out; } - if (filter == nullptr) { - releaseFilterInput(schema_arr, schema_bytes, condition_arr, condition_bytes, env); - return 0; - } - // store the result in a map holder = std::shared_ptr(new FilterHolder(schema_ptr, std::move(filter))); module_id = filter_modules_.Insert(holder); @@ -1023,19 +994,6 @@ static jlong BuildFilter(JNIEnv* env, jbyteArray schema_arr, jbyteArray conditio return module_id; } -JNIEXPORT jlong JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildFilter( - JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray condition_arr, - jlong configuration_id) { - return BuildFilter(env, schema_arr, condition_arr, configuration_id, false); -} - -JNIEXPORT jlong JNICALL -Java_org_apache_arrow_gandiva_evaluator_JniWrapper_buildFilterFromCache( - JNIEnv* env, jobject obj, jbyteArray schema_arr, jbyteArray condition_arr, - jlong configuration_id) { - return BuildFilter(env, schema_arr, condition_arr, configuration_id, true); -} - JNIEXPORT jint JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_evaluateFilter( JNIEnv* env, jobject cls, jlong module_id, jint num_rows, jlongArray buf_addrs, jlongArray buf_sizes, jint jselection_vector_type, jlong out_buf_addr, diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java index 03203369a4..06577a516e 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Optional; import org.apache.arrow.gandiva.exceptions.EvaluatorClosedException; import org.apache.arrow.gandiva.exceptions.GandivaException; import org.apache.arrow.gandiva.expression.ArrowTypeHelper; @@ -114,20 +113,6 @@ public static Filter make(Schema schema, Condition condition, boolean optimize) */ public static Filter make(Schema schema, Condition condition, long configurationId) throws GandivaException { - return makeInternal(schema, condition, configurationId, false); - } - - /** Constructs an evaluator from cached object code, without compiling on a miss. */ - public static Optional tryMakeFromCache( - Schema schema, Condition condition, ConfigurationBuilder.ConfigOptions options) - throws GandivaException { - return Optional.ofNullable( - makeInternal(schema, condition, JniLoader.getConfiguration(options), true)); - } - - private static Filter makeInternal( - Schema schema, Condition condition, long configurationId, boolean cacheOnly) - throws GandivaException { // Invoke the JNI layer to create the LLVM module representing the filter. GandivaTypes.Condition conditionBuf = condition.toProtobuf(); GandivaTypes.Schema schemaBuf = ArrowTypeHelper.arrowSchemaToProtobuf(schema); @@ -137,13 +122,7 @@ private static Filter makeInternal( // No lock here, deliberately -- see the equivalent comment in Projector.make(). The // duplicate-LLVM-symbol race from GH-601 is fixed in Gandiva's native Filter::Make(), so this // no longer needs to be serialized. - long moduleId = - cacheOnly - ? wrapper.buildFilterFromCache(schemaBytes, conditionBytes, configurationId) - : wrapper.buildFilter(schemaBytes, conditionBytes, configurationId); - if (cacheOnly && moduleId == 0) { - return null; - } + long moduleId = wrapper.buildFilter(schemaBytes, conditionBytes, configurationId); logger.debug("Created module for the filter with id {}", moduleId); return new Filter(wrapper, moduleId, schema); } diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java index 8b05148147..acb25ebae2 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java @@ -36,11 +36,6 @@ public class JniWrapper { * @param configId Configuration to gandiva. * @return A moduleId that is passed to the evaluateProjector() and closeProjector() methods */ - /** Returns zero on an object-cache miss without compiling. */ - native long buildProjectorFromCache( - byte[] schemaBuf, byte[] exprListBuf, int selectionVectorType, long configId) - throws GandivaException; - native long buildProjector( byte[] schemaBuf, byte[] exprListBuf, int selectionVectorType, long configId) throws GandivaException; @@ -91,10 +86,6 @@ native void evaluateProjector( * @param configId Configuration to gandiva. * @return A moduleId that is passed to the evaluateFilter() and closeFilter() methods */ - /** Returns zero on an object-cache miss without compiling. */ - native long buildFilterFromCache(byte[] schemaBuf, byte[] conditionBuf, long configId) - throws GandivaException; - native long buildFilter(byte[] schemaBuf, byte[] conditionBuf, long configId) throws GandivaException; diff --git a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java index b0e01234e5..13fc040f34 100644 --- a/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java +++ b/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Optional; import org.apache.arrow.gandiva.exceptions.EvaluatorClosedException; import org.apache.arrow.gandiva.exceptions.GandivaException; import org.apache.arrow.gandiva.exceptions.UnsupportedTypeException; @@ -195,35 +194,6 @@ public static Projector make( SelectionVectorType selectionVectorType, long configurationId) throws GandivaException { - return makeInternal(schema, exprs, selectionVectorType, configurationId, false); - } - - /** Constructs an evaluator from cached object code, without compiling on a miss. */ - public static Optional tryMakeFromCache( - Schema schema, List exprs, ConfigurationBuilder.ConfigOptions options) - throws GandivaException { - return tryMakeFromCache(schema, exprs, SelectionVectorType.SV_NONE, options); - } - - /** Constructs a cached evaluator for the specified selection-vector mode. */ - public static Optional tryMakeFromCache( - Schema schema, - List exprs, - SelectionVectorType selectionVectorType, - ConfigurationBuilder.ConfigOptions options) - throws GandivaException { - return Optional.ofNullable( - makeInternal( - schema, exprs, selectionVectorType, JniLoader.getConfiguration(options), true)); - } - - private static Projector makeInternal( - Schema schema, - List exprs, - SelectionVectorType selectionVectorType, - long configurationId, - boolean cacheOnly) - throws GandivaException { // serialize the schema and the list of expressions as a protobuf GandivaTypes.ExpressionList.Builder builder = GandivaTypes.ExpressionList.newBuilder(); for (ExpressionTree expr : exprs) { @@ -243,14 +213,8 @@ private static Projector makeInternal( // in the arrow C++ tree. Concurrent buildProjector() calls are safe against a Gandiva that // contains that fix; do not re-add a lock here without first checking the native side. long moduleId = - cacheOnly - ? wrapper.buildProjectorFromCache( - schemaBytes, exprBytes, selectionVectorType.getNumber(), configurationId) - : wrapper.buildProjector( - schemaBytes, exprBytes, selectionVectorType.getNumber(), configurationId); - if (cacheOnly && moduleId == 0) { - return null; - } + wrapper.buildProjector( + schemaBytes, exprBytes, selectionVectorType.getNumber(), configurationId); logger.debug("Created module for the projector with id {}", moduleId); return new Projector(wrapper, moduleId, schema, exprs.size()); } diff --git a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java b/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java deleted file mode 100644 index 69c1babe25..0000000000 --- a/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/CacheOnlyEvaluatorTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.arrow.gandiva.evaluator; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import org.apache.arrow.gandiva.expression.TreeBuilder; -import org.apache.arrow.gandiva.ipc.GandivaTypes.SelectionVectorType; -import org.apache.arrow.vector.types.pojo.ArrowType; -import org.apache.arrow.vector.types.pojo.Field; -import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.jupiter.api.Test; - -class CacheOnlyEvaluatorTest extends BaseEvaluatorTest { - @Test - void projectorMissDoesNotCompileAndOptimizationKeysAreDistinct() throws Exception { - Field input = Field.nullable("java_cache_only_project", new ArrowType.Int(32, true)); - Schema schema = new Schema(List.of(input)); - var expr = - TreeBuilder.makeExpression( - TreeBuilder.makeFunction( - "add", - List.of(TreeBuilder.makeField(input), TreeBuilder.makeField(input)), - input.getType()), - Field.nullable("result", input.getType())); - var o3 = new ConfigurationBuilder.ConfigOptions().withOptimize(true); - var o0 = new ConfigurationBuilder.ConfigOptions().withOptimize(false); - assertTrue(Projector.tryMakeFromCache(schema, List.of(expr), o3).isEmpty()); - assertTrue(Projector.tryMakeFromCache(schema, List.of(expr), o3).isEmpty()); - var o0Evaluator = Projector.make(schema, List.of(expr), o0); - try { - assertTrue(Projector.tryMakeFromCache(schema, List.of(expr), o3).isEmpty()); - } finally { - o0Evaluator.close(); - } - var o3Evaluator = Projector.make(schema, List.of(expr), o3); - try { - var hit = Projector.tryMakeFromCache(schema, List.of(expr), o3); - assertTrue(hit.isPresent()); - hit.orElseThrow().close(); - assertTrue( - Projector.tryMakeFromCache(schema, List.of(expr), SelectionVectorType.SV_INT16, o3) - .isEmpty()); - assertTrue( - Projector.tryMakeFromCache( - schema, - List.of(expr), - new ConfigurationBuilder.ConfigOptions().withOptimize(true).withTargetCPU(false)) - .isEmpty()); - } finally { - o3Evaluator.close(); - } - } - - @Test - void filterMissDoesNotCompileAndOptimizationKeysAreDistinct() throws Exception { - Field input = Field.nullable("java_cache_only_filter", new ArrowType.Int(32, true)); - Schema schema = new Schema(List.of(input)); - var condition = - TreeBuilder.makeCondition( - TreeBuilder.makeFunction( - "less_than", - List.of(TreeBuilder.makeField(input), TreeBuilder.makeLiteral(7)), - new ArrowType.Bool())); - var o3 = new ConfigurationBuilder.ConfigOptions().withOptimize(true); - var o0 = new ConfigurationBuilder.ConfigOptions().withOptimize(false); - assertTrue(Filter.tryMakeFromCache(schema, condition, o3).isEmpty()); - assertTrue(Filter.tryMakeFromCache(schema, condition, o3).isEmpty()); - var o0Evaluator = Filter.make(schema, condition, o0); - try { - assertTrue(Filter.tryMakeFromCache(schema, condition, o3).isEmpty()); - } finally { - o0Evaluator.close(); - } - var o3Evaluator = Filter.make(schema, condition, o3); - try { - var hit = Filter.tryMakeFromCache(schema, condition, o3); - assertTrue(hit.isPresent()); - hit.orElseThrow().close(); - } finally { - o3Evaluator.close(); - } - } -}