From f627ddd09f340d44aadba5bc7193930f78ff83d3 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 23 Apr 2026 15:04:51 +0200 Subject: [PATCH 1/2] HIVE-14609: HS2 cannot drop a function whose associated jar file has been removed --- .../function/AbstractFunctionAnalyzer.java | 5 + .../function/drop/DropFunctionAnalyzer.java | 36 ++++- .../drop/TestDropFunctionAnalyzer.java | 141 ++++++++++++++++++ 3 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java index 4d8cae0a7caa..f52b3a1879c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.Entity.Type; +import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -44,6 +45,10 @@ public AbstractFunctionAnalyzer(QueryState queryState) throws SemanticException super(queryState); } + public AbstractFunctionAnalyzer(QueryState queryState, Hive db) throws SemanticException { + super(queryState, db); + } + /** * Add write entities to the semantic analyzer to restrict function creation to privileged users. */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/drop/DropFunctionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/drop/DropFunctionAnalyzer.java index 23e76d897713..5b814579fc0a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/drop/DropFunctionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/drop/DropFunctionAnalyzer.java @@ -24,10 +24,13 @@ import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.FunctionUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; import org.apache.hadoop.hive.ql.ddl.function.AbstractFunctionAnalyzer; import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -41,6 +44,14 @@ public DropFunctionAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } + public DropFunctionAnalyzer(QueryState queryState, Hive db) throws SemanticException { + super(queryState, db); + } + + protected FunctionInfo getFunctionInfo(String functionName) throws SemanticException { + return FunctionRegistry.getFunctionInfo(functionName); + } + @Override public void analyzeInternal(ASTNode root) throws SemanticException { String functionName = root.getChild(0).getText(); @@ -48,12 +59,18 @@ public void analyzeInternal(ASTNode root) throws SemanticException { boolean throwException = !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROP_IGNORES_NON_EXISTENT); boolean isTemporary = (root.getFirstChildWithType(HiveParser.TOK_TEMPORARY) != null); - FunctionInfo info = FunctionRegistry.getFunctionInfo(functionName); + FunctionInfo info = getFunctionInfo(functionName); if (info == null) { - if (throwException) { + // getFunctionInfo returns null when the function's JAR resource cannot be loaded (e.g. the + // HDFS file was deleted). For permanent functions fall back to a direct metastore lookup so + // that an orphaned definition can still be removed without the JAR being present. + if (!isTemporary && functionExistsInMetastore(functionName)) { + LOG.warn("Function {} has unavailable resources; proceeding with drop using metastore metadata only.", + functionName); + } else if (throwException) { throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(functionName)); } else { - return; // Fail silently + return; } } else if (info.isBuiltIn()) { throw new SemanticException(ErrorMsg.DROP_NATIVE_FUNCTION.getMsg(functionName)); @@ -62,6 +79,17 @@ public void analyzeInternal(ASTNode root) throws SemanticException { DropFunctionDesc desc = new DropFunctionDesc(functionName, isTemporary, null); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); - addEntities(functionName, info.getClassName(), isTemporary, null); + String className = info != null ? info.getClassName() : null; + addEntities(functionName, className, isTemporary, null); + } + + private boolean functionExistsInMetastore(String functionName) { + try { + String[] parts = FunctionUtils.getQualifiedFunctionNameParts(functionName.toLowerCase()); + db.getFunction(parts[0], parts[1]); + return true; + } catch (HiveException e) { + return false; + } } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java new file mode 100644 index 000000000000..4a40c871a551 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.function.drop; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Collections; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfForTest; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.exec.FunctionInfo; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for DropFunctionAnalyzer focusing on the case where the function's JAR resource is + * unavailable (e.g. deleted from HDFS after the function was registered). + */ +public class TestDropFunctionAnalyzer { + + private HiveConf conf; + + @BeforeEach + void setUp() throws Exception { + conf = new HiveConfForTest(getClass()); + SessionState.start(conf); + } + + @AfterEach + void tearDown() throws Exception { + SessionState ss = SessionState.get(); + if (ss != null) { + ss.close(); + } + } + + /** Simulates JAR-unavailable by returning null from getFunctionInfo. */ + private DropFunctionAnalyzer analyzerWithUnavailableJar(Hive mockDb) throws SemanticException { + QueryState queryState = QueryState.getNewQueryState(conf, null); + queryState.setCommandType(HiveOperation.DROPFUNCTION); + return new DropFunctionAnalyzer(queryState, mockDb) { + @Override + protected FunctionInfo getFunctionInfo(String functionName) throws SemanticException { + return null; + } + }; + } + + /** + * When getFunctionInfo returns null (JAR unavailable) but the function still exists in the + * metastore, DROP FUNCTION must proceed and emit a drop task so the orphaned definition is + * actually removed. + */ + @Test + void testDropSucceedsWhenJarUnavailableButFunctionInMetastore() throws Exception { + Hive mockDb = mock(Hive.class); + Function msFunction = new Function("dummy", "default", "com.example.DummyUDF", + "user", PrincipalType.USER, 0, FunctionType.JAVA, Collections.emptyList()); + Database msDatabase = new Database("default", "", "/tmp", Collections.emptyMap()); + + when(mockDb.getFunction("default", "dummy")).thenReturn(msFunction); + when(mockDb.getDatabase("default")).thenReturn(msDatabase); + when(mockDb.getDatabase(any(), eq("default"))).thenReturn(msDatabase); + + DropFunctionAnalyzer analyzer = analyzerWithUnavailableJar(mockDb); + analyzer.analyzeInternal(parse("drop function dummy")); + + assertEquals(1, analyzer.getRootTasks().size(), "Expected one DROP task even when JAR is unavailable"); + } + + /** + * When the function does not exist in either the session registry or the metastore, DROP FUNCTION + * (without IF EXISTS) must surface the error to the client. + */ + @Test + void testDropThrowsWhenFunctionNotInMetastore() throws Exception { + conf.setBoolVar(ConfVars.DROP_IGNORES_NON_EXISTENT, false); + Hive mockDb = mock(Hive.class); + when(mockDb.getFunction(anyString(), anyString())).thenThrow(new HiveException("not found")); + + DropFunctionAnalyzer analyzer = analyzerWithUnavailableJar(mockDb); + assertThrows(SemanticException.class, () -> analyzer.analyzeInternal(parse("drop function dummy")), + "Expected SemanticException when function not in registry or metastore"); + } + + /** + * DROP FUNCTION IF EXISTS must silently succeed (no task, no exception) when the function is + * absent from both the session registry and the metastore. + */ + @Test + void testDropIfExistsSilentWhenFunctionAbsent() throws Exception { + Hive mockDb = mock(Hive.class); + when(mockDb.getFunction(anyString(), anyString())).thenThrow(new HiveException("not found")); + + DropFunctionAnalyzer analyzer = analyzerWithUnavailableJar(mockDb); + analyzer.analyzeInternal(parse("drop function if exists dummy")); + + assertEquals(0, analyzer.getRootTasks().size(), "Expected no tasks when function is absent and IF EXISTS is set"); + } + + private ASTNode parse(String sql) throws Exception { + return ParseUtils.parse(sql, new Context(conf)); + } +} From f21a420ae0ed77bac22d0770bc8ad35aab17158c Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 23 Apr 2026 17:26:33 +0200 Subject: [PATCH 2/2] sonarqube --- .../hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java | 2 +- .../hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java index f52b3a1879c8..ee90b1af1050 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java @@ -45,7 +45,7 @@ public AbstractFunctionAnalyzer(QueryState queryState) throws SemanticException super(queryState); } - public AbstractFunctionAnalyzer(QueryState queryState, Hive db) throws SemanticException { + protected AbstractFunctionAnalyzer(QueryState queryState, Hive db) throws SemanticException { super(queryState, db); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java index 4a40c871a551..f4d3968333f4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/ddl/function/drop/TestDropFunctionAnalyzer.java @@ -53,12 +53,12 @@ * Tests for DropFunctionAnalyzer focusing on the case where the function's JAR resource is * unavailable (e.g. deleted from HDFS after the function was registered). */ -public class TestDropFunctionAnalyzer { +class TestDropFunctionAnalyzer { private HiveConf conf; @BeforeEach - void setUp() throws Exception { + void setUp() { conf = new HiveConfForTest(getClass()); SessionState.start(conf); }