ADH-4718: Support spark connect (from pan3793)

.github/workflows/master.yml

@@ -77,7 +77,7 @@ jobs:
             comment: 'verify-on-spark-3.4-binary'
           - java: 17
             spark: '3.5'
-            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0-preview1 -Dspark.archive.name=spark-4.0.0-preview1-bin-hadoop3.tgz'
+            spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://dist.apache.org/repos/dist/dev/spark/v4.0.0-preview2-rc1-bin -Dspark.archive.name=spark-4.0.0-preview2-bin-hadoop3.tgz'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-4.0-binary'
     env:

HOW-TO.md

@@ -0,0 +1,85 @@
+<!--
+- Licensed to the Apache Software Foundation (ASF) under one or more
+- contributor license agreements.  See the NOTICE file distributed with
+- this work for additional information regarding copyright ownership.
+- The ASF licenses this file to You under the Apache License, Version 2.0
+- (the "License"); you may not use this file except in compliance with
+- the License.  You may obtain a copy of the License at
+-
+-   http://www.apache.org/licenses/LICENSE-2.0
+-
+- Unless required by applicable law or agreed to in writing, software
+- distributed under the License is distributed on an "AS IS" BASIS,
+- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+- See the License for the specific language governing permissions and
+- limitations under the License.
+-->
+
+Kyuubi Connect
+==============
+
+### Source Code under development
+
+```
+git clone https://github.com/pan3793/kyuubi.git -b kyuubi-next kyuubi-next
+```
+
+### Requirements
+
+- `JAVA_HOME` points to Java 17
+- `SPARK_HOME` points to `/path/of/spark-4.0.0-preview2-bin-hadoop3`
+
+### Run
+
+#### Run within IDEA
+
+- Run `build/mvn clean install -DskipTests` to build the project and produce the Spark engine jar
+- Run `kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala` using IDEA
+
+You can set `SPARK_HOME`, `KYUUBI_CONF_DIR` in the Run/Debug Configuration Dialog
+
+![img](docs/imgs/idea_run_debug_configurations_dialog.png)
+
+#### Run within Terminal
+
+```
+build/dist
+cd dist
+bin/kyuubi run --conf kyuubi.frontend.grpc.bind.port=10999
+```
+
+The gRPC service listens 10999 by default.
+
+### Connect to Kyuubi Connect
+
+Spark Connect Scala client (Requires: Java 17, Spark 4.0.0-preview2)
+```
+cd /path/of/spark-4.0.0-preview2-bin-hadoop3
+bin/spark-shell --remote sc://H27212-MAC-01.local:10999 --user_id chengpan --user_name chengpan
+```
+
+PySpark Connect client (Requires: Python >=3.9)
+```
+pip install pyspark-connect==4.0.0.dev2
+pyspark --remote sc://H27212-MAC-01.local:10999 --user_id chengpan --user_name chengpan
+```
+
+Run examples
+```
+Welcome to
+      ____              __
+     / __/__  ___ _____/ /__
+    _\ \/ _ \/ _ `/ __/  '_/
+   /___/ .__/\_,_/_/ /_/\_\   version 4.0.0-preview2
+      /_/
+
+Type in expressions to have them evaluated.
+Spark session available as 'spark'.
+
+scala> spark.sql("select 1").show()
++---+
+|  1|
++---+
+|  1|
++---+
+```

build/dist

@@ -260,11 +260,11 @@ for arg in "$@"; do
 done
 
 # shellcheck disable=SC2050
-if [ "$SCALA_VERSION" = "2.12" ]; then
-  EXTRA_SPARK_ENGINE_BUILD_COMMAND=("$MVN" install $MVN_DIST_OPT ${FILTERED_ARGS[@]} -Pscala-2.13 -pl :kyuubi-spark-sql-engine_2.13 -am)
-else
-  EXTRA_SPARK_ENGINE_BUILD_COMMAND=("$MVN" install $MVN_DIST_OPT ${FILTERED_ARGS[@]} -pl :kyuubi-spark-sql-engine_2.12 -am)
-fi
+#if [ "$SCALA_VERSION" = "2.12" ]; then
+#  EXTRA_SPARK_ENGINE_BUILD_COMMAND=("$MVN" install $MVN_DIST_OPT ${FILTERED_ARGS[@]} -Pscala-2.13 -pl :kyuubi-spark-sql-engine_2.13 -am)
+#else
+#  EXTRA_SPARK_ENGINE_BUILD_COMMAND=("$MVN" install $MVN_DIST_OPT ${FILTERED_ARGS[@]} -pl :kyuubi-spark-sql-engine_2.12 -am)
+#fi
 
 # shellcheck disable=SC2145
 echo -e "\$ ${EXTRA_SPARK_ENGINE_BUILD_COMMAND[@]}\n"
@@ -318,7 +318,8 @@ cp "$KYUUBI_HOME/externals/kyuubi-flink-sql-engine/target/kyuubi-flink-sql-engin
 
 # Copy spark engines
 # shellcheck disable=SC2045
-for scala_version in 2.12 2.13; do
+#for scala_version in 2.12 2.13; do
+for scala_version in 2.13; do
   cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engine_${scala_version}-${VERSION}.jar" "$DISTDIR/externals/engines/spark/"
 done
 

extensions/spark/kyuubi-spark-connector-hive/src/test/scala/org/apache/kyuubi/spark/connector/hive/command/DropNamespaceSuite.scala

@@ -63,7 +63,8 @@ trait DropNamespaceSuiteBase extends DDLCommandTestUtils {
       sql(s"DROP NAMESPACE $catalogName.unknown")
     }.getMessage
     assert(message.contains(s"'unknown' not found") ||
-      message.contains(s"The schema `unknown` cannot be found"))
+      message.contains(s"The schema `unknown` cannot be found") ||
+      message.contains("SCHEMA_NOT_FOUND"))
   }
 
   test("drop non-empty namespace with a non-cascading mode") {

externals/kyuubi-spark-sql-engine/pom.xml

@@ -87,6 +87,13 @@
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-connect_${scala.binary.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkGrpcBackendService.scala

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.engine.spark
+
+import org.apache.spark.sql.SparkSession
+
+import org.apache.kyuubi.engine.spark.session.SparkSQLSessionManager
+import org.apache.kyuubi.service.AbstractBackendService
+import org.apache.kyuubi.session.SessionManager
+
+/**
+ * A [[org.apache.kyuubi.service.BackendService]] constructed
+ * with [[SparkSession]] which give it the ability to talk with
+ * Spark and let Spark do all the rest heavy work :)
+ *
+ *  @param name  Service Name
+ *  @param spark A [[SparkSession]] instance
+ *               that this backend service holds to run [[org.apache.kyuubi.operation.Operation]]s.
+ */
+class SparkGrpcBackendService(name: String, spark: SparkSession)
+  extends AbstractBackendService(name) {
+
+  def this(spark: SparkSession) = this(classOf[SparkGrpcBackendService].getSimpleName, spark)
+
+  override val sessionManager: SessionManager = new SparkSQLSessionManager(spark)
+
+  def sparkSession: SparkSession = spark
+}