[Draft] Master catalog spi review 16

be/src/common/config.cpp

@@ -1543,7 +1543,10 @@ DEFINE_mInt64(s3_put_token_limit, "0");
 DEFINE_mInt64(s3_rate_limiter_log_interval, "1000");
 DEFINE_Validator(s3_rate_limiter_log_interval, [](int64_t config) -> bool { return config >= 0; });
 
-DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors");
+// The dir TrinoConnectorPluginLoader loads Trino's own plugins from, used verbatim. Keep the default
+// in sync with FE Config.trino_connector_plugin_dir: FE and BE load the same plugins and an operator
+// who leaves both untouched expects both to find them.
+DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/trino_plugins");
 
 // ca_cert_file is in this path by default, Normally no modification is required
 // ca cert default path is different from different OS

be/src/io/fs/connectivity/s3_connectivity_tester.cpp

@@ -24,6 +24,10 @@ namespace doris::io {
 
 Status S3ConnectivityTester::test(const std::map<std::string, std::string>& properties) {
     auto it = properties.find(TEST_LOCATION);
+    if (it == properties.end()) {
+        return Status::InvalidArgument("Missing '{}' property in S3 connectivity test request",
+                                       TEST_LOCATION);
+    }
     S3URI s3_uri(it->second);
     RETURN_IF_ERROR(s3_uri.parse());
 

build.sh

@@ -1041,7 +1041,11 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     mkdir -p "${DORIS_OUTPUT}/fe/conf/ssl"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/jdbc_drivers/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_udf/"
-    mkdir -p "${DORIS_OUTPUT}/fe/plugins/connectors/"
+    # Drop point for the trino-connector's own Trino plugins. Deliberately NOT the legacy
+    # plugins/connectors/: that name is still read as a fallback for deployments upgrading from
+    # <= 2.1.8, so a fresh install must not create it (an empty dir would be harmless, but the
+    # one-letter gap to the plugins/connector/ tree above is not).
+    mkdir -p "${DORIS_OUTPUT}/fe/plugins/trino_plugins/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/"
 
@@ -1081,6 +1085,23 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     done
     unset CONN_PLUGIN_DIR conn_module conn_plugin_target conn_module_dir conn_zip
 
+    # RC-4: self-contain the paimon connector plugin for OSS. The connector sets
+    # fs.oss.impl=com.aliyun.jindodata.oss.JindoOssFileSystem; that impl lives in the jindofs jars,
+    # which are packaged from thirdparty by post-build.sh into fe/lib/jindofs (NOT a maven artifact).
+    # The plugin runs child-first, so without its OWN copy JindoOssFileSystem resolves from the parent
+    # 'app' classloader and cannot be cast to the plugin's child-loaded org.apache.hadoop.fs.FileSystem.
+    # Copy the jindofs jars into the paimon plugin lib so JindoOssFileSystem loads child-first alongside
+    # the plugin's own hadoop FileSystem (same self-contained intent as the bundled hadoop-aws/S3A).
+    # Naturally gated: a no-op unless jindofs was packaged (--jindofs / DISABLE_BUILD_JINDOFS=OFF).
+    # CAVEAT (docker-gated, enablePaimonTest=true): jindo-core ships a native lib that can bind to only one
+    # classloader per JVM, so this is safe only while no concurrent non-paimon path loads jindo from
+    # fe/lib/jindofs in the same FE process.
+    PAIMON_CONN_LIB="${DORIS_OUTPUT}/fe/plugins/connector/paimon/lib"
+    if [[ -d "${PAIMON_CONN_LIB}" && -d "${DORIS_OUTPUT}/fe/lib/jindofs" ]]; then
+        cp -p "${DORIS_OUTPUT}/fe/lib/jindofs/"*.jar "${PAIMON_CONN_LIB}/" 2>/dev/null || true
+    fi
+    unset PAIMON_CONN_LIB
+
     if [ "${TARGET_SYSTEM}" = "Darwin" ] || [ "${TARGET_SYSTEM}" = "Linux" ]; then
       mkdir -p "${DORIS_OUTPUT}/fe/arthas"
       rm -rf "${DORIS_OUTPUT}/fe/arthas/*"
@@ -1249,7 +1270,8 @@ EOF
     mkdir -p "${DORIS_OUTPUT}/be/plugins/jdbc_drivers/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/java_udf/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/python_udf/"
-    mkdir -p "${DORIS_OUTPUT}/be/plugins/connectors/"
+    # Mirrors the FE drop point above; the BE JNI scanner loads the same Trino plugins independently.
+    mkdir -p "${DORIS_OUTPUT}/be/plugins/trino_plugins/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/hadoop_conf/"
     mkdir -p "${DORIS_OUTPUT}/be/plugins/java_extensions/"
     cp -r -p "${DORIS_HOME}/be/src/udf/python/python_server.py" "${DORIS_OUTPUT}/be/plugins/python_udf/"

docker/thirdparties/hive-regression-local-env.md

@@ -0,0 +1,162 @@
+# Hive Regression Local Environment
+
+This document records the local Docker and regression configuration needed to run:
+
+```bash
+regression-test/suites/external_table_p0/hive/
+```
+
+## Required Thirdparty Components
+
+Run Hive2 and Hive3 through the thirdparty launcher:
+
+```bash
+./docker/thirdparties/run-thirdparties-docker.sh -c hive2,hive3 --hive-mode refresh
+```
+
+The Hive startup script may also start `mysql` automatically. Both
+`docker-compose/hive/hive-2x_settings.env` and
+`docker-compose/hive/hive-3x_settings.env` default `JFS_CLUSTER_META` to:
+
+```bash
+mysql://root:123456@(127.0.0.1:3316)/juicefs_meta
+```
+
+Because of that setting, `run-thirdparties-docker.sh` treats local MySQL 5.7 as
+an implicit dependency for Hive.
+
+Each Hive version starts its own compose services:
+
+- Hive2: `hive2-server`, `hive2-metastore`, `hadoop2-namenode`,
+  `hadoop2-datanode`, `hive2-metastore-postgresql`
+- Hive3: `hive3-server`, `hive3-metastore`, `hadoop3-namenode`,
+  `hadoop3-datanode`, `hive3-metastore-postgresql`
+- MySQL 5.7: required by the default JuiceFS metadata URI when using the local
+  `127.0.0.1:3316` endpoint
+
+The Hive p0 directory does not require these components by default:
+
+- `pg`
+- `iceberg`
+- `minio`
+- `hudi`
+- `kerberos`
+- `ranger`
+
+`test_external_catalog_hive.groovy` has a Ranger-specific branch guarded by
+`enableRangerTest`; if this option is not set to `true`, that branch is skipped.
+
+## Current Running Containers Observed
+
+The following relevant containers were running when checked with
+`sudo docker ps`:
+
+- Hive3 stack: `hive3-server`, `hive3-metastore`, `hadoop3-datanode`,
+  `hadoop3-namenode`, `hive3-metastore-postgresql`; all healthy
+- Hive2 stack: `hive2-server`, `hive2-metastore`, `hadoop2-datanode`,
+  `hadoop2-namenode`, `hive2-metastore-postgresql`; all healthy
+- MySQL 5.7: `mysql-doris--syt--mysql_57-1`, healthy, published as
+  `127.0.0.1:3316 -> 3306`
+
+Other Doris thirdparty containers were also running, but they are not required
+for `external_table_p0/hive/`:
+
+- PostgreSQL 14 on `5442`
+- Iceberg REST, Spark, Postgres, and MinIO
+- Oracle on `1521`
+- ClickHouse on `8123`
+- SQLServer on `1433`
+- OceanBase on `2881`
+
+The observed Hive2/Hive3 and MySQL compose projects came from another checkout
+under `/mnt/disk2/suyiteng/doris`, not from this worktree. PostgreSQL and
+Iceberg compose projects came from this worktree.
+
+## Regression Configuration
+
+The Hive p0 suite directly reads these keys from
+`regression-test/conf/regression-conf.groovy`:
+
+```groovy
+enableHiveTest
+externalEnvIp
+hive2HmsPort
+hive2HdfsPort
+hive3HmsPort
+hive3HdfsPort
+hdfsUser
+```
+
+It also uses framework helpers backed by:
+
+```groovy
+brokerName
+hdfsPasswd
+hdfsFs
+```
+
+For local Hive2/Hive3 Docker with default ports, the relevant values are:
+
+```groovy
+enableHiveTest = true
+
+hive2HmsPort = 9083
+hive2HdfsPort = 8020
+hive2ServerPort = 10000
+hive2PgPort = 5432
+
+hive3HmsPort = 9383
+hive3HdfsPort = 8320
+hive3ServerPort = 13000
+hive3PgPort = 5732
+
+hdfsUser = "doris-test"
+brokerName = "broker_name"
+hdfsPasswd = ""
+```
+
+`externalEnvIp` must be the address that Doris BE can use to reach the Hive
+Docker services. For a fully local setup this is usually:
+
+```groovy
+externalEnvIp = "127.0.0.1"
+```
+
+If BE runs in a context where `127.0.0.1` is not the Docker host, use the host
+IP selected by `run-thirdparties-docker.sh` through `IP_HOST`.
+
+## Config Gaps Noted In regression-conf.groovy.bak
+
+`regression-test/conf/regression-conf.groovy.bak` already contains the core Hive
+keys listed above. The notable follow-ups are:
+
+- `externalEnvIp` was set to `172.20.32.136`; verify this is reachable from BE,
+  or replace it with `127.0.0.1` for local host-mode Docker.
+- FE/JDBC addresses in the file used `9033`, `9022`, and `8033`; align these
+  with the current Doris worktree cluster ports before running regression.
+- `enableRangerTest` is not present. This is acceptable for the Hive p0 suite
+  because the Ranger-specific branch is guarded and skipped unless explicitly
+  enabled.
+
+## Thirdparty Script Settings
+
+Before starting Hive through `run-thirdparties-docker.sh`, set a unique
+`CONTAINER_UID` in:
+
+```bash
+docker/thirdparties/custom_settings.env
+```
+
+The default `CONTAINER_UID="doris--"` is rejected by the startup script.
+
+Hive baseline restore also depends on these settings:
+
+```bash
+s3Endpoint
+s3BucketName
+HIVE_BASELINE_VERSION
+HIVE_BASELINE_TARBALL_CACHE
+```
+
+If baseline download is unavailable, place the matching tarball manually under
+`HIVE_BASELINE_TARBALL_CACHE`.

fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java

@@ -20,8 +20,8 @@
 import org.apache.doris.common.classloader.ThreadClassLoaderContext;
 import org.apache.doris.common.jni.JniScanner;
 import org.apache.doris.common.jni.vec.ColumnType;
-import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
-import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
+import org.apache.doris.kerberos.PreExecutionAuthenticator;
+import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;

fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableJniScanner.java

@@ -21,8 +21,8 @@
 import org.apache.doris.common.jni.JniScanner;
 import org.apache.doris.common.jni.vec.ColumnType;
 import org.apache.doris.common.jni.vec.ColumnValue;
-import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
-import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
+import org.apache.doris.kerberos.PreExecutionAuthenticator;
+import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
 
 import com.google.common.base.Preconditions;
 import org.apache.iceberg.FileScanTask;
@@ -110,8 +110,12 @@ protected int getNext() throws IOException {
                 }
                 StructLike row = reader.next();
                 for (int i = 0; i < requiredFieldCount; i++) {
-                    // FE keeps the fields requested by BE at the start of the Iceberg projection.
-                    // FileScanTask.schema() is not the row schema for every DataTask implementation.
+                    // Read positionally: FE (IcebergScanPlanProvider.doPlanSystemTableScan) projects the
+                    // metadata-table scan to exactly the BE-requested fields, in required_fields order, so the
+                    // i-th projected row field is the i-th required field. Do NOT index via scanTask.schema():
+                    // for a metadata StaticDataTask, schema() returns the FULL table schema while rows() yields a
+                    // narrowed StructProjection, so a full-schema ordinal overruns the projected row (upstream
+                    // #65262 -- reverting this to a by-name/schema() lookup reintroduces ArrayIndexOutOfBounds).
                     Object value = row.get(i, Object.class);
                     ColumnValue columnValue = new IcebergSysTableColumnValue(value, timezone);
                     appendData(i, columnValue);

fe/be-java-extensions/java-common/pom.xml

@@ -34,6 +34,11 @@ under the License.
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.doris</groupId>
+            <artifactId>fe-kerberos</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.doris</groupId>
             <artifactId>fe-common</artifactId>

fe/fe-common/src/main/java/org/apache/doris/common/maxcompute/MCUtils.java → fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MCUtils.java

@@ -15,7 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.common.maxcompute;
+package org.apache.doris.maxcompute;
+
+import org.apache.doris.common.maxcompute.MCProperties;
 
 import com.aliyun.auth.credentials.Credential;
 import com.aliyun.auth.credentials.provider.EcsRamRoleCredentialProvider;

fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java

@@ -19,7 +19,6 @@
 
 import org.apache.doris.common.jni.JniScanner;
 import org.apache.doris.common.jni.vec.ColumnType;
-import org.apache.doris.common.maxcompute.MCUtils;
 
 import com.aliyun.odps.Odps;
 import com.aliyun.odps.table.configuration.CompressionCodec;

fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java

@@ -21,7 +21,6 @@
 import org.apache.doris.common.jni.vec.VectorColumn;
 import org.apache.doris.common.jni.vec.VectorTable;
 import org.apache.doris.common.maxcompute.MCProperties;
-import org.apache.doris.common.maxcompute.MCUtils;
 
 import com.aliyun.odps.Odps;
 import com.aliyun.odps.OdpsType;

fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java

@@ -20,8 +20,8 @@
 import org.apache.doris.common.jni.JniScanner;
 import org.apache.doris.common.jni.vec.ColumnType;
 import org.apache.doris.common.jni.vec.TableSchema;
-import org.apache.doris.common.security.authentication.PreExecutionAuthenticator;
-import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache;
+import org.apache.doris.kerberos.PreExecutionAuthenticator;
+import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
 
 import com.google.common.base.Preconditions;
 import org.apache.paimon.CoreOptions;
@@ -50,6 +50,7 @@
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -271,6 +272,12 @@ static int getFieldIndex(List<String> fieldNames, String fieldName) {
     }
 
     private List<Predicate> getPredicates() {
+        // Backstop for a missing paimon_predicate param (scan with no pushed-down filter): a null here means
+        // "no filter", not an error. Guard the unconditional deserialize so the JNI reader never NPEs on
+        // deserialize(null) ("encodedStr is null"). The FE producer also always emits an (empty) predicate now.
+        if (paimonPredicate == null) {
+            return Collections.emptyList();
+        }
         List<Predicate> predicates = PaimonUtils.deserialize(paimonPredicate);
         if (LOG.isDebugEnabled()) {
             LOG.debug("predicates:{}", predicates);

fe/be-java-extensions/preload-extensions/pom.xml

@@ -62,6 +62,18 @@ under the License.
             <artifactId>commons-io</artifactId>
             <version>${commons-io.version}</version>
         </dependency>
+        <!-- Runtime-only: Hive-using JNI scanners (paimon-hive-connector / hudi / HiveConf)
+             need org.apache.commons.lang.StringUtils (commons-lang 2.x) at runtime. It used to
+             arrive transitively via fe-common's odps-sdk-core; after P4-T09 (a53f2b17b8d) made
+             fe-common odps-free it was evicted from this shared preload classpath, breaking every
+             scanner with NoClassDefFoundError. Restore it here (version managed by fe/pom.xml) so
+             it lands in the preload-extensions runtime classpath, the shared parent of all JNI
+             scanners, while staying out of java-udf's own jar. -->
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <scope>runtime</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-memory-unsafe</artifactId>