branch-4.1: [improvement](regression) Reduce external regression memory

docker/thirdparties/docker-compose/kerberos/.gitignore

@@ -0,0 +1,8 @@
+# Generated by run-thirdparties-docker.sh.
+/conf/kerberos1/*.conf
+/conf/kerberos1/*.xml
+/conf/kerberos2/*.conf
+/conf/kerberos2/*.xml
+/data/
+/two-kerberos-hives/*.conf
+/two-kerberos-hives/*.keytab

docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh → docker/thirdparties/docker-compose/kerberos/Dockerfile

@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,25 +15,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -euo pipefail
+FROM apache/hive:3.1.3
 
-if test $# -gt 0; then
-    echo "$0 does not accept arguments" >&2
-    exit 32
-fi
+USER root
 
-# Supervisord is not running
-if ! test -f /tmp/supervisor.sock; then
-    exit 0
-fi
+RUN apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        krb5-admin-server \
+        krb5-kdc \
+        krb5-user \
+    && rm -rf /var/lib/apt/lists/*
 
-# Check if all Hadoop services are running
-FAILED=$(supervisorctl status | grep -v RUNNING || true)
+COPY entrypoint-hive-master.sh /opt/doris/entrypoint.sh
+COPY health-checks/health.sh /opt/doris/health.sh
 
-if [ "$FAILED" == "" ]; then
-  echo "All services are running"
-  exit 0
-else
-  echo "Some of the services are failing: ${FAILED}"
-  exit 1
-fi
+RUN chmod 755 /opt/doris/entrypoint.sh /opt/doris/health.sh
+
+ENTRYPOINT ["/opt/doris/entrypoint.sh"]

docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf

@@ -33,15 +33,9 @@
 [realms]
  LABS.TERADATA.COM = {
   kdc = hadoop-master:5588
-  admin_server = hadoop-master:5749
- }
- OTHERLABS.TERADATA.COM = {
-  kdc = hadoop-master:5589
-  admin_server = hadoop-master:5750
  }
  OTHERREALM.COM = {
   kdc = hadoop-master-2:6688
-  admin_server = hadoop-master-2:6749
  }
 
 [domain_realm]

docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml → docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0"?>
 <!--
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
@@ -18,12 +18,37 @@ specific language governing permissions and limitations
 under the License.
 -->
 <configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://${HOST}:${FS_PORT}</value>
+    </property>
+    <property>
+        <name>hadoop.security.authentication</name>
+        <value>kerberos</value>
+    </property>
+    <property>
+        <name>hadoop.security.authorization</name>
+        <value>true</value>
+    </property>
     <property>
         <name>hadoop.security.auth_to_local</name>
         <value>
-            RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*//
+            RULE:[2:$1@$0](.*@LABS.TERADATA.COM)s/@.*//
             RULE:[2:$1@$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
+            RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*//
             DEFAULT
         </value>
     </property>
+    <property>
+        <name>hadoop.proxyuser.hive.hosts</name>
+        <value>*</value>
+    </property>
+    <property>
+        <name>hadoop.proxyuser.hive.users</name>
+        <value>*</value>
+    </property>
+    <property>
+        <name>hadoop.security.token.service.use_ip</name>
+        <value>false</value>
+    </property>
 </configuration>

docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl

@@ -0,0 +1,113 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>file:///data/hdfs/name</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>file:///data/hdfs/data</value>
+    </property>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+    <property>
+        <name>dfs.permissions.enabled</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>dfs.block.access.token.enable</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>dfs.namenode.kerberos.principal</name>
+        <value>hdfs/${HOST}@${REALM}</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-bind-host</name>
+        <value>0.0.0.0</value>
+    </property>
+    <property>
+        <name>dfs.namenode.keytab.file</name>
+        <value>/data/keytabs/hdfs.keytab</value>
+    </property>
+    <property>
+        <name>dfs.datanode.kerberos.principal</name>
+        <value>hdfs/${HOST}@${REALM}</value>
+    </property>
+    <property>
+        <name>dfs.datanode.keytab.file</name>
+        <value>/data/keytabs/hdfs.keytab</value>
+    </property>
+    <property>
+        <name>dfs.namenode.kerberos.internal.spnego.principal</name>
+        <value>HTTP/${HOST}@${REALM}</value>
+    </property>
+    <property>
+        <name>dfs.web.authentication.kerberos.principal</name>
+        <value>HTTP/${HOST}@${REALM}</value>
+    </property>
+    <property>
+        <name>dfs.web.authentication.kerberos.keytab</name>
+        <value>/data/keytabs/spnego.keytab</value>
+    </property>
+    <property>
+        <name>dfs.data.transfer.protection</name>
+        <value>authentication</value>
+    </property>
+    <property>
+        <name>dfs.http.policy</name>
+        <value>HTTP_ONLY</value>
+    </property>
+    <property>
+        <name>ignore.secure.ports.for.testing</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>dfs.datanode.address</name>
+        <value>0.0.0.0:${DFS_DN_PORT}</value>
+    </property>
+    <property>
+        <name>dfs.datanode.http.address</name>
+        <value>0.0.0.0:${DFS_DN_HTTP_PORT}</value>
+    </property>
+    <property>
+        <name>dfs.datanode.ipc.address</name>
+        <value>0.0.0.0:${DFS_DN_IPC_PORT}</value>
+    </property>
+    <property>
+        <name>dfs.datanode.hostname</name>
+        <value>${HOST}</value>
+    </property>
+    <property>
+        <name>dfs.client.use.datanode.hostname</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address</name>
+        <value>0.0.0.0:${DFS_NN_HTTP_PORT}</value>
+    </property>
+</configuration>

docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl

@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<configuration>
+    <property>
+        <name>javax.jdo.option.ConnectionURL</name>
+        <value>jdbc:derby:;databaseName=/data/metastore/metastore_db;create=true</value>
+    </property>
+    <property>
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+    </property>
+    <property>
+        <name>datanucleus.autoCreateSchema</name>
+        <value>false</value>
+    </property>
+    <property>
+        <name>hive.metastore.schema.verification</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hive.metastore.uris</name>
+        <value>thrift://${HOST}:${HMS_PORT}</value>
+    </property>
+    <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>hdfs://${HOST}:${FS_PORT}/user/hive/warehouse</value>
+    </property>
+    <property>
+        <name>hive.metastore.sasl.enabled</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hive.metastore.kerberos.principal</name>
+        <value>hive/${HOST}@${REALM}</value>
+    </property>
+    <property>
+        <name>hive.metastore.kerberos.keytab.file</name>
+        <value>/data/keytabs/hive.keytab</value>
+    </property>
+    <property>
+        <name>hive.metastore.execute.setugi</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>metastore.storage.schema.reader.impl</name>
+        <value>org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader</value>
+    </property>
+</configuration>

docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh → docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl

@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,5 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM
-beeline -u "jdbc:hive2://localhost:16000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;"
+[kdcdefaults]
+ kdc_ports = ${KDC_PORT}
+ kdc_tcp_ports = ${KDC_PORT}
+
+[realms]
+ ${REALM} = {
+  database_name = /data/kdc/principal
+  key_stash_file = /data/kdc/.k5.${REALM}
+  max_life = 24h
+  max_renewable_life = 7d
+  supported_enctypes = aes128-cts-hmac-sha1-96:normal
+ }