diff --git a/docker/thirdparties/docker-compose/kerberos/.gitignore b/docker/thirdparties/docker-compose/kerberos/.gitignore new file mode 100644 index 00000000000000..c7e96926b15ca5 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/.gitignore @@ -0,0 +1,8 @@ +# Generated by run-thirdparties-docker.sh. +/conf/kerberos1/*.conf +/conf/kerberos1/*.xml +/conf/kerberos2/*.conf +/conf/kerberos2/*.xml +/data/ +/two-kerberos-hives/*.conf +/two-kerberos-hives/*.keytab diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh b/docker/thirdparties/docker-compose/kerberos/Dockerfile old mode 100755 new mode 100644 similarity index 61% rename from docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh rename to docker/thirdparties/docker-compose/kerberos/Dockerfile index 77df431d85ac3b..88723e1313b999 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh +++ b/docker/thirdparties/docker-compose/kerberos/Dockerfile @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,25 +15,20 @@ # specific language governing permissions and limitations # under the License. -set -euo pipefail +FROM apache/hive:3.1.3 -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi +USER root -# Supervisord is not running -if ! test -f /tmp/supervisor.sock; then - exit 0 -fi +RUN apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + krb5-admin-server \ + krb5-kdc \ + krb5-user \ + && rm -rf /var/lib/apt/lists/* -# Check if all Hadoop services are running -FAILED=$(supervisorctl status | grep -v RUNNING || true) +COPY entrypoint-hive-master.sh /opt/doris/entrypoint.sh +COPY health-checks/health.sh /opt/doris/health.sh -if [ "$FAILED" == "" ]; then - echo "All services are running" - exit 0 -else - echo "Some of the services are failing: ${FAILED}" - exit 1 -fi +RUN chmod 755 /opt/doris/entrypoint.sh /opt/doris/health.sh + +ENTRYPOINT ["/opt/doris/entrypoint.sh"] diff --git a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf index 83fe29c2cb2fa0..c7d41612a5359e 100644 --- a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf +++ b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf @@ -33,15 +33,9 @@ [realms] LABS.TERADATA.COM = { kdc = hadoop-master:5588 - admin_server = hadoop-master:5749 - } - OTHERLABS.TERADATA.COM = { - kdc = hadoop-master:5589 - admin_server = hadoop-master:5750 } OTHERREALM.COM = { kdc = hadoop-master-2:6688 - admin_server = hadoop-master-2:6749 } [domain_realm] diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh deleted file mode 100755 index ec2dc074e705cd..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# test whether OVERRIDES_DIR is set -if [[ -n "${OVERRIDES_DIR+x}" ]]; then - echo "The OVERRIDES_DIR (${OVERRIDES_DIR}) support is disabled as it was deemed unused." >&2 - echo "It is being removed." >&2 - exit 16 -fi - -if test -e /overrides; then - find /overrides >&2 - echo "The /overrides handling is disabled as it was deemed unused." >&2 - echo "It is being removed." >&2 - exit 17 -fi diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh deleted file mode 100755 index 93c6e385effe1f..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi - -set -x - -HADOOP_INIT_D=${HADOOP_INIT_D:-/etc/hadoop-init.d/} - -echo "Applying hadoop init.d scripts from ${HADOOP_INIT_D}" -if test -d "${HADOOP_INIT_D}"; then - for init_script in "${HADOOP_INIT_D}"*; do - chmod a+x "${init_script}" - "${init_script}" - done -fi - -trap exit INT - -echo "Running services with supervisord" -rm -rf /etc/supervisord.d/socks-proxy.conf -rm -rf /etc/supervisord.d/sshd.conf - -supervisord -c /etc/supervisord.conf diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml b/docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl old mode 100755 new mode 100644 similarity index 58% rename from docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml rename to docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl index c0ce38e3cdc1bd..c5625a53ef639d --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml +++ b/docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl @@ -1,4 +1,4 @@ - + + + fs.defaultFS + hdfs://${HOST}:${FS_PORT} + + + hadoop.security.authentication + kerberos + + + hadoop.security.authorization + true + hadoop.security.auth_to_local - RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// + RULE:[2:$1@$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:$1@$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// DEFAULT + + hadoop.proxyuser.hive.hosts + * + + + hadoop.proxyuser.hive.users + * + + + hadoop.security.token.service.use_ip + false + diff --git a/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl b/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl new file mode 100644 index 00000000000000..b56a18f5ca57d4 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl @@ -0,0 +1,113 @@ + + + + + dfs.namenode.name.dir + file:///data/hdfs/name + + + dfs.datanode.data.dir + file:///data/hdfs/data + + + dfs.replication + 1 + + + dfs.permissions.enabled + false + + + dfs.namenode.datanode.registration.ip-hostname-check + false + + + dfs.block.access.token.enable + true + + + dfs.namenode.kerberos.principal + hdfs/${HOST}@${REALM} + + + dfs.namenode.rpc-bind-host + 0.0.0.0 + + + dfs.namenode.keytab.file + /data/keytabs/hdfs.keytab + + + dfs.datanode.kerberos.principal + hdfs/${HOST}@${REALM} + + + dfs.datanode.keytab.file + /data/keytabs/hdfs.keytab + + + dfs.namenode.kerberos.internal.spnego.principal + HTTP/${HOST}@${REALM} + + + dfs.web.authentication.kerberos.principal + HTTP/${HOST}@${REALM} + + + dfs.web.authentication.kerberos.keytab + /data/keytabs/spnego.keytab + + + dfs.data.transfer.protection + authentication + + + dfs.http.policy + HTTP_ONLY + + + ignore.secure.ports.for.testing + true + + + dfs.datanode.address + 0.0.0.0:${DFS_DN_PORT} + + + dfs.datanode.http.address + 0.0.0.0:${DFS_DN_HTTP_PORT} + + + dfs.datanode.ipc.address + 0.0.0.0:${DFS_DN_IPC_PORT} + + + dfs.datanode.hostname + ${HOST} + + + dfs.client.use.datanode.hostname + true + + + dfs.namenode.http-address + 0.0.0.0:${DFS_NN_HTTP_PORT} + + diff --git a/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl b/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl new file mode 100644 index 00000000000000..2b4b41e20a7341 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl @@ -0,0 +1,65 @@ + + + + + javax.jdo.option.ConnectionURL + jdbc:derby:;databaseName=/data/metastore/metastore_db;create=true + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + datanucleus.autoCreateSchema + false + + + hive.metastore.schema.verification + true + + + hive.metastore.uris + thrift://${HOST}:${HMS_PORT} + + + hive.metastore.warehouse.dir + hdfs://${HOST}:${FS_PORT}/user/hive/warehouse + + + hive.metastore.sasl.enabled + true + + + hive.metastore.kerberos.principal + hive/${HOST}@${REALM} + + + hive.metastore.kerberos.keytab.file + /data/keytabs/hive.keytab + + + hive.metastore.execute.setugi + true + + + metastore.storage.schema.reader.impl + org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader + + diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh b/docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl old mode 100755 new mode 100644 similarity index 74% rename from docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh rename to docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl index 7545969bc47d65..0b678db1538447 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh +++ b/docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,5 +15,15 @@ # specific language governing permissions and limitations # under the License. -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -beeline -u "jdbc:hive2://localhost:16000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;" \ No newline at end of file +[kdcdefaults] + kdc_ports = ${KDC_PORT} + kdc_tcp_ports = ${KDC_PORT} + +[realms] + ${REALM} = { + database_name = /data/kdc/principal + key_stash_file = /data/kdc/.k5.${REALM} + max_life = 24h + max_renewable_life = 7d + supported_enctypes = aes128-cts-hmac-sha1-96:normal + } diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl deleted file mode 100644 index e16c70e16db817..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[kdcdefaults] - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - -[realms] - LABS.TERADATA.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT1} - kdc_tcp_listen = ${KDC_PORT1} - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - } - - OTHERLABS.TERADATA.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5-other.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT2} - kdc_tcp_listen = ${KDC_PORT2} - kdc_ports = ${KDC_PORT2} - kdc_tcp_ports = ${KDC_PORT2} - kadmind_port = ${KADMIND_PORT2} - kpasswd_port = ${KPASSWD_PORT2} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl deleted file mode 100644 index 1edf2bb8fd05fe..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[logging] - default = FILE:/var/log/krb5libs.log - kdc = FILE:/var/log/krb5kdc.log - admin_server = FILE:/var/log/kadmind.log - -[libdefaults] - default_realm = LABS.TERADATA.COM - dns_lookup_realm = false - dns_lookup_kdc = false - forwardable = true - allow_weak_crypto = true - -[realms] - LABS.TERADATA.COM = { - kdc = ${HOST}:${KDC_PORT1} - admin_server = ${HOST}:${KADMIND_PORT1} - } - OTHERLABS.TERADATA.COM = { - kdc = ${HOST}:${KDC_PORT2} - admin_server = ${HOST}:${KADMIND_PORT2} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl deleted file mode 100644 index 61b4994ad5c1ac..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[kdcdefaults] - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - - -[realms] - OTHERREALM.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT1} - kdc_tcp_listen = ${KDC_PORT1} - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl similarity index 81% rename from docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl rename to docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl index c817dbdd79753d..84d8707184e1bc 100644 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl +++ b/docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -19,17 +18,18 @@ [logging] default = FILE:/var/log/krb5libs.log kdc = FILE:/var/log/krb5kdc.log - admin_server = FILE:/var/log/kadmind.log [libdefaults] - default_realm = OTHERREALM.COM + default_realm = ${REALM} dns_lookup_realm = false dns_lookup_kdc = false forwardable = true - allow_weak_crypto = true + udp_preference_limit = 1 [realms] - OTHERREALM.COM = { - kdc = ${HOST}:${KDC_PORT1} - admin_server = ${HOST}:${KADMIND_PORT1} - } \ No newline at end of file + ${REALM} = { + kdc = 127.0.0.1:${KDC_PORT} + } + +[domain_realm] + ${HOST} = ${REALM} diff --git a/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl deleted file mode 100644 index e91c65c100463f..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[mysqld] -port=${MYSQL_PORT} -datadir=/var/lib/mysql -socket=/var/lib/mysql/mysql.sock -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links=0 -# Settings user and group are ignored when systemd is used. -# If you need to run mysqld under a different user or group, -# customize your systemd unit file for mariadb according to the -# instructions in http://fedoraproject.org/wiki/Systemd - -[mysqld_safe] -log-error=/var/log/mariadb/mariadb.log -pid-file=/var/run/mariadb/mariadb.pid - -# -# include all files from the config directory -# -!includedir /etc/my.cnf.d - diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh index b6eeb381fabfd1..5dd4adf46d458e 100755 --- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh +++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh @@ -15,87 +15,125 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + set -euo pipefail -source /usr/local/common/hive-configure.sh -source /usr/local/common/event-hook.sh - -echo "Configuring hive" -configure /etc/hive/conf/hive-site.xml hive HIVE_SITE_CONF -configure /etc/hive/conf/hiveserver2-site.xml hive HIVE_SITE_CONF -configure /etc/hadoop/conf/core-site.xml core CORE_CONF -configure /etc/hadoop/conf/hdfs-site.xml hdfs HDFS_CONF -configure /etc/hadoop/conf/yarn-site.xml yarn YARN_CONF -configure /etc/hadoop/conf/mapred-site.xml mapred MAPRED_CONF -configure /etc/hive/conf/beeline-site.xml beeline BEELINE_SITE_CONF - -echo "Copying kerberos keytabs to keytabs/" -mkdir -p /etc/hadoop-init.d/ - -if [ "$1" == "1" ]; then - cp /etc/trino/conf/* /keytabs/ -elif [ "$1" == "2" ]; then - cp /etc/trino/conf/hive-presto-master.keytab /keytabs/other-hive-presto-master.keytab - cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab -else - echo "Invalid index parameter. Exiting." - exit 1 -fi -cd /usr/hdp/3.1.0.0-78/hive/auxlib -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jdom-1.1.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-core-3.4.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ecs-4.2.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ram-3.0.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-sts-3.0.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-core-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-core-linux-el7-aarch64-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-sdk-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aws-java-sdk-bundle-1.11.375.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-huaweicloud-3.1.1-hw-54.5.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-cos-3.1.0-8.3.22.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/cos_api-bundle-5.6.244.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aws-3.2.1.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/paimon-hive-connector-3.1-1.3-SNAPSHOT.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/gcs-connector-hadoop3-2.2.24-shaded.jar - - -/usr/local/hadoop-run.sh & - -# check healthy hear -echo "Waiting for hadoop to be healthy" - -for i in {1..60}; do - if /usr/local/health.sh; then - echo "Hadoop is healthy" + +: "${HOST:?}" +: "${REALM:?}" +: "${KDC_PORT:?}" +: "${FS_PORT:?}" +: "${HMS_PORT:?}" +: "${HIVE_CLIENT_KEYTAB:?}" +: "${PRESTO_CLIENT_KEYTAB:?}" + +export HADOOP_CONF_DIR=/opt/doris/conf +export HIVE_CONF_DIR=/opt/doris/conf +export KRB5_KDC_PROFILE=/opt/doris/conf/kdc.conf +export PATH="/usr/local/sbin:/usr/sbin:${PATH}" +export HADOOP_CLIENT_OPTS="-Xms192m -Xmx320m" + +readonly HDFS_PRINCIPAL="hdfs/${HOST}@${REALM}" +readonly HTTP_PRINCIPAL="HTTP/${HOST}@${REALM}" +readonly HIVE_PRINCIPAL="hive/${HOST}@${REALM}" +readonly HIVE_CLIENT_PRINCIPAL="hive/presto-master.docker.cluster@${REALM}" +readonly PRESTO_CLIENT_PRINCIPAL="presto-server/presto-master.docker.cluster@${REALM}" + +declare -a SERVICE_PIDS=() + +stop_services() { + kill "${SERVICE_PIDS[@]}" 2>/dev/null || true + wait "${SERVICE_PIDS[@]}" 2>/dev/null || true +} + +trap stop_services EXIT INT TERM + +start_service() { + "$@" & + SERVICE_PIDS+=("$!") +} + +wait_for_port() { + local host=$1 + local port=$2 + local service=$3 + + for _ in {1..120}; do + if (exec 3<>"/dev/tcp/${host}/${port}") 2>/dev/null; then + exec 3>&- + exec 3<&- + return + fi + sleep 1 + done + + echo "Timed out waiting for ${service} on ${host}:${port}" >&2 + return 1 +} + +create_keytab() { + local principal=$1 + local keytab=$2 + + kadmin.local -r "${REALM}" -q "addprinc -randkey ${principal}" + kadmin.local -r "${REALM}" -q "ktadd -k ${keytab} ${principal}" +} + +report_stage() { + echo "DORIS_KERBEROS_STAGE=$1" +} + +report_stage "prepare-data" +mkdir -p /data/hdfs/data /data/hdfs/name /data/kdc /data/keytabs /data/metastore /keytabs +rm -rf /tmp/hadoop-server-conf +cp -R "${HADOOP_CONF_DIR}" /tmp/hadoop-server-conf +sed -i "s#hdfs://${HOST}:${FS_PORT}#hdfs://127.0.0.1:${FS_PORT}#" \ + /tmp/hadoop-server-conf/core-site.xml + +report_stage "initialize-kdc" +kdb5_util create -s -r "${REALM}" -P doris-kerberos-test +create_keytab "${HDFS_PRINCIPAL}" /data/keytabs/hdfs.keytab +create_keytab "${HTTP_PRINCIPAL}" /data/keytabs/spnego.keytab +create_keytab "${HIVE_PRINCIPAL}" /data/keytabs/hive.keytab +create_keytab "${HIVE_CLIENT_PRINCIPAL}" "/keytabs/${HIVE_CLIENT_KEYTAB}" +create_keytab "${PRESTO_CLIENT_PRINCIPAL}" "/keytabs/${PRESTO_CLIENT_KEYTAB}" +chmod 644 /keytabs/*.keytab + +report_stage "start-kdc" +start_service krb5kdc -n -r "${REALM}" +wait_for_port 127.0.0.1 "${KDC_PORT}" "Kerberos KDC" + +report_stage "start-namenode" +export HDFS_NAMENODE_OPTS="-Xms128m -Xmx256m" +export HDFS_DATANODE_OPTS="-Xms96m -Xmx192m" +hdfs namenode -format -force -nonInteractive +start_service hdfs namenode +wait_for_port "${HOST}" "${FS_PORT}" "HDFS NameNode" + +report_stage "start-datanode" +start_service env HADOOP_CONF_DIR=/tmp/hadoop-server-conf hdfs datanode +wait_for_port "${HOST}" "${DFS_DN_PORT}" "HDFS DataNode" + +report_stage "wait-for-datanode-registration" +export KRB5CCNAME=FILE:/tmp/hdfs-admin.ccache +kinit -kt /data/keytabs/hdfs.keytab "${HDFS_PRINCIPAL}" +for _ in {1..120}; do + if hdfs dfsadmin -Dfs.defaultFS="hdfs://127.0.0.1:${FS_PORT}" -report 2>/dev/null \ + | grep -q '^Live datanodes (1):'; then break fi - echo "Hadoop is not healthy yet. Retrying in 60 seconds..." - sleep 5 + sleep 1 done +hdfs dfsadmin -Dfs.defaultFS="hdfs://127.0.0.1:${FS_PORT}" -report \ + | grep -q '^Live datanodes (1):' +kdestroy + +report_stage "initialize-hive-metastore" +schematool -dbType derby -initSchema +start_service hive --service metastore -p "${HMS_PORT}" +wait_for_port "${HOST}" "${HMS_PORT}" "Hive Metastore" + +touch /tmp/SUCCESS +echo "Minimal Kerberos HDFS and Hive Metastore environment is ready" +echo "DORIS_KERBEROS_READY" -if [ $i -eq 60 ]; then - echo "Hadoop did not become healthy after 60 attempts. Exiting." - exit 1 -fi - -echo "Init kerberos test data" - -if [ "$1" == "1" ]; then - kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -elif [ "$1" == "2" ]; then - kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -else - echo "Invalid index parameter. Exiting." - exit 1 -fi -hive -f /usr/local/sql/create_kerberos_hive_table.sql -if [[ ${enablePaimonHms} == "true" ]]; then - echo "Creating Paimon HMS catalog and table" - hadoop fs -put /tmp/paimon_data/* /user/hive/warehouse/ - # Render the OSS location directly to avoid passing oss_bucket as a Hive session conf. - paimon_hql=$(mktemp /tmp/create_paimon_hive_table.XXXXXX.hql) - sed "s|__OSS_BUCKET__|${OSSBucket}|g" /usr/local/sql/create_paimon_hive_table.hql > "${paimon_hql}" - hive -f "${paimon_hql}" - rm -f "${paimon_hql}" -fi - -exec_success_hook +wait -n "${SERVICE_PIDS[@]}" diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl index 67cecd8ba61421..792d0c0491da97 100644 --- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl +++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl @@ -1,96 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://127.0.0.1:${MYSQL_PORT}/metastore -HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver -HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=root -HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=root -HIVE_SITE_CONF_datanucleus_autoCreateSchema=false -HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT} -HIVE_SITE_CONF_hive_metastore_uris=thrift://${IP_HOST}:${HMS_PORT} -HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 -HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT} -HIVE_SITE_CONF_hive_server2_webui_port=0 -HIVE_SITE_CONF_hive_compactor_initiator_on=true -HIVE_SITE_CONF_hive_compactor_worker_threads=2 -HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader -BEELINE_SITE_CONF_beeline_hs2_jdbc_url_tcpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive -BEELINE_SITE_CONF_beeline_hs2_jdbc_url_httpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive - - -CORE_CONF_fs_defaultFS=hdfs://${HOST}:${FS_PORT} -CORE_CONF_hadoop_http_staticuser_user=root -CORE_CONF_hadoop_proxyuser_hue_hosts=* -CORE_CONF_hadoop_proxyuser_hue_groups=* - -HDFS_CONF_dfs_webhdfs_enabled=true -HDFS_CONF_dfs_permissions_enabled=false -HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false -HDFS_CONF_dfs_datanode_address=${HOST}:${DFS_DN_PORT} -HDFS_CONF_dfs_datanode_http_address=${HOST}:${DFS_DN_HTTP_PORT} -HDFS_CONF_dfs_datanode_ipc_address=${HOST}:${DFS_DN_IPC_PORT} -HDFS_CONF_dfs_namenode_http___address=${HOST}:${DFS_NN_HTTP_PORT} -YARN_CONF_yarn_log___aggregation___enable=true -YARN_CONF_yarn_resourcemanager_recovery_enabled=true -YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore -YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate -YARN_CONF_yarn_nodemanager_remote___app___log___dir=/var/log/hadoop-yarn/apps -YARN_CONF_yarn_log_server_url=http://${HOST}:${YARM_LOG_SERVER_PORT}/jobhistory/logs -YARN_CONF_yarn_timeline___service_enabled=false -YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true -YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true -YARN_CONF_yarn_resourcemanager_hostname=${HOST} -MAPRED_CONF_mapreduce_shuffle_port=${MAPREDUCE_SHUFFLE_PORT} -YARN_CONF_yarn_timeline___service_hostname=${HOST} -YARN_CONF_yarn_resourcemanager_address=${HOST}:${YARN_RM_PORT} -YARN_CONF_yarn_resourcemanager_scheduler_address=${HOST}:${YARN_RM_SCHEDULER_PORT} -YARN_CONF_yarn_resourcemanager_resource___tracker_address=${HOST}:${YARN_RM_TRACKER_PORT} -YARN_CONF_yarn_resourcemanager_admin_address=${HOST}:${YARN_RM_ADMIN_PORT} -YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT} -YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT} -YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT} +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem -HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem -HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk} -HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk} -HIVE_SITE_CONF_fs_s3a_endpoint=${AWSEndpoint} -HIVE_SITE_CONF_fs_AbstractFileSystem_obs_impl=org.apache.hadoop.fs.obs.OBS -HIVE_SITE_CONF_fs_obs_access_key=${OBSAk} -HIVE_SITE_CONF_fs_obs_secret_key=${OBSSk} -HIVE_SITE_CONF_fs_obs_endpoint=${OBSEndpoint} -HIVE_SITE_CONF_fs_cosn_credentials_provider=org.apache.hadoop.fs.auth.SimpleCredentialProvider -HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk} -HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk} -HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion} -HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem -HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem -HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN -HIVE_SITE_CONF_fs_oss_impl=com.aliyun.jindodata.oss.JindoOssFileSystem -HIVE_SITE_CONF_fs_AbstractFileSystem_oss_impl=com.aliyun.jindodata.oss.JindoOSS -HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk} -HIVE_SITE_CONF_fs_oss_accessKeySecret=${OSSSk} -HIVE_SITE_CONF_fs_oss_endpoint=${OSSEndpoint} -OSSBucket=${OSSBucket} -HIVE_SITE_CONF_fs_AbstractFileSystem_gs_impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS -HIVE_SITE_CONF_fs_gs_project_id=${GCSProjectId} -HIVE_SITE_CONF_google_cloud_auth_service_account_enable=true -HIVE_SITE_CONF_fs_gs_auth_service_account_email=${GCSAccountEmail} -HIVE_SITE_CONF_fs_gs_auth_service_account_private_key_id=${GCSAccountPrivateKeyId} -HIVE_SITE_CONF_fs_gs_auth_service_account_private_key=${GCSAccountPrivateKey} -HIVE_SITE_CONF_fs_gs_proxy_address=${GCSProxyAddress} -enablePaimonHms=${enablePaimonHms} +HOST=${HOST} +REALM=${REALM} +KDC_PORT=${KDC_PORT} +FS_PORT=${FS_PORT} +HMS_PORT=${HMS_PORT} +DFS_DN_PORT=${DFS_DN_PORT} +DFS_DN_HTTP_PORT=${DFS_DN_HTTP_PORT} +DFS_DN_IPC_PORT=${DFS_DN_IPC_PORT} +DFS_NN_HTTP_PORT=${DFS_NN_HTTP_PORT} +HIVE_CLIENT_KEYTAB=${HIVE_CLIENT_KEYTAB} +PRESTO_CLIENT_KEYTAB=${PRESTO_CLIENT_KEYTAB} +HADOOP_CONF_DIR=/opt/doris/conf +HIVE_CONF_DIR=/opt/doris/conf +KRB5_CONFIG=/etc/krb5.conf diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh index 515f37e36ac9e3..07cd7edb379f3b 100755 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh @@ -18,17 +18,11 @@ set -euo pipefail -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi - -set -x - -HEALTH_D=${HEALTH_D:-/etc/health.d/} - -if test -d "${HEALTH_D}"; then - for health_script in "${HEALTH_D}"/*; do - "${health_script}" &>> /var/log/container-health.log || exit 1 - done -fi +test -f /tmp/SUCCESS +kinit -kt /data/keytabs/hive.keytab "hive/${HOST}@${REALM}" +exec 3<>"/dev/tcp/${HOST}/${FS_PORT}" +exec 3>&- +exec 3<&- +exec 3<>"/dev/tcp/${HOST}/${HMS_PORT}" +exec 3>&- +exec 3<&- diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh deleted file mode 100755 index ab464b5233b31a..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -beeline -u "jdbc:hive2://localhost:15000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;" \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl index c253c06f3e488b..d21496516afac4 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl +++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl @@ -14,59 +14,41 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -version: "3" + +x-kerberos-image: &kerberos-image + image: doris-kerberos-minimal:${CONTAINER_UID} + build: + context: . + dockerfile: Dockerfile + network: host + +x-kerberos-service: &kerberos-service + <<: *kerberos-image + network_mode: "host" + mem_limit: 1536m + stop_grace_period: 30s + services: hive-krb1: - image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized_96 + <<: *kerberos-service container_name: doris-${CONTAINER_UID}-kerberos1 + hostname: hadoop-master volumes: - - ../common:/usr/local/common + - ./conf/kerberos1:/opt/doris/conf:ro + - ./conf/kerberos1/krb5.conf:/etc/krb5.conf:ro + - ./data/kerberos1:/data - ./two-kerberos-hives:/keytabs - - ./sql:/usr/local/sql - - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - - ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh - - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh - - ./conf/kerberos1/my.cnf:/etc/my.cnf - - ./conf/kerberos1/kdc.conf:/var/kerberos/krb5kdc/kdc.conf - - ./conf/kerberos1/krb5.conf:/etc/krb5.conf - - ./paimon_data:/tmp/paimon_data - hostname: hadoop-master - entrypoint: /usr/local/entrypoint-hive-master.sh 1 - healthcheck: - test: ["CMD", "ls", "/tmp/SUCCESS"] - interval: 5s - timeout: 10s - retries: 120 - network_mode: "host" env_file: - ./hadoop-hive-1.env + hive-krb2: - image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized-2_96 + <<: *kerberos-service container_name: doris-${CONTAINER_UID}-kerberos2 hostname: hadoop-master-2 volumes: - - ../common:/usr/local/common + - ./conf/kerberos2:/opt/doris/conf:ro + - ./conf/kerberos2/krb5.conf:/etc/krb5.conf:ro + - ./data/kerberos2:/data - ./two-kerberos-hives:/keytabs - - ./sql:/usr/local/sql - - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - - ./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh - - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh - - ./conf/kerberos2/my.cnf:/etc/my.cnf - - ./conf/kerberos2/kdc.conf:/var/kerberos/krb5kdc/kdc.conf - - ./conf/kerberos2/krb5.conf:/etc/krb5.conf - - ./paimon_data:/tmp/paimon_data - entrypoint: /usr/local/entrypoint-hive-master.sh 2 - healthcheck: - test: ["CMD", "ls", "/tmp/SUCCESS"] - interval: 5s - timeout: 10s - retries: 120 - network_mode: "host" env_file: - - ./hadoop-hive-2.env \ No newline at end of file + - ./hadoop-hive-2.env diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env index 4d71155cbb2b88..0e019dfcdd9f18 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env +++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,54 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Change this to a specific string. -# Do not use "_" or other sepcial characters, only number and alphabeta. -# NOTICE: change this uid will modify hive-*.yaml - export HOST="hadoop-master" +export REALM="LABS.TERADATA.COM" +export KDC_PORT=5588 export FS_PORT=8520 export HMS_PORT=9583 -export HS_PORT=15000 -export MYSQL_PORT=3356 export DFS_DN_PORT=9566 export DFS_DN_HTTP_PORT=9564 export DFS_DN_IPC_PORT=9567 export DFS_NN_HTTP_PORT=9570 -export YARM_LOG_SERVER_PORT=8588 -export YARN_RM_PORT=8532 -export YARN_RM_SCHEDULER_PORT=8530 -export YARN_RM_TRACKER_PORT=8531 -export YARN_RM_ADMIN_PORT=8533 -export YARN_RM_WEBAPP_PORT=8589 -export YARN_NM_LOCAL_PORT=8540 -export YARN_NM_WEBAPP_PORT=8542 -export MAPREDUCE_SHUFFLE_PORT=13562 -export KADMIND_PORT=5464 -export KDC_PORT1=5588 -export KDC_PORT2=5589 -export KADMIND_PORT1=5749 -export KADMIND_PORT2=5750 -export KPASSWD_PORT1=5464 -export KPASSWD_PORT2=5465 - -# prepare for paimon hms test,control load paimon hms data or not -export enablePaimonHms="false" -# hms on s3/oss/obs/cos -export AWSAk="*****************" -export AWSSk="*****************" -export AWSEndpoint="s3.ap-northeast-1.amazonaws.com" -export OSSAk="*****************" -export OSSSk="*****************" -export OSSEndpoint="oss-cn-beijing.aliyuncs.com" -export OSSBucket="${OSSBucket:-doris-regression-bj}" -export OBSAk="*****************" -export OBSSk="*****************" -export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com" -export COSAk="*****************" -export COSSk="*****************" -export COSRegion="ap-beijing" -export GCSProjectId="" -export GCSAccountEmail="" -export GCSAccountPrivateKeyId="" -export GCSAccountPrivateKey="" -export GCSProxyAddress="" +export HIVE_CLIENT_KEYTAB="hive-presto-master.keytab" +export PRESTO_CLIENT_KEYTAB="presto-server.keytab" diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env index e26a3297a756c5..a41c7f7ccc137b 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env +++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,50 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Change this to a specific string. -# Do not use "_" or other sepcial characters, only number and alphabeta. -# NOTICE: change this uid will modify hive-*.yaml - export HOST="hadoop-master-2" +export REALM="OTHERREALM.COM" +export KDC_PORT=6688 export FS_PORT=8620 export HMS_PORT=9683 -export HS_PORT=16000 -export MYSQL_PORT=3366 export DFS_DN_PORT=9666 export DFS_DN_HTTP_PORT=9664 export DFS_DN_IPC_PORT=9667 export DFS_NN_HTTP_PORT=9670 -export YARM_LOG_SERVER_PORT=8688 -export YARN_RM_PORT=8632 -export YARN_RM_SCHEDULER_PORT=8630 -export YARN_RM_TRACKER_PORT=8631 -export YARN_RM_ADMIN_PORT=8633 -export YARN_RM_WEBAPP_PORT=8689 -export YARN_NM_LOCAL_PORT=8640 -export YARN_NM_WEBAPP_PORT=8642 -export MAPREDUCE_SHUFFLE_PORT=13662 -export KDC_PORT1=6688 -export KADMIND_PORT1=6749 -export KPASSWD_PORT1=6464 - -# prepare for paimon hms test,control load paimon hms data or not -export enablePaimonHms="false" -# hms on s3/oss/obs/cos -export AWSAk="*****************" -export AWSSk="*****************" -export AWSEndpoint="s3.ap-northeast-1.amazonaws.com" -export OSSAk="*****************" -export OSSSk="*****************" -export OSSEndpoint="oss-cn-beijing.aliyuncs.com" -export OSSBucket="${OSSBucket:-doris-regression-bj}" -export OBSAk="*****************" -export OBSSk="*****************" -export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com" -export COSAk="*****************" -export COSSk="*****************" -export COSRegion="ap-beijing" -export GCSProjectId="" -export GCSAccountEmail="" -export GCSAccountPrivateKeyId="" -export GCSAccountPrivateKey="" -export GCSProxyAddress="" +export HIVE_CLIENT_KEYTAB="other-hive-presto-master.keytab" +export PRESTO_CLIENT_KEYTAB="other-presto-server.keytab" diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql b/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql deleted file mode 100644 index ecf58e88158b12..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `test_krb_hive_db`; -CREATE TABLE IF NOT EXISTS `test_krb_hive_db`.`test_krb_hive_tbl`( - `id_key` int, - `string_key` string, - `rate_val` double, - `comment` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; - -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(1, 'a', 3.16, 'cc0'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(2, 'b', 41.2, 'cc1'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(3, 'c', 6.2, 'cc2'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(4, 'd', 1.4, 'cc3'); diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql b/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql deleted file mode 100644 index 367296490e63da..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE DATABASE IF NOT EXISTS hdfs_db; - -USE hdfs_db; - -CREATE EXTERNAL TABLE external_test_table - STORED BY 'org.apache.paimon.hive.PaimonStorageHandler' -LOCATION 'hdfs:///user/hive/warehouse/hdfs_db.db/external_test_table'; - -CREATE DATABASE IF NOT EXISTS ali_db; - -USE ali_db; - -CREATE EXTERNAL TABLE external_test_table - STORED BY 'org.apache.paimon.hive.PaimonStorageHandler' -LOCATION 'oss://__OSS_BUCKET__/regression/paimon_warehouse/ali_db.db/hive_test_table'; diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml deleted file mode 100755 index 4541c1328aea16..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - fs.default.name - hdfs://hadoop-master-2:9000 - - diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 46e4bd83768831..2e7cc9d56f23ef 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -674,34 +674,150 @@ start_lakesoul() { fi } +dump_kerberos_container_state() { + local container="$1" + + echo "===== ${container} state =====" >&2 + sudo docker inspect --format \ + 'status={{.State.Status}} running={{.State.Running}} exitCode={{.State.ExitCode}} oomKilled={{.State.OOMKilled}} error={{.State.Error}}' \ + "${container}" >&2 || true + echo "===== ${container} logs (tail -200) =====" >&2 + sudo docker logs --tail 200 "${container}" >&2 || true +} + +cleanup_kerberos_ready_wait() { + local wait_pid="$1" + + [[ -n "${wait_pid}" ]] || return 0 + sudo kill -TERM -- "-${wait_pid}" >/dev/null 2>&1 || true + wait "${wait_pid}" >/dev/null 2>&1 || true +} + +wait_for_kerberos_ready() { + local container="$1" + local wait_pid="" + local wait_status=0 + + echo "Waiting for ${container} readiness event" + setsid sudo timeout --signal=TERM --kill-after=5s 20m bash -c ' + log_dir=$(mktemp -d) + mkfifo "${log_dir}/container.log" + docker logs --follow "$1" >"${log_dir}/container.log" 2>&1 & + log_pid=$! + cleanup() { + kill -KILL "${log_pid}" 2>/dev/null || true + wait "${log_pid}" 2>/dev/null || true + rm -rf "${log_dir}" + } + trap cleanup EXIT + while IFS= read -r line; do + if [[ "${line}" == "DORIS_KERBEROS_READY" ]]; then + echo "${line}" + exit 0 + fi + done <"${log_dir}/container.log" + exit 1 + ' _ "${container}" & + wait_pid=$! + trap 'cleanup_kerberos_ready_wait "${wait_pid}"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + if ! wait "${wait_pid}"; then + wait_status=1 + fi + wait_pid="" + trap - EXIT TERM INT + if [[ "${wait_status}" -ne 0 ]]; then + echo "ERROR: timed out or container exited before ${container} became ready" >&2 + dump_kerberos_container_state "${container}" + return 1 + fi +} + +cleanup_kerberos_readiness_jobs() { + local pid + + for pid in "$@"; do + kill "${pid}" >/dev/null 2>&1 || true + done + for pid in "$@"; do + wait "${pid}" >/dev/null 2>&1 || true + done +} + +validate_kerberos_container() { + local container="$1" + + echo "Running one-shot readiness validation for ${container}" + if ! sudo docker exec "${container}" /opt/doris/health.sh; then + echo "ERROR: one-shot readiness validation failed for ${container}" >&2 + dump_kerberos_container_state "${container}" + return 1 + fi +} + start_kerberos() { echo "RUN_KERBEROS" + local KERBEROS_DIR="${ROOT}/docker-compose/kerberos" + local -a containers=( + "doris-${CONTAINER_UID}-kerberos1" + "doris-${CONTAINER_UID}-kerberos2" + ) + local -a readiness_pids=() + local readiness_status=0 + local container + local pid + export CONTAINER_UID=${CONTAINER_UID} - envsubst <"${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl >"${ROOT}"/docker-compose/kerberos/kerberos.yaml - sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh - sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh + envsubst <"${KERBEROS_DIR}/kerberos.yaml.tpl" >"${KERBEROS_DIR}/kerberos.yaml" + mkdir -p "${KERBEROS_DIR}/conf/kerberos1" "${KERBEROS_DIR}/conf/kerberos2" \ + "${KERBEROS_DIR}/two-kerberos-hives" for i in {1..2}; do - . "${ROOT}"/docker-compose/kerberos/kerberos${i}_settings.env - envsubst <"${ROOT}"/docker-compose/kerberos/hadoop-hive.env.tpl >"${ROOT}"/docker-compose/kerberos/hadoop-hive-${i}.env - envsubst <"${ROOT}"/docker-compose/kerberos/conf/my.cnf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/my.cnf - envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf - envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf + . "${KERBEROS_DIR}/kerberos${i}_settings.env" + envsubst <"${KERBEROS_DIR}/hadoop-hive.env.tpl" >"${KERBEROS_DIR}/hadoop-hive-${i}.env" + for config in kdc.conf krb5.conf core-site.xml hdfs-site.xml hive-site.xml; do + envsubst <"${KERBEROS_DIR}/conf/${config}.tpl" >"${KERBEROS_DIR}/conf/kerberos${i}/${config}" + done done sudo chmod a+w /etc/hosts - sudo sed -i "1i${IP_HOST} hadoop-master" /etc/hosts - sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts - sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml down - sudo rm -rf "${ROOT}"/docker-compose/kerberos/data + if ! awk -v ip="${IP_HOST}" '$1 == ip && $2 == "hadoop-master" { found = 1 } END { exit !found }' /etc/hosts; then + sudo sed -i "1i${IP_HOST} hadoop-master" /etc/hosts + fi + if ! awk -v ip="${IP_HOST}" '$1 == ip && $2 == "hadoop-master-2" { found = 1 } END { exit !found }' /etc/hosts; then + sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts + fi + register_stack_metadata "kerberos" "${KERBEROS_DIR}/kerberos.yaml" "" + compose_cmd "${KERBEROS_DIR}/kerberos.yaml" "" down --remove-orphans + sudo rm -rf "${KERBEROS_DIR}/data" if [[ "${STOP}" -ne 1 ]]; then echo "PREPARE KERBEROS DATA" - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.keytab - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.jks - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.conf - sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml up --remove-orphans --wait -d - sudo rm -df /keytabs - sudo ln -s "${ROOT}"/docker-compose/kerberos/two-kerberos-hives /keytabs - sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /keytabs/krb5.conf - sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /etc/krb5.conf + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.keytab + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.jks + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.conf + compose_cmd "${KERBEROS_DIR}/kerberos.yaml" "" up --build --remove-orphans -d + trap 'cleanup_kerberos_readiness_jobs "${readiness_pids[@]}"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + for container in "${containers[@]}"; do + wait_for_kerberos_ready "${container}" & + readiness_pids+=("$!") + done + for pid in "${readiness_pids[@]}"; do + if ! wait "${pid}"; then + readiness_status=1 + fi + done + readiness_pids=() + trap - EXIT TERM INT + if [[ "${readiness_status}" -ne 0 ]]; then + return 1 + fi + for container in "${containers[@]}"; do + validate_kerberos_container "${container}" + done + sudo ln -sfn "${KERBEROS_DIR}/two-kerberos-hives" /keytabs + sudo cp "${KERBEROS_DIR}/common/conf/doris-krb5.conf" /keytabs/krb5.conf + sudo cp "${KERBEROS_DIR}/common/conf/doris-krb5.conf" /etc/krb5.conf sleep 2 fi } diff --git a/regression-test/data/external_table_p0/es/test_es_flatten_type.out b/regression-test/data/external_table_p2/es/test_es_flatten_type.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_flatten_type.out rename to regression-test/data/external_table_p2/es/test_es_flatten_type.out diff --git a/regression-test/data/external_table_p0/es/test_es_keyword_array_type.out b/regression-test/data/external_table_p2/es/test_es_keyword_array_type.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_keyword_array_type.out rename to regression-test/data/external_table_p2/es/test_es_keyword_array_type.out diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p2/es/test_es_query.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_query.out rename to regression-test/data/external_table_p2/es/test_es_query.out diff --git a/regression-test/data/external_table_p0/es/test_es_query_nereids.out b/regression-test/data/external_table_p2/es/test_es_query_nereids.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_query_nereids.out rename to regression-test/data/external_table_p2/es/test_es_query_nereids.out diff --git a/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out b/regression-test/data/external_table_p2/es/test_es_query_no_http_url.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_query_no_http_url.out rename to regression-test/data/external_table_p2/es/test_es_query_no_http_url.out diff --git a/regression-test/data/external_table_p0/jdbc/test_db2_jdbc_catalog.out b/regression-test/data/external_table_p2/jdbc/test_db2_jdbc_catalog.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/test_db2_jdbc_catalog.out rename to regression-test/data/external_table_p2/jdbc/test_db2_jdbc_catalog.out diff --git a/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out b/regression-test/data/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out rename to regression-test/data/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.out diff --git a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out b/regression-test/data/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out rename to regression-test/data/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.out diff --git a/regression-test/data/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.out b/regression-test/data/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.out rename to regression-test/data/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.out diff --git a/regression-test/data/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.out b/regression-test/data/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.out similarity index 100% rename from regression-test/data/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.out rename to regression-test/data/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.out diff --git a/regression-test/data/mtmv_p0/test_es_mtmv.out b/regression-test/data/mtmv_p2/test_es_mtmv.out similarity index 100% rename from regression-test/data/mtmv_p0/test_es_mtmv.out rename to regression-test/data/mtmv_p2/test_es_mtmv.out diff --git a/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy b/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy index 5979c651a8d173..0acc531b494f9c 100644 --- a/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy +++ b/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("ibmdb2.md", "p0,external,db2,external_docker,external_docker_db2") { +suite("ibmdb2.md", "p2,external,db2,external_docker,external_docker_db2") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "db2_catalog_md"; diff --git a/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy b/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy index dd8c3284ef1985..c17820fbb39112 100644 --- a/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy +++ b/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("oceanbase.md", "p0,external,oceanbase,external_docker,external_docker_oceanbase") { +suite("oceanbase.md", "p2,external,oceanbase,external_docker,external_docker_oceanbase") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "oceanbase_catalog_md"; diff --git a/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy b/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy index 0ba7e146ae9a50..89ce6909dfee89 100644 --- a/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy +++ b/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("sqlserver.md", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("sqlserver.md", "p2,external,sqlserver,external_docker,external_docker_sqlserver") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "sqlserver_catalog_md"; diff --git a/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy b/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy index 62c515d84ab086..262b7a28ee9bf0 100644 --- a/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy +++ b/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy @@ -138,6 +138,9 @@ suite("test_information_schema_timezone", "p0,external,hive,kerberos,external_do "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy index 4e2f34fb9b6dce..88010ce570785d 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy @@ -130,59 +130,6 @@ suite("test_jdbc_row_count", "p0,external,mysql,external_docker,external_docker_ assertEquals("1", result[0][2]) sql """drop catalog ${catalog_name}""" - // Test sqlserver - catalog_name = "test_sqlserver_jdbc_row_count"; - driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mssql-jdbc-11.2.3.jre8.jar" - String sqlserver_port = context.config.otherConfigs.get("sqlserver_2022_port"); - sql """drop catalog if exists ${catalog_name} """ - sql """ create catalog if not exists ${catalog_name} properties( - "type"="jdbc", - "user"="sa", - "password"="Doris123456", - "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", - "driver_url" = "${driver_url}", - "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - );""" - sql """use ${catalog_name}.dbo""" - result = sql """show table stats student""" - Thread.sleep(1000) - for (int i = 0; i < 60; i++) { - result = sql """show table stats student"""; - if (result[0][2] != "-1") { - break; - } - logger.info("Table row count not ready yet. Wait 1 second.") - Thread.sleep(1000) - } - assertEquals("3", result[0][2]) - sql """drop catalog ${catalog_name}""" - - // Test sqlserver lower - catalog_name = "test_sqlserver_lower_jdbc_row_count"; - sql """drop catalog if exists ${catalog_name} """ - sql """ create catalog if not exists ${catalog_name} properties( - "type"="jdbc", - "user"="sa", - "password"="Doris123456", - "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", - "driver_url" = "${driver_url}", - "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver", - "lower_case_meta_names" = "true" - );""" - sql """use ${catalog_name}.dbo""" - result = sql """show table stats test_lower""" - Thread.sleep(1000) - for (int i = 0; i < 60; i++) { - result = sql """show table stats test_lower"""; - if (result[0][2] != "-1") { - break; - } - logger.info("Table row count not ready yet. Wait 1 second.") - Thread.sleep(1000) - } - assertEquals("1", result[0][2]) - sql """drop catalog ${catalog_name}""" - // Test oracle catalog_name = "test_oracle_jdbc_row_count"; String oracle_port = context.config.otherConfigs.get("oracle_11_port"); @@ -242,4 +189,3 @@ suite("test_jdbc_row_count", "p0,external,mysql,external_docker,external_docker_ sql """drop catalog ${catalog_name}""" } } - diff --git a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy index 9228b7788b66d4..484b698d27cab9 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy @@ -44,7 +44,11 @@ suite("test_iceberg_hadoop_catalog_kerberos", "p0,external,kerberos,external_doc "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.kerberos.keytab.login.autorenewal.enabled" = "false", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication" ); """ diff --git a/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy index 2c4da8b2d20162..4c01c156464024 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy @@ -47,11 +47,16 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.kerberos.keytab.login.autorenewal.enabled" = "false", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication" ); """ sql """ switch ${hms_catalog_name} """ + sql """ create database if not exists test_krb_hive_db """ sql """ use test_krb_hive_db """ sql """ drop table if exists ${test_tbl_name}""" sql """ @@ -76,6 +81,10 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa ) with HDFS ( "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -94,6 +103,10 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa FORMAT AS CSV PROPERTIES( "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -115,6 +128,10 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa "hadoop.username" = "doris", "format" = "csv", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index 820a4321f8ba37..9b9b5af37ae10d 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -37,6 +37,10 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", @@ -48,52 +52,69 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa ); """ sql """ switch hms_kerberos """ + sql """ CREATE DATABASE IF NOT EXISTS test_single_krb_hive_db """ + sql """ USE test_single_krb_hive_db """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl """ + sql """ + CREATE TABLE test_krb_hive_tbl ( + id_key INT, + string_key STRING, + rate_val DOUBLE, + comment STRING + ) ENGINE = hive + PROPERTIES ("file_format" = "parquet") + """ + sql """ + INSERT INTO test_krb_hive_tbl VALUES + (1, 'a', 3.16, 'cc0'), + (2, 'b', 41.2, 'cc1'), + (3, 'c', 6.2, 'cc2'), + (4, 'd', 1.4, 'cc3') + """ sql """ show databases """ - order_qt_q01 """ select * from hms_kerberos.test_krb_hive_db.test_krb_hive_tbl """ + order_qt_q01 """ select * from hms_kerberos.test_single_krb_hive_db.test_krb_hive_tbl """ sql """drop catalog hms_kerberos;""" - try { - sql """drop catalog if exists hms_kerberos_hadoop_err1;""" - sql """ - CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 - PROPERTIES ( - "type" = "hms", - "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", - "hadoop.security.authentication" = "kerberos", - "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", - "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// - RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// - RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// - DEFAULT", - "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" - ); - """ - sql """ switch hms_kerberos_hadoop_err1 """ + sql """drop catalog if exists hms_kerberos_hadoop_err1;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// + RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// + DEFAULT", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" + ); + """ + sql """ switch hms_kerberos_hadoop_err1 """ + test { sql """ show databases """ - } catch (Exception e) { - logger.info(e.toString()) - // caused by a warning msg if enable sasl on hive but "hive.metastore.sasl.enabled" is not true: - // "set_ugi() not successful, Likely cause: new client talking to old server. Continuing without it." - assertTrue(e.toString().contains("thrift.transport.TTransportException")) + exception "thrift.transport.TTransportException" } - try { - sql """drop catalog if exists hms_kerberos_hadoop_err2;""" - sql """ - CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 - PROPERTIES ( - "type" = "hms", - "hive.metastore.sasl.enabled " = "true", - "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" - ); - """ - sql """ switch hms_kerberos_hadoop_err2 """ + sql """drop catalog if exists hms_kerberos_hadoop_err2;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 + PROPERTIES ( + "type" = "hms", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", + "hive.metastore.kerberos.principal" = "hive/invalid-host@LABS.TERADATA.COM" + ); + """ + sql """ switch hms_kerberos_hadoop_err2 """ + test { sql """ show databases """ - } catch (Exception e) { - // org.apache.thrift.transport.TTransportException: GSS initiate failed - assertTrue(e.toString().contains("Could not connect to meta store using any of the URIs provided. Most recent failure: shade.doris.hive.org.apache.thrift.transport.TTransportException: GSS initiate failed")) + exception "GSS initiate failed" } // try { diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index b6b0d9a73ae7c8..87ea09560b7163 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -42,6 +42,10 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -61,6 +65,10 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9683", "fs.defaultFS" = "hdfs://${externalEnvIp}:8620", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master-2@OTHERREALM.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", @@ -72,6 +80,32 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d ); """ + def initializeFixture = { String catalogName -> + sql """ switch ${catalogName} """ + sql """ CREATE DATABASE IF NOT EXISTS test_krb_hive_db """ + sql """ USE test_krb_hive_db """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl """ + sql """ + CREATE TABLE test_krb_hive_tbl ( + id_key INT, + string_key STRING, + rate_val DOUBLE, + comment STRING + ) ENGINE = hive + PROPERTIES ("file_format" = "parquet") + """ + sql """ + INSERT INTO test_krb_hive_tbl VALUES + (1, 'a', 3.16, 'cc0'), + (2, 'b', 41.2, 'cc1'), + (3, 'c', 6.2, 'cc2'), + (4, 'd', 1.4, 'cc3') + """ + } + + initializeFixture(hms_catalog_name) + initializeFixture("other_${hms_catalog_name}") + // 1. catalogA sql """switch ${hms_catalog_name};""" logger.info("switched to catalog " + hms_catalog_name) diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy index 7c0a79ee7748b4..25ca5ebf6da87a 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy @@ -75,6 +75,9 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext } def hdfsNonXmlParams = "\"fs.defaultFS\" = \"hdfs://${externalEnvIp}:8520\",\n" + + "\"dfs.namenode.kerberos.principal\" = \"hdfs/hadoop-master@LABS.TERADATA.COM\",\n" + + "\"dfs.client.use.datanode.hostname\" = \"true\",\n" + + "\"hadoop.security.token.service.use_ip\" = \"false\",\n" + "\"hadoop.kerberos.min.seconds.before.relogin\" = \"5\",\n" + "\"hadoop.security.authentication\" = \"kerberos\",\n" + "\"hadoop.kerberos.principal\"=\"hive/presto-master.docker.cluster@LABS.TERADATA.COM\",\n" + @@ -316,4 +319,4 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext export_hdfs("hdfs://${externalEnvIp}:8520", hdfsNonXmlParams) -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p0/es/test_es_catalog_http_open_api.groovy b/regression-test/suites/external_table_p2/es/test_es_catalog_http_open_api.groovy similarity index 97% rename from regression-test/suites/external_table_p0/es/test_es_catalog_http_open_api.groovy rename to regression-test/suites/external_table_p2/es/test_es_catalog_http_open_api.groovy index 9eafacacbb11a2..67d7bbc95b7dd5 100644 --- a/regression-test/suites/external_table_p0/es/test_es_catalog_http_open_api.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_catalog_http_open_api.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_catalog_http_open_api", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_catalog_http_open_api", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_flatten_type.groovy b/regression-test/suites/external_table_p2/es/test_es_flatten_type.groovy similarity index 96% rename from regression-test/suites/external_table_p0/es/test_es_flatten_type.groovy rename to regression-test/suites/external_table_p2/es/test_es_flatten_type.groovy index 7428dd386bcca1..024933ebb31092 100644 --- a/regression-test/suites/external_table_p0/es/test_es_flatten_type.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_flatten_type.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_flatten_type", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_flatten_type", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy b/regression-test/suites/external_table_p2/es/test_es_keyword_array_type.groovy similarity index 98% rename from regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy rename to regression-test/suites/external_table_p2/es/test_es_keyword_array_type.groovy index 2c153c940f90f1..82f07c19e40afb 100644 --- a/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_keyword_array_type.groovy @@ -20,7 +20,7 @@ // is an array, Doris should not throw an error. Instead, it should serialize // the array to a JSON string representation. -suite("test_es_keyword_array_type", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_keyword_array_type", "p2,external,es,external_docker,external_docker_es") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p2/es/test_es_query.groovy similarity index 99% rename from regression-test/suites/external_table_p0/es/test_es_query.groovy rename to regression-test/suites/external_table_p2/es/test_es_query.groovy index 4994018a4c0ee3..bfe2ee7170bd01 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_query", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy b/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy similarity index 99% rename from regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy rename to regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy index 2a6c9c1527d080..230c05e7784053 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_query_nereids", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_query_nereids", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_query_no_http_url.groovy b/regression-test/suites/external_table_p2/es/test_es_query_no_http_url.groovy similarity index 98% rename from regression-test/suites/external_table_p0/es/test_es_query_no_http_url.groovy rename to regression-test/suites/external_table_p2/es/test_es_query_no_http_url.groovy index f5219c1509e2dd..1e749df7119adf 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query_no_http_url.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query_no_http_url.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_query_no_http_url", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_query_no_http_url", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_db2_jdbc_catalog.groovy similarity index 99% rename from regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy rename to regression-test/suites/external_table_p2/jdbc/test_db2_jdbc_catalog.groovy index bfd6bdb720e83d..eeef434e87e091 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_db2_jdbc_catalog.groovy @@ -20,7 +20,7 @@ import java.sql.Connection import java.sql.DriverManager import java.net.URL -suite("test_db2_jdbc_catalog", "p0,external,db2,external_docker,external_docker_db2") { +suite("test_db2_jdbc_catalog", "p2,external") { qt_sql """select current_catalog()""" String enabled = context.config.otherConfigs.get("enableJdbcTest") diff --git a/regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy similarity index 95% rename from regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy rename to regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy index 3d07434b561ac8..7ebc54cd60bc11 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_oceanbase_jdbc_catalog", "p0,external,oceanbase,external_docker,external_docker_oceanbase") { +suite("test_oceanbase_jdbc_catalog", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest"); String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String s3_endpoint = getS3Endpoint() diff --git a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.groovy similarity index 98% rename from regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy rename to regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.groovy index 05545c18b1b48b..bfddab549536fe 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sqlserver_jdbc_catalog", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("test_sqlserver_jdbc_catalog", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest"); String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String s3_endpoint = getS3Endpoint() diff --git a/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy new file mode 100644 index 00000000000000..24eb42c4991e5f --- /dev/null +++ b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_sqlserver_jdbc_row_count", "p2,external") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + logger.info("enabled " + enabled) + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mssql-jdbc-11.2.3.jre8.jar" + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String catalog_name = "test_sqlserver_jdbc_row_count"; + String sqlserver_port = context.config.otherConfigs.get("sqlserver_2022_port"); + sql """drop catalog if exists ${catalog_name} """ + sql """ create catalog if not exists ${catalog_name} properties( + "type"="jdbc", + "user"="sa", + "password"="Doris123456", + "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", + "driver_url" = "${driver_url}", + "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + );""" + sql """use ${catalog_name}.dbo""" + def result = sql """show table stats student""" + Thread.sleep(1000) + for (int i = 0; i < 60; i++) { + result = sql """show table stats student"""; + if (result[0][2] != "-1") { + break; + } + logger.info("Table row count not ready yet. Wait 1 second.") + Thread.sleep(1000) + } + assertEquals("3", result[0][2]) + sql """drop catalog ${catalog_name}""" + + catalog_name = "test_sqlserver_lower_jdbc_row_count"; + sql """drop catalog if exists ${catalog_name} """ + sql """ create catalog if not exists ${catalog_name} properties( + "type"="jdbc", + "user"="sa", + "password"="Doris123456", + "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", + "driver_url" = "${driver_url}", + "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "lower_case_meta_names" = "true" + );""" + sql """use ${catalog_name}.dbo""" + result = sql """show table stats test_lower""" + Thread.sleep(1000) + for (int i = 0; i < 60; i++) { + result = sql """show table stats test_lower"""; + if (result[0][2] != "-1") { + break; + } + logger.info("Table row count not ready yet. Wait 1 second.") + Thread.sleep(1000) + } + assertEquals("1", result[0][2]) + sql """drop catalog ${catalog_name}""" + } +} diff --git a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.groovy b/regression-test/suites/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.groovy similarity index 96% rename from regression-test/suites/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.groovy rename to regression-test/suites/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.groovy index cfd878802c71f1..af86285bed7ebf 100644 --- a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.groovy +++ b/regression-test/suites/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sqlserver_all_types_select", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("test_sqlserver_all_types_select", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String s3_endpoint = getS3Endpoint() diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy index 49f4945f60e656..01b98dc02b2973 100644 --- a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy @@ -122,6 +122,9 @@ suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -129,6 +132,9 @@ suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") { String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hdfs.authentication.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -227,4 +233,3 @@ suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") { testQuery(paimon_hms_catalog_properties + paimon_fs_gcs_support + gcs_warehouse_properties + gcs_storage_s3_properties, "support_gcs_s3", "gcs_db") } - diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy index 6092ed9bd3be38..62771e46ebcfbb 100644 --- a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy +++ b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy @@ -424,6 +424,9 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -431,6 +434,9 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { """ String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hdfs.authentication.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -484,8 +490,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { String db_location = "obs://${obs_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + obs_storage_properties, "hive_hms_obs_test", db_location) testQueryAndInsert(hms_properties + obs_region_param + obs_storage_properties, "hive_hms_obs_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_new", db_location) //OBS - Partition table tests db_location = "obs://${obs_parent_path}/hive/hms/partition/" + System.currentTimeMillis() @@ -498,8 +502,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { db_location = "gs://${gcs_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + gcs_storage_old_properties, "hive_hms_gcs_test", db_location) testQueryAndInsert(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_test_new", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + gcs_storage_old_properties, "hive_hms_on_gcs_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + gcs_storage_new_properties, "hive_hms_on_gcs_kerberos_new", db_location) //GCS - Insert overwrite tests db_location = "gs://${gcs_parent_path}/hive/hms/overwrite/" + System.currentTimeMillis() testInsertOverwrite(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_overwrite_test", db_location) @@ -509,8 +511,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { db_location = "cosn://${cos_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + cos_storage_properties, "hive_hms_cos_test", db_location) testQueryAndInsert(hms_properties + cos_region_param + cos_storage_properties, "hive_hms_cos_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_new", db_location) //COS - Partition table tests db_location = "cosn://${cos_parent_path}/hive/hms/partition/" + System.currentTimeMillis() @@ -526,8 +526,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { db_location = "oss://${oss_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + oss_storage_properties, "hive_hms_oss_test", db_location) testQueryAndInsert(hms_properties + oss_region_param + oss_storage_properties, "hive_hms_oss_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + oss_storage_properties, "hive_hms_on_oss_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + oss_storage_properties, "hive_hms_on_oss_kerberos_new", db_location) //OSS - Partition table tests (fix for partition path scheme mismatch) db_location = "oss://${oss_parent_path}/hive/hms/partition/" + System.currentTimeMillis() diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy index 41785f7fe1a08c..fa3cff25572ee1 100644 --- a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy +++ b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy @@ -477,6 +477,9 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "io-impl" = "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -485,6 +488,9 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "io-impl" = "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -543,10 +549,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property hmsTestQueryAndInsert(hms_prop + warehouse + oss_region_param + oss_storage_properties, "iceberg_hms_on_oss") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_old") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new") warehouse = """ 'warehouse' = 'oss://${oss_bucket_endpoint_parent_path}/iceberg-hms-warehouse', """ @@ -560,10 +562,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property + warehouse + obs_storage_properties, "iceberg_hms_on_obs") hmsTestQueryAndInsert(hms_prop+ warehouse + obs_region_param + obs_storage_properties, "iceberg_hms_on_obs") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_old") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_new") /*--------HMS on GCS-----------*/ if(context.config.otherConfigs.get("enableGCS")){ @@ -575,10 +573,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property hmsTestQueryAndInsert(hms_prop+ warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_new") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_new") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_old") } /*--------HMS on COS-----------*/ @@ -593,9 +587,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property 'warehouse' = 'cos://${cos_parent_path}/iceberg-hms-cos-warehouse', """ hmsTestQueryAndInsert(hms_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos") - //kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_old") - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_new") /*--------HMS on S3-----------*/ diff --git a/regression-test/suites/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.groovy b/regression-test/suites/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.groovy similarity index 96% rename from regression-test/suites/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.groovy rename to regression-test/suites/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.groovy index 9973aaae2fb678..1a3f144adb4422 100644 --- a/regression-test/suites/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.groovy +++ b/regression-test/suites/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_trino_sqlserver", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("test_trino_sqlserver", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest") String enabled_trino_connector = context.config.otherConfigs.get("enableTrinoConnectorTest") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/mtmv_p0/test_es_mtmv.groovy b/regression-test/suites/mtmv_p2/test_es_mtmv.groovy similarity index 95% rename from regression-test/suites/mtmv_p0/test_es_mtmv.groovy rename to regression-test/suites/mtmv_p2/test_es_mtmv.groovy index e7c72289e17bf3..8dc757561fe65b 100644 --- a/regression-test/suites/mtmv_p0/test_es_mtmv.groovy +++ b/regression-test/suites/mtmv_p2/test_es_mtmv.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_mtmv", "p0,external,es,external_docker,external_docker_hive") { +suite("test_es_mtmv", "p2,external,es,external_docker,external_docker_hive") { String enabled = context.config.otherConfigs.get("enableEsTest") logger.info("enabled: " + enabled) String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -25,7 +25,7 @@ suite("test_es_mtmv", "p0,external,es,external_docker,external_docker_hive") { if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "es_mtmv_catalog"; String mvName = "test_es_mtmv" - String dbName = "regression_test_mtmv_p0" + String dbName = "regression_test_mtmv_p2" String esDb = "default_db" String esTable = "test1" sql """drop catalog if exists ${catalog_name} """ @@ -60,4 +60,3 @@ suite("test_es_mtmv", "p0,external,es,external_docker,external_docker_hive") { sql """ drop catalog if exists ${catalog_name} """ } } -