From 5a2b92399d0bb91f26c303d5722c997664cf9590 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 13 Jul 2026 15:10:18 +0800 Subject: [PATCH 1/2] [test](regression) Move memory-intensive external cases to P2 (#65459) Problem Summary: The P0 external regression pipeline currently starts several memory-intensive Docker services for relatively low-priority compatibility coverage. This PR reduces the P0 resource footprint by moving the following service-specific cases and expected results to P2: - DB2 - SQL Server / MSSQL, including the SQL Server Trino connector and row-count coverage - OceanBase - Elasticsearch 5/6/7/8, including ES MTMV coverage The SQL Server row-count checks are split from the mixed P0 JDBC suite so the existing MySQL, PostgreSQL, and Oracle P0 coverage remains unchanged. The ES MTMV database reference is updated to match its new P2 directory. Kerberos was evaluated separately but remains required in P0 for safety. The earlier Kerberos move has been fully reverted, so the cumulative diff of this PR contains no Kerberos file changes. This PR only changes regression-suite grouping and paths. It does not change Docker service configuration, runtime behavior, query behavior, or expected test results. (cherry picked from commit 394740cc350b0c1fb082d2fc5040d9617c2da0fc) --- .../es/test_es_flatten_type.out | 0 .../es/test_es_keyword_array_type.out | 0 .../es/test_es_query.out | 0 .../es/test_es_query_nereids.out | 0 .../es/test_es_query_no_http_url.out | 0 .../jdbc/test_db2_jdbc_catalog.out | 0 .../jdbc/test_oceanbase_jdbc_catalog.out | 0 .../jdbc/test_sqlserver_jdbc_catalog.out | 0 .../test_sqlserver_all_types_select.out | 0 .../jdbc/test_trino_sqlserver.out | 0 .../{mtmv_p0 => mtmv_p2}/test_es_mtmv.out | 0 .../suites/doc/external/jdbc/ibmdb2.md.groovy | 2 +- .../doc/external/jdbc/oceanbase.md.groovy | 2 +- .../doc/external/jdbc/sqlserver.md.groovy | 2 +- .../jdbc/test_jdbc_row_count.groovy | 54 ------------- .../es/test_es_catalog_http_open_api.groovy | 2 +- .../es/test_es_flatten_type.groovy | 2 +- .../es/test_es_keyword_array_type.groovy | 2 +- .../es/test_es_query.groovy | 2 +- .../es/test_es_query_nereids.groovy | 2 +- .../es/test_es_query_no_http_url.groovy | 2 +- .../jdbc/test_db2_jdbc_catalog.groovy | 2 +- .../jdbc/test_oceanbase_jdbc_catalog.groovy | 2 +- .../jdbc/test_sqlserver_jdbc_catalog.groovy | 2 +- .../jdbc/test_sqlserver_jdbc_row_count.groovy | 76 +++++++++++++++++++ .../test_sqlserver_all_types_select.groovy | 2 +- .../jdbc/test_trino_sqlserver.groovy | 2 +- .../{mtmv_p0 => mtmv_p2}/test_es_mtmv.groovy | 5 +- 28 files changed, 92 insertions(+), 71 deletions(-) rename regression-test/data/{external_table_p0 => external_table_p2}/es/test_es_flatten_type.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/es/test_es_keyword_array_type.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/es/test_es_query.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/es/test_es_query_nereids.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/es/test_es_query_no_http_url.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/jdbc/test_db2_jdbc_catalog.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/jdbc/test_oceanbase_jdbc_catalog.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/jdbc/test_sqlserver_jdbc_catalog.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/jdbc/type_test/select/test_sqlserver_all_types_select.out (100%) rename regression-test/data/{external_table_p0 => external_table_p2}/trino_connector/jdbc/test_trino_sqlserver.out (100%) rename regression-test/data/{mtmv_p0 => mtmv_p2}/test_es_mtmv.out (100%) rename regression-test/suites/{external_table_p0 => external_table_p2}/es/test_es_catalog_http_open_api.groovy (97%) rename regression-test/suites/{external_table_p0 => external_table_p2}/es/test_es_flatten_type.groovy (96%) rename regression-test/suites/{external_table_p0 => external_table_p2}/es/test_es_keyword_array_type.groovy (98%) rename regression-test/suites/{external_table_p0 => external_table_p2}/es/test_es_query.groovy (99%) rename regression-test/suites/{external_table_p0 => external_table_p2}/es/test_es_query_nereids.groovy (99%) rename regression-test/suites/{external_table_p0 => external_table_p2}/es/test_es_query_no_http_url.groovy (98%) rename regression-test/suites/{external_table_p0 => external_table_p2}/jdbc/test_db2_jdbc_catalog.groovy (99%) rename regression-test/suites/{external_table_p0 => external_table_p2}/jdbc/test_oceanbase_jdbc_catalog.groovy (95%) rename regression-test/suites/{external_table_p0 => external_table_p2}/jdbc/test_sqlserver_jdbc_catalog.groovy (98%) create mode 100644 regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy rename regression-test/suites/{external_table_p0 => external_table_p2}/jdbc/type_test/select/test_sqlserver_all_types_select.groovy (96%) rename regression-test/suites/{external_table_p0 => external_table_p2}/trino_connector/jdbc/test_trino_sqlserver.groovy (96%) rename regression-test/suites/{mtmv_p0 => mtmv_p2}/test_es_mtmv.groovy (95%) diff --git a/regression-test/data/external_table_p0/es/test_es_flatten_type.out b/regression-test/data/external_table_p2/es/test_es_flatten_type.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_flatten_type.out rename to regression-test/data/external_table_p2/es/test_es_flatten_type.out diff --git a/regression-test/data/external_table_p0/es/test_es_keyword_array_type.out b/regression-test/data/external_table_p2/es/test_es_keyword_array_type.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_keyword_array_type.out rename to regression-test/data/external_table_p2/es/test_es_keyword_array_type.out diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p2/es/test_es_query.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_query.out rename to regression-test/data/external_table_p2/es/test_es_query.out diff --git a/regression-test/data/external_table_p0/es/test_es_query_nereids.out b/regression-test/data/external_table_p2/es/test_es_query_nereids.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_query_nereids.out rename to regression-test/data/external_table_p2/es/test_es_query_nereids.out diff --git a/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out b/regression-test/data/external_table_p2/es/test_es_query_no_http_url.out similarity index 100% rename from regression-test/data/external_table_p0/es/test_es_query_no_http_url.out rename to regression-test/data/external_table_p2/es/test_es_query_no_http_url.out diff --git a/regression-test/data/external_table_p0/jdbc/test_db2_jdbc_catalog.out b/regression-test/data/external_table_p2/jdbc/test_db2_jdbc_catalog.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/test_db2_jdbc_catalog.out rename to regression-test/data/external_table_p2/jdbc/test_db2_jdbc_catalog.out diff --git a/regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out b/regression-test/data/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.out rename to regression-test/data/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.out diff --git a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out b/regression-test/data/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out rename to regression-test/data/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.out diff --git a/regression-test/data/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.out b/regression-test/data/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.out similarity index 100% rename from regression-test/data/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.out rename to regression-test/data/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.out diff --git a/regression-test/data/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.out b/regression-test/data/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.out similarity index 100% rename from regression-test/data/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.out rename to regression-test/data/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.out diff --git a/regression-test/data/mtmv_p0/test_es_mtmv.out b/regression-test/data/mtmv_p2/test_es_mtmv.out similarity index 100% rename from regression-test/data/mtmv_p0/test_es_mtmv.out rename to regression-test/data/mtmv_p2/test_es_mtmv.out diff --git a/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy b/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy index 5979c651a8d173..0acc531b494f9c 100644 --- a/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy +++ b/regression-test/suites/doc/external/jdbc/ibmdb2.md.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("ibmdb2.md", "p0,external,db2,external_docker,external_docker_db2") { +suite("ibmdb2.md", "p2,external,db2,external_docker,external_docker_db2") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "db2_catalog_md"; diff --git a/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy b/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy index dd8c3284ef1985..c17820fbb39112 100644 --- a/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy +++ b/regression-test/suites/doc/external/jdbc/oceanbase.md.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("oceanbase.md", "p0,external,oceanbase,external_docker,external_docker_oceanbase") { +suite("oceanbase.md", "p2,external,oceanbase,external_docker,external_docker_oceanbase") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "oceanbase_catalog_md"; diff --git a/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy b/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy index 0ba7e146ae9a50..89ce6909dfee89 100644 --- a/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy +++ b/regression-test/suites/doc/external/jdbc/sqlserver.md.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("sqlserver.md", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("sqlserver.md", "p2,external,sqlserver,external_docker,external_docker_sqlserver") { String enabled = context.config.otherConfigs.get("enableJdbcTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "sqlserver_catalog_md"; diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy index 4e2f34fb9b6dce..88010ce570785d 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_row_count.groovy @@ -130,59 +130,6 @@ suite("test_jdbc_row_count", "p0,external,mysql,external_docker,external_docker_ assertEquals("1", result[0][2]) sql """drop catalog ${catalog_name}""" - // Test sqlserver - catalog_name = "test_sqlserver_jdbc_row_count"; - driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mssql-jdbc-11.2.3.jre8.jar" - String sqlserver_port = context.config.otherConfigs.get("sqlserver_2022_port"); - sql """drop catalog if exists ${catalog_name} """ - sql """ create catalog if not exists ${catalog_name} properties( - "type"="jdbc", - "user"="sa", - "password"="Doris123456", - "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", - "driver_url" = "${driver_url}", - "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - );""" - sql """use ${catalog_name}.dbo""" - result = sql """show table stats student""" - Thread.sleep(1000) - for (int i = 0; i < 60; i++) { - result = sql """show table stats student"""; - if (result[0][2] != "-1") { - break; - } - logger.info("Table row count not ready yet. Wait 1 second.") - Thread.sleep(1000) - } - assertEquals("3", result[0][2]) - sql """drop catalog ${catalog_name}""" - - // Test sqlserver lower - catalog_name = "test_sqlserver_lower_jdbc_row_count"; - sql """drop catalog if exists ${catalog_name} """ - sql """ create catalog if not exists ${catalog_name} properties( - "type"="jdbc", - "user"="sa", - "password"="Doris123456", - "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", - "driver_url" = "${driver_url}", - "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver", - "lower_case_meta_names" = "true" - );""" - sql """use ${catalog_name}.dbo""" - result = sql """show table stats test_lower""" - Thread.sleep(1000) - for (int i = 0; i < 60; i++) { - result = sql """show table stats test_lower"""; - if (result[0][2] != "-1") { - break; - } - logger.info("Table row count not ready yet. Wait 1 second.") - Thread.sleep(1000) - } - assertEquals("1", result[0][2]) - sql """drop catalog ${catalog_name}""" - // Test oracle catalog_name = "test_oracle_jdbc_row_count"; String oracle_port = context.config.otherConfigs.get("oracle_11_port"); @@ -242,4 +189,3 @@ suite("test_jdbc_row_count", "p0,external,mysql,external_docker,external_docker_ sql """drop catalog ${catalog_name}""" } } - diff --git a/regression-test/suites/external_table_p0/es/test_es_catalog_http_open_api.groovy b/regression-test/suites/external_table_p2/es/test_es_catalog_http_open_api.groovy similarity index 97% rename from regression-test/suites/external_table_p0/es/test_es_catalog_http_open_api.groovy rename to regression-test/suites/external_table_p2/es/test_es_catalog_http_open_api.groovy index 9eafacacbb11a2..67d7bbc95b7dd5 100644 --- a/regression-test/suites/external_table_p0/es/test_es_catalog_http_open_api.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_catalog_http_open_api.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_catalog_http_open_api", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_catalog_http_open_api", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_flatten_type.groovy b/regression-test/suites/external_table_p2/es/test_es_flatten_type.groovy similarity index 96% rename from regression-test/suites/external_table_p0/es/test_es_flatten_type.groovy rename to regression-test/suites/external_table_p2/es/test_es_flatten_type.groovy index 7428dd386bcca1..024933ebb31092 100644 --- a/regression-test/suites/external_table_p0/es/test_es_flatten_type.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_flatten_type.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_flatten_type", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_flatten_type", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy b/regression-test/suites/external_table_p2/es/test_es_keyword_array_type.groovy similarity index 98% rename from regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy rename to regression-test/suites/external_table_p2/es/test_es_keyword_array_type.groovy index 2c153c940f90f1..82f07c19e40afb 100644 --- a/regression-test/suites/external_table_p0/es/test_es_keyword_array_type.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_keyword_array_type.groovy @@ -20,7 +20,7 @@ // is an array, Doris should not throw an error. Instead, it should serialize // the array to a JSON string representation. -suite("test_es_keyword_array_type", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_keyword_array_type", "p2,external,es,external_docker,external_docker_es") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p2/es/test_es_query.groovy similarity index 99% rename from regression-test/suites/external_table_p0/es/test_es_query.groovy rename to regression-test/suites/external_table_p2/es/test_es_query.groovy index 4994018a4c0ee3..bfe2ee7170bd01 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_query", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy b/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy similarity index 99% rename from regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy rename to regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy index 2a6c9c1527d080..230c05e7784053 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query_nereids.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query_nereids.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_query_nereids", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_query_nereids", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/es/test_es_query_no_http_url.groovy b/regression-test/suites/external_table_p2/es/test_es_query_no_http_url.groovy similarity index 98% rename from regression-test/suites/external_table_p0/es/test_es_query_no_http_url.groovy rename to regression-test/suites/external_table_p2/es/test_es_query_no_http_url.groovy index f5219c1509e2dd..1e749df7119adf 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query_no_http_url.groovy +++ b/regression-test/suites/external_table_p2/es/test_es_query_no_http_url.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_query_no_http_url", "p0,external,es,external_docker,external_docker_es") { +suite("test_es_query_no_http_url", "p2,external") { String enabled = context.config.otherConfigs.get("enableEsTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_db2_jdbc_catalog.groovy similarity index 99% rename from regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy rename to regression-test/suites/external_table_p2/jdbc/test_db2_jdbc_catalog.groovy index bfd6bdb720e83d..eeef434e87e091 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_db2_jdbc_catalog.groovy @@ -20,7 +20,7 @@ import java.sql.Connection import java.sql.DriverManager import java.net.URL -suite("test_db2_jdbc_catalog", "p0,external,db2,external_docker,external_docker_db2") { +suite("test_db2_jdbc_catalog", "p2,external") { qt_sql """select current_catalog()""" String enabled = context.config.otherConfigs.get("enableJdbcTest") diff --git a/regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy similarity index 95% rename from regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy rename to regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy index 3d07434b561ac8..7ebc54cd60bc11 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oceanbase_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_oceanbase_jdbc_catalog.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_oceanbase_jdbc_catalog", "p0,external,oceanbase,external_docker,external_docker_oceanbase") { +suite("test_oceanbase_jdbc_catalog", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest"); String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String s3_endpoint = getS3Endpoint() diff --git a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.groovy similarity index 98% rename from regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy rename to regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.groovy index 05545c18b1b48b..bfddab549536fe 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_catalog.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sqlserver_jdbc_catalog", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("test_sqlserver_jdbc_catalog", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest"); String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String s3_endpoint = getS3Endpoint() diff --git a/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy new file mode 100644 index 00000000000000..24eb42c4991e5f --- /dev/null +++ b/regression-test/suites/external_table_p2/jdbc/test_sqlserver_jdbc_row_count.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_sqlserver_jdbc_row_count", "p2,external") { + String enabled = context.config.otherConfigs.get("enableJdbcTest") + logger.info("enabled " + enabled) + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mssql-jdbc-11.2.3.jre8.jar" + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String catalog_name = "test_sqlserver_jdbc_row_count"; + String sqlserver_port = context.config.otherConfigs.get("sqlserver_2022_port"); + sql """drop catalog if exists ${catalog_name} """ + sql """ create catalog if not exists ${catalog_name} properties( + "type"="jdbc", + "user"="sa", + "password"="Doris123456", + "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", + "driver_url" = "${driver_url}", + "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + );""" + sql """use ${catalog_name}.dbo""" + def result = sql """show table stats student""" + Thread.sleep(1000) + for (int i = 0; i < 60; i++) { + result = sql """show table stats student"""; + if (result[0][2] != "-1") { + break; + } + logger.info("Table row count not ready yet. Wait 1 second.") + Thread.sleep(1000) + } + assertEquals("3", result[0][2]) + sql """drop catalog ${catalog_name}""" + + catalog_name = "test_sqlserver_lower_jdbc_row_count"; + sql """drop catalog if exists ${catalog_name} """ + sql """ create catalog if not exists ${catalog_name} properties( + "type"="jdbc", + "user"="sa", + "password"="Doris123456", + "jdbc_url" = "jdbc:sqlserver://${externalEnvIp}:${sqlserver_port};encrypt=false;databaseName=doris_test;", + "driver_url" = "${driver_url}", + "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "lower_case_meta_names" = "true" + );""" + sql """use ${catalog_name}.dbo""" + result = sql """show table stats test_lower""" + Thread.sleep(1000) + for (int i = 0; i < 60; i++) { + result = sql """show table stats test_lower"""; + if (result[0][2] != "-1") { + break; + } + logger.info("Table row count not ready yet. Wait 1 second.") + Thread.sleep(1000) + } + assertEquals("1", result[0][2]) + sql """drop catalog ${catalog_name}""" + } +} diff --git a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.groovy b/regression-test/suites/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.groovy similarity index 96% rename from regression-test/suites/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.groovy rename to regression-test/suites/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.groovy index cfd878802c71f1..af86285bed7ebf 100644 --- a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_sqlserver_all_types_select.groovy +++ b/regression-test/suites/external_table_p2/jdbc/type_test/select/test_sqlserver_all_types_select.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sqlserver_all_types_select", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("test_sqlserver_all_types_select", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String s3_endpoint = getS3Endpoint() diff --git a/regression-test/suites/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.groovy b/regression-test/suites/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.groovy similarity index 96% rename from regression-test/suites/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.groovy rename to regression-test/suites/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.groovy index 9973aaae2fb678..1a3f144adb4422 100644 --- a/regression-test/suites/external_table_p0/trino_connector/jdbc/test_trino_sqlserver.groovy +++ b/regression-test/suites/external_table_p2/trino_connector/jdbc/test_trino_sqlserver.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_trino_sqlserver", "p0,external,sqlserver,external_docker,external_docker_sqlserver") { +suite("test_trino_sqlserver", "p2,external") { String enabled = context.config.otherConfigs.get("enableJdbcTest") String enabled_trino_connector = context.config.otherConfigs.get("enableTrinoConnectorTest") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") diff --git a/regression-test/suites/mtmv_p0/test_es_mtmv.groovy b/regression-test/suites/mtmv_p2/test_es_mtmv.groovy similarity index 95% rename from regression-test/suites/mtmv_p0/test_es_mtmv.groovy rename to regression-test/suites/mtmv_p2/test_es_mtmv.groovy index e7c72289e17bf3..8dc757561fe65b 100644 --- a/regression-test/suites/mtmv_p0/test_es_mtmv.groovy +++ b/regression-test/suites/mtmv_p2/test_es_mtmv.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_es_mtmv", "p0,external,es,external_docker,external_docker_hive") { +suite("test_es_mtmv", "p2,external,es,external_docker,external_docker_hive") { String enabled = context.config.otherConfigs.get("enableEsTest") logger.info("enabled: " + enabled) String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") @@ -25,7 +25,7 @@ suite("test_es_mtmv", "p0,external,es,external_docker,external_docker_hive") { if (enabled != null && enabled.equalsIgnoreCase("true")) { String catalog_name = "es_mtmv_catalog"; String mvName = "test_es_mtmv" - String dbName = "regression_test_mtmv_p0" + String dbName = "regression_test_mtmv_p2" String esDb = "default_db" String esTable = "test1" sql """drop catalog if exists ${catalog_name} """ @@ -60,4 +60,3 @@ suite("test_es_mtmv", "p0,external,es,external_docker,external_docker_hive") { sql """ drop catalog if exists ${catalog_name} """ } } - From 62d244f8e34d2e6b6151d508ff98e97b42cf7b26 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 20 Jul 2026 14:32:23 +0800 Subject: [PATCH 2/2] [improvement](regression) Reduce Kerberos test environment memory (#65564) Problem Summary: The external regression pipeline currently runs two heavyweight Kerberos containers. Each container starts HiveServer2, YARN ResourceManager/NodeManager, MariaDB, and other processes that the Doris Kerberos cases do not use, resulting in roughly 4.3 GB of combined memory usage. This PR replaces those images with a minimal reusable environment based on Apache Hive 3.1.3. Each realm now runs only: - a native MIT Kerberos KDC - one HDFS NameNode and DataNode - one Kerberos-enabled Hive Metastore backed by embedded Derby The existing realm names, ports, service principals, client principals, and keytab filenames remain unchanged. JVM heaps are explicitly bounded, and each container has a 1.5 GB memory limit. HDFS listeners bind independently from their advertised Kerberos hostnames, and startup now waits until each DataNode is registered with its NameNode. The container no longer owns Hive test fixtures. `test_single_hive_kerberos` and `test_two_hive_kerberos` now create their test database, Hive table, and four rows through Doris. `test_non_catalog_kerberos` also creates its database so it no longer depends on suite execution order. Existing expected output files remain unchanged. (cherry picked from commit 52e154530ff72b53d205f77d845a3072e44ea5f1) --- .../docker-compose/kerberos/.gitignore | 8 + .../supervisorctl-check.sh => Dockerfile} | 32 ++- .../kerberos/common/conf/doris-krb5.conf | 6 - .../common/hadoop/apply-config-overrides.sh | 31 --- .../kerberos/common/hadoop/hadoop-run.sh | 44 ---- .../core-site.xml.tpl} | 29 ++- .../kerberos/conf/hdfs-site.xml.tpl | 113 ++++++++++ .../kerberos/conf/hive-site.xml.tpl | 65 ++++++ .../kdc.conf.tpl} | 15 +- .../kerberos/conf/kerberos1/kdc.conf.tpl | 50 ----- .../kerberos/conf/kerberos1/krb5.conf.tpl | 39 ---- .../kerberos/conf/kerberos2/kdc.conf.tpl | 38 ---- .../conf/{kerberos2 => }/krb5.conf.tpl | 16 +- .../docker-compose/kerberos/conf/my.cnf.tpl | 38 ---- .../kerberos/entrypoint-hive-master.sh | 196 +++++++++++------- .../kerberos/hadoop-hive.env.tpl | 121 +++-------- .../kerberos/health-checks/health.sh | 22 +- .../health-checks/hive-health-check.sh | 20 -- .../docker-compose/kerberos/kerberos.yaml.tpl | 68 +++--- .../kerberos/kerberos1_settings.env | 50 +---- .../kerberos/kerberos2_settings.env | 46 +--- .../sql/create_kerberos_hive_table.sql | 17 -- .../kerberos/sql/create_paimon_hive_table.hql | 15 -- .../hive2-default-fs-site.xml | 25 --- .../thirdparties/run-thirdparties-docker.sh | 156 ++++++++++++-- .../test_information_schema_timezone.groovy | 3 + ...est_iceberg_hadoop_catalog_kerberos.groovy | 6 +- .../kerberos/test_non_catalog_kerberos.groovy | 19 +- .../kerberos/test_single_hive_kerberos.groovy | 99 +++++---- .../kerberos/test_two_hive_kerberos.groovy | 34 +++ .../hdfs_all_test.groovy | 5 +- .../paimon/test_paimon_hms_catalog.groovy | 7 +- .../hive_on_hms_and_dlf.groovy | 14 +- ...eberg_on_hms_and_filesystem_and_dlf.groovy | 21 +- 34 files changed, 712 insertions(+), 756 deletions(-) create mode 100644 docker/thirdparties/docker-compose/kerberos/.gitignore rename docker/thirdparties/docker-compose/kerberos/{health-checks/supervisorctl-check.sh => Dockerfile} (61%) mode change 100755 => 100644 delete mode 100755 docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh delete mode 100755 docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh rename docker/thirdparties/docker-compose/kerberos/{two-kerberos-hives/auth-to-local.xml => conf/core-site.xml.tpl} (58%) mode change 100755 => 100644 create mode 100644 docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl create mode 100644 docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl rename docker/thirdparties/docker-compose/kerberos/{health-checks/hive-health-check-2.sh => conf/kdc.conf.tpl} (74%) mode change 100755 => 100644 delete mode 100644 docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl delete mode 100644 docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl delete mode 100644 docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl rename docker/thirdparties/docker-compose/kerberos/conf/{kerberos2 => }/krb5.conf.tpl (81%) delete mode 100644 docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl delete mode 100755 docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh delete mode 100644 docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql delete mode 100644 docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql delete mode 100755 docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml diff --git a/docker/thirdparties/docker-compose/kerberos/.gitignore b/docker/thirdparties/docker-compose/kerberos/.gitignore new file mode 100644 index 00000000000000..c7e96926b15ca5 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/.gitignore @@ -0,0 +1,8 @@ +# Generated by run-thirdparties-docker.sh. +/conf/kerberos1/*.conf +/conf/kerberos1/*.xml +/conf/kerberos2/*.conf +/conf/kerberos2/*.xml +/data/ +/two-kerberos-hives/*.conf +/two-kerberos-hives/*.keytab diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh b/docker/thirdparties/docker-compose/kerberos/Dockerfile old mode 100755 new mode 100644 similarity index 61% rename from docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh rename to docker/thirdparties/docker-compose/kerberos/Dockerfile index 77df431d85ac3b..88723e1313b999 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/supervisorctl-check.sh +++ b/docker/thirdparties/docker-compose/kerberos/Dockerfile @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,25 +15,20 @@ # specific language governing permissions and limitations # under the License. -set -euo pipefail +FROM apache/hive:3.1.3 -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi +USER root -# Supervisord is not running -if ! test -f /tmp/supervisor.sock; then - exit 0 -fi +RUN apt-get -o Acquire::Retries=3 -o APT::Update::Error-Mode=any update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + krb5-admin-server \ + krb5-kdc \ + krb5-user \ + && rm -rf /var/lib/apt/lists/* -# Check if all Hadoop services are running -FAILED=$(supervisorctl status | grep -v RUNNING || true) +COPY entrypoint-hive-master.sh /opt/doris/entrypoint.sh +COPY health-checks/health.sh /opt/doris/health.sh -if [ "$FAILED" == "" ]; then - echo "All services are running" - exit 0 -else - echo "Some of the services are failing: ${FAILED}" - exit 1 -fi +RUN chmod 755 /opt/doris/entrypoint.sh /opt/doris/health.sh + +ENTRYPOINT ["/opt/doris/entrypoint.sh"] diff --git a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf index 83fe29c2cb2fa0..c7d41612a5359e 100644 --- a/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf +++ b/docker/thirdparties/docker-compose/kerberos/common/conf/doris-krb5.conf @@ -33,15 +33,9 @@ [realms] LABS.TERADATA.COM = { kdc = hadoop-master:5588 - admin_server = hadoop-master:5749 - } - OTHERLABS.TERADATA.COM = { - kdc = hadoop-master:5589 - admin_server = hadoop-master:5750 } OTHERREALM.COM = { kdc = hadoop-master-2:6688 - admin_server = hadoop-master-2:6749 } [domain_realm] diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh deleted file mode 100755 index ec2dc074e705cd..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/apply-config-overrides.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# test whether OVERRIDES_DIR is set -if [[ -n "${OVERRIDES_DIR+x}" ]]; then - echo "The OVERRIDES_DIR (${OVERRIDES_DIR}) support is disabled as it was deemed unused." >&2 - echo "It is being removed." >&2 - exit 16 -fi - -if test -e /overrides; then - find /overrides >&2 - echo "The /overrides handling is disabled as it was deemed unused." >&2 - echo "It is being removed." >&2 - exit 17 -fi diff --git a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh b/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh deleted file mode 100755 index 93c6e385effe1f..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/common/hadoop/hadoop-run.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi - -set -x - -HADOOP_INIT_D=${HADOOP_INIT_D:-/etc/hadoop-init.d/} - -echo "Applying hadoop init.d scripts from ${HADOOP_INIT_D}" -if test -d "${HADOOP_INIT_D}"; then - for init_script in "${HADOOP_INIT_D}"*; do - chmod a+x "${init_script}" - "${init_script}" - done -fi - -trap exit INT - -echo "Running services with supervisord" -rm -rf /etc/supervisord.d/socks-proxy.conf -rm -rf /etc/supervisord.d/sshd.conf - -supervisord -c /etc/supervisord.conf diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml b/docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl old mode 100755 new mode 100644 similarity index 58% rename from docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml rename to docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl index c0ce38e3cdc1bd..c5625a53ef639d --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/auth-to-local.xml +++ b/docker/thirdparties/docker-compose/kerberos/conf/core-site.xml.tpl @@ -1,4 +1,4 @@ - + + + fs.defaultFS + hdfs://${HOST}:${FS_PORT} + + + hadoop.security.authentication + kerberos + + + hadoop.security.authorization + true + hadoop.security.auth_to_local - RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// + RULE:[2:$1@$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:$1@$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:$1@$0](.*@OTHERREALM.COM)s/@.*// DEFAULT + + hadoop.proxyuser.hive.hosts + * + + + hadoop.proxyuser.hive.users + * + + + hadoop.security.token.service.use_ip + false + diff --git a/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl b/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl new file mode 100644 index 00000000000000..b56a18f5ca57d4 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/hdfs-site.xml.tpl @@ -0,0 +1,113 @@ + + + + + dfs.namenode.name.dir + file:///data/hdfs/name + + + dfs.datanode.data.dir + file:///data/hdfs/data + + + dfs.replication + 1 + + + dfs.permissions.enabled + false + + + dfs.namenode.datanode.registration.ip-hostname-check + false + + + dfs.block.access.token.enable + true + + + dfs.namenode.kerberos.principal + hdfs/${HOST}@${REALM} + + + dfs.namenode.rpc-bind-host + 0.0.0.0 + + + dfs.namenode.keytab.file + /data/keytabs/hdfs.keytab + + + dfs.datanode.kerberos.principal + hdfs/${HOST}@${REALM} + + + dfs.datanode.keytab.file + /data/keytabs/hdfs.keytab + + + dfs.namenode.kerberos.internal.spnego.principal + HTTP/${HOST}@${REALM} + + + dfs.web.authentication.kerberos.principal + HTTP/${HOST}@${REALM} + + + dfs.web.authentication.kerberos.keytab + /data/keytabs/spnego.keytab + + + dfs.data.transfer.protection + authentication + + + dfs.http.policy + HTTP_ONLY + + + ignore.secure.ports.for.testing + true + + + dfs.datanode.address + 0.0.0.0:${DFS_DN_PORT} + + + dfs.datanode.http.address + 0.0.0.0:${DFS_DN_HTTP_PORT} + + + dfs.datanode.ipc.address + 0.0.0.0:${DFS_DN_IPC_PORT} + + + dfs.datanode.hostname + ${HOST} + + + dfs.client.use.datanode.hostname + true + + + dfs.namenode.http-address + 0.0.0.0:${DFS_NN_HTTP_PORT} + + diff --git a/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl b/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl new file mode 100644 index 00000000000000..2b4b41e20a7341 --- /dev/null +++ b/docker/thirdparties/docker-compose/kerberos/conf/hive-site.xml.tpl @@ -0,0 +1,65 @@ + + + + + javax.jdo.option.ConnectionURL + jdbc:derby:;databaseName=/data/metastore/metastore_db;create=true + + + javax.jdo.option.ConnectionDriverName + org.apache.derby.jdbc.EmbeddedDriver + + + datanucleus.autoCreateSchema + false + + + hive.metastore.schema.verification + true + + + hive.metastore.uris + thrift://${HOST}:${HMS_PORT} + + + hive.metastore.warehouse.dir + hdfs://${HOST}:${FS_PORT}/user/hive/warehouse + + + hive.metastore.sasl.enabled + true + + + hive.metastore.kerberos.principal + hive/${HOST}@${REALM} + + + hive.metastore.kerberos.keytab.file + /data/keytabs/hive.keytab + + + hive.metastore.execute.setugi + true + + + metastore.storage.schema.reader.impl + org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader + + diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh b/docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl old mode 100755 new mode 100644 similarity index 74% rename from docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh rename to docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl index 7545969bc47d65..0b678db1538447 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check-2.sh +++ b/docker/thirdparties/docker-compose/kerberos/conf/kdc.conf.tpl @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,5 +15,15 @@ # specific language governing permissions and limitations # under the License. -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -beeline -u "jdbc:hive2://localhost:16000/default;principal=hive/hadoop-master-2@OTHERREALM.COM" -e "show databases;" \ No newline at end of file +[kdcdefaults] + kdc_ports = ${KDC_PORT} + kdc_tcp_ports = ${KDC_PORT} + +[realms] + ${REALM} = { + database_name = /data/kdc/principal + key_stash_file = /data/kdc/.k5.${REALM} + max_life = 24h + max_renewable_life = 7d + supported_enctypes = aes128-cts-hmac-sha1-96:normal + } diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl deleted file mode 100644 index e16c70e16db817..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/kdc.conf.tpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[kdcdefaults] - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - -[realms] - LABS.TERADATA.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT1} - kdc_tcp_listen = ${KDC_PORT1} - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - } - - OTHERLABS.TERADATA.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5-other.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5-other.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT2} - kdc_tcp_listen = ${KDC_PORT2} - kdc_ports = ${KDC_PORT2} - kdc_tcp_ports = ${KDC_PORT2} - kadmind_port = ${KADMIND_PORT2} - kpasswd_port = ${KPASSWD_PORT2} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl deleted file mode 100644 index 1edf2bb8fd05fe..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos1/krb5.conf.tpl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[logging] - default = FILE:/var/log/krb5libs.log - kdc = FILE:/var/log/krb5kdc.log - admin_server = FILE:/var/log/kadmind.log - -[libdefaults] - default_realm = LABS.TERADATA.COM - dns_lookup_realm = false - dns_lookup_kdc = false - forwardable = true - allow_weak_crypto = true - -[realms] - LABS.TERADATA.COM = { - kdc = ${HOST}:${KDC_PORT1} - admin_server = ${HOST}:${KADMIND_PORT1} - } - OTHERLABS.TERADATA.COM = { - kdc = ${HOST}:${KDC_PORT2} - admin_server = ${HOST}:${KADMIND_PORT2} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl deleted file mode 100644 index 61b4994ad5c1ac..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/kdc.conf.tpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[kdcdefaults] - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - - -[realms] - OTHERREALM.COM = { - acl_file = /var/kerberos/krb5kdc/kadm5.acl - dict_file = /usr/share/dict/words - admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab - supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal - kdc_listen = ${KDC_PORT1} - kdc_tcp_listen = ${KDC_PORT1} - kdc_ports = ${KDC_PORT1} - kdc_tcp_ports = ${KDC_PORT1} - kadmind_port = ${KADMIND_PORT1} - kpasswd_port = ${KPASSWD_PORT1} - } \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl similarity index 81% rename from docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl rename to docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl index c817dbdd79753d..84d8707184e1bc 100644 --- a/docker/thirdparties/docker-compose/kerberos/conf/kerberos2/krb5.conf.tpl +++ b/docker/thirdparties/docker-compose/kerberos/conf/krb5.conf.tpl @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -19,17 +18,18 @@ [logging] default = FILE:/var/log/krb5libs.log kdc = FILE:/var/log/krb5kdc.log - admin_server = FILE:/var/log/kadmind.log [libdefaults] - default_realm = OTHERREALM.COM + default_realm = ${REALM} dns_lookup_realm = false dns_lookup_kdc = false forwardable = true - allow_weak_crypto = true + udp_preference_limit = 1 [realms] - OTHERREALM.COM = { - kdc = ${HOST}:${KDC_PORT1} - admin_server = ${HOST}:${KADMIND_PORT1} - } \ No newline at end of file + ${REALM} = { + kdc = 127.0.0.1:${KDC_PORT} + } + +[domain_realm] + ${HOST} = ${REALM} diff --git a/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl b/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl deleted file mode 100644 index e91c65c100463f..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/conf/my.cnf.tpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[mysqld] -port=${MYSQL_PORT} -datadir=/var/lib/mysql -socket=/var/lib/mysql/mysql.sock -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links=0 -# Settings user and group are ignored when systemd is used. -# If you need to run mysqld under a different user or group, -# customize your systemd unit file for mariadb according to the -# instructions in http://fedoraproject.org/wiki/Systemd - -[mysqld_safe] -log-error=/var/log/mariadb/mariadb.log -pid-file=/var/run/mariadb/mariadb.pid - -# -# include all files from the config directory -# -!includedir /etc/my.cnf.d - diff --git a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh index b6eeb381fabfd1..5dd4adf46d458e 100755 --- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh +++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh @@ -15,87 +15,125 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + set -euo pipefail -source /usr/local/common/hive-configure.sh -source /usr/local/common/event-hook.sh - -echo "Configuring hive" -configure /etc/hive/conf/hive-site.xml hive HIVE_SITE_CONF -configure /etc/hive/conf/hiveserver2-site.xml hive HIVE_SITE_CONF -configure /etc/hadoop/conf/core-site.xml core CORE_CONF -configure /etc/hadoop/conf/hdfs-site.xml hdfs HDFS_CONF -configure /etc/hadoop/conf/yarn-site.xml yarn YARN_CONF -configure /etc/hadoop/conf/mapred-site.xml mapred MAPRED_CONF -configure /etc/hive/conf/beeline-site.xml beeline BEELINE_SITE_CONF - -echo "Copying kerberos keytabs to keytabs/" -mkdir -p /etc/hadoop-init.d/ - -if [ "$1" == "1" ]; then - cp /etc/trino/conf/* /keytabs/ -elif [ "$1" == "2" ]; then - cp /etc/trino/conf/hive-presto-master.keytab /keytabs/other-hive-presto-master.keytab - cp /etc/trino/conf/presto-server.keytab /keytabs/other-presto-server.keytab -else - echo "Invalid index parameter. Exiting." - exit 1 -fi -cd /usr/hdp/3.1.0.0-78/hive/auxlib -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jdom-1.1.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-core-3.4.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ecs-4.2.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-ram-3.0.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aliyun-java-sdk-sts-3.0.0.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-core-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-core-linux-el7-aarch64-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/jindo-sdk-6.3.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/aws-java-sdk-bundle-1.11.375.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-huaweicloud-3.1.1-hw-54.5.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-cos-3.1.0-8.3.22.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/cos_api-bundle-5.6.244.4.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aws-3.2.1.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/paimon-hive-connector-3.1-1.3-SNAPSHOT.jar -curl -O https://s3BucketName.s3Endpoint/regression/docker/hive3/gcs-connector-hadoop3-2.2.24-shaded.jar - - -/usr/local/hadoop-run.sh & - -# check healthy hear -echo "Waiting for hadoop to be healthy" - -for i in {1..60}; do - if /usr/local/health.sh; then - echo "Hadoop is healthy" + +: "${HOST:?}" +: "${REALM:?}" +: "${KDC_PORT:?}" +: "${FS_PORT:?}" +: "${HMS_PORT:?}" +: "${HIVE_CLIENT_KEYTAB:?}" +: "${PRESTO_CLIENT_KEYTAB:?}" + +export HADOOP_CONF_DIR=/opt/doris/conf +export HIVE_CONF_DIR=/opt/doris/conf +export KRB5_KDC_PROFILE=/opt/doris/conf/kdc.conf +export PATH="/usr/local/sbin:/usr/sbin:${PATH}" +export HADOOP_CLIENT_OPTS="-Xms192m -Xmx320m" + +readonly HDFS_PRINCIPAL="hdfs/${HOST}@${REALM}" +readonly HTTP_PRINCIPAL="HTTP/${HOST}@${REALM}" +readonly HIVE_PRINCIPAL="hive/${HOST}@${REALM}" +readonly HIVE_CLIENT_PRINCIPAL="hive/presto-master.docker.cluster@${REALM}" +readonly PRESTO_CLIENT_PRINCIPAL="presto-server/presto-master.docker.cluster@${REALM}" + +declare -a SERVICE_PIDS=() + +stop_services() { + kill "${SERVICE_PIDS[@]}" 2>/dev/null || true + wait "${SERVICE_PIDS[@]}" 2>/dev/null || true +} + +trap stop_services EXIT INT TERM + +start_service() { + "$@" & + SERVICE_PIDS+=("$!") +} + +wait_for_port() { + local host=$1 + local port=$2 + local service=$3 + + for _ in {1..120}; do + if (exec 3<>"/dev/tcp/${host}/${port}") 2>/dev/null; then + exec 3>&- + exec 3<&- + return + fi + sleep 1 + done + + echo "Timed out waiting for ${service} on ${host}:${port}" >&2 + return 1 +} + +create_keytab() { + local principal=$1 + local keytab=$2 + + kadmin.local -r "${REALM}" -q "addprinc -randkey ${principal}" + kadmin.local -r "${REALM}" -q "ktadd -k ${keytab} ${principal}" +} + +report_stage() { + echo "DORIS_KERBEROS_STAGE=$1" +} + +report_stage "prepare-data" +mkdir -p /data/hdfs/data /data/hdfs/name /data/kdc /data/keytabs /data/metastore /keytabs +rm -rf /tmp/hadoop-server-conf +cp -R "${HADOOP_CONF_DIR}" /tmp/hadoop-server-conf +sed -i "s#hdfs://${HOST}:${FS_PORT}#hdfs://127.0.0.1:${FS_PORT}#" \ + /tmp/hadoop-server-conf/core-site.xml + +report_stage "initialize-kdc" +kdb5_util create -s -r "${REALM}" -P doris-kerberos-test +create_keytab "${HDFS_PRINCIPAL}" /data/keytabs/hdfs.keytab +create_keytab "${HTTP_PRINCIPAL}" /data/keytabs/spnego.keytab +create_keytab "${HIVE_PRINCIPAL}" /data/keytabs/hive.keytab +create_keytab "${HIVE_CLIENT_PRINCIPAL}" "/keytabs/${HIVE_CLIENT_KEYTAB}" +create_keytab "${PRESTO_CLIENT_PRINCIPAL}" "/keytabs/${PRESTO_CLIENT_KEYTAB}" +chmod 644 /keytabs/*.keytab + +report_stage "start-kdc" +start_service krb5kdc -n -r "${REALM}" +wait_for_port 127.0.0.1 "${KDC_PORT}" "Kerberos KDC" + +report_stage "start-namenode" +export HDFS_NAMENODE_OPTS="-Xms128m -Xmx256m" +export HDFS_DATANODE_OPTS="-Xms96m -Xmx192m" +hdfs namenode -format -force -nonInteractive +start_service hdfs namenode +wait_for_port "${HOST}" "${FS_PORT}" "HDFS NameNode" + +report_stage "start-datanode" +start_service env HADOOP_CONF_DIR=/tmp/hadoop-server-conf hdfs datanode +wait_for_port "${HOST}" "${DFS_DN_PORT}" "HDFS DataNode" + +report_stage "wait-for-datanode-registration" +export KRB5CCNAME=FILE:/tmp/hdfs-admin.ccache +kinit -kt /data/keytabs/hdfs.keytab "${HDFS_PRINCIPAL}" +for _ in {1..120}; do + if hdfs dfsadmin -Dfs.defaultFS="hdfs://127.0.0.1:${FS_PORT}" -report 2>/dev/null \ + | grep -q '^Live datanodes (1):'; then break fi - echo "Hadoop is not healthy yet. Retrying in 60 seconds..." - sleep 5 + sleep 1 done +hdfs dfsadmin -Dfs.defaultFS="hdfs://127.0.0.1:${FS_PORT}" -report \ + | grep -q '^Live datanodes (1):' +kdestroy + +report_stage "initialize-hive-metastore" +schematool -dbType derby -initSchema +start_service hive --service metastore -p "${HMS_PORT}" +wait_for_port "${HOST}" "${HMS_PORT}" "Hive Metastore" + +touch /tmp/SUCCESS +echo "Minimal Kerberos HDFS and Hive Metastore environment is ready" +echo "DORIS_KERBEROS_READY" -if [ $i -eq 60 ]; then - echo "Hadoop did not become healthy after 60 attempts. Exiting." - exit 1 -fi - -echo "Init kerberos test data" - -if [ "$1" == "1" ]; then - kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -elif [ "$1" == "2" ]; then - kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master-2@OTHERREALM.COM -else - echo "Invalid index parameter. Exiting." - exit 1 -fi -hive -f /usr/local/sql/create_kerberos_hive_table.sql -if [[ ${enablePaimonHms} == "true" ]]; then - echo "Creating Paimon HMS catalog and table" - hadoop fs -put /tmp/paimon_data/* /user/hive/warehouse/ - # Render the OSS location directly to avoid passing oss_bucket as a Hive session conf. - paimon_hql=$(mktemp /tmp/create_paimon_hive_table.XXXXXX.hql) - sed "s|__OSS_BUCKET__|${OSSBucket}|g" /usr/local/sql/create_paimon_hive_table.hql > "${paimon_hql}" - hive -f "${paimon_hql}" - rm -f "${paimon_hql}" -fi - -exec_success_hook +wait -n "${SERVICE_PIDS[@]}" diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl index 67cecd8ba61421..792d0c0491da97 100644 --- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl +++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl @@ -1,96 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:mysql://127.0.0.1:${MYSQL_PORT}/metastore -HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=com.mysql.jdbc.Driver -HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=root -HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=root -HIVE_SITE_CONF_datanucleus_autoCreateSchema=false -HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT} -HIVE_SITE_CONF_hive_metastore_uris=thrift://${IP_HOST}:${HMS_PORT} -HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 -HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT} -HIVE_SITE_CONF_hive_server2_webui_port=0 -HIVE_SITE_CONF_hive_compactor_initiator_on=true -HIVE_SITE_CONF_hive_compactor_worker_threads=2 -HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader -BEELINE_SITE_CONF_beeline_hs2_jdbc_url_tcpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive -BEELINE_SITE_CONF_beeline_hs2_jdbc_url_httpUrl=jdbc:hive2://${HOST}:${HS_PORT}/default;user=hdfs;password=hive - - -CORE_CONF_fs_defaultFS=hdfs://${HOST}:${FS_PORT} -CORE_CONF_hadoop_http_staticuser_user=root -CORE_CONF_hadoop_proxyuser_hue_hosts=* -CORE_CONF_hadoop_proxyuser_hue_groups=* - -HDFS_CONF_dfs_webhdfs_enabled=true -HDFS_CONF_dfs_permissions_enabled=false -HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false -HDFS_CONF_dfs_datanode_address=${HOST}:${DFS_DN_PORT} -HDFS_CONF_dfs_datanode_http_address=${HOST}:${DFS_DN_HTTP_PORT} -HDFS_CONF_dfs_datanode_ipc_address=${HOST}:${DFS_DN_IPC_PORT} -HDFS_CONF_dfs_namenode_http___address=${HOST}:${DFS_NN_HTTP_PORT} -YARN_CONF_yarn_log___aggregation___enable=true -YARN_CONF_yarn_resourcemanager_recovery_enabled=true -YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore -YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate -YARN_CONF_yarn_nodemanager_remote___app___log___dir=/var/log/hadoop-yarn/apps -YARN_CONF_yarn_log_server_url=http://${HOST}:${YARM_LOG_SERVER_PORT}/jobhistory/logs -YARN_CONF_yarn_timeline___service_enabled=false -YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true -YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true -YARN_CONF_yarn_resourcemanager_hostname=${HOST} -MAPRED_CONF_mapreduce_shuffle_port=${MAPREDUCE_SHUFFLE_PORT} -YARN_CONF_yarn_timeline___service_hostname=${HOST} -YARN_CONF_yarn_resourcemanager_address=${HOST}:${YARN_RM_PORT} -YARN_CONF_yarn_resourcemanager_scheduler_address=${HOST}:${YARN_RM_SCHEDULER_PORT} -YARN_CONF_yarn_resourcemanager_resource___tracker_address=${HOST}:${YARN_RM_TRACKER_PORT} -YARN_CONF_yarn_resourcemanager_admin_address=${HOST}:${YARN_RM_ADMIN_PORT} -YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT} -YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT} -YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT} +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem -HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem -HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk} -HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk} -HIVE_SITE_CONF_fs_s3a_endpoint=${AWSEndpoint} -HIVE_SITE_CONF_fs_AbstractFileSystem_obs_impl=org.apache.hadoop.fs.obs.OBS -HIVE_SITE_CONF_fs_obs_access_key=${OBSAk} -HIVE_SITE_CONF_fs_obs_secret_key=${OBSSk} -HIVE_SITE_CONF_fs_obs_endpoint=${OBSEndpoint} -HIVE_SITE_CONF_fs_cosn_credentials_provider=org.apache.hadoop.fs.auth.SimpleCredentialProvider -HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk} -HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk} -HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion} -HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem -HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem -HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN -HIVE_SITE_CONF_fs_oss_impl=com.aliyun.jindodata.oss.JindoOssFileSystem -HIVE_SITE_CONF_fs_AbstractFileSystem_oss_impl=com.aliyun.jindodata.oss.JindoOSS -HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk} -HIVE_SITE_CONF_fs_oss_accessKeySecret=${OSSSk} -HIVE_SITE_CONF_fs_oss_endpoint=${OSSEndpoint} -OSSBucket=${OSSBucket} -HIVE_SITE_CONF_fs_AbstractFileSystem_gs_impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS -HIVE_SITE_CONF_fs_gs_project_id=${GCSProjectId} -HIVE_SITE_CONF_google_cloud_auth_service_account_enable=true -HIVE_SITE_CONF_fs_gs_auth_service_account_email=${GCSAccountEmail} -HIVE_SITE_CONF_fs_gs_auth_service_account_private_key_id=${GCSAccountPrivateKeyId} -HIVE_SITE_CONF_fs_gs_auth_service_account_private_key=${GCSAccountPrivateKey} -HIVE_SITE_CONF_fs_gs_proxy_address=${GCSProxyAddress} -enablePaimonHms=${enablePaimonHms} +HOST=${HOST} +REALM=${REALM} +KDC_PORT=${KDC_PORT} +FS_PORT=${FS_PORT} +HMS_PORT=${HMS_PORT} +DFS_DN_PORT=${DFS_DN_PORT} +DFS_DN_HTTP_PORT=${DFS_DN_HTTP_PORT} +DFS_DN_IPC_PORT=${DFS_DN_IPC_PORT} +DFS_NN_HTTP_PORT=${DFS_NN_HTTP_PORT} +HIVE_CLIENT_KEYTAB=${HIVE_CLIENT_KEYTAB} +PRESTO_CLIENT_KEYTAB=${PRESTO_CLIENT_KEYTAB} +HADOOP_CONF_DIR=/opt/doris/conf +HIVE_CONF_DIR=/opt/doris/conf +KRB5_CONFIG=/etc/krb5.conf diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh index 515f37e36ac9e3..07cd7edb379f3b 100755 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh +++ b/docker/thirdparties/docker-compose/kerberos/health-checks/health.sh @@ -18,17 +18,11 @@ set -euo pipefail -if test $# -gt 0; then - echo "$0 does not accept arguments" >&2 - exit 32 -fi - -set -x - -HEALTH_D=${HEALTH_D:-/etc/health.d/} - -if test -d "${HEALTH_D}"; then - for health_script in "${HEALTH_D}"/*; do - "${health_script}" &>> /var/log/container-health.log || exit 1 - done -fi +test -f /tmp/SUCCESS +kinit -kt /data/keytabs/hive.keytab "hive/${HOST}@${REALM}" +exec 3<>"/dev/tcp/${HOST}/${FS_PORT}" +exec 3>&- +exec 3<&- +exec 3<>"/dev/tcp/${HOST}/${HMS_PORT}" +exec 3>&- +exec 3<&- diff --git a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh b/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh deleted file mode 100755 index ab464b5233b31a..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/health-checks/hive-health-check.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -kinit -kt /etc/hive/conf/hive.keytab hive/hadoop-master@LABS.TERADATA.COM -beeline -u "jdbc:hive2://localhost:15000/default;principal=hive/hadoop-master@LABS.TERADATA.COM" -e "show databases;" \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl index c253c06f3e488b..d21496516afac4 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl +++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl @@ -14,59 +14,41 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -version: "3" + +x-kerberos-image: &kerberos-image + image: doris-kerberos-minimal:${CONTAINER_UID} + build: + context: . + dockerfile: Dockerfile + network: host + +x-kerberos-service: &kerberos-service + <<: *kerberos-image + network_mode: "host" + mem_limit: 1536m + stop_grace_period: 30s + services: hive-krb1: - image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized_96 + <<: *kerberos-service container_name: doris-${CONTAINER_UID}-kerberos1 + hostname: hadoop-master volumes: - - ../common:/usr/local/common + - ./conf/kerberos1:/opt/doris/conf:ro + - ./conf/kerberos1/krb5.conf:/etc/krb5.conf:ro + - ./data/kerberos1:/data - ./two-kerberos-hives:/keytabs - - ./sql:/usr/local/sql - - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - - ./health-checks/hive-health-check.sh:/etc/health.d/hive-health-check.sh - - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh - - ./conf/kerberos1/my.cnf:/etc/my.cnf - - ./conf/kerberos1/kdc.conf:/var/kerberos/krb5kdc/kdc.conf - - ./conf/kerberos1/krb5.conf:/etc/krb5.conf - - ./paimon_data:/tmp/paimon_data - hostname: hadoop-master - entrypoint: /usr/local/entrypoint-hive-master.sh 1 - healthcheck: - test: ["CMD", "ls", "/tmp/SUCCESS"] - interval: 5s - timeout: 10s - retries: 120 - network_mode: "host" env_file: - ./hadoop-hive-1.env + hive-krb2: - image: doristhirdpartydocker/trinodb:hdp3.1-hive-kerberized-2_96 + <<: *kerberos-service container_name: doris-${CONTAINER_UID}-kerberos2 hostname: hadoop-master-2 volumes: - - ../common:/usr/local/common + - ./conf/kerberos2:/opt/doris/conf:ro + - ./conf/kerberos2/krb5.conf:/etc/krb5.conf:ro + - ./data/kerberos2:/data - ./two-kerberos-hives:/keytabs - - ./sql:/usr/local/sql - - ./common/hadoop/apply-config-overrides.sh:/etc/hadoop-init.d/00-apply-config-overrides.sh - - ./common/hadoop/hadoop-run.sh:/usr/local/hadoop-run.sh - - ./health-checks/health.sh:/usr/local/health.sh - - ./health-checks/supervisorctl-check.sh:/etc/health.d/supervisorctl-check.sh - - ./health-checks/hive-health-check-2.sh:/etc/health.d/hive-health-check-2.sh - - ./entrypoint-hive-master.sh:/usr/local/entrypoint-hive-master.sh - - ./conf/kerberos2/my.cnf:/etc/my.cnf - - ./conf/kerberos2/kdc.conf:/var/kerberos/krb5kdc/kdc.conf - - ./conf/kerberos2/krb5.conf:/etc/krb5.conf - - ./paimon_data:/tmp/paimon_data - entrypoint: /usr/local/entrypoint-hive-master.sh 2 - healthcheck: - test: ["CMD", "ls", "/tmp/SUCCESS"] - interval: 5s - timeout: 10s - retries: 120 - network_mode: "host" env_file: - - ./hadoop-hive-2.env \ No newline at end of file + - ./hadoop-hive-2.env diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env index 4d71155cbb2b88..0e019dfcdd9f18 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env +++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,54 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Change this to a specific string. -# Do not use "_" or other sepcial characters, only number and alphabeta. -# NOTICE: change this uid will modify hive-*.yaml - export HOST="hadoop-master" +export REALM="LABS.TERADATA.COM" +export KDC_PORT=5588 export FS_PORT=8520 export HMS_PORT=9583 -export HS_PORT=15000 -export MYSQL_PORT=3356 export DFS_DN_PORT=9566 export DFS_DN_HTTP_PORT=9564 export DFS_DN_IPC_PORT=9567 export DFS_NN_HTTP_PORT=9570 -export YARM_LOG_SERVER_PORT=8588 -export YARN_RM_PORT=8532 -export YARN_RM_SCHEDULER_PORT=8530 -export YARN_RM_TRACKER_PORT=8531 -export YARN_RM_ADMIN_PORT=8533 -export YARN_RM_WEBAPP_PORT=8589 -export YARN_NM_LOCAL_PORT=8540 -export YARN_NM_WEBAPP_PORT=8542 -export MAPREDUCE_SHUFFLE_PORT=13562 -export KADMIND_PORT=5464 -export KDC_PORT1=5588 -export KDC_PORT2=5589 -export KADMIND_PORT1=5749 -export KADMIND_PORT2=5750 -export KPASSWD_PORT1=5464 -export KPASSWD_PORT2=5465 - -# prepare for paimon hms test,control load paimon hms data or not -export enablePaimonHms="false" -# hms on s3/oss/obs/cos -export AWSAk="*****************" -export AWSSk="*****************" -export AWSEndpoint="s3.ap-northeast-1.amazonaws.com" -export OSSAk="*****************" -export OSSSk="*****************" -export OSSEndpoint="oss-cn-beijing.aliyuncs.com" -export OSSBucket="${OSSBucket:-doris-regression-bj}" -export OBSAk="*****************" -export OBSSk="*****************" -export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com" -export COSAk="*****************" -export COSSk="*****************" -export COSRegion="ap-beijing" -export GCSProjectId="" -export GCSAccountEmail="" -export GCSAccountPrivateKeyId="" -export GCSAccountPrivateKey="" -export GCSProxyAddress="" +export HIVE_CLIENT_KEYTAB="hive-presto-master.keytab" +export PRESTO_CLIENT_KEYTAB="presto-server.keytab" diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env index e26a3297a756c5..a41c7f7ccc137b 100644 --- a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env +++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,50 +16,14 @@ # specific language governing permissions and limitations # under the License. -# Change this to a specific string. -# Do not use "_" or other sepcial characters, only number and alphabeta. -# NOTICE: change this uid will modify hive-*.yaml - export HOST="hadoop-master-2" +export REALM="OTHERREALM.COM" +export KDC_PORT=6688 export FS_PORT=8620 export HMS_PORT=9683 -export HS_PORT=16000 -export MYSQL_PORT=3366 export DFS_DN_PORT=9666 export DFS_DN_HTTP_PORT=9664 export DFS_DN_IPC_PORT=9667 export DFS_NN_HTTP_PORT=9670 -export YARM_LOG_SERVER_PORT=8688 -export YARN_RM_PORT=8632 -export YARN_RM_SCHEDULER_PORT=8630 -export YARN_RM_TRACKER_PORT=8631 -export YARN_RM_ADMIN_PORT=8633 -export YARN_RM_WEBAPP_PORT=8689 -export YARN_NM_LOCAL_PORT=8640 -export YARN_NM_WEBAPP_PORT=8642 -export MAPREDUCE_SHUFFLE_PORT=13662 -export KDC_PORT1=6688 -export KADMIND_PORT1=6749 -export KPASSWD_PORT1=6464 - -# prepare for paimon hms test,control load paimon hms data or not -export enablePaimonHms="false" -# hms on s3/oss/obs/cos -export AWSAk="*****************" -export AWSSk="*****************" -export AWSEndpoint="s3.ap-northeast-1.amazonaws.com" -export OSSAk="*****************" -export OSSSk="*****************" -export OSSEndpoint="oss-cn-beijing.aliyuncs.com" -export OSSBucket="${OSSBucket:-doris-regression-bj}" -export OBSAk="*****************" -export OBSSk="*****************" -export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com" -export COSAk="*****************" -export COSSk="*****************" -export COSRegion="ap-beijing" -export GCSProjectId="" -export GCSAccountEmail="" -export GCSAccountPrivateKeyId="" -export GCSAccountPrivateKey="" -export GCSProxyAddress="" +export HIVE_CLIENT_KEYTAB="other-hive-presto-master.keytab" +export PRESTO_CLIENT_KEYTAB="other-presto-server.keytab" diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql b/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql deleted file mode 100644 index ecf58e88158b12..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/sql/create_kerberos_hive_table.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `test_krb_hive_db`; -CREATE TABLE IF NOT EXISTS `test_krb_hive_db`.`test_krb_hive_tbl`( - `id_key` int, - `string_key` string, - `rate_val` double, - `comment` string) -ROW FORMAT SERDE - 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' -STORED AS INPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' -OUTPUTFORMAT - 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'; - -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(1, 'a', 3.16, 'cc0'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(2, 'b', 41.2, 'cc1'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(3, 'c', 6.2, 'cc2'); -INSERT INTO test_krb_hive_db.test_krb_hive_tbl values(4, 'd', 1.4, 'cc3'); diff --git a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql b/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql deleted file mode 100644 index 367296490e63da..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE DATABASE IF NOT EXISTS hdfs_db; - -USE hdfs_db; - -CREATE EXTERNAL TABLE external_test_table - STORED BY 'org.apache.paimon.hive.PaimonStorageHandler' -LOCATION 'hdfs:///user/hive/warehouse/hdfs_db.db/external_test_table'; - -CREATE DATABASE IF NOT EXISTS ali_db; - -USE ali_db; - -CREATE EXTERNAL TABLE external_test_table - STORED BY 'org.apache.paimon.hive.PaimonStorageHandler' -LOCATION 'oss://__OSS_BUCKET__/regression/paimon_warehouse/ali_db.db/hive_test_table'; diff --git a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml b/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml deleted file mode 100755 index 4541c1328aea16..00000000000000 --- a/docker/thirdparties/docker-compose/kerberos/two-kerberos-hives/hive2-default-fs-site.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - fs.default.name - hdfs://hadoop-master-2:9000 - - diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 46e4bd83768831..2e7cc9d56f23ef 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -674,34 +674,150 @@ start_lakesoul() { fi } +dump_kerberos_container_state() { + local container="$1" + + echo "===== ${container} state =====" >&2 + sudo docker inspect --format \ + 'status={{.State.Status}} running={{.State.Running}} exitCode={{.State.ExitCode}} oomKilled={{.State.OOMKilled}} error={{.State.Error}}' \ + "${container}" >&2 || true + echo "===== ${container} logs (tail -200) =====" >&2 + sudo docker logs --tail 200 "${container}" >&2 || true +} + +cleanup_kerberos_ready_wait() { + local wait_pid="$1" + + [[ -n "${wait_pid}" ]] || return 0 + sudo kill -TERM -- "-${wait_pid}" >/dev/null 2>&1 || true + wait "${wait_pid}" >/dev/null 2>&1 || true +} + +wait_for_kerberos_ready() { + local container="$1" + local wait_pid="" + local wait_status=0 + + echo "Waiting for ${container} readiness event" + setsid sudo timeout --signal=TERM --kill-after=5s 20m bash -c ' + log_dir=$(mktemp -d) + mkfifo "${log_dir}/container.log" + docker logs --follow "$1" >"${log_dir}/container.log" 2>&1 & + log_pid=$! + cleanup() { + kill -KILL "${log_pid}" 2>/dev/null || true + wait "${log_pid}" 2>/dev/null || true + rm -rf "${log_dir}" + } + trap cleanup EXIT + while IFS= read -r line; do + if [[ "${line}" == "DORIS_KERBEROS_READY" ]]; then + echo "${line}" + exit 0 + fi + done <"${log_dir}/container.log" + exit 1 + ' _ "${container}" & + wait_pid=$! + trap 'cleanup_kerberos_ready_wait "${wait_pid}"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + if ! wait "${wait_pid}"; then + wait_status=1 + fi + wait_pid="" + trap - EXIT TERM INT + if [[ "${wait_status}" -ne 0 ]]; then + echo "ERROR: timed out or container exited before ${container} became ready" >&2 + dump_kerberos_container_state "${container}" + return 1 + fi +} + +cleanup_kerberos_readiness_jobs() { + local pid + + for pid in "$@"; do + kill "${pid}" >/dev/null 2>&1 || true + done + for pid in "$@"; do + wait "${pid}" >/dev/null 2>&1 || true + done +} + +validate_kerberos_container() { + local container="$1" + + echo "Running one-shot readiness validation for ${container}" + if ! sudo docker exec "${container}" /opt/doris/health.sh; then + echo "ERROR: one-shot readiness validation failed for ${container}" >&2 + dump_kerberos_container_state "${container}" + return 1 + fi +} + start_kerberos() { echo "RUN_KERBEROS" + local KERBEROS_DIR="${ROOT}/docker-compose/kerberos" + local -a containers=( + "doris-${CONTAINER_UID}-kerberos1" + "doris-${CONTAINER_UID}-kerberos2" + ) + local -a readiness_pids=() + local readiness_status=0 + local container + local pid + export CONTAINER_UID=${CONTAINER_UID} - envsubst <"${ROOT}"/docker-compose/kerberos/kerberos.yaml.tpl >"${ROOT}"/docker-compose/kerberos/kerberos.yaml - sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh - sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/kerberos/entrypoint-hive-master.sh + envsubst <"${KERBEROS_DIR}/kerberos.yaml.tpl" >"${KERBEROS_DIR}/kerberos.yaml" + mkdir -p "${KERBEROS_DIR}/conf/kerberos1" "${KERBEROS_DIR}/conf/kerberos2" \ + "${KERBEROS_DIR}/two-kerberos-hives" for i in {1..2}; do - . "${ROOT}"/docker-compose/kerberos/kerberos${i}_settings.env - envsubst <"${ROOT}"/docker-compose/kerberos/hadoop-hive.env.tpl >"${ROOT}"/docker-compose/kerberos/hadoop-hive-${i}.env - envsubst <"${ROOT}"/docker-compose/kerberos/conf/my.cnf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/my.cnf - envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/kdc.conf - envsubst <"${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf.tpl > "${ROOT}"/docker-compose/kerberos/conf/kerberos${i}/krb5.conf + . "${KERBEROS_DIR}/kerberos${i}_settings.env" + envsubst <"${KERBEROS_DIR}/hadoop-hive.env.tpl" >"${KERBEROS_DIR}/hadoop-hive-${i}.env" + for config in kdc.conf krb5.conf core-site.xml hdfs-site.xml hive-site.xml; do + envsubst <"${KERBEROS_DIR}/conf/${config}.tpl" >"${KERBEROS_DIR}/conf/kerberos${i}/${config}" + done done sudo chmod a+w /etc/hosts - sudo sed -i "1i${IP_HOST} hadoop-master" /etc/hosts - sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts - sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml down - sudo rm -rf "${ROOT}"/docker-compose/kerberos/data + if ! awk -v ip="${IP_HOST}" '$1 == ip && $2 == "hadoop-master" { found = 1 } END { exit !found }' /etc/hosts; then + sudo sed -i "1i${IP_HOST} hadoop-master" /etc/hosts + fi + if ! awk -v ip="${IP_HOST}" '$1 == ip && $2 == "hadoop-master-2" { found = 1 } END { exit !found }' /etc/hosts; then + sudo sed -i "1i${IP_HOST} hadoop-master-2" /etc/hosts + fi + register_stack_metadata "kerberos" "${KERBEROS_DIR}/kerberos.yaml" "" + compose_cmd "${KERBEROS_DIR}/kerberos.yaml" "" down --remove-orphans + sudo rm -rf "${KERBEROS_DIR}/data" if [[ "${STOP}" -ne 1 ]]; then echo "PREPARE KERBEROS DATA" - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.keytab - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.jks - rm -rf "${ROOT}"/docker-compose/kerberos/two-kerberos-hives/*.conf - sudo docker compose -f "${ROOT}"/docker-compose/kerberos/kerberos.yaml up --remove-orphans --wait -d - sudo rm -df /keytabs - sudo ln -s "${ROOT}"/docker-compose/kerberos/two-kerberos-hives /keytabs - sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /keytabs/krb5.conf - sudo cp "${ROOT}"/docker-compose/kerberos/common/conf/doris-krb5.conf /etc/krb5.conf + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.keytab + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.jks + rm -rf "${KERBEROS_DIR}"/two-kerberos-hives/*.conf + compose_cmd "${KERBEROS_DIR}/kerberos.yaml" "" up --build --remove-orphans -d + trap 'cleanup_kerberos_readiness_jobs "${readiness_pids[@]}"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + for container in "${containers[@]}"; do + wait_for_kerberos_ready "${container}" & + readiness_pids+=("$!") + done + for pid in "${readiness_pids[@]}"; do + if ! wait "${pid}"; then + readiness_status=1 + fi + done + readiness_pids=() + trap - EXIT TERM INT + if [[ "${readiness_status}" -ne 0 ]]; then + return 1 + fi + for container in "${containers[@]}"; do + validate_kerberos_container "${container}" + done + sudo ln -sfn "${KERBEROS_DIR}/two-kerberos-hives" /keytabs + sudo cp "${KERBEROS_DIR}/common/conf/doris-krb5.conf" /keytabs/krb5.conf + sudo cp "${KERBEROS_DIR}/common/conf/doris-krb5.conf" /etc/krb5.conf sleep 2 fi } diff --git a/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy b/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy index 62c515d84ab086..262b7a28ee9bf0 100644 --- a/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy +++ b/regression-test/suites/external_table_p0/info_schema_db/test_information_schema_timezone.groovy @@ -138,6 +138,9 @@ suite("test_information_schema_timezone", "p0,external,hive,kerberos,external_do "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", diff --git a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy index 9228b7788b66d4..484b698d27cab9 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_iceberg_hadoop_catalog_kerberos.groovy @@ -44,7 +44,11 @@ suite("test_iceberg_hadoop_catalog_kerberos", "p0,external,kerberos,external_doc "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.kerberos.keytab.login.autorenewal.enabled" = "false", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication" ); """ diff --git a/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy index 2c4da8b2d20162..4c01c156464024 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy @@ -47,11 +47,16 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.kerberos.keytab.login.autorenewal.enabled" = "false", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication" ); """ sql """ switch ${hms_catalog_name} """ + sql """ create database if not exists test_krb_hive_db """ sql """ use test_krb_hive_db """ sql """ drop table if exists ${test_tbl_name}""" sql """ @@ -76,6 +81,10 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa ) with HDFS ( "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -94,6 +103,10 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa FORMAT AS CSV PROPERTIES( "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// @@ -115,6 +128,10 @@ suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,externa "hadoop.username" = "doris", "format" = "csv", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index 820a4321f8ba37..9b9b5af37ae10d 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -37,6 +37,10 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", @@ -48,52 +52,69 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa ); """ sql """ switch hms_kerberos """ + sql """ CREATE DATABASE IF NOT EXISTS test_single_krb_hive_db """ + sql """ USE test_single_krb_hive_db """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl """ + sql """ + CREATE TABLE test_krb_hive_tbl ( + id_key INT, + string_key STRING, + rate_val DOUBLE, + comment STRING + ) ENGINE = hive + PROPERTIES ("file_format" = "parquet") + """ + sql """ + INSERT INTO test_krb_hive_tbl VALUES + (1, 'a', 3.16, 'cc0'), + (2, 'b', 41.2, 'cc1'), + (3, 'c', 6.2, 'cc2'), + (4, 'd', 1.4, 'cc3') + """ sql """ show databases """ - order_qt_q01 """ select * from hms_kerberos.test_krb_hive_db.test_krb_hive_tbl """ + order_qt_q01 """ select * from hms_kerberos.test_single_krb_hive_db.test_krb_hive_tbl """ sql """drop catalog hms_kerberos;""" - try { - sql """drop catalog if exists hms_kerberos_hadoop_err1;""" - sql """ - CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 - PROPERTIES ( - "type" = "hms", - "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", - "hadoop.security.authentication" = "kerberos", - "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", - "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// - RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// - RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// - DEFAULT", - "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" - ); - """ - sql """ switch hms_kerberos_hadoop_err1 """ + sql """drop catalog if exists hms_kerberos_hadoop_err1;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err1 + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*// + RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*// + DEFAULT", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" + ); + """ + sql """ switch hms_kerberos_hadoop_err1 """ + test { sql """ show databases """ - } catch (Exception e) { - logger.info(e.toString()) - // caused by a warning msg if enable sasl on hive but "hive.metastore.sasl.enabled" is not true: - // "set_ugi() not successful, Likely cause: new client talking to old server. Continuing without it." - assertTrue(e.toString().contains("thrift.transport.TTransportException")) + exception "thrift.transport.TTransportException" } - try { - sql """drop catalog if exists hms_kerberos_hadoop_err2;""" - sql """ - CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 - PROPERTIES ( - "type" = "hms", - "hive.metastore.sasl.enabled " = "true", - "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", - "fs.defaultFS" = "hdfs://${externalEnvIp}:8520" - ); - """ - sql """ switch hms_kerberos_hadoop_err2 """ + sql """drop catalog if exists hms_kerberos_hadoop_err2;""" + sql """ + CREATE CATALOG IF NOT EXISTS hms_kerberos_hadoop_err2 + PROPERTIES ( + "type" = "hms", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", + "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", + "hive.metastore.kerberos.principal" = "hive/invalid-host@LABS.TERADATA.COM" + ); + """ + sql """ switch hms_kerberos_hadoop_err2 """ + test { sql """ show databases """ - } catch (Exception e) { - // org.apache.thrift.transport.TTransportException: GSS initiate failed - assertTrue(e.toString().contains("Could not connect to meta store using any of the URIs provided. Most recent failure: shade.doris.hive.org.apache.thrift.transport.TTransportException: GSS initiate failed")) + exception "GSS initiate failed" } // try { diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index b6b0d9a73ae7c8..87ea09560b7163 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -42,6 +42,10 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9583", "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -61,6 +65,10 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "type" = "hms", "hive.metastore.uris" = "thrift://${externalEnvIp}:9683", "fs.defaultFS" = "hdfs://${externalEnvIp}:8620", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master-2@OTHERREALM.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", + "dfs.data.transfer.protection" = "authentication", "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", @@ -72,6 +80,32 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d ); """ + def initializeFixture = { String catalogName -> + sql """ switch ${catalogName} """ + sql """ CREATE DATABASE IF NOT EXISTS test_krb_hive_db """ + sql """ USE test_krb_hive_db """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl """ + sql """ + CREATE TABLE test_krb_hive_tbl ( + id_key INT, + string_key STRING, + rate_val DOUBLE, + comment STRING + ) ENGINE = hive + PROPERTIES ("file_format" = "parquet") + """ + sql """ + INSERT INTO test_krb_hive_tbl VALUES + (1, 'a', 3.16, 'cc0'), + (2, 'b', 41.2, 'cc1'), + (3, 'c', 6.2, 'cc2'), + (4, 'd', 1.4, 'cc3') + """ + } + + initializeFixture(hms_catalog_name) + initializeFixture("other_${hms_catalog_name}") + // 1. catalogA sql """switch ${hms_catalog_name};""" logger.info("switched to catalog " + hms_catalog_name) diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy index 7c0a79ee7748b4..25ca5ebf6da87a 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy @@ -75,6 +75,9 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext } def hdfsNonXmlParams = "\"fs.defaultFS\" = \"hdfs://${externalEnvIp}:8520\",\n" + + "\"dfs.namenode.kerberos.principal\" = \"hdfs/hadoop-master@LABS.TERADATA.COM\",\n" + + "\"dfs.client.use.datanode.hostname\" = \"true\",\n" + + "\"hadoop.security.token.service.use_ip\" = \"false\",\n" + "\"hadoop.kerberos.min.seconds.before.relogin\" = \"5\",\n" + "\"hadoop.security.authentication\" = \"kerberos\",\n" + "\"hadoop.kerberos.principal\"=\"hive/presto-master.docker.cluster@LABS.TERADATA.COM\",\n" + @@ -316,4 +319,4 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext export_hdfs("hdfs://${externalEnvIp}:8520", hdfsNonXmlParams) -} \ No newline at end of file +} diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy index 49f4945f60e656..01b98dc02b2973 100644 --- a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy @@ -122,6 +122,9 @@ suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -129,6 +132,9 @@ suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") { String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hdfs.authentication.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -227,4 +233,3 @@ suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") { testQuery(paimon_hms_catalog_properties + paimon_fs_gcs_support + gcs_warehouse_properties + gcs_storage_s3_properties, "support_gcs_s3", "gcs_db") } - diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy index 6092ed9bd3be38..62771e46ebcfbb 100644 --- a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy +++ b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy @@ -424,6 +424,9 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -431,6 +434,9 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { """ String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hdfs.authentication.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab" @@ -484,8 +490,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { String db_location = "obs://${obs_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + obs_storage_properties, "hive_hms_obs_test", db_location) testQueryAndInsert(hms_properties + obs_region_param + obs_storage_properties, "hive_hms_obs_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_new", db_location) //OBS - Partition table tests db_location = "obs://${obs_parent_path}/hive/hms/partition/" + System.currentTimeMillis() @@ -498,8 +502,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { db_location = "gs://${gcs_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + gcs_storage_old_properties, "hive_hms_gcs_test", db_location) testQueryAndInsert(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_test_new", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + gcs_storage_old_properties, "hive_hms_on_gcs_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + gcs_storage_new_properties, "hive_hms_on_gcs_kerberos_new", db_location) //GCS - Insert overwrite tests db_location = "gs://${gcs_parent_path}/hive/hms/overwrite/" + System.currentTimeMillis() testInsertOverwrite(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_overwrite_test", db_location) @@ -509,8 +511,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { db_location = "cosn://${cos_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + cos_storage_properties, "hive_hms_cos_test", db_location) testQueryAndInsert(hms_properties + cos_region_param + cos_storage_properties, "hive_hms_cos_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_new", db_location) //COS - Partition table tests db_location = "cosn://${cos_parent_path}/hive/hms/partition/" + System.currentTimeMillis() @@ -526,8 +526,6 @@ suite("hive_on_hms_and_dlf", "p2,external,new_catalog_property") { db_location = "oss://${oss_parent_path}/hive/hms/" + System.currentTimeMillis() testQueryAndInsert(hms_properties + oss_storage_properties, "hive_hms_oss_test", db_location) testQueryAndInsert(hms_properties + oss_region_param + oss_storage_properties, "hive_hms_oss_test_region", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + oss_storage_properties, "hive_hms_on_oss_kerberos_old", db_location) - testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + oss_storage_properties, "hive_hms_on_oss_kerberos_new", db_location) //OSS - Partition table tests (fix for partition path scheme mismatch) db_location = "oss://${oss_parent_path}/hive/hms/partition/" + System.currentTimeMillis() diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy index 41785f7fe1a08c..fa3cff25572ee1 100644 --- a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy +++ b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy @@ -477,6 +477,9 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property // kerberos String hdfs_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "hadoop.security.authentication" = "kerberos", "io-impl" = "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -485,6 +488,9 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property String hdfs_new_kerberos_properties = """ "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", + "dfs.namenode.kerberos.principal" = "hdfs/hadoop-master@LABS.TERADATA.COM", + "dfs.client.use.datanode.hostname" = "true", + "hadoop.security.token.service.use_ip" = "false", "io-impl" = "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO", "hdfs.authentication.type" = "kerberos", "hdfs.authentication.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", @@ -543,10 +549,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property hmsTestQueryAndInsert(hms_prop + warehouse + oss_region_param + oss_storage_properties, "iceberg_hms_on_oss") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_old") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new") warehouse = """ 'warehouse' = 'oss://${oss_bucket_endpoint_parent_path}/iceberg-hms-warehouse', """ @@ -560,10 +562,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property + warehouse + obs_storage_properties, "iceberg_hms_on_obs") hmsTestQueryAndInsert(hms_prop+ warehouse + obs_region_param + obs_storage_properties, "iceberg_hms_on_obs") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_old") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_new") /*--------HMS on GCS-----------*/ if(context.config.otherConfigs.get("enableGCS")){ @@ -575,10 +573,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property hmsTestQueryAndInsert(hms_prop+ warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_new") - //new kerberos - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_new") - //old kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_old") } /*--------HMS on COS-----------*/ @@ -593,9 +587,6 @@ suite("iceberg_on_hms_and_filesystem_and_dlf", "p2,external,new_catalog_property 'warehouse' = 'cos://${cos_parent_path}/iceberg-hms-cos-warehouse', """ hmsTestQueryAndInsert(hms_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos") - //kerberos - hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_old") - hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_new") /*--------HMS on S3-----------*/