[feature](cloud) Add Alibaba Cloud OSS C++ SDK stack - Part 2

be/CMakeLists.txt

@@ -216,6 +216,12 @@ if(BUILD_AZURE STREQUAL "ON")
     add_definitions(-DUSE_AZURE)
 endif()
 
+# Set if use native OSS SDK or not
+option(BUILD_OSS "ON for building native Alibaba OSS support for BE or OFF for not" OFF)
+message(STATUS "build oss: ${BUILD_OSS}")
+if(BUILD_OSS STREQUAL "ON")
+    add_definitions(-DUSE_OSS)
+endif()
 
 set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
 

be/cmake/thirdparty.cmake

@@ -157,6 +157,19 @@ if(BUILD_AZURE STREQUAL "ON")
     add_thirdparty(azure-storage-common)
 endif()
 
+if(NOT BUILD_OSS STREQUAL "OFF")
+    # STS SDK V2 for AssumeRole (must be before OSS SDK)
+    add_thirdparty(darabonba_core)
+    add_thirdparty(alibabacloud_credentials)
+    add_thirdparty(alibabacloud_open_api_v2)
+    add_thirdparty(alibabacloud_sts20150401)
+    # OSS SDK
+    add_thirdparty(alibabacloud-oss-cpp-sdk)
+    add_thirdparty(apr)
+    add_thirdparty(apr-util LIBNAME "lib/libaprutil.a")
+    add_thirdparty(uuid LIB64)
+endif()
+
 add_thirdparty(minizip LIB64)
 add_thirdparty(simdjson LIB64)
 add_thirdparty(idn LIB64)

build.sh

@@ -80,6 +80,7 @@ Usage: $0 <options>
     DISABLE_BE_JAVA_EXTENSIONS  If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hadoop-hudi-scanner,jdbc-scanner and so on Default is OFF.
     DISABLE_JAVA_CHECK_STYLE    If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE.
     DISABLE_BUILD_AZURE         If set DISABLE_BUILD_AZURE=ON, it will not build azure into BE.
+    DISABLE_BUILD_OSS           If set DISABLE_BUILD_OSS=ON, it will not build the native OSS C++ SDK into BE.
     DISABLE_BUILD_JUICEFS       If set DISABLE_BUILD_JUICEFS=OFF, it will package juicefs-hadoop jar into FE/BE output. Default is ON (skip).
     DISABLE_BUILD_JINDOFS       If set DISABLE_BUILD_JINDOFS=OFF, it will package jindofs jars into FE/BE output. Default is ON (skip).
     EXTRA_FE_MODULES            Optional FE feature modules in feature=module_path format, separated by commas.
@@ -292,6 +293,7 @@ PARAMETER_COUNT="$#"
 PARAMETER_FLAG=0
 DENABLE_CLANG_COVERAGE='OFF'
 BUILD_AZURE='ON'
+BUILD_OSS='ON'
 BUILD_UI=1
 if [[ "$#" == 1 ]]; then
     # default
@@ -590,6 +592,10 @@ if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]
     BUILD_AZURE='OFF'
 fi
 
+if [[ "$(echo "${DISABLE_BUILD_OSS}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then
+    BUILD_OSS='OFF'
+fi
+
 if [[ "$(echo "${DISABLE_BUILD_JINDOFS}" | tr '[:lower:]' '[:upper:]')" == "OFF" ]]; then
     BUILD_JINDOFS='ON'
 else
@@ -841,6 +847,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
         -DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
         -DDORIS_JAVA_HOME="${JAVA_HOME}" \
         -DBUILD_AZURE="${BUILD_AZURE}" \
+        -DBUILD_OSS="${BUILD_OSS}" \
         -DENABLE_DYNAMIC_ARCH="${ENABLE_DYNAMIC_ARCH}" \
         -DFAISS_ENABLE_GPU="${FAISS_ENABLE_GPU:-OFF}" \
         "${BE_EXTRA_CMAKE_ARGS[@]}" \
@@ -884,6 +891,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
         -DUSE_JEMALLOC="${USE_JEMALLOC}" \
         -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
         -DBUILD_AZURE="${BUILD_AZURE}" \
+        -DBUILD_OSS="${BUILD_OSS}" \
         -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
         -DENABLE_DYNAMIC_ARCH="${ENABLE_DYNAMIC_ARCH}" \
         "${CLOUD_EXTRA_CMAKE_ARGS[@]}" \

cloud/CMakeLists.txt

@@ -362,6 +362,12 @@ if(BUILD_AZURE STREQUAL "ON")
     add_definitions(-DUSE_AZURE)
 endif()
 
+option(BUILD_OSS "ON for building native Alibaba OSS support or OFF for not" OFF)
+message(STATUS "build oss: ${BUILD_OSS}")
+if(BUILD_OSS STREQUAL "ON")
+    add_definitions(-DUSE_OSS)
+endif()
+
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     set(ASAN_LIBS -static-libasan)
     set(LSAN_LIBS -static-liblsan)

cloud/cmake/thirdparty.cmake

@@ -119,6 +119,18 @@ if(BUILD_AZURE STREQUAL "ON")
 endif()
 # end azure libs
 
+# begin oss libs
+if(NOT BUILD_OSS STREQUAL "OFF")
+    add_thirdparty(darabonba_core)
+    add_thirdparty(alibabacloud_credentials)
+    add_thirdparty(alibabacloud_open_api_v2)
+    add_thirdparty(alibabacloud_sts20150401)
+    add_thirdparty(alibabacloud-oss-cpp-sdk)
+    add_thirdparty(apr)
+    add_thirdparty(apr-util LIBNAME "lib/libaprutil.a")
+endif()
+# end oss libs
+
 add_thirdparty(gtest NOTADD)
 add_thirdparty(gtest_main NOTADD)
 add_thirdparty(gmock NOTADD)

fe/fe-filesystem/fe-filesystem-oss/pom.xml

@@ -58,6 +58,18 @@ under the License.
             <artifactId>aliyun-sdk-oss</artifactId>
             <version>${aliyun-sdk-oss.version}</version>
         </dependency>
+        <!-- Explicit: aliyun-java-sdk-core used directly in AssumeRole and ECS credential providers -->
+        <dependency>
+            <groupId>com.aliyun</groupId>
+            <artifactId>aliyun-java-sdk-core</artifactId>
+            <version>4.7.6</version>
+        </dependency>
+        <!-- Explicit: aliyun-java-sdk-sts not transitively pulled by aliyun-sdk-oss -->
+        <dependency>
+            <groupId>com.aliyun</groupId>
+            <artifactId>aliyun-java-sdk-sts</artifactId>
+            <version>3.1.3</version>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/AliyunCredentialsBridge.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.filesystem.oss;
+
+import com.aliyun.oss.common.auth.Credentials;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentials;
+import com.aliyuncs.auth.AlibabaCloudCredentials;
+import com.aliyuncs.auth.AlibabaCloudCredentialsProvider;
+import com.aliyuncs.auth.BasicSessionCredentials;
+
+/**
+ * Bridges {@link AlibabaCloudCredentialsProvider} (aliyuncs SDK) to
+ * {@link CredentialsProvider} (OSS SDK) so SDK-managed providers such as
+ * {@code OIDCCredentialsProvider} and {@code STSAssumeRoleSessionCredentialsProvider}
+ * can be passed directly to {@code OSSClientBuilder.build()}.
+ *
+ * <p>The aliyuncs providers handle credential caching and refresh internally;
+ * this class only bridges the credential types.
+ */
+class AliyunCredentialsBridge implements CredentialsProvider {
+
+    private final AlibabaCloudCredentialsProvider delegate;
+
+    AliyunCredentialsBridge(AlibabaCloudCredentialsProvider delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void setCredentials(Credentials credentials) {
+    }
+
+    @Override
+    public Credentials getCredentials() {
+        try {
+            AlibabaCloudCredentials c = delegate.getCredentials();
+            // Temporary credentials (STS/RRSA/ECS) return BasicSessionCredentials with a token
+            if (c instanceof BasicSessionCredentials) {
+                String token = ((BasicSessionCredentials) c).getSessionToken();
+                return new DefaultCredentials(c.getAccessKeyId(), c.getAccessKeySecret(), token);
+            }
+            return new DefaultCredentials(c.getAccessKeyId(), c.getAccessKeySecret());
+        } catch (Exception e) {
+            throw new IllegalStateException("Failed to obtain credentials: " + e.getMessage(), e);
+        }
+    }
+}

fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssCredentialsProviderType.java

@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.filesystem.oss;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Credential provider mode for OSS filesystem access.
+ *
+ * <p>Mirrors {@code S3CredentialsProviderType} for consistency across providers.
+ */
+public enum OssCredentialsProviderType {
+
+    /**
+     * Try in order: INSTANCE_PROFILE → OIDC (if env configured) → ENV → static ak/sk → anonymous.
+     */
+    DEFAULT("DEFAULT"),
+
+    /**
+     * ECS instance metadata (100.100.100.200). Role name from {@code oss.ecs_ram_role_name}
+     * or auto-discovered from the metadata base URL when not set.
+     */
+    INSTANCE_PROFILE("INSTANCE_PROFILE"),
+
+    /**
+     * RRSA / Kubernetes pod identity via STS AssumeRoleWithOIDC.
+     * Role ARN and token file from {@code oss.oidc_provider_arn}/{@code oss.oidc_token_file}
+     * or from env vars {@code ALIBABA_CLOUD_ROLE_ARN} / {@code ALIBABA_CLOUD_OIDC_TOKEN_FILE}
+     * / {@code ALIBABA_CLOUD_OIDC_PROVIDER_ARN}.
+     */
+    OIDC("OIDC"),
+
+    /**
+     * Environment variables: {@code OSS_ACCESS_KEY_ID} and {@code OSS_ACCESS_KEY_SECRET}.
+     */
+    ENV("ENV"),
+
+    /** No credentials — public buckets only. */
+    ANONYMOUS("ANONYMOUS");
+
+    private final String mode;
+
+    OssCredentialsProviderType(String mode) {
+        this.mode = mode;
+    }
+
+    public String getMode() {
+        return mode;
+    }
+
+    public static OssCredentialsProviderType fromString(String value) {
+        if (StringUtils.isBlank(value)) {
+            return DEFAULT;
+        }
+        String normalized = value.trim().toUpperCase().replace('-', '_');
+        switch (normalized) {
+            case "INSTANCE_PROFILE":
+            case "ECS":
+            case "ECS_RAM_ROLE":
+                return INSTANCE_PROFILE;
+            case "OIDC":
+            case "RRSA":
+            case "WEB_IDENTITY":
+                return OIDC;
+            case "ENV":
+            case "ENVIRONMENT":
+                return ENV;
+            case "ANONYMOUS":
+                return ANONYMOUS;
+            case "DEFAULT":
+                return DEFAULT;
+            default:
+                throw new IllegalArgumentException(
+                        "Unsupported oss.credentials_provider: " + value);
+        }
+    }
+}

fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java

@@ -62,6 +62,9 @@ public final class OssFileSystemProperties
     public static final String FORCE_PARSING_BY_STANDARD_URI =
             "oss.force_parsing_by_standard_uri";
 
+    public static final String CREDENTIALS_PROVIDER = "oss.credentials_provider";
+
+    public static final String DEFAULT_CREDENTIALS_PROVIDER = "DEFAULT";
     public static final String DEFAULT_MAX_CONNECTIONS = "100";
     public static final String DEFAULT_REQUEST_TIMEOUT_MS = "10000";
     public static final String DEFAULT_CONNECTION_TIMEOUT_MS = "10000";
@@ -149,11 +152,36 @@ public final class OssFileSystemProperties
             description = "The OSS role ARN for AssumeRole access.")
     private String roleArn = "";
 
+    @ConnectorProperty(names = {"oss.role_session_name"},
+            required = false,
+            description = "Session name used for STS AssumeRole; default 'doris-session'.")
+    private String roleSessionName = "doris-session";
+
     @ConnectorProperty(names = {"AWS_EXTERNAL_ID"},
             required = false,
             description = "The external ID for AssumeRole trust policy.")
     private String externalId = "";
 
+    @ConnectorProperty(names = {"oss.ecs_ram_role_name"},
+            required = false,
+            description = "ECS-attached RAM role name; mutually exclusive with oss.role_arn.")
+    private String ecsRamRoleName = "";
+
+    @ConnectorProperty(names = {"oss.oidc_provider_arn"},
+            required = false,
+            description = "OIDC provider ARN for RRSA (Kubernetes pod identity).")
+    private String oidcProviderArn = "";
+
+    @ConnectorProperty(names = {"oss.oidc_token_file"},
+            required = false,
+            description = "Path to OIDC token file mounted by Kubernetes.")
+    private String oidcTokenFile = "";
+
+    @ConnectorProperty(names = {CREDENTIALS_PROVIDER, "AWS_CREDENTIALS_PROVIDER_TYPE"},
+            required = false,
+            description = "Credential provider mode: DEFAULT, INSTANCE_PROFILE, OIDC, ENV, ANONYMOUS.")
+    private String credentialsProvider = DEFAULT_CREDENTIALS_PROVIDER;
+
     @ConnectorProperty(names = {"AWS_ROOT_PATH"},
             required = false,
             description = "The root path prefix inside the bucket.")
@@ -178,6 +206,14 @@ public static OssFileSystemProperties of(Map<String, String> properties) {
     @Override
     public void validate() {
         new ParamRules()
+                .mutuallyExclusive(roleArn, ecsRamRoleName,
+                        "OSS_ROLE_ARN and oss.ecs_ram_role_name are mutually exclusive.")
+                .mutuallyExclusive(ecsRamRoleName, oidcProviderArn,
+                        "oss.ecs_ram_role_name and oss.oidc_provider_arn are mutually exclusive.")
+                .requireTogether(new String[] {oidcProviderArn, oidcTokenFile},
+                        "oss.oidc_provider_arn and oss.oidc_token_file must be set together.")
+                .check(this::hasUnsupportedCredentialsProviderType,
+                        "Unsupported oss.credentials_provider: " + credentialsProvider)
                 .requireTogether(new String[] {accessKey, secretKey},
                         "Both the access key and the secret key must be set.")
                 .check(() -> StringUtils.isBlank(region),
@@ -313,6 +349,35 @@ public String getRoleArn() {
         return roleArn;
     }
 
+    public String getRoleSessionName() {
+        return roleSessionName;
+    }
+
+    public String getEcsRamRoleName() {
+        return ecsRamRoleName;
+    }
+
+    public String getOidcProviderArn() {
+        return oidcProviderArn;
+    }
+
+    public String getOidcTokenFile() {
+        return oidcTokenFile;
+    }
+
+    public OssCredentialsProviderType getCredentialsProviderType() {
+        return OssCredentialsProviderType.fromString(credentialsProvider);
+    }
+
+    private boolean hasUnsupportedCredentialsProviderType() {
+        try {
+            getCredentialsProviderType();
+            return false;
+        } catch (IllegalArgumentException e) {
+            return true;
+        }
+    }
+
     @Override
     public String getExternalId() {
         return externalId;