From 25e389847049906c14d3d6bbf80f5c7ed876b390 Mon Sep 17 00:00:00 2001 From: Sivarajan Narayanan Date: Sun, 19 Jul 2026 01:21:06 +0530 Subject: [PATCH 1/2] [improve](fs/oss) add ECS RAM role, AssumeRole, RRSA, and provider-type credential support --- fe/fe-filesystem/fe-filesystem-oss/pom.xml | 12 ++ .../oss/AliyunCredentialsBridge.java | 62 +++++++++ .../oss/OssCredentialsProviderType.java | 92 +++++++++++++ .../oss/OssFileSystemProperties.java | 65 +++++++++ .../doris/filesystem/oss/OssObjStorage.java | 95 +++++++++++++ .../oss/OssCredentialProviderTest.java | 118 +++++++++++++++++ .../oss/OssFileSystemPropertiesTest.java | 125 ++++++++++++++++++ 7 files changed, 569 insertions(+) create mode 100644 fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/AliyunCredentialsBridge.java create mode 100644 fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssCredentialsProviderType.java create mode 100644 fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssCredentialProviderTest.java diff --git a/fe/fe-filesystem/fe-filesystem-oss/pom.xml b/fe/fe-filesystem/fe-filesystem-oss/pom.xml index a6b96473c02df5..4469b6c182f2e0 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/pom.xml +++ b/fe/fe-filesystem/fe-filesystem-oss/pom.xml @@ -58,6 +58,18 @@ under the License. aliyun-sdk-oss ${aliyun-sdk-oss.version} + + + com.aliyun + aliyun-java-sdk-core + 4.7.6 + + + + com.aliyun + aliyun-java-sdk-sts + 3.1.3 + org.junit.jupiter junit-jupiter-api diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/AliyunCredentialsBridge.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/AliyunCredentialsBridge.java new file mode 100644 index 00000000000000..84794aacc26bf2 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/AliyunCredentialsBridge.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; +import com.aliyuncs.auth.AlibabaCloudCredentials; +import com.aliyuncs.auth.AlibabaCloudCredentialsProvider; +import com.aliyuncs.auth.BasicSessionCredentials; + +/** + * Bridges {@link AlibabaCloudCredentialsProvider} (aliyuncs SDK) to + * {@link CredentialsProvider} (OSS SDK) so SDK-managed providers such as + * {@code OIDCCredentialsProvider} and {@code STSAssumeRoleSessionCredentialsProvider} + * can be passed directly to {@code OSSClientBuilder.build()}. + * + *

The aliyuncs providers handle credential caching and refresh internally; + * this class only bridges the credential types. + */ +class AliyunCredentialsBridge implements CredentialsProvider { + + private final AlibabaCloudCredentialsProvider delegate; + + AliyunCredentialsBridge(AlibabaCloudCredentialsProvider delegate) { + this.delegate = delegate; + } + + @Override + public void setCredentials(Credentials credentials) { + } + + @Override + public Credentials getCredentials() { + try { + AlibabaCloudCredentials c = delegate.getCredentials(); + // Temporary credentials (STS/RRSA/ECS) return BasicSessionCredentials with a token + if (c instanceof BasicSessionCredentials) { + String token = ((BasicSessionCredentials) c).getSessionToken(); + return new DefaultCredentials(c.getAccessKeyId(), c.getAccessKeySecret(), token); + } + return new DefaultCredentials(c.getAccessKeyId(), c.getAccessKeySecret()); + } catch (Exception e) { + throw new IllegalStateException("Failed to obtain credentials: " + e.getMessage(), e); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssCredentialsProviderType.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssCredentialsProviderType.java new file mode 100644 index 00000000000000..39e173683adae1 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssCredentialsProviderType.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.oss; + +import org.apache.commons.lang3.StringUtils; + +/** + * Credential provider mode for OSS filesystem access. + * + *

Mirrors {@code S3CredentialsProviderType} for consistency across providers. + */ +public enum OssCredentialsProviderType { + + /** + * Try in order: INSTANCE_PROFILE → OIDC (if env configured) → ENV → static ak/sk → anonymous. + */ + DEFAULT("DEFAULT"), + + /** + * ECS instance metadata (100.100.100.200). Role name from {@code oss.ecs_ram_role_name} + * or auto-discovered from the metadata base URL when not set. + */ + INSTANCE_PROFILE("INSTANCE_PROFILE"), + + /** + * RRSA / Kubernetes pod identity via STS AssumeRoleWithOIDC. + * Role ARN and token file from {@code oss.oidc_provider_arn}/{@code oss.oidc_token_file} + * or from env vars {@code ALIBABA_CLOUD_ROLE_ARN} / {@code ALIBABA_CLOUD_OIDC_TOKEN_FILE} + * / {@code ALIBABA_CLOUD_OIDC_PROVIDER_ARN}. + */ + OIDC("OIDC"), + + /** + * Environment variables: {@code OSS_ACCESS_KEY_ID} and {@code OSS_ACCESS_KEY_SECRET}. + */ + ENV("ENV"), + + /** No credentials — public buckets only. */ + ANONYMOUS("ANONYMOUS"); + + private final String mode; + + OssCredentialsProviderType(String mode) { + this.mode = mode; + } + + public String getMode() { + return mode; + } + + public static OssCredentialsProviderType fromString(String value) { + if (StringUtils.isBlank(value)) { + return DEFAULT; + } + String normalized = value.trim().toUpperCase().replace('-', '_'); + switch (normalized) { + case "INSTANCE_PROFILE": + case "ECS": + case "ECS_RAM_ROLE": + return INSTANCE_PROFILE; + case "OIDC": + case "RRSA": + case "WEB_IDENTITY": + return OIDC; + case "ENV": + case "ENVIRONMENT": + return ENV; + case "ANONYMOUS": + return ANONYMOUS; + case "DEFAULT": + return DEFAULT; + default: + throw new IllegalArgumentException( + "Unsupported oss.credentials_provider: " + value); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java index 663b5596c242cd..1b664876c2e84b 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssFileSystemProperties.java @@ -62,6 +62,9 @@ public final class OssFileSystemProperties public static final String FORCE_PARSING_BY_STANDARD_URI = "oss.force_parsing_by_standard_uri"; + public static final String CREDENTIALS_PROVIDER = "oss.credentials_provider"; + + public static final String DEFAULT_CREDENTIALS_PROVIDER = "DEFAULT"; public static final String DEFAULT_MAX_CONNECTIONS = "100"; public static final String DEFAULT_REQUEST_TIMEOUT_MS = "10000"; public static final String DEFAULT_CONNECTION_TIMEOUT_MS = "10000"; @@ -149,11 +152,36 @@ public final class OssFileSystemProperties description = "The OSS role ARN for AssumeRole access.") private String roleArn = ""; + @ConnectorProperty(names = {"oss.role_session_name"}, + required = false, + description = "Session name used for STS AssumeRole; default 'doris-session'.") + private String roleSessionName = "doris-session"; + @ConnectorProperty(names = {"AWS_EXTERNAL_ID"}, required = false, description = "The external ID for AssumeRole trust policy.") private String externalId = ""; + @ConnectorProperty(names = {"oss.ecs_ram_role_name"}, + required = false, + description = "ECS-attached RAM role name; mutually exclusive with oss.role_arn.") + private String ecsRamRoleName = ""; + + @ConnectorProperty(names = {"oss.oidc_provider_arn"}, + required = false, + description = "OIDC provider ARN for RRSA (Kubernetes pod identity).") + private String oidcProviderArn = ""; + + @ConnectorProperty(names = {"oss.oidc_token_file"}, + required = false, + description = "Path to OIDC token file mounted by Kubernetes.") + private String oidcTokenFile = ""; + + @ConnectorProperty(names = {CREDENTIALS_PROVIDER, "AWS_CREDENTIALS_PROVIDER_TYPE"}, + required = false, + description = "Credential provider mode: DEFAULT, INSTANCE_PROFILE, OIDC, ENV, ANONYMOUS.") + private String credentialsProvider = DEFAULT_CREDENTIALS_PROVIDER; + @ConnectorProperty(names = {"AWS_ROOT_PATH"}, required = false, description = "The root path prefix inside the bucket.") @@ -178,6 +206,14 @@ public static OssFileSystemProperties of(Map properties) { @Override public void validate() { new ParamRules() + .mutuallyExclusive(roleArn, ecsRamRoleName, + "OSS_ROLE_ARN and oss.ecs_ram_role_name are mutually exclusive.") + .mutuallyExclusive(ecsRamRoleName, oidcProviderArn, + "oss.ecs_ram_role_name and oss.oidc_provider_arn are mutually exclusive.") + .requireTogether(new String[] {oidcProviderArn, oidcTokenFile}, + "oss.oidc_provider_arn and oss.oidc_token_file must be set together.") + .check(this::hasUnsupportedCredentialsProviderType, + "Unsupported oss.credentials_provider: " + credentialsProvider) .requireTogether(new String[] {accessKey, secretKey}, "Both the access key and the secret key must be set.") .check(() -> StringUtils.isBlank(region), @@ -313,6 +349,35 @@ public String getRoleArn() { return roleArn; } + public String getRoleSessionName() { + return roleSessionName; + } + + public String getEcsRamRoleName() { + return ecsRamRoleName; + } + + public String getOidcProviderArn() { + return oidcProviderArn; + } + + public String getOidcTokenFile() { + return oidcTokenFile; + } + + public OssCredentialsProviderType getCredentialsProviderType() { + return OssCredentialsProviderType.fromString(credentialsProvider); + } + + private boolean hasUnsupportedCredentialsProviderType() { + try { + getCredentialsProviderType(); + return false; + } catch (IllegalArgumentException e) { + return true; + } + } + @Override public String getExternalId() { return externalId; diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java index cfb1da6853270c..8f370bb2859531 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/main/java/org/apache/doris/filesystem/oss/OssObjStorage.java @@ -176,9 +176,53 @@ private static boolean isVirtualHostCompatible(String bucket) { protected OSS buildOssClient(boolean pathStyle) throws IOException { String endpoint = properties.getEndpoint(); + OssCredentialsProviderType type = properties.getCredentialsProviderType(); + switch (type) { + case INSTANCE_PROFILE: + return new OSSClientBuilder().build(endpoint, + new com.aliyun.oss.common.auth.EcsRamRoleCredentialsProvider( + resolveEcsRoleName()), + clientConfiguration(pathStyle)); + case OIDC: + return new OSSClientBuilder().build(endpoint, + new AliyunCredentialsBridge(buildOidcProvider()), + clientConfiguration(pathStyle)); + case ENV: + return new OSSClientBuilder().build(endpoint, + new com.aliyun.oss.common.auth.EnvironmentVariableCredentialsProvider(), + clientConfiguration(pathStyle)); + case ANONYMOUS: + return new OSSClientBuilder().build(endpoint, anonymousCredentialsProvider(), + anonymousClientConfiguration(pathStyle)); + default: + break; // DEFAULT — fall through to property-driven chain below + } + // DEFAULT: property-driven chain + if (hasText(properties.getEcsRamRoleName())) { + return new OSSClientBuilder().build(endpoint, + new com.aliyun.oss.common.auth.EcsRamRoleCredentialsProvider( + properties.getEcsRamRoleName()), + clientConfiguration(pathStyle)); + } + if (hasText(properties.getOidcProviderArn())) { + return new OSSClientBuilder().build(endpoint, + new AliyunCredentialsBridge(buildOidcProvider()), + clientConfiguration(pathStyle)); + } + if (hasText(properties.getRoleArn())) { + return new OSSClientBuilder().build(endpoint, + new AliyunCredentialsBridge(buildAssumeRoleProvider()), + clientConfiguration(pathStyle)); + } + // existing ak/sk / anonymous logic — unchanged String accessKey = properties.getAccessKey(); String secretKey = properties.getSecretKey(); if (!hasText(accessKey)) { + if (hasText(System.getenv("OSS_ACCESS_KEY_ID"))) { + return new OSSClientBuilder().build(endpoint, + new com.aliyun.oss.common.auth.EnvironmentVariableCredentialsProvider(), + clientConfiguration(pathStyle)); + } return new OSSClientBuilder().build(endpoint, anonymousCredentialsProvider(), anonymousClientConfiguration(pathStyle)); } @@ -190,6 +234,57 @@ protected OSS buildOssClient(boolean pathStyle) throws IOException { return new OSSClientBuilder().build(endpoint, accessKey, secretKey, config); } + // protected so tests can override + protected String discoverEcsRoleName() throws IOException { + java.net.URL url = new java.net.URL( + "http://100.100.100.200/latest/meta-data/ram/security-credentials/"); + java.net.HttpURLConnection conn = (java.net.HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(3000); + conn.setReadTimeout(3000); + try (java.io.InputStream in = conn.getInputStream()) { + return new String(in.readAllBytes()).trim().split("\n")[0].trim(); + } finally { + conn.disconnect(); + } + } + + private String resolveEcsRoleName() throws IOException { + String name = properties.getEcsRamRoleName(); + return hasText(name) ? name : discoverEcsRoleName(); + } + + private com.aliyuncs.auth.AlibabaCloudCredentialsProvider buildOidcProvider() { + String roleArn = hasText(properties.getRoleArn()) + ? properties.getRoleArn() : System.getenv("ALIBABA_CLOUD_ROLE_ARN"); + String providerArn = hasText(properties.getOidcProviderArn()) + ? properties.getOidcProviderArn() + : System.getenv("ALIBABA_CLOUD_OIDC_PROVIDER_ARN"); + String tokenFile = hasText(properties.getOidcTokenFile()) + ? properties.getOidcTokenFile() + : System.getenv("ALIBABA_CLOUD_OIDC_TOKEN_FILE"); + return com.aliyuncs.auth.OIDCCredentialsProvider.builder() + .roleArn(roleArn) + .oidcProviderArn(providerArn) + .oidcTokenFilePath(tokenFile) + .roleSessionName(properties.getRoleSessionName()) + .stsRegionId(properties.getRegion()) + .build(); + } + + private com.aliyuncs.auth.AlibabaCloudCredentialsProvider buildAssumeRoleProvider() { + com.aliyuncs.auth.STSAssumeRoleSessionCredentialsProvider.Builder builder = + com.aliyuncs.auth.STSAssumeRoleSessionCredentialsProvider.builder() + .accessKeyId(properties.getAccessKey()) + .accessKeySecret(properties.getSecretKey()) + .roleArn(properties.getRoleArn()) + .roleSessionName(properties.getRoleSessionName()) + .stsRegionId(properties.getRegion()); + if (hasText(properties.getExternalId())) { + builder.externalId(properties.getExternalId()); + } + return builder.build(); + } + @Override public RemoteObjects listObjects(String remotePath, String continuationToken) throws IOException { return listObjectsWithOptions(remotePath, ObjectListOptions.builder() diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssCredentialProviderTest.java b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssCredentialProviderTest.java new file mode 100644 index 00000000000000..edbe9245d72428 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssCredentialProviderTest.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.oss; + +import com.aliyun.oss.common.auth.Credentials; +import com.aliyuncs.auth.AlibabaCloudCredentials; +import com.aliyuncs.auth.BasicSessionCredentials; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Tests for credential bridging and provider selection in OssObjStorage. + * No real OSS credentials required — all external calls are overridden. + */ +class OssCredentialProviderTest { + + // ── AliyunCredentialsBridge ─────────────────────────────────────────────── + + @Test + void bridge_wrapsSessionCredentialsWithToken() { + BasicSessionCredentials alibabaCreds = + new BasicSessionCredentials("session-ak", "session-sk", "session-token"); + + AliyunCredentialsBridge bridge = new AliyunCredentialsBridge(() -> alibabaCreds); + + Credentials ossCreds = bridge.getCredentials(); + Assertions.assertEquals("session-ak", ossCreds.getAccessKeyId()); + Assertions.assertEquals("session-sk", ossCreds.getSecretAccessKey()); + Assertions.assertEquals("session-token", ossCreds.getSecurityToken()); + } + + @Test + void bridge_wrapsBasicCredentialsWithoutToken() { + AlibabaCloudCredentials alibabaCreds = new com.aliyuncs.auth.BasicCredentials("ak", "sk"); + + AliyunCredentialsBridge bridge = new AliyunCredentialsBridge(() -> alibabaCreds); + + Credentials ossCreds = bridge.getCredentials(); + Assertions.assertEquals("ak", ossCreds.getAccessKeyId()); + Assertions.assertEquals("sk", ossCreds.getSecretAccessKey()); + Assertions.assertTrue(ossCreds.getSecurityToken() == null + || ossCreds.getSecurityToken().isEmpty()); + } + + @Test + void bridge_wrapsProviderException() { + AliyunCredentialsBridge bridge = new AliyunCredentialsBridge(() -> { + throw new com.aliyuncs.exceptions.ClientException("network error"); + }); + + Assertions.assertThrows(IllegalStateException.class, bridge::getCredentials); + } + + // ── OssObjStorage — INSTANCE_PROFILE mode auto-discovery ───────────────── + + @Test + void objStorage_instanceProfileMode_callsDiscoverEcsRoleName() { + OssFileSystemProperties props = OssFileSystemProperties.of(java.util.Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.credentials_provider", "instance_profile")); + + String[] captured = {null}; + OssObjStorage storage = new OssObjStorage(props) { + @Override + protected String discoverEcsRoleName() { + captured[0] = "discovered"; + return "AutoRole"; + } + }; + + try { + storage.getClient(); + } catch (Exception ignored) { + // expected — no real endpoint in tests + } + Assertions.assertEquals("discovered", captured[0], + "discoverEcsRoleName() should be called when no role name is set"); + } + + @Test + void objStorage_instanceProfileMode_usesConfiguredRoleNameWithoutDiscovery() { + OssFileSystemProperties props = OssFileSystemProperties.of(java.util.Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.ecs_ram_role_name", "ExplicitRole", + "oss.credentials_provider", "instance_profile")); + + String[] captured = {null}; + OssObjStorage storage = new OssObjStorage(props) { + @Override + protected String discoverEcsRoleName() { + captured[0] = "should-not-be-called"; + return "AutoRole"; + } + }; + + try { + storage.getClient(); + } catch (Exception ignored) { + // expected + } + Assertions.assertNull(captured[0], "discoverEcsRoleName() should NOT be called when role name is set"); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemPropertiesTest.java index a4d7a11c6ce962..005bac4e5835f4 100644 --- a/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-oss/src/test/java/org/apache/doris/filesystem/oss/OssFileSystemPropertiesTest.java @@ -180,4 +180,129 @@ void objStorageDoesNotExposeLegacyToS3PropsTranslator() { Assertions.assertNotEquals("toS3Props", method.getName()); } } + + @Test + void bind_roleArnAndRoleSessionName() { + OssFileSystemProperties p = OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.access_key", "ak", + "oss.secret_key", "sk", + "oss.role_arn", "acs:ram::123:role/r", + "oss.role_session_name", "my-session", + "oss.external_id", "ext-id")); + + Assertions.assertEquals("acs:ram::123:role/r", p.getRoleArn()); + Assertions.assertEquals("my-session", p.getRoleSessionName()); + Assertions.assertEquals("ext-id", p.getExternalId()); + } + + @Test + void bind_roleSessionName_defaultsToDorisSession() { + OssFileSystemProperties p = OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.access_key", "ak", + "oss.secret_key", "sk", + "oss.role_arn", "acs:ram::123:role/r")); + + Assertions.assertEquals("doris-session", p.getRoleSessionName()); + } + + @Test + void bind_ecsRamRoleName() { + OssFileSystemProperties p = OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.ecs_ram_role_name", "my-ecs-role")); + + Assertions.assertEquals("my-ecs-role", p.getEcsRamRoleName()); + Assertions.assertEquals("", p.getAccessKey()); + } + + @Test + void validate_roleArnAndEcsRamRoleAreMutuallyExclusive() { + Assertions.assertThrows(IllegalArgumentException.class, () -> + OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.access_key", "ak", + "oss.secret_key", "sk", + "oss.role_arn", "acs:ram::123:role/r", + "oss.ecs_ram_role_name", "my-role"))); + } + + @Test + void validate_legacyRoleArnAliasesResolved() { + OssFileSystemProperties p = OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.access_key", "ak", + "oss.secret_key", "sk", + "OSS_ROLE_ARN", "acs:ram::123:role/legacy")); + + Assertions.assertEquals("acs:ram::123:role/legacy", p.getRoleArn()); + } + + @Test + void bind_oidcRrsaProperties() { + OssFileSystemProperties p = OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "OSS_ROLE_ARN", "acs:ram::123:role/r", + "oss.oidc_provider_arn", "acs:ram::123:oidc-provider/p", + "oss.oidc_token_file", "/var/run/secrets/token", + "oss.role_session_name", "doris-k8s")); + + Assertions.assertEquals("acs:ram::123:role/r", p.getRoleArn()); + Assertions.assertEquals("acs:ram::123:oidc-provider/p", p.getOidcProviderArn()); + Assertions.assertEquals("/var/run/secrets/token", p.getOidcTokenFile()); + Assertions.assertEquals("doris-k8s", p.getRoleSessionName()); + } + + @Test + void validate_oidcProviderArnRequiresTokenFileTogether() { + Assertions.assertThrows(IllegalArgumentException.class, () -> + OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.oidc_provider_arn", "acs:ram::123:oidc-provider/p" + // oss.oidc_token_file missing — must be set together + ))); + } + + @Test + void validate_ecsRamRoleAndOidcAreMutuallyExclusive() { + Assertions.assertThrows(IllegalArgumentException.class, () -> + OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.ecs_ram_role_name", "my-role", + "oss.oidc_provider_arn", "acs:ram::123:oidc-provider/p", + "oss.oidc_token_file", "/var/run/secrets/token"))); + } + + @Test + void bind_credentialsProviderType_instanceProfile() { + OssFileSystemProperties p = OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.credentials_provider", "instance_profile")); + + Assertions.assertEquals(OssCredentialsProviderType.INSTANCE_PROFILE, + p.getCredentialsProviderType()); + } + + @Test + void bind_credentialsProviderType_aliases() { + Assertions.assertEquals(OssCredentialsProviderType.INSTANCE_PROFILE, + OssCredentialsProviderType.fromString("ecs")); + Assertions.assertEquals(OssCredentialsProviderType.OIDC, + OssCredentialsProviderType.fromString("rrsa")); + Assertions.assertEquals(OssCredentialsProviderType.OIDC, + OssCredentialsProviderType.fromString("WEB_IDENTITY")); + Assertions.assertEquals(OssCredentialsProviderType.DEFAULT, + OssCredentialsProviderType.fromString("")); + Assertions.assertEquals(OssCredentialsProviderType.DEFAULT, + OssCredentialsProviderType.fromString(null)); + } + + @Test + void validate_unsupportedCredentialsProvider() { + Assertions.assertThrows(IllegalArgumentException.class, () -> + OssFileSystemProperties.of(Map.of( + "oss.endpoint", "https://oss-cn-hangzhou.aliyuncs.com", + "oss.credentials_provider", "INVALID_TYPE"))); + } } From c3ad4f0ebc9686b2504751ef499c814dc9ee119c Mon Sep 17 00:00:00 2001 From: Sivarajan Narayanan Date: Sun, 19 Jul 2026 02:21:53 +0530 Subject: [PATCH 2/2] [build] add Alibaba Cloud OSS C++ SDK stack as optional thirdparty --- be/CMakeLists.txt | 6 + be/cmake/thirdparty.cmake | 13 +++ build.sh | 8 ++ cloud/CMakeLists.txt | 6 + cloud/cmake/thirdparty.cmake | 12 ++ thirdparty/build-thirdparty.sh | 103 ++++++++++++++++++ thirdparty/download-thirdparty.sh | 38 +++++++ .../patches/tea-cpp-doris-oss-fix.patch | 63 +++++++++++ thirdparty/vars.sh | 51 +++++++++ 9 files changed, 300 insertions(+) create mode 100644 thirdparty/patches/tea-cpp-doris-oss-fix.patch diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index b1ac9e05957fcb..03b221b1dd1cb4 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -216,6 +216,12 @@ if(BUILD_AZURE STREQUAL "ON") add_definitions(-DUSE_AZURE) endif() +# Set if use native OSS SDK or not +option(BUILD_OSS "ON for building native Alibaba OSS support for BE or OFF for not" OFF) +message(STATUS "build oss: ${BUILD_OSS}") +if(BUILD_OSS STREQUAL "ON") + add_definitions(-DUSE_OSS) +endif() set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools") diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 92dffd98da875b..aed2ed847db2cd 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -157,6 +157,19 @@ if(BUILD_AZURE STREQUAL "ON") add_thirdparty(azure-storage-common) endif() +if(NOT BUILD_OSS STREQUAL "OFF") + # STS SDK V2 for AssumeRole (must be before OSS SDK) + add_thirdparty(darabonba_core) + add_thirdparty(alibabacloud_credentials) + add_thirdparty(alibabacloud_open_api_v2) + add_thirdparty(alibabacloud_sts20150401) + # OSS SDK + add_thirdparty(alibabacloud-oss-cpp-sdk) + add_thirdparty(apr) + add_thirdparty(apr-util LIBNAME "lib/libaprutil.a") + add_thirdparty(uuid LIB64) +endif() + add_thirdparty(minizip LIB64) add_thirdparty(simdjson LIB64) add_thirdparty(idn LIB64) diff --git a/build.sh b/build.sh index b10d789892376a..bc8cb789d1f320 100755 --- a/build.sh +++ b/build.sh @@ -80,6 +80,7 @@ Usage: $0 DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hadoop-hudi-scanner,jdbc-scanner and so on Default is OFF. DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE. DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not build azure into BE. + DISABLE_BUILD_OSS If set DISABLE_BUILD_OSS=ON, it will not build the native OSS C++ SDK into BE. DISABLE_BUILD_JUICEFS If set DISABLE_BUILD_JUICEFS=OFF, it will package juicefs-hadoop jar into FE/BE output. Default is ON (skip). DISABLE_BUILD_JINDOFS If set DISABLE_BUILD_JINDOFS=OFF, it will package jindofs jars into FE/BE output. Default is ON (skip). EXTRA_FE_MODULES Optional FE feature modules in feature=module_path format, separated by commas. @@ -292,6 +293,7 @@ PARAMETER_COUNT="$#" PARAMETER_FLAG=0 DENABLE_CLANG_COVERAGE='OFF' BUILD_AZURE='ON' +BUILD_OSS='ON' BUILD_UI=1 if [[ "$#" == 1 ]]; then # default @@ -590,6 +592,10 @@ if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]] BUILD_AZURE='OFF' fi +if [[ "$(echo "${DISABLE_BUILD_OSS}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then + BUILD_OSS='OFF' +fi + if [[ "$(echo "${DISABLE_BUILD_JINDOFS}" | tr '[:lower:]' '[:upper:]')" == "OFF" ]]; then BUILD_JINDOFS='ON' else @@ -841,6 +847,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \ -DDORIS_JAVA_HOME="${JAVA_HOME}" \ -DBUILD_AZURE="${BUILD_AZURE}" \ + -DBUILD_OSS="${BUILD_OSS}" \ -DENABLE_DYNAMIC_ARCH="${ENABLE_DYNAMIC_ARCH}" \ -DFAISS_ENABLE_GPU="${FAISS_ENABLE_GPU:-OFF}" \ "${BE_EXTRA_CMAKE_ARGS[@]}" \ @@ -884,6 +891,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then -DUSE_JEMALLOC="${USE_JEMALLOC}" \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -DBUILD_AZURE="${BUILD_AZURE}" \ + -DBUILD_OSS="${BUILD_OSS}" \ -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \ -DENABLE_DYNAMIC_ARCH="${ENABLE_DYNAMIC_ARCH}" \ "${CLOUD_EXTRA_CMAKE_ARGS[@]}" \ diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index d4e0b3466598e2..89987496126405 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -362,6 +362,12 @@ if(BUILD_AZURE STREQUAL "ON") add_definitions(-DUSE_AZURE) endif() +option(BUILD_OSS "ON for building native Alibaba OSS support or OFF for not" OFF) +message(STATUS "build oss: ${BUILD_OSS}") +if(BUILD_OSS STREQUAL "ON") + add_definitions(-DUSE_OSS) +endif() + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") set(ASAN_LIBS -static-libasan) set(LSAN_LIBS -static-liblsan) diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index a383b263b9c63d..aeecbea9b075de 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -119,6 +119,18 @@ if(BUILD_AZURE STREQUAL "ON") endif() # end azure libs +# begin oss libs +if(NOT BUILD_OSS STREQUAL "OFF") + add_thirdparty(darabonba_core) + add_thirdparty(alibabacloud_credentials) + add_thirdparty(alibabacloud_open_api_v2) + add_thirdparty(alibabacloud_sts20150401) + add_thirdparty(alibabacloud-oss-cpp-sdk) + add_thirdparty(apr) + add_thirdparty(apr-util LIBNAME "lib/libaprutil.a") +endif() +# end oss libs + add_thirdparty(gtest NOTADD) add_thirdparty(gtest_main NOTADD) add_thirdparty(gmock NOTADD) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index e6126df77cc34b..52dd21356c47b8 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -76,6 +76,7 @@ else fi BUILD_AZURE="ON" +BUILD_OSS="ON" while true; do case "$1" in @@ -127,6 +128,10 @@ if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]] BUILD_AZURE='OFF' fi +if [[ "$(echo "${DISABLE_BUILD_OSS}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then + BUILD_OSS='OFF' +fi + echo "Get params: PARALLEL -- ${PARALLEL} CLEAN -- ${CLEAN} @@ -1920,6 +1925,99 @@ build_ali_sdk() { "${BUILD_SYSTEM}" install } +build_apr() { + check_if_source_exist "${APR_SOURCE}" + cd "${TP_SOURCE_DIR}/${APR_SOURCE}" + CPPFLAGS="-I${TP_INCLUDE_DIR}" LDFLAGS="-L${TP_LIB_DIR}" \ + CC=gcc ./configure --enable-layout=classic --prefix="${TP_INSTALL_DIR}" \ + --disable-shared --enable-static + make -j "${PARALLEL}" + make install +} + +build_apr_util() { + check_if_source_exist "${APR_UTIL_SOURCE}" + cd "${TP_SOURCE_DIR}/${APR_UTIL_SOURCE}" + CPPFLAGS="-I${TP_INCLUDE_DIR}" LDFLAGS="-L${TP_LIB_DIR}" \ + CC=gcc ./configure --enable-layout=classic --prefix="${TP_INSTALL_DIR}" \ + --disable-shared --enable-static \ + --with-apr="${TP_INSTALL_DIR}" + make -j "${PARALLEL}" + make install +} + +_build_oss_cmake_lib() { + local src_var="$1" + local src_dir="${TP_SOURCE_DIR}/${!src_var}" + check_if_source_exist "${!src_var}" + cd "${src_dir}" + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + "${CMAKE_CMD}" -G "${GENERATOR}" \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" \ + -DBUILD_PRODUCT=core \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ + -DTP_INSTALL_DIR="${TP_INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS=OFF \ + -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \ + -DCURL_LIBRARY="${TP_LIB_DIR}/libcurl.a" \ + -DCURL_INCLUDE_DIR="${TP_INCLUDE_DIR}" \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + .. + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install +} + +build_tea_cpp() { + if [[ "${BUILD_OSS}" == "OFF" ]]; then return; fi + _build_oss_cmake_lib TEA_CPP_SOURCE +} + +build_credentials_cpp() { + if [[ "${BUILD_OSS}" == "OFF" ]]; then return; fi + _build_oss_cmake_lib CREDENTIALS_CPP_SOURCE +} + +build_openapi_v2() { + if [[ "${BUILD_OSS}" == "OFF" ]]; then return; fi + _build_oss_cmake_lib OPENAPI_V2_SOURCE +} + +build_sts_20150401() { + if [[ "${BUILD_OSS}" == "OFF" ]]; then return; fi + _build_oss_cmake_lib STS_20150401_SOURCE +} + +build_oss_cpp() { + if [[ "${BUILD_OSS}" == "OFF" ]]; then return; fi + build_apr + build_apr_util + check_if_source_exist "${OSS_CPP_SOURCE}" + cd "${TP_SOURCE_DIR}/${OSS_CPP_SOURCE}" + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + "${CMAKE_CMD}" -G "${GENERATOR}" \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" \ + -DBUILD_PRODUCT=core \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ + -DTP_INSTALL_DIR="${TP_INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_SAMPLE=OFF \ + -DBUILD_TESTS=OFF \ + -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}" \ + -DCMAKE_CXX_FLAGS="-fvisibility=hidden -D_GLIBCXX_USE_CXX11_ABI=1" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + .. + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install +} + # base64 build_base64() { check_if_source_exist "${BASE64_SOURCE}" @@ -2269,6 +2367,11 @@ if [[ "${#packages[@]}" -eq 0 ]]; then libdeflate streamvbyte ali_sdk + tea_cpp + credentials_cpp + openapi_v2 + sts_20150401 + oss_cpp base64 azure brotli diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index b4d3e5b42d744c..8821418185abd8 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -729,6 +729,44 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi +# patch tea-cpp for OpenSSL 1.1.1 compatibility (EVP_MD_CTX_get0_md → EVP_MD_CTX_md) +if [[ " ${TP_ARCHIVES[*]} " =~ " TEA_CPP " ]]; then + cd "${TP_SOURCE_DIR}/${TEA_CPP_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + echo "Applying OpenSSL 1.1.1 compatibility fix to tea-cpp..." + sed -i.bak 's/EVP_MD_CTX_get0_md/EVP_MD_CTX_md/g' include/darabonba/signature/RSASigner.hpp + touch "${PATCHED_MARK}" + fi + cd - + echo "Finished patching ${TEA_CPP_SOURCE}" +fi + +# patch tea-cpp: CURLOPT_CAINFO support + CURL error propagation (required for OSS STS SSL) +if [[ " ${TP_ARCHIVES[*]} " =~ " TEA_CPP " ]]; then + cd "${TP_SOURCE_DIR}/${TEA_CPP_SOURCE}" + if [[ ! -f "patched_mark_doris_oss_fix" ]]; then + echo "Applying OSS SSL/error fix to tea-cpp..." + patch -p1 <"${TP_PATCH_DIR}/tea-cpp-doris-oss-fix.patch" + touch "patched_mark_doris_oss_fix" + fi + cd - + echo "Finished patching ${TEA_CPP_SOURCE} (doris-oss-fix)" +fi + +# patch credentials-cpp for OpenSSL 1.1.1 compatibility (EVP_MD_CTX_get0_md → EVP_MD_CTX_md) +if [[ " ${TP_ARCHIVES[*]} " =~ " CREDENTIALS_CPP " ]]; then + cd "${TP_SOURCE_DIR}/${CREDENTIALS_CPP_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + echo "Applying OpenSSL 1.1.1 compatibility fix to credentials-cpp..." + if [[ -f "src/credentials/providers/RsaKeyPairCredentialsProvider.cpp" ]]; then + sed -i.bak 's/EVP_MD_CTX_get0_md/EVP_MD_CTX_md/g' src/credentials/providers/RsaKeyPairCredentialsProvider.cpp + fi + touch "${PATCHED_MARK}" + fi + cd - + echo "Finished patching ${CREDENTIALS_CPP_SOURCE}" +fi + # patch paimon-cpp if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" diff --git a/thirdparty/patches/tea-cpp-doris-oss-fix.patch b/thirdparty/patches/tea-cpp-doris-oss-fix.patch new file mode 100644 index 00000000000000..29e4ee37c5511f --- /dev/null +++ b/thirdparty/patches/tea-cpp-doris-oss-fix.patch @@ -0,0 +1,63 @@ +--- a/include/darabonba/Core.hpp ++++ b/include/darabonba/Core.hpp +@@ -57,4 +57,5 @@ + std::string https_proxy; // Per-request HTTPS proxy + std::string no_proxy; // Per-request no-proxy list + std::string credential; // Per-request credentials ++ std::string ca_cert; // Per-request CA cert file path + }; +--- a/src/darabonba/Core.cpp ++++ b/src/darabonba/Core.cpp +@@ -126,6 +126,9 @@ + if (runtime.contains("credential")) { + config.credential = runtime["credential"].get(); + } ++ if (runtime.contains("ca")) { ++ config.ca_cert = runtime["ca"].get(); ++ } + } + + return config; +@@ -221,6 +224,7 @@ + request_runtime["httpProxy"] = request_config.http_proxy; + request_runtime["httpsProxy"] = request_config.https_proxy; + request_runtime["noProxy"] = request_config.no_proxy; ++ request_runtime["ca"] = request_config.ca_cert; + + return it->second->makeRequest(request, request_runtime); + } +--- a/src/darabonba/http/MCurlHttpClient.cpp ++++ b/src/darabonba/http/MCurlHttpClient.cpp +@@ -30,6 +30,10 @@ + } else { + curl_easy_setopt(easyHandle, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(easyHandle, CURLOPT_SSL_VERIFYHOST, + 2L); // 2L for full verification ++ std::string caFile = options.value("ca", std::string {}); ++ if (!caFile.empty()) { ++ curl_easy_setopt(easyHandle, CURLOPT_CAINFO, caFile.c_str()); ++ } + } + // timeout +@@ -154,7 +158,21 @@ + while ((msg = curl_multi_info_read(mCurl_, &msgs_in_queue)) != nullptr) { + if (msg->msg == CURLMSG_DONE) { + auto easyHandle = msg->easy_handle; ++ auto curlResult = msg->data.result; + auto curlStorage = std::move(runningCurl_[easyHandle]); + runningCurl_.erase(easyHandle); + ++ if (curlResult != CURLE_OK) { ++ const char* errorStr = curl_easy_strerror(curlResult); ++ std::string errorMsg = std::string("CURL error: ") + errorStr; ++ ++ curl_slist_free_all(curlStorage->reqHeader); ++ curl_multi_remove_handle(mCurl_, easyHandle); ++ curl_easy_cleanup(easyHandle); ++ ++ curlStorage->promise->set_exception( ++ std::make_exception_ptr(std::runtime_error(errorMsg))); ++ continue; ++ } ++ + auto body = dynamic_cast( diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index f8d18a3a2bea16..6249a5db32ba9b 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -579,6 +579,51 @@ LANCE_C_NAME="lance-c-v0.1.2.tar.gz" LANCE_C_SOURCE="lance-c-0.1.2" LANCE_C_MD5SUM="eb6ec9bc63fa5245864282f24b521d0b" +# Alibaba Cloud OSS C++ SDK stack +# Required for native OSS data-plane access in BE (enable with USE_OSS=ON, disable with DISABLE_BUILD_OSS=ON) + +# APR — required by aliyun-oss-cpp-sdk +APR_DOWNLOAD="https://dlcdn.apache.org//apr/apr-1.7.6.tar.gz" +APR_NAME="apr-1.7.6.tar.gz" +APR_SOURCE="apr-1.7.6" +APR_MD5SUM="44f27088e423da7cefc1a9717b910d85" + +# APR-Util — required by aliyun-oss-cpp-sdk +APR_UTIL_DOWNLOAD="https://dlcdn.apache.org//apr/apr-util-1.6.3.tar.gz" +APR_UTIL_NAME="apr-util-1.6.3.tar.gz" +APR_UTIL_SOURCE="apr-util-1.6.3" +APR_UTIL_MD5SUM="b2b6fb440548869dc228535e339f619b" + +# tea-cpp — Alibaba darabonba transport core (MIT) +TEA_CPP_DOWNLOAD="https://github.com/aliyun/tea-cpp/archive/b063160af903fe8385f98a95b77a4a8804e09816.tar.gz" +TEA_CPP_NAME="tea-cpp-b063160.tar.gz" +TEA_CPP_SOURCE="tea-cpp-b063160af903fe8385f98a95b77a4a8804e09816" +TEA_CPP_MD5SUM="aeccc8ef5766f0001f81c6cee659776c" + +# credentials-cpp — Alibaba Cloud credential providers (Apache 2.0) +CREDENTIALS_CPP_DOWNLOAD="https://github.com/aliyun/credentials-cpp/archive/f7584e1827b4672d237448f6c8fe348602edf1ae.tar.gz" +CREDENTIALS_CPP_NAME="credentials-cpp-f7584e1.tar.gz" +CREDENTIALS_CPP_SOURCE="credentials-cpp-f7584e1827b4672d237448f6c8fe348602edf1ae" +CREDENTIALS_CPP_MD5SUM="9ece163713695d54aebabc73f388aef4" + +# alibabacloud-open-api-v2 — Alibaba Cloud SDK v2 core (Apache 2.0) +OPENAPI_V2_DOWNLOAD="https://github.com/alibabacloud-sdk-cpp/alibabacloud-open-api-v2/archive/b915b2f8673e4c20aa88c3099f28c654128ee20d.tar.gz" +OPENAPI_V2_NAME="alibabacloud-open-api-v2-b915b2f.tar.gz" +OPENAPI_V2_SOURCE="alibabacloud-open-api-v2-b915b2f8673e4c20aa88c3099f28c654128ee20d" +OPENAPI_V2_MD5SUM="7a1e3307f4b33c6e61810f2c06f74d11" + +# sts-20150401 — STS AssumeRole SDK (Apache 2.0) +STS_20150401_DOWNLOAD="https://github.com/alibabacloud-sdk-cpp/sts-20150401/archive/refs/tags/1.0.9.tar.gz" +STS_20150401_NAME="sts-20150401-1.0.9.tar.gz" +STS_20150401_SOURCE="sts-20150401-1.0.9" +STS_20150401_MD5SUM="ff8a75c6a8f83b8dd694cfa29171ffc7" + +# aliyun-oss-cpp-sdk — Alibaba Cloud OSS C++ SDK (Apache 2.0) +OSS_CPP_DOWNLOAD="https://github.com/aliyun/aliyun-oss-cpp-sdk/archive/refs/tags/1.10.1.tar.gz" +OSS_CPP_NAME="aliyun-oss-cpp-sdk-1.10.1.tar.gz" +OSS_CPP_SOURCE="aliyun-oss-cpp-sdk-1.10.1" +OSS_CPP_MD5SUM="7b1628a678a47148e6c25c88ecf2b5f6" + # all thirdparties which need to be downloaded is set in array TP_ARCHIVES export TP_ARCHIVES=( 'LIBEVENT' @@ -667,6 +712,12 @@ export TP_ARCHIVES=( 'LANCE_C' ) +# OSS C++ SDK stack — added only when OSS native SDK build is enabled +if [[ "$(echo "${DISABLE_BUILD_OSS}" | tr '[:lower:]' '[:upper:]')" != "ON" ]]; then + read -r -a TP_ARCHIVES <<<"${TP_ARCHIVES[*]} APR APR_UTIL TEA_CPP CREDENTIALS_CPP OPENAPI_V2 STS_20150401 OSS_CPP" + export TP_ARCHIVES +fi + if [[ "$(uname -s)" == 'Darwin' ]]; then #binutils BINUTILS_DOWNLOAD='https://ftp.nluug.nl/pub/gnu/binutils/binutils-2.39.tar.gz'