From eb78bf29db8660edc28439179b5f761868d15b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Oct 2026 10:47:01 +0100 Subject: [PATCH 1/9] GH-55258: [C++] Move Azure to Filesystem Registry --- cpp/src/arrow/filesystem/azurefs.cc | 11 +++++++++++ cpp/src/arrow/filesystem/azurefs_test.cc | 18 ++++++++++++++++++ cpp/src/arrow/filesystem/filesystem.cc | 13 ------------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index 20b0a655d8e2..7e97d74b4ccb 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -3561,4 +3561,15 @@ Result AzureFileSystem::PathFromUri(const std::string& uri_string) return path; } +static Result> AzureFileSystemFactory( + const Uri& uri, const io::IOContext& io_context, std::string* out_path) { + ARROW_ASSIGN_OR_RAISE(auto options, AzureOptions::FromUri(uri, out_path)); + return AzureFileSystem::Make(options, io_context); +} + +FileSystemRegistrar kAzureFileSystemRegistrar[]{ + ARROW_REGISTER_FILESYSTEM("abfs", AzureFileSystemFactory, {}), + ARROW_REGISTER_FILESYSTEM("abfss", AzureFileSystemFactory, {}), +}; + } // namespace arrow::fs diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 250e3572bd9b..f5adbb8e1f2d 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -509,6 +509,24 @@ TEST(AzureFileSystem, InitializingFilesystemWithoutAccountNameFails) { ASSERT_RAISES(Invalid, AzureFileSystem::Make(options)); } +TEST(AzureFileSystem, FileSystemFromUriRegistry) { + std::string path; + for (const auto& scheme : {"abfs", "abfss"}) { + ASSERT_OK_AND_ASSIGN( + auto fs, FileSystemFromUri(std::string(scheme) + + "://account.blob.core.windows.net/container/dir/" + "blob?credential_kind=anonymous", + &path)); + EXPECT_EQ(fs->type_name(), "abfs"); + EXPECT_EQ(path, "container/dir/blob"); + } + EXPECT_RAISES_WITH_MESSAGE_THAT( + NotImplemented, ::testing::HasSubstr("Filesystem factory does not support"), + FileSystemFromUriAndOptions( + "abfs://account.blob.core.windows.net/container?credential_kind=anonymous", + {{"foo", std::string{"bar"}}})); +} + TEST(AzureFileSystem, InitializeWithDefaultCredential) { AzureOptions options; options.account_name = "dummy-account-name"; diff --git a/cpp/src/arrow/filesystem/filesystem.cc b/cpp/src/arrow/filesystem/filesystem.cc index c15bc06681d6..6b49f05eb0b1 100644 --- a/cpp/src/arrow/filesystem/filesystem.cc +++ b/cpp/src/arrow/filesystem/filesystem.cc @@ -25,9 +25,6 @@ #include "arrow/util/config.h" #include "arrow/filesystem/filesystem.h" -#ifdef ARROW_AZURE -# include "arrow/filesystem/azurefs.h" -#endif #ifdef ARROW_HDFS # include "arrow/filesystem/hdfs.h" #endif @@ -909,16 +906,6 @@ Result> FileSystemFromUriReal( scheme, "', got ", options.size(), " option(s)"); } - if (scheme == "abfs" || scheme == "abfss") { -#ifdef ARROW_AZURE - ARROW_ASSIGN_OR_RAISE(auto options, AzureOptions::FromUri(uri, out_path)); - return AzureFileSystem::Make(options, io_context); -#else - return Status::NotImplemented( - "Got Azure Blob File System URI but Arrow compiled without Azure Blob File " - "System support"); -#endif - } if (scheme == "hdfs" || scheme == "viewfs") { #ifdef ARROW_HDFS ARROW_ASSIGN_OR_RAISE(auto options, HdfsOptions::FromUri(uri)); From 6238191e42d64a48a7f38d10fdeecc20885c80fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 5 Nov 2026 17:39:51 +0100 Subject: [PATCH 2/9] Initial move of Azure to its own module --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 17 +++-- cpp/src/arrow/ArrowAzureConfig.cmake.in | 44 +++++++++++++ cpp/src/arrow/CMakeLists.txt | 72 +++++++++++++++------ cpp/src/arrow/arrow-azure.pc.in | 30 +++++++++ cpp/src/arrow/filesystem/CMakeLists.txt | 21 +++++- cpp/src/arrow/filesystem/azure_visibility.h | 49 ++++++++++++++ cpp/src/arrow/filesystem/azurefs.h | 5 +- cpp/src/arrow/filesystem/azurefs_internal.h | 4 +- cpp/src/arrow/filesystem/azurefs_module.cc | 18 ++++++ 9 files changed, 233 insertions(+), 27 deletions(-) create mode 100644 cpp/src/arrow/ArrowAzureConfig.cmake.in create mode 100644 cpp/src/arrow/arrow-azure.pc.in create mode 100644 cpp/src/arrow/filesystem/azure_visibility.h create mode 100644 cpp/src/arrow/filesystem/azurefs_module.cc diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f4aa19bab59b..d21cd3781b81 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4443,13 +4443,19 @@ function(build_azure_sdk) endfunction() if(ARROW_WITH_AZURE_SDK) - resolve_dependency(Azure REQUIRED_VERSION 1.10.2) + resolve_dependency(Azure + ARROW_CMAKE_PACKAGE_NAME + ArrowAzure + ARROW_PC_PACKAGE_NAME + arrow-azure + REQUIRED_VERSION + 1.10.2) set(AZURE_SDK_LINK_LIBRARIES Azure::azure-storage-files-datalake Azure::azure-storage-blobs Azure::azure-identity) if(AZURE_SDK_VENDORED AND NOT WIN32) - find_curl(ARROW) + find_curl(ARROW_AZURE) find_package(LibXml2 REQUIRED) - list(APPEND ARROW_SYSTEM_DEPENDENCIES LibXml2) + list(APPEND ARROW_AZURE_SYSTEM_DEPENDENCIES LibXml2) endif() endif() @@ -4464,12 +4470,13 @@ if(ARROW_BUILD_STATIC) if(ARROW_AZURE AND AZURE_SDK_VENDORED AND NOT WIN32) - arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl" ARROW) + arrow_append_pc_system_library("libcurl" ARROW_AZURE_CURL_PC "-lcurl" ARROW_AZURE) endif() if(ARROW_AZURE AND AZURE_SDK_VENDORED AND NOT WIN32) - arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2" ARROW) + arrow_append_pc_system_library("libxml-2.0" ARROW_AZURE_LIBXML2_PC "-lxml2" + ARROW_AZURE) endif() if(ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED" diff --git a/cpp/src/arrow/ArrowAzureConfig.cmake.in b/cpp/src/arrow/ArrowAzureConfig.cmake.in new file mode 100644 index 000000000000..c6257ee70ad9 --- /dev/null +++ b/cpp/src/arrow/ArrowAzureConfig.cmake.in @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This config sets the following variables in your project:: +# +# ArrowAzure_FOUND - true if Arrow Azure found on the system +# +# This config sets the following targets in your project:: +# +# ArrowAzure::arrow_azure_shared - for linked as shared library if shared library is built +# ArrowAzure::arrow_azure_static - for linked as static library if static library is built + +@PACKAGE_INIT@ + +set(ARROW_AZURE_SYSTEM_DEPENDENCIES "@ARROW_AZURE_SYSTEM_DEPENDENCIES@") + +include(CMakeFindDependencyMacro) +find_dependency(Arrow CONFIG) + +if(ARROW_BUILD_STATIC) + arrow_find_dependencies("${ARROW_AZURE_SYSTEM_DEPENDENCIES}") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/ArrowAzureTargets.cmake") + +arrow_keep_backward_compatibility(ArrowAzure arrow_azure) + +check_required_components(ArrowAzure) + +arrow_show_details(ArrowAzure ARROW_AZURE) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 6ceca297be3d..815de5578c1f 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -33,16 +33,6 @@ set(ARROW_SHARED_PRIVATE_LINK_LIBS) set(ARROW_SHARED_INSTALL_INTERFACE_LIBS) set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) -if(ARROW_AZURE) - if(Azure_SOURCE STREQUAL "SYSTEM") - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES}) - elseif(Azure_SOURCE STREQUAL "BUNDLED") - if(NOT WIN32) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl LibXml2::LibXml2) - endif() - endif() -endif() - if(ARROW_USE_OPENSSL) list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) endif() @@ -964,9 +954,61 @@ if(ARROW_FILESYSTEM) filesystem/util_internal.cc) if(ARROW_AZURE) - list(APPEND ARROW_FILESYSTEM_SRCS filesystem/azurefs.cc) - set_source_files_properties(filesystem/azurefs.cc + set(ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) + if(Azure_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES}) + elseif(Azure_SOURCE STREQUAL "BUNDLED") + if(NOT WIN32) + list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl + LibXml2::LibXml2) + endif() + endif() + set(ARROW_AZURE_LIB_SRCS filesystem/azurefs_module.cc filesystem/azurefs.cc) + set_source_files_properties(filesystem/azurefs_module.cc filesystem/azurefs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + # If libarrow_azure.a is only built, "pkg-config --cflags --libs + # arrow-azure" outputs build flags for static linking not shared + # linking. ARROW_AZURE_PC_* except ARROW_AZURE_PC_*_PRIVATE are for the + # static linking case. + if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC) + string(APPEND ARROW_AZURE_PC_CFLAGS "${ARROW_AZURE_PC_CFLAGS_PRIVATE}") + set(ARROW_AZURE_PC_CFLAGS_PRIVATE "") + set(ARROW_AZURE_PC_LIBS "${ARROW_AZURE_PC_LIBS_PRIVATE}") + set(ARROW_AZURE_PC_LIBS_PRIVATE "") + set(ARROW_AZURE_PC_REQUIRES "${ARROW_AZURE_PC_REQUIRES_PRIVATE}") + set(ARROW_AZURE_PC_REQUIRES_PRIVATE "") + else() + set(ARROW_AZURE_PC_LIBS "") + set(ARROW_AZURE_PC_REQUIRES "") + endif() + add_arrow_lib(arrow_azure + CMAKE_PACKAGE_NAME + ArrowAzure + PKG_CONFIG_NAME + arrow-azure + SOURCES + ${ARROW_AZURE_LIB_SRCS} + SHARED_LINK_LIBS + arrow_shared + SHARED_PRIVATE_LINK_LIBS + ${AZURE_SDK_LINK_LIBRARIES} + SHARED_INSTALL_INTERFACE_LIBS + Arrow::arrow_shared + STATIC_LINK_LIBS + arrow_static + ${AZURE_SDK_LINK_LIBRARIES} + STATIC_INSTALL_INTERFACE_LIBS + ${ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_LINK_FLAGS + ${ARROW_VERSION_SCRIPT_FLAGS} + OUTPUTS + ARROW_AZURE_LIBRARIES) + foreach(LIB_TARGET ${ARROW_AZURE_LIBRARIES}) + target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_AZURE_EXPORTING) + endforeach() + if(ARROW_BUILD_STATIC AND WIN32) + target_compile_definitions(arrow_azure_static PUBLIC ARROW_AZURE_STATIC) + endif() endif() if(ARROW_GCS) set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) @@ -1085,12 +1127,6 @@ if(ARROW_FILESYSTEM) endif() arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS}) - if(ARROW_AZURE) - foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS}) - target_link_libraries(${ARROW_FILESYSTEM_TARGET} - PRIVATE ${AZURE_SDK_LINK_LIBRARIES}) - endforeach() - endif() if(ARROW_HDFS) foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS}) target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE arrow::hadoop) diff --git a/cpp/src/arrow/arrow-azure.pc.in b/cpp/src/arrow/arrow-azure.pc.in new file mode 100644 index 000000000000..dca6f722e4f6 --- /dev/null +++ b/cpp/src/arrow/arrow-azure.pc.in @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +prefix=@CMAKE_INSTALL_PREFIX@ +includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@ +libdir=@ARROW_PKG_CONFIG_LIBDIR@ + +Name: Apache Arrow Azure +Description: Apache Arrow's Azure filesystem implementation. +Version: @ARROW_VERSION@ +Requires: arrow@ARROW_AZURE_PC_REQUIRES@ +Requires.private:@ARROW_AZURE_PC_REQUIRES_PRIVATE@ +Libs: -L${libdir} -larrow_azure@ARROW_AZURE_PC_LIBS@ +Libs.private:@ARROW_AZURE_PC_LIBS_PRIVATE@ +Cflags:@ARROW_AZURE_PC_CFLAGS@ +Cflags.private:@ARROW_AZURE_PC_CFLAGS_PRIVATE@ diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 154c2161938a..0e34b165926e 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -75,11 +75,30 @@ if(ARROW_GCS) endif() if(ARROW_AZURE) + set(ARROW_AZURE_TEST_EXTRA_LINK_LIBS) + if(ARROW_TEST_LINKAGE STREQUAL "shared") + list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS arrow_azure_shared) + else() + list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS arrow_azure_static) + endif() + list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS ${AZURE_SDK_LINK_LIBRARIES}) add_arrow_test(azurefs_test EXTRA_LABELS filesystem EXTRA_LINK_LIBS - ${AZURE_SDK_LINK_LIBRARIES}) + ${ARROW_AZURE_TEST_EXTRA_LINK_LIBS}) + # TODO: + #if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED) + # add_arrow_test(azurefs_module_test + # SOURCES + # azurefs_module_test.cc + # EXTRA_LABELS + # filesystem + # DEFINITIONS + # ARROW_AZURE_LIBPATH="$" + # EXTRA_DEPENDENCIES + # arrow_azure_shared) + #endif() endif() if(ARROW_S3) diff --git a/cpp/src/arrow/filesystem/azure_visibility.h b/cpp/src/arrow/filesystem/azure_visibility.h new file mode 100644 index 000000000000..aa62b5ba53c1 --- /dev/null +++ b/cpp/src/arrow/filesystem/azure_visibility.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(_WIN32) || defined(__CYGWIN__) +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4251) +# else +# pragma GCC diagnostic ignored "-Wattributes" +# endif + +# ifdef ARROW_AZURE_STATIC +# define ARROW_AZURE_EXPORT +# elif defined(ARROW_AZURE_EXPORTING) +# define ARROW_AZURE_EXPORT __declspec(dllexport) +# else +# define ARROW_AZURE_EXPORT __declspec(dllimport) +# endif + +# define ARROW_AZURE_NO_EXPORT + +# if defined(_MSC_VER) +# pragma warning(pop) +# endif + +#else // Not Windows +# ifndef ARROW_AZURE_EXPORT +# define ARROW_AZURE_EXPORT __attribute__((visibility("default"))) +# endif +# ifndef ARROW_AZURE_NO_EXPORT +# define ARROW_AZURE_NO_EXPORT __attribute__((visibility("hidden"))) +# endif +#endif diff --git a/cpp/src/arrow/filesystem/azurefs.h b/cpp/src/arrow/filesystem/azurefs.h index d21bcd08df81..dc70d2b06c34 100644 --- a/cpp/src/arrow/filesystem/azurefs.h +++ b/cpp/src/arrow/filesystem/azurefs.h @@ -21,6 +21,7 @@ #include #include +#include "arrow/filesystem/azure_visibility.h" #include "arrow/filesystem/filesystem.h" #include "arrow/util/macros.h" #include "arrow/util/uri.h" @@ -59,7 +60,7 @@ class TestAzureOptions; /// - `AZURE_FEDERATED_TOKEN_FILE` /// /// Functions are provided for explicit configuration of credentials if that is preferred. -struct ARROW_EXPORT AzureOptions { +struct ARROW_AZURE_EXPORT AzureOptions { friend class TestAzureOptions; /// \brief The name of the Azure Storage Account being accessed. @@ -247,7 +248,7 @@ struct ARROW_EXPORT AzureOptions { /// [2]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage /// [3]: /// https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-known-issues -class ARROW_EXPORT AzureFileSystem : public FileSystem { +class ARROW_AZURE_EXPORT AzureFileSystem : public FileSystem { private: class Impl; std::unique_ptr impl_; diff --git a/cpp/src/arrow/filesystem/azurefs_internal.h b/cpp/src/arrow/filesystem/azurefs_internal.h index c6d730fcb938..836ce825ecb3 100644 --- a/cpp/src/arrow/filesystem/azurefs_internal.h +++ b/cpp/src/arrow/filesystem/azurefs_internal.h @@ -17,6 +17,7 @@ #pragma once +#include "arrow/filesystem/azure_visibility.h" #include "arrow/result.h" namespace Azure::Storage::Files::DataLake { @@ -70,7 +71,8 @@ enum class HierarchicalNamespaceSupport { /// account. /// \return kEnabled/kDisabled/kContainerNotFound (kUnknown is never /// returned). -ARROW_EXPORT Result CheckIfHierarchicalNamespaceIsEnabled( +ARROW_AZURE_EXPORT Result +CheckIfHierarchicalNamespaceIsEnabled( const Azure::Storage::Files::DataLake::DataLakeFileSystemClient& adlfs_client, const arrow::fs::AzureOptions& options); diff --git a/cpp/src/arrow/filesystem/azurefs_module.cc b/cpp/src/arrow/filesystem/azurefs_module.cc new file mode 100644 index 000000000000..d91413138b45 --- /dev/null +++ b/cpp/src/arrow/filesystem/azurefs_module.cc @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/filesystem/filesystem_library.h" From 9bc9f3e03ec93509d8568b1659f07a247270e72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 6 Nov 2026 12:29:40 +0100 Subject: [PATCH 3/9] Add bindings for python and R --- python/CMakeLists.txt | 16 ++++++++++++++++ r/configure | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 750f0f06f576..b263c189e212 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -730,6 +730,18 @@ if(PYARROW_BUILD_AZURE) if(NOT ARROW_AZURE) message(FATAL_ERROR "You must build Arrow C++ with ARROW_AZURE=ON") endif() + find_package(ArrowAzure REQUIRED) + if(ARROW_BUILD_SHARED) + if(PYARROW_BUNDLE_ARROW_CPP) + bundle_arrow_lib(${ARROW_AZURE_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) + if(MSVC) + bundle_arrow_import_lib(${ARROW_AZURE_IMPORT_LIB}) + endif() + endif() + set(AZURE_LINK_LIBS ArrowAzure::arrow_azure_shared) + else() + set(AZURE_LINK_LIBS ArrowAzure::arrow_azure_static) + endif() list(APPEND CYTHON_EXTENSIONS _azurefs) endif() @@ -1066,6 +1078,10 @@ if(PYARROW_BUILD_PARQUET) endif() endif() +if(PYARROW_BUILD_AZURE) + target_link_libraries(_azurefs PRIVATE ${AZURE_LINK_LIBS}) +endif() + if(PYARROW_BUILD_GCS) target_link_libraries(_gcsfs PRIVATE ${GCS_LINK_LIBS}) endif() diff --git a/r/configure b/r/configure index e5fa53d6ab98..3ea4831c8bf7 100755 --- a/r/configure +++ b/r/configure @@ -365,6 +365,10 @@ add_feature_flags () { fi if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" + PKG_CONFIG_NAMES_FEATURES="$PKG_CONFIG_NAMES_FEATURES arrow-azure" + PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_azure $PKG_LIBS_FEATURES_WITHOUT_PC" + # NOTE: arrow_azure is assumed to have the same -L flag as arrow + # so there is no need to add its location to PKG_DIRS PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2" fi if arrow_built_with ARROW_GCS; then From 7f2881536728f94d6f69e074ce38bcf38c7fedbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 6 Nov 2026 13:34:13 +0100 Subject: [PATCH 4/9] Add azure module test --- cpp/src/arrow/filesystem/CMakeLists.txt | 23 ++++---- .../arrow/filesystem/azurefs_module_test.cc | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 cpp/src/arrow/filesystem/azurefs_module_test.cc diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 0e34b165926e..9c2772a99d36 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -87,18 +87,17 @@ if(ARROW_AZURE) filesystem EXTRA_LINK_LIBS ${ARROW_AZURE_TEST_EXTRA_LINK_LIBS}) - # TODO: - #if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED) - # add_arrow_test(azurefs_module_test - # SOURCES - # azurefs_module_test.cc - # EXTRA_LABELS - # filesystem - # DEFINITIONS - # ARROW_AZURE_LIBPATH="$" - # EXTRA_DEPENDENCIES - # arrow_azure_shared) - #endif() + if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED) + add_arrow_test(azurefs_module_test + SOURCES + azurefs_module_test.cc + EXTRA_LABELS + filesystem + DEFINITIONS + ARROW_AZURE_LIBPATH="$" + EXTRA_DEPENDENCIES + arrow_azure_shared) + endif() endif() if(ARROW_S3) diff --git a/cpp/src/arrow/filesystem/azurefs_module_test.cc b/cpp/src/arrow/filesystem/azurefs_module_test.cc new file mode 100644 index 000000000000..7317385dd5d5 --- /dev/null +++ b/cpp/src/arrow/filesystem/azurefs_module_test.cc @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "arrow/filesystem/filesystem.h" +#include "arrow/testing/gtest_util.h" + +namespace arrow::fs { + +class RegistrationTestEnvironment : public ::testing::Environment { + public: + void SetUp() override { ASSERT_OK(LoadFileSystemFactories(ARROW_AZURE_LIBPATH)); } +}; + +auto* lib_env = ::testing::AddGlobalTestEnvironment(new RegistrationTestEnvironment); + +TEST(AzureModuleTest, FromUri) { + std::string path; + for (const auto* scheme : {"abfs", "abfss"}) { + ASSERT_OK_AND_ASSIGN( + auto fs, FileSystemFromUri(std::string(scheme) + + "://account.blob.core.windows.net/container/dir/" + "blob?credential_kind=anonymous", + &path)); + EXPECT_EQ(fs->type_name(), "abfs"); + EXPECT_EQ(path, "container/dir/blob"); + } +} + +TEST(AzureModuleTest, FromUriAndOptionsNotImplemented) { + EXPECT_RAISES_WITH_MESSAGE_THAT( + NotImplemented, ::testing::HasSubstr("Filesystem factory does not support"), + FileSystemFromUriAndOptions( + "abfs://account.blob.core.windows.net/container?credential_kind=anonymous", + {{"foo", std::string{"bar"}}})); +} + +} // namespace arrow::fs From 053abc2bc2a3b7ecc3c78025e7282921586b08b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 9 Nov 2026 12:30:20 +0100 Subject: [PATCH 5/9] Add Linux packaging split for azure --- .../apache-arrow/debian/control.in | 25 +++++++++++++++ .../debian/libarrow-azure-2700.install | 1 + .../debian/libarrow-azure-dev.install | 4 +++ .../apache-arrow/yum/arrow.spec.in | 32 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install create mode 100644 dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index 5609724a56df..a945973a89b8 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -155,6 +155,19 @@ Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for GCS filesystem module. +Package: libarrow-azure-2700 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow2700 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Azure filesystem module. + Package: libarrow-flight2700 Section: libs Architecture: any @@ -289,6 +302,18 @@ Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for GCS filesystem module. +Package: libarrow-azure-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-azure-2700 (= ${binary:Version}), + libarrow-dev (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Azure filesystem module. + Package: libarrow-flight-dev Section: libdevel Architecture: any diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install new file mode 100644 index 000000000000..fcc16570a402 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_azure.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install new file mode 100644 index 000000000000..3209f6d9ed85 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install @@ -0,0 +1,4 @@ +usr/lib/*/cmake/ArrowAzure/ +usr/lib/*/libarrow_azure.a +usr/lib/*/libarrow_azure.so +usr/lib/*/pkgconfig/arrow-azure.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 946198442f66..c6e6bfec6153 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -392,6 +392,38 @@ Libraries and header files for Apache Arrow GCS filesystem. %{_libdir}/pkgconfig/arrow-gcs.pc %endif +%package -n %{name}%{so_version}-azure-libs +Summary: C++ library for the Azure filesystem +License: Apache-2.0 +Requires: %{name}%{so_version}-libs = %{version}-%{release} + +%description -n %{name}%{so_version}-azure-libs +This package contains the libraries for Apache Arrow Azure filesystem. + +%files -n %{name}%{so_version}-azure-libs +%defattr(-,root,root,-) +%doc README.md +%license LICENSE.txt NOTICE.txt +%{_libdir}/libarrow_azure.so.* + +%package azure-devel +Summary: Libraries and header files for Apache Arrow Azure filesystem +License: Apache-2.0 +Requires: %{name}%{so_version}-azure-libs = %{version}-%{release} +Requires: %{name}-devel = %{version}-%{release} + +%description azure-devel +Libraries and header files for Apache Arrow Azure filesystem. + +%files azure-devel +%defattr(-,root,root,-) +%doc README.md +%license LICENSE.txt NOTICE.txt +%{_libdir}/cmake/ArrowAzure/ +%{_libdir}/libarrow_azure.a +%{_libdir}/libarrow_azure.so +%{_libdir}/pkgconfig/arrow-azure.pc + %package -n %{name}%{so_version}-acero-libs Summary: C++ library to execute a query in streaming License: Apache-2.0 From 417c470e0231a76faa364f176e1f325b45a2cc38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Nov 2026 08:37:14 +0100 Subject: [PATCH 6/9] Some more changes to docs, meson build and validations --- ci/scripts/python_wheel_xlinux_build.sh | 3 +++ cpp/src/arrow/filesystem/meson.build | 1 + docs/source/cpp/build_system.rst | 2 ++ 3 files changed, 6 insertions(+) diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index 8931bfbfa9e0..a64ca50b61e8 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -27,6 +27,9 @@ function check_arrow_visibility { if [[ "${ARROW_GCS}" == "ON" ]]; then libs+=(libarrow_gcs.so) fi + if [[ "${ARROW_AZURE}" == "ON" ]]; then + libs+=(libarrow_azure.so) + fi for lib in "${libs[@]}"; do nm --demangle --dynamic "/tmp/arrow-dist/lib/${lib}" done > nm_arrow.log diff --git a/cpp/src/arrow/filesystem/meson.build b/cpp/src/arrow/filesystem/meson.build index e2eaa116a710..91183e257897 100644 --- a/cpp/src/arrow/filesystem/meson.build +++ b/cpp/src/arrow/filesystem/meson.build @@ -18,6 +18,7 @@ install_headers( [ 'api.h', + 'azure_visibility.h', 'azurefs.h', 'filesystem.h', 'filesystem_library.h', diff --git a/docs/source/cpp/build_system.rst b/docs/source/cpp/build_system.rst index 800631b6e098..6a61f5152d6d 100644 --- a/docs/source/cpp/build_system.rst +++ b/docs/source/cpp/build_system.rst @@ -115,6 +115,7 @@ This is the list of available packages: * ArrowCUDA * ArrowAcero +* ArrowAzure * ArrowCompute * ArrowDataset * ArrowFlight @@ -186,6 +187,7 @@ Available packages The Arrow C++ provides a pkg-config package for each module. Here are all available packages: + * ``arrow-azure`` * ``arrow-csv`` * ``arrow-cuda`` * ``arrow-dataset`` From 0dfcc22c51d6ea9bb67346263e6153f922726406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 Nov 2026 11:16:28 +0100 Subject: [PATCH 7/9] Add ARROW_EXAMPLE_AZURE_STATIC_CHECK test --- ci/scripts/cpp_test.sh | 4 +++- cpp/examples/minimal_build/CMakeLists.txt | 17 +++++++++++++++++ cpp/examples/minimal_build/example.cc | 11 +++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index e74a832f0d4d..3243bc788295 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -146,6 +146,7 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \ cmake \ -S "${source_dir}/examples/minimal_build" \ -B "${build_dir}/examples/minimal_build" \ + -DARROW_EXAMPLE_AZURE_STATIC_CHECK=ON \ -DARROW_EXAMPLE_GCS_STATIC_CHECK=ON \ -DARROW_EXAMPLE_S3_STATIC_CHECK=ON \ -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" @@ -157,7 +158,8 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \ # Test static filesystem linking for static_example in arrow-example-s3-static \ arrow-example-gcs-static \ - arrow-example-s3-gcs-static; do + arrow-example-s3-gcs-static \ + arrow-example-azure-static; do example="${build_dir}/examples/minimal_build/${static_example}" if [ -x "${example}" ]; then PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \ diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index c56ddb0b40ef..ad5fe03383d0 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -94,3 +94,20 @@ elseif(ARROW_EXAMPLE_GCS_STATIC_CHECK) message(STATUS "Skipping GCS static examples: requires ARROW_GCS and Arrow::arrow_static" ) endif() + +option(ARROW_EXAMPLE_AZURE_STATIC_CHECK "Verify Azure and static linking" OFF) +if(ARROW_EXAMPLE_AZURE_STATIC_CHECK + AND ARROW_AZURE + AND TARGET Arrow::arrow_static) + find_package(ArrowAzure REQUIRED) + if(NOT TARGET ArrowAzure::arrow_azure_static) + message(FATAL_ERROR "ArrowAzure::arrow_azure_static not found") + endif() + add_executable(arrow-example-azure-static example.cc) + target_compile_definitions(arrow-example-azure-static + PRIVATE ARROW_EXAMPLE_AZURE_STATIC_CHECK) + target_link_libraries(arrow-example-azure-static PRIVATE ArrowAzure::arrow_azure_static) +elseif(ARROW_EXAMPLE_AZURE_STATIC_CHECK) + message(STATUS "Skipping Azure static example: requires ARROW_AZURE and Arrow::arrow_static" + ) +endif() diff --git a/cpp/examples/minimal_build/example.cc b/cpp/examples/minimal_build/example.cc index 3652bc2e4341..feaaa719a8cd 100644 --- a/cpp/examples/minimal_build/example.cc +++ b/cpp/examples/minimal_build/example.cc @@ -16,6 +16,9 @@ // under the License. #include +#ifdef ARROW_EXAMPLE_AZURE_STATIC_CHECK +# include +#endif #ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK # include #endif @@ -72,6 +75,14 @@ Status RunMain(int argc, char** argv) { arrow::fs::GcsFileSystem::Make(arrow::fs::GcsOptions::Anonymous()).status()); #endif +#ifdef ARROW_EXAMPLE_AZURE_STATIC_CHECK + std::cerr << "* Verify Azure filesystem construction" << std::endl; + arrow::fs::AzureOptions azure_options; + azure_options.account_name = "example"; + ARROW_RETURN_NOT_OK(azure_options.ConfigureAnonymousCredential()); + ARROW_RETURN_NOT_OK(arrow::fs::AzureFileSystem::Make(azure_options).status()); +#endif + return Status::OK(); } From 256029334d12b3d4846372445f6b2c1f2d68a881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 Nov 2026 11:48:23 +0100 Subject: [PATCH 8/9] Add missing PC flags --- cpp/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 22bd1028b95e..fe661e37227d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -493,6 +493,15 @@ endif() set(ARROW_GCS_PC_LIBS_PRIVATE "") set(ARROW_GCS_PC_REQUIRES_PRIVATE "") +# For arrow-azure.pc. +set(ARROW_AZURE_PC_CFLAGS "") +set(ARROW_AZURE_PC_CFLAGS_PRIVATE "") +if(ARROW_BUILD_STATIC) + string(APPEND ARROW_AZURE_PC_CFLAGS_PRIVATE " -DARROW_AZURE_STATIC") +endif() +set(ARROW_AZURE_PC_LIBS_PRIVATE "") +set(ARROW_AZURE_PC_REQUIRES_PRIVATE "") + # For arrow-s3.pc. set(ARROW_S3_PC_CFLAGS "") set(ARROW_S3_PC_CFLAGS_PRIVATE "") From 79a44fdfe9d15f55006b2dbbecc5fd7f0ea643cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 Nov 2026 13:56:29 +0100 Subject: [PATCH 9/9] Move Azure and its dependencies to ARROW_AZURE_BUNDLED_STATIC_LIBS --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 19 +++++------ cpp/examples/minimal_build/README.md | 5 +-- cpp/src/arrow/ArrowAzureConfig.cmake.in | 32 +++++++++++++++++++ cpp/src/arrow/CMakeLists.txt | 29 ++++++++++++++++- .../debian/libarrow-azure-dev.install | 2 +- .../apache-arrow/yum/arrow.spec.in | 2 +- docs/source/developers/cpp/building.rst | 4 +++ r/configure | 6 +++- 8 files changed, 84 insertions(+), 15 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d21cd3781b81..03fe8744d4c0 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -24,6 +24,7 @@ processorcount(NPROC) # including allocators like jemalloc and mimalloc set(ARROW_BUNDLED_STATIC_LIBS) set(ARROW_GCS_BUNDLED_STATIC_LIBS) +set(ARROW_AZURE_BUNDLED_STATIC_LIBS) # Accumulate all system dependencies to provide suitable static link # parameters to the third party libraries. @@ -4428,15 +4429,12 @@ function(build_azure_sdk) set(AZURE_SDK_VENDORED TRUE PARENT_SCOPE) - list(PREPEND - ARROW_BUNDLED_STATIC_LIBS - Azure::azure-core - Azure::azure-identity - Azure::azure-storage-blobs - Azure::azure-storage-common - Azure::azure-storage-files-datalake) - set(ARROW_BUNDLED_STATIC_LIBS - ${ARROW_BUNDLED_STATIC_LIBS} + set(ARROW_AZURE_BUNDLED_STATIC_LIBS + Azure::azure-storage-files-datalake + Azure::azure-storage-blobs + Azure::azure-storage-common + Azure::azure-identity + Azure::azure-core PARENT_SCOPE) list(POP_BACK CMAKE_MESSAGE_INDENT) @@ -4471,6 +4469,8 @@ if(ARROW_BUILD_STATIC) AND AZURE_SDK_VENDORED AND NOT WIN32) arrow_append_pc_system_library("libcurl" ARROW_AZURE_CURL_PC "-lcurl" ARROW_AZURE) + arrow_append_pc_system_library("openssl" ARROW_AZURE_OPENSSL_PC "-lssl -lcrypto" + ARROW_AZURE) endif() if(ARROW_AZURE AND AZURE_SDK_VENDORED @@ -4524,3 +4524,4 @@ endif() message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}") message(STATUS "GCS bundled static libraries: ${ARROW_GCS_BUNDLED_STATIC_LIBS}") +message(STATUS "Azure bundled static libraries: ${ARROW_AZURE_BUNDLED_STATIC_LIBS}") diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index ca76c33cd70e..f2b09a3baecc 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -87,8 +87,9 @@ docker compose run --rm static-system-dependency ### Static filesystem examples -Enable `ARROW_EXAMPLE_S3_STATIC_CHECK` or `ARROW_EXAMPLE_GCS_STATIC_CHECK` -to test static linking. Enable both to also test S3 and GCS together. +Enable `ARROW_EXAMPLE_S3_STATIC_CHECK`, `ARROW_EXAMPLE_GCS_STATIC_CHECK` or +`ARROW_EXAMPLE_AZURE_STATIC_CHECK` to test static linking. Enable both S3 and +GCS to also test them together. Run the examples from this directory. [docker-compose]: https://docs.docker.com/compose/ diff --git a/cpp/src/arrow/ArrowAzureConfig.cmake.in b/cpp/src/arrow/ArrowAzureConfig.cmake.in index c6257ee70ad9..18966df3a73f 100644 --- a/cpp/src/arrow/ArrowAzureConfig.cmake.in +++ b/cpp/src/arrow/ArrowAzureConfig.cmake.in @@ -27,6 +27,7 @@ @PACKAGE_INIT@ set(ARROW_AZURE_SYSTEM_DEPENDENCIES "@ARROW_AZURE_SYSTEM_DEPENDENCIES@") +set(ARROW_AZURE_BUNDLED_STATIC_LIBS "@ARROW_AZURE_BUNDLED_STATIC_LIBS@") include(CMakeFindDependencyMacro) find_dependency(Arrow CONFIG) @@ -37,6 +38,37 @@ endif() include("${CMAKE_CURRENT_LIST_DIR}/ArrowAzureTargets.cmake") +if(TARGET ArrowAzure::arrow_azure_static + AND ARROW_AZURE_BUNDLED_STATIC_LIBS + AND NOT TARGET ArrowAzure::arrow_azure_bundled_dependencies) + add_library(ArrowAzure::arrow_azure_bundled_dependencies STATIC IMPORTED) + get_target_property(arrow_azure_bundled_dependencies_link_libs + ArrowAzure::arrow_azure_static INTERFACE_LINK_LIBRARIES) + list(REMOVE_ITEM arrow_azure_bundled_dependencies_link_libs + ArrowAzure::arrow_azure_bundled_dependencies Arrow::arrow_static) + if(NOT WIN32) + list(APPEND arrow_azure_bundled_dependencies_link_libs OpenSSL::SSL OpenSSL::Crypto) + endif() + target_link_libraries(ArrowAzure::arrow_azure_bundled_dependencies + INTERFACE ${arrow_azure_bundled_dependencies_link_libs}) + get_target_property(arrow_azure_static_configurations ArrowAzure::arrow_azure_static + IMPORTED_CONFIGURATIONS) + foreach(CONFIGURATION ${arrow_azure_static_configurations}) + string(TOUPPER "${CONFIGURATION}" CONFIGURATION) + get_target_property(arrow_azure_static_location ArrowAzure::arrow_azure_static + IMPORTED_LOCATION_${CONFIGURATION}) + get_filename_component(arrow_azure_lib_dir "${arrow_azure_static_location}" + DIRECTORY) + set_property(TARGET ArrowAzure::arrow_azure_bundled_dependencies + APPEND + PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION}) + set_target_properties(ArrowAzure::arrow_azure_bundled_dependencies + PROPERTIES IMPORTED_LOCATION_${CONFIGURATION} + "${arrow_azure_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_azure_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + endforeach() +endif() + arrow_keep_backward_compatibility(ArrowAzure arrow_azure) check_required_components(ArrowAzure) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 815de5578c1f..43bdf98be2bd 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -955,10 +955,37 @@ if(ARROW_FILESYSTEM) if(ARROW_AZURE) set(ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) + if(ARROW_BUILD_STATIC AND ARROW_AZURE_BUNDLED_STATIC_LIBS) + arrow_car(_FIRST_AZURE_LIB ${ARROW_AZURE_BUNDLED_STATIC_LIBS}) + arrow_cdr(_OTHER_AZURE_LIBS ${ARROW_AZURE_BUNDLED_STATIC_LIBS}) + arrow_create_merged_static_lib(arrow_azure_bundled_dependencies + NAME + arrow_azure_bundled_dependencies + ROOT + ${_FIRST_AZURE_LIB} + TO_MERGE + ${_OTHER_AZURE_LIBS}) + get_target_property(arrow_azure_bundled_dependencies_path + arrow_azure_bundled_dependencies IMPORTED_LOCATION) + install(FILES ${arrow_azure_bundled_dependencies_path} ${INSTALL_IS_OPTIONAL} + DESTINATION ${CMAKE_INSTALL_LIBDIR}) + string(PREPEND ARROW_AZURE_PC_LIBS_PRIVATE " -larrow_azure_bundled_dependencies") + list(PREPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS + ArrowAzure::arrow_azure_bundled_dependencies) + endif() if(Azure_SOURCE STREQUAL "SYSTEM") list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES}) elseif(Azure_SOURCE STREQUAL "BUNDLED") - if(NOT WIN32) + if(WIN32) + list(APPEND + ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS + bcrypt + crypt32 + webservices + winhttp) + string(APPEND ARROW_AZURE_PC_LIBS_PRIVATE + " -lbcrypt -lcrypt32 -lwebservices -lwinhttp") + else() list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl LibXml2::LibXml2) endif() diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install index 3209f6d9ed85..52ccd5ed4984 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install @@ -1,4 +1,4 @@ usr/lib/*/cmake/ArrowAzure/ -usr/lib/*/libarrow_azure.a +usr/lib/*/libarrow_azure*.a usr/lib/*/libarrow_azure.so usr/lib/*/pkgconfig/arrow-azure.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index c6e6bfec6153..8afa17eabda5 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -420,7 +420,7 @@ Libraries and header files for Apache Arrow Azure filesystem. %doc README.md %license LICENSE.txt NOTICE.txt %{_libdir}/cmake/ArrowAzure/ -%{_libdir}/libarrow_azure.a +%{_libdir}/libarrow_azure*.a %{_libdir}/libarrow_azure.so %{_libdir}/pkgconfig/arrow-azure.pc diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index 8e40b047fd53..ec005237d48d 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -618,6 +618,10 @@ With ``-DARROW_GCS=ON``, a bundled google-cloud-cpp goes into a separate ``arrow_gcs_bundled_dependencies`` library. The ``ArrowGcs::arrow_gcs_static`` CMake target includes it automatically, otherwise link it right after ``arrow_gcs``. +With ``-DARROW_AZURE=ON``, a bundled Azure SDK for C++ goes into a separate +``arrow_azure_bundled_dependencies`` library. The ``ArrowAzure::arrow_azure_static`` +CMake target includes it automatically, otherwise link it right after ``arrow_azure``. + On Linux and macOS, if your application does not link to the ``pthread`` library already, you must include ``-pthread`` in your linker setup. In CMake this can be accomplished with the ``Threads`` built-in package: diff --git a/r/configure b/r/configure index 3ea4831c8bf7..5e0fae39124e 100755 --- a/r/configure +++ b/r/configure @@ -366,7 +366,11 @@ add_feature_flags () { if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" PKG_CONFIG_NAMES_FEATURES="$PKG_CONFIG_NAMES_FEATURES arrow-azure" - PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_azure $PKG_LIBS_FEATURES_WITHOUT_PC" + if [ -n "$(find "$LIB_DIR" -name 'libarrow_azure_bundled_dependencies.*')" ]; then + PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_azure -larrow_azure_bundled_dependencies $PKG_LIBS_FEATURES_WITHOUT_PC" + else + PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_azure $PKG_LIBS_FEATURES_WITHOUT_PC" + fi # NOTE: arrow_azure is assumed to have the same -L flag as arrow # so there is no need to add its location to PKG_DIRS PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2"