Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ci/scripts/cpp_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \
cmake \
-S "${source_dir}/examples/minimal_build" \
-B "${build_dir}/examples/minimal_build" \
-DARROW_EXAMPLE_AZURE_STATIC_CHECK=ON \
-DARROW_EXAMPLE_GCS_STATIC_CHECK=ON \
-DARROW_EXAMPLE_S3_STATIC_CHECK=ON \
-DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
Expand All @@ -157,7 +158,8 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \
# Test static filesystem linking
for static_example in arrow-example-s3-static \
arrow-example-gcs-static \
arrow-example-s3-gcs-static; do
arrow-example-s3-gcs-static \
arrow-example-azure-static; do
example="${build_dir}/examples/minimal_build/${static_example}"
if [ -x "${example}" ]; then
PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
Expand Down
3 changes: 3 additions & 0 deletions ci/scripts/python_wheel_xlinux_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ function check_arrow_visibility {
if [[ "${ARROW_GCS}" == "ON" ]]; then
libs+=(libarrow_gcs.so)
fi
if [[ "${ARROW_AZURE}" == "ON" ]]; then
libs+=(libarrow_azure.so)
fi
for lib in "${libs[@]}"; do
nm --demangle --dynamic "/tmp/arrow-dist/lib/${lib}"
done > nm_arrow.log
Expand Down
9 changes: 9 additions & 0 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,15 @@ endif()
set(ARROW_GCS_PC_LIBS_PRIVATE "")
set(ARROW_GCS_PC_REQUIRES_PRIVATE "")

# For arrow-azure.pc.
set(ARROW_AZURE_PC_CFLAGS "")
set(ARROW_AZURE_PC_CFLAGS_PRIVATE "")
if(ARROW_BUILD_STATIC)
string(APPEND ARROW_AZURE_PC_CFLAGS_PRIVATE " -DARROW_AZURE_STATIC")
endif()
set(ARROW_AZURE_PC_LIBS_PRIVATE "")
set(ARROW_AZURE_PC_REQUIRES_PRIVATE "")

# For arrow-s3.pc.
set(ARROW_S3_PC_CFLAGS "")
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
Expand Down
36 changes: 22 additions & 14 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ processorcount(NPROC)
# including allocators like jemalloc and mimalloc
set(ARROW_BUNDLED_STATIC_LIBS)
set(ARROW_GCS_BUNDLED_STATIC_LIBS)
set(ARROW_AZURE_BUNDLED_STATIC_LIBS)

# Accumulate all system dependencies to provide suitable static link
# parameters to the third party libraries.
Expand Down Expand Up @@ -4428,28 +4429,31 @@ function(build_azure_sdk)
set(AZURE_SDK_VENDORED
TRUE
PARENT_SCOPE)
list(PREPEND
ARROW_BUNDLED_STATIC_LIBS
Azure::azure-core
Azure::azure-identity
Azure::azure-storage-blobs
Azure::azure-storage-common
Azure::azure-storage-files-datalake)
set(ARROW_BUNDLED_STATIC_LIBS
${ARROW_BUNDLED_STATIC_LIBS}
set(ARROW_AZURE_BUNDLED_STATIC_LIBS
Azure::azure-storage-files-datalake
Azure::azure-storage-blobs
Azure::azure-storage-common
Azure::azure-identity
Azure::azure-core
PARENT_SCOPE)

list(POP_BACK CMAKE_MESSAGE_INDENT)
endfunction()

if(ARROW_WITH_AZURE_SDK)
resolve_dependency(Azure REQUIRED_VERSION 1.10.2)
resolve_dependency(Azure
ARROW_CMAKE_PACKAGE_NAME
ArrowAzure
ARROW_PC_PACKAGE_NAME
arrow-azure
REQUIRED_VERSION
1.10.2)
set(AZURE_SDK_LINK_LIBRARIES Azure::azure-storage-files-datalake
Azure::azure-storage-blobs Azure::azure-identity)
if(AZURE_SDK_VENDORED AND NOT WIN32)
find_curl(ARROW)
find_curl(ARROW_AZURE)
find_package(LibXml2 REQUIRED)
list(APPEND ARROW_SYSTEM_DEPENDENCIES LibXml2)
list(APPEND ARROW_AZURE_SYSTEM_DEPENDENCIES LibXml2)
endif()
endif()

Expand All @@ -4464,12 +4468,15 @@ if(ARROW_BUILD_STATIC)
if(ARROW_AZURE
AND AZURE_SDK_VENDORED
AND NOT WIN32)
arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl" ARROW)
arrow_append_pc_system_library("libcurl" ARROW_AZURE_CURL_PC "-lcurl" ARROW_AZURE)
arrow_append_pc_system_library("openssl" ARROW_AZURE_OPENSSL_PC "-lssl -lcrypto"
ARROW_AZURE)
endif()
if(ARROW_AZURE
AND AZURE_SDK_VENDORED
AND NOT WIN32)
arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2" ARROW)
arrow_append_pc_system_library("libxml-2.0" ARROW_AZURE_LIBXML2_PC "-lxml2"
ARROW_AZURE)
endif()
if(ARROW_GCS
AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED"
Expand Down Expand Up @@ -4517,3 +4524,4 @@ endif()

message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")
message(STATUS "GCS bundled static libraries: ${ARROW_GCS_BUNDLED_STATIC_LIBS}")
message(STATUS "Azure bundled static libraries: ${ARROW_AZURE_BUNDLED_STATIC_LIBS}")
17 changes: 17 additions & 0 deletions cpp/examples/minimal_build/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,20 @@ elseif(ARROW_EXAMPLE_GCS_STATIC_CHECK)
message(STATUS "Skipping GCS static examples: requires ARROW_GCS and Arrow::arrow_static"
)
endif()

option(ARROW_EXAMPLE_AZURE_STATIC_CHECK "Verify Azure and static linking" OFF)
if(ARROW_EXAMPLE_AZURE_STATIC_CHECK
AND ARROW_AZURE
AND TARGET Arrow::arrow_static)
find_package(ArrowAzure REQUIRED)
if(NOT TARGET ArrowAzure::arrow_azure_static)
message(FATAL_ERROR "ArrowAzure::arrow_azure_static not found")
endif()
add_executable(arrow-example-azure-static example.cc)
target_compile_definitions(arrow-example-azure-static
PRIVATE ARROW_EXAMPLE_AZURE_STATIC_CHECK)
target_link_libraries(arrow-example-azure-static PRIVATE ArrowAzure::arrow_azure_static)
elseif(ARROW_EXAMPLE_AZURE_STATIC_CHECK)
message(STATUS "Skipping Azure static example: requires ARROW_AZURE and Arrow::arrow_static"
)
endif()
5 changes: 3 additions & 2 deletions cpp/examples/minimal_build/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,9 @@ docker compose run --rm static-system-dependency

### Static filesystem examples

Enable `ARROW_EXAMPLE_S3_STATIC_CHECK` or `ARROW_EXAMPLE_GCS_STATIC_CHECK`
to test static linking. Enable both to also test S3 and GCS together.
Enable `ARROW_EXAMPLE_S3_STATIC_CHECK`, `ARROW_EXAMPLE_GCS_STATIC_CHECK` or
`ARROW_EXAMPLE_AZURE_STATIC_CHECK` to test static linking. Enable both S3 and
GCS to also test them together.
Run the examples from this directory.

[docker-compose]: https://docs.docker.com/compose/
11 changes: 11 additions & 0 deletions cpp/examples/minimal_build/example.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
// under the License.

#include <arrow/csv/api.h>
#ifdef ARROW_EXAMPLE_AZURE_STATIC_CHECK
# include <arrow/filesystem/azurefs.h>
#endif
#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK
# include <arrow/filesystem/gcsfs.h>
#endif
Expand Down Expand Up @@ -72,6 +75,14 @@ Status RunMain(int argc, char** argv) {
arrow::fs::GcsFileSystem::Make(arrow::fs::GcsOptions::Anonymous()).status());
#endif

#ifdef ARROW_EXAMPLE_AZURE_STATIC_CHECK
std::cerr << "* Verify Azure filesystem construction" << std::endl;
arrow::fs::AzureOptions azure_options;
azure_options.account_name = "example";
ARROW_RETURN_NOT_OK(azure_options.ConfigureAnonymousCredential());
ARROW_RETURN_NOT_OK(arrow::fs::AzureFileSystem::Make(azure_options).status());
#endif

return Status::OK();
}

Expand Down
76 changes: 76 additions & 0 deletions cpp/src/arrow/ArrowAzureConfig.cmake.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This config sets the following variables in your project::
#
# ArrowAzure_FOUND - true if Arrow Azure found on the system
#
# This config sets the following targets in your project::
#
# ArrowAzure::arrow_azure_shared - for linked as shared library if shared library is built
# ArrowAzure::arrow_azure_static - for linked as static library if static library is built

@PACKAGE_INIT@

set(ARROW_AZURE_SYSTEM_DEPENDENCIES "@ARROW_AZURE_SYSTEM_DEPENDENCIES@")
set(ARROW_AZURE_BUNDLED_STATIC_LIBS "@ARROW_AZURE_BUNDLED_STATIC_LIBS@")

include(CMakeFindDependencyMacro)
find_dependency(Arrow CONFIG)

if(ARROW_BUILD_STATIC)
arrow_find_dependencies("${ARROW_AZURE_SYSTEM_DEPENDENCIES}")
endif()

include("${CMAKE_CURRENT_LIST_DIR}/ArrowAzureTargets.cmake")

if(TARGET ArrowAzure::arrow_azure_static
AND ARROW_AZURE_BUNDLED_STATIC_LIBS
AND NOT TARGET ArrowAzure::arrow_azure_bundled_dependencies)
add_library(ArrowAzure::arrow_azure_bundled_dependencies STATIC IMPORTED)
get_target_property(arrow_azure_bundled_dependencies_link_libs
ArrowAzure::arrow_azure_static INTERFACE_LINK_LIBRARIES)
list(REMOVE_ITEM arrow_azure_bundled_dependencies_link_libs
ArrowAzure::arrow_azure_bundled_dependencies Arrow::arrow_static)
if(NOT WIN32)
list(APPEND arrow_azure_bundled_dependencies_link_libs OpenSSL::SSL OpenSSL::Crypto)
endif()
target_link_libraries(ArrowAzure::arrow_azure_bundled_dependencies
INTERFACE ${arrow_azure_bundled_dependencies_link_libs})
get_target_property(arrow_azure_static_configurations ArrowAzure::arrow_azure_static
IMPORTED_CONFIGURATIONS)
foreach(CONFIGURATION ${arrow_azure_static_configurations})
string(TOUPPER "${CONFIGURATION}" CONFIGURATION)
get_target_property(arrow_azure_static_location ArrowAzure::arrow_azure_static
IMPORTED_LOCATION_${CONFIGURATION})
get_filename_component(arrow_azure_lib_dir "${arrow_azure_static_location}"
DIRECTORY)
set_property(TARGET ArrowAzure::arrow_azure_bundled_dependencies
APPEND
PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION})
set_target_properties(ArrowAzure::arrow_azure_bundled_dependencies
PROPERTIES IMPORTED_LOCATION_${CONFIGURATION}
"${arrow_azure_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_azure_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
endforeach()
endif()

arrow_keep_backward_compatibility(ArrowAzure arrow_azure)

check_required_components(ArrowAzure)

arrow_show_details(ArrowAzure ARROW_AZURE)
99 changes: 81 additions & 18 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,6 @@ set(ARROW_SHARED_PRIVATE_LINK_LIBS)
set(ARROW_SHARED_INSTALL_INTERFACE_LIBS)
set(ARROW_STATIC_INSTALL_INTERFACE_LIBS)

if(ARROW_AZURE)
if(Azure_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES})
elseif(Azure_SOURCE STREQUAL "BUNDLED")
if(NOT WIN32)
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl LibXml2::LibXml2)
endif()
endif()
endif()

if(ARROW_USE_OPENSSL)
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS})
endif()
Expand Down Expand Up @@ -964,9 +954,88 @@ if(ARROW_FILESYSTEM)
filesystem/util_internal.cc)

if(ARROW_AZURE)
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/azurefs.cc)
set_source_files_properties(filesystem/azurefs.cc
set(ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
if(ARROW_BUILD_STATIC AND ARROW_AZURE_BUNDLED_STATIC_LIBS)
arrow_car(_FIRST_AZURE_LIB ${ARROW_AZURE_BUNDLED_STATIC_LIBS})
arrow_cdr(_OTHER_AZURE_LIBS ${ARROW_AZURE_BUNDLED_STATIC_LIBS})
arrow_create_merged_static_lib(arrow_azure_bundled_dependencies
NAME
arrow_azure_bundled_dependencies
ROOT
${_FIRST_AZURE_LIB}
TO_MERGE
${_OTHER_AZURE_LIBS})
get_target_property(arrow_azure_bundled_dependencies_path
arrow_azure_bundled_dependencies IMPORTED_LOCATION)
install(FILES ${arrow_azure_bundled_dependencies_path} ${INSTALL_IS_OPTIONAL}
DESTINATION ${CMAKE_INSTALL_LIBDIR})
string(PREPEND ARROW_AZURE_PC_LIBS_PRIVATE " -larrow_azure_bundled_dependencies")
list(PREPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS
ArrowAzure::arrow_azure_bundled_dependencies)
endif()
if(Azure_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES})
elseif(Azure_SOURCE STREQUAL "BUNDLED")
if(WIN32)
list(APPEND
ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS
bcrypt
crypt32
webservices
winhttp)
string(APPEND ARROW_AZURE_PC_LIBS_PRIVATE
" -lbcrypt -lcrypt32 -lwebservices -lwinhttp")
else()
list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl
LibXml2::LibXml2)
endif()
endif()
set(ARROW_AZURE_LIB_SRCS filesystem/azurefs_module.cc filesystem/azurefs.cc)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as GCS. GCS is tracked in #55284 , which covers GCS. @kou, should I create one for Azure for C GLib?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you do it, it's helpful. Or I can create it later. :-)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

set_source_files_properties(filesystem/azurefs_module.cc filesystem/azurefs.cc
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
# If libarrow_azure.a is only built, "pkg-config --cflags --libs
# arrow-azure" outputs build flags for static linking not shared
# linking. ARROW_AZURE_PC_* except ARROW_AZURE_PC_*_PRIVATE are for the
# static linking case.
if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
string(APPEND ARROW_AZURE_PC_CFLAGS "${ARROW_AZURE_PC_CFLAGS_PRIVATE}")
set(ARROW_AZURE_PC_CFLAGS_PRIVATE "")
set(ARROW_AZURE_PC_LIBS "${ARROW_AZURE_PC_LIBS_PRIVATE}")
set(ARROW_AZURE_PC_LIBS_PRIVATE "")
set(ARROW_AZURE_PC_REQUIRES "${ARROW_AZURE_PC_REQUIRES_PRIVATE}")
set(ARROW_AZURE_PC_REQUIRES_PRIVATE "")
else()
set(ARROW_AZURE_PC_LIBS "")
set(ARROW_AZURE_PC_REQUIRES "")
endif()
add_arrow_lib(arrow_azure
CMAKE_PACKAGE_NAME
ArrowAzure
PKG_CONFIG_NAME
arrow-azure
SOURCES
${ARROW_AZURE_LIB_SRCS}
SHARED_LINK_LIBS
arrow_shared
SHARED_PRIVATE_LINK_LIBS
${AZURE_SDK_LINK_LIBRARIES}
SHARED_INSTALL_INTERFACE_LIBS
Arrow::arrow_shared
STATIC_LINK_LIBS
arrow_static
${AZURE_SDK_LINK_LIBRARIES}
STATIC_INSTALL_INTERFACE_LIBS
${ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS}
SHARED_LINK_FLAGS
${ARROW_VERSION_SCRIPT_FLAGS}
OUTPUTS
ARROW_AZURE_LIBRARIES)
foreach(LIB_TARGET ${ARROW_AZURE_LIBRARIES})
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_AZURE_EXPORTING)
endforeach()
if(ARROW_BUILD_STATIC AND WIN32)
target_compile_definitions(arrow_azure_static PUBLIC ARROW_AZURE_STATIC)
endif()
endif()
if(ARROW_GCS)
set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
Expand Down Expand Up @@ -1085,12 +1154,6 @@ if(ARROW_FILESYSTEM)
endif()

arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
if(ARROW_AZURE)
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
target_link_libraries(${ARROW_FILESYSTEM_TARGET}
PRIVATE ${AZURE_SDK_LINK_LIBRARIES})
endforeach()
endif()
if(ARROW_HDFS)
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE arrow::hadoop)
Expand Down
Loading
Loading