Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions ci/scripts/cpp_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -146,19 +146,24 @@ if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \
cmake \
-S "${source_dir}/examples/minimal_build" \
-B "${build_dir}/examples/minimal_build" \
-DARROW_EXAMPLE_GCS_STATIC_CHECK=ON \
-DARROW_EXAMPLE_S3_STATIC_CHECK=ON \
-DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
cmake --build "${build_dir}/examples/minimal_build"
pushd "${source_dir}/examples/minimal_build"
# PATH= is for Windows.
PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
"${build_dir}/examples/minimal_build/arrow-example"
# Test static linking with S3
s3_example="${build_dir}/examples/minimal_build/arrow-example-s3-static"
if [ -x "${s3_example}" ]; then
PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
"${s3_example}"
fi
# Test static filesystem linking
for static_example in arrow-example-s3-static \
arrow-example-gcs-static \
arrow-example-s3-gcs-static; do
example="${build_dir}/examples/minimal_build/${static_example}"
if [ -x "${example}" ]; then
PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
"${example}"
fi
done
popd
fi

Expand Down
20 changes: 12 additions & 8 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@
include(ProcessorCount)
processorcount(NPROC)

# Accumulate all bundled targets and we will splice them together later as
# Accumulate core bundled targets and we will splice them together later as
# libarrow_bundled_dependencies.a so that third party libraries have something
# usable to create statically-linked builds with some BUNDLED dependencies,
# including allocators like jemalloc and mimalloc
set(ARROW_BUNDLED_STATIC_LIBS)
set(ARROW_GCS_BUNDLED_STATIC_LIBS)

# Accumulate all system dependencies to provide suitable static link
# parameters to the third party libraries.
Expand Down Expand Up @@ -1942,6 +1943,9 @@ function(build_absl)
# When ABSL_ENABLE_INSTALL is ON, the real target is "time" not "absl_time"
# Cannot use set_property on alias targets (absl::time is an alias)
target_link_libraries(time INTERFACE ${CoreFoundation})
set(ARROW_PC_LIBS_PRIVATE
"${ARROW_PC_LIBS_PRIVATE} -framework CoreFoundation"
PARENT_SCOPE)
endif()

list(PREPEND
Expand Down Expand Up @@ -3860,13 +3864,8 @@ function(build_google_cloud_cpp_storage)
# 141MB -> 79MB
file(REMOVE_RECURSE "${google_cloud_cpp_SOURCE_DIR}/ci")

list(PREPEND
ARROW_BUNDLED_STATIC_LIBS
google-cloud-cpp::storage
google-cloud-cpp::rest_internal
google-cloud-cpp::common)
set(ARROW_BUNDLED_STATIC_LIBS
${ARROW_BUNDLED_STATIC_LIBS}
set(ARROW_GCS_BUNDLED_STATIC_LIBS
google-cloud-cpp::storage google-cloud-cpp::rest_internal google-cloud-cpp::common
Comment on lines +3867 to +3868

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am still trying to get my head around why this is an issue but

list(PREPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LINK_LIBRARIES})

isn't and we don't seem to require an ARROW_S3_BUNDLED_STATIC_LIBS

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, as this is green and fixes a real issue I think we should merge. We should probably create ARROW_S3_BUNDLED_STATIC_LIBS as this might be just coincidence that is not currently failing but we can do that afterwards.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PARENT_SCOPE)

list(POP_BACK CMAKE_MESSAGE_INDENT)
Expand Down Expand Up @@ -4453,6 +4452,10 @@ endif()
if(ARROW_BUILD_STATIC)
if(ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
arrow_append_pc_system_library("libcurl" ARROW_GCS_CURL_PC "-lcurl" ARROW_GCS)
if(NOT WIN32)
arrow_append_pc_system_library("openssl" ARROW_GCS_OPENSSL_PC "-lssl -lcrypto"
ARROW_GCS)
endif()
endif()
if(ARROW_AZURE
AND AZURE_SDK_VENDORED
Expand Down Expand Up @@ -4509,3 +4512,4 @@ if(ARROW_FLIGHT_SQL_ODBC)
endif()

message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")
message(STATUS "GCS bundled static libraries: ${ARROW_GCS_BUNDLED_STATIC_LIBS}")
27 changes: 27 additions & 0 deletions cpp/examples/minimal_build/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,30 @@ elseif(ARROW_EXAMPLE_S3_STATIC_CHECK)
message(STATUS "Skipping arrow-example-s3-static: "
"requires ARROW_S3 and Arrow::arrow_static")
endif()

option(ARROW_EXAMPLE_GCS_STATIC_CHECK "Verify GCS and static linking" OFF)
if(ARROW_EXAMPLE_GCS_STATIC_CHECK
AND ARROW_GCS
AND TARGET Arrow::arrow_static)
find_package(ArrowGcs REQUIRED)
if(NOT TARGET ArrowGcs::arrow_gcs_static)
message(FATAL_ERROR "ArrowGcs::arrow_gcs_static not found")
endif()
add_executable(arrow-example-gcs-static example.cc)
target_compile_definitions(arrow-example-gcs-static
PRIVATE ARROW_EXAMPLE_GCS_STATIC_CHECK)
target_link_libraries(arrow-example-gcs-static PRIVATE ArrowGcs::arrow_gcs_static)

if(ARROW_EXAMPLE_S3_STATIC_CHECK AND TARGET ArrowS3::arrow_s3_static)
add_executable(arrow-example-s3-gcs-static example.cc)
target_compile_definitions(arrow-example-s3-gcs-static
PRIVATE ARROW_EXAMPLE_S3_STATIC_CHECK
ARROW_EXAMPLE_GCS_STATIC_CHECK)
target_link_libraries(arrow-example-s3-gcs-static PRIVATE ArrowS3::arrow_s3_static
ArrowGcs::arrow_gcs_static)
endif()

elseif(ARROW_EXAMPLE_GCS_STATIC_CHECK)
message(STATUS "Skipping GCS static examples: requires ARROW_GCS and Arrow::arrow_static"
)
endif()
6 changes: 6 additions & 0 deletions cpp/examples/minimal_build/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,4 +85,10 @@ system. To run this configuration, set
docker compose run --rm static-system-dependency
```

### Static filesystem examples

Enable `ARROW_EXAMPLE_S3_STATIC_CHECK` or `ARROW_EXAMPLE_GCS_STATIC_CHECK`
to test static linking. Enable both to also test S3 and GCS together.
Run the examples from this directory.

[docker-compose]: https://docs.docker.com/compose/
9 changes: 9 additions & 0 deletions cpp/examples/minimal_build/example.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
// under the License.

#include <arrow/csv/api.h>
#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK
# include <arrow/filesystem/gcsfs.h>
#endif
#ifdef ARROW_EXAMPLE_S3_STATIC_CHECK
# include <arrow/filesystem/s3fs.h>
#endif
Expand Down Expand Up @@ -63,6 +66,12 @@ Status RunMain(int argc, char** argv) {
ARROW_RETURN_NOT_OK(arrow::fs::EnsureS3Finalized());
#endif

#ifdef ARROW_EXAMPLE_GCS_STATIC_CHECK
std::cerr << "* Verify GCS filesystem construction" << std::endl;
ARROW_RETURN_NOT_OK(
arrow::fs::GcsFileSystem::Make(arrow::fs::GcsOptions::Anonymous()).status());
#endif

return Status::OK();
}

Expand Down
17 changes: 14 additions & 3 deletions cpp/src/arrow/ArrowConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,26 @@ if(TARGET Arrow::arrow_static AND NOT TARGET Arrow::arrow_bundled_dependencies)
)
endforeach()

if(ARROW_BUNDLED_STATIC_LIBS)
get_target_property(arrow_bundled_dependencies_link_libs Arrow::arrow_static
INTERFACE_LINK_LIBRARIES)
list(REMOVE_ITEM arrow_bundled_dependencies_link_libs
Arrow::arrow_bundled_dependencies)
target_link_libraries(Arrow::arrow_bundled_dependencies
INTERFACE ${arrow_bundled_dependencies_link_libs})
endif()

# CMP0057: Support new if() IN_LIST operator.
# https://cmake.org/cmake/help/latest/policy/CMP0057.html
cmake_policy(PUSH)
cmake_policy(SET CMP0057 NEW)
if(APPLE AND ("aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS
OR "absl::time" IN_LIST ARROW_BUNDLED_STATIC_LIBS))
find_library(CORE_FOUNDATION CoreFoundation)
target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${CORE_FOUNDATION})
endif()
if("aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS)
if(APPLE)
find_library(CORE_FOUNDATION CoreFoundation)
target_link_libraries(Arrow::arrow_bundled_dependencies
INTERFACE ${CORE_FOUNDATION})
find_library(NETWORK Network)
target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${NETWORK})
find_library(SECURITY Security)
Expand Down
35 changes: 35 additions & 0 deletions cpp/src/arrow/ArrowGcsConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
@PACKAGE_INIT@

set(ARROW_GCS_SYSTEM_DEPENDENCIES "@ARROW_GCS_SYSTEM_DEPENDENCIES@")
set(ARROW_GCS_BUNDLED_STATIC_LIBS "@ARROW_GCS_BUNDLED_STATIC_LIBS@")

include(CMakeFindDependencyMacro)
find_dependency(Arrow CONFIG)
Expand All @@ -37,6 +38,40 @@ endif()

include("${CMAKE_CURRENT_LIST_DIR}/ArrowGcsTargets.cmake")

if(TARGET ArrowGcs::arrow_gcs_static
AND ARROW_GCS_BUNDLED_STATIC_LIBS
AND NOT TARGET ArrowGcs::arrow_gcs_bundled_dependencies)
add_library(ArrowGcs::arrow_gcs_bundled_dependencies STATIC IMPORTED)
get_target_property(arrow_gcs_bundled_dependencies_link_libs ArrowGcs::arrow_gcs_static
INTERFACE_LINK_LIBRARIES)
list(REMOVE_ITEM arrow_gcs_bundled_dependencies_link_libs
ArrowGcs::arrow_gcs_bundled_dependencies Arrow::arrow_static)
if("@absl_SOURCE@" STREQUAL "BUNDLED")
list(APPEND arrow_gcs_bundled_dependencies_link_libs
Arrow::arrow_bundled_dependencies)
endif()
if(NOT WIN32)
list(APPEND arrow_gcs_bundled_dependencies_link_libs OpenSSL::SSL OpenSSL::Crypto)
endif()
target_link_libraries(ArrowGcs::arrow_gcs_bundled_dependencies
INTERFACE ${arrow_gcs_bundled_dependencies_link_libs})
get_target_property(arrow_gcs_static_configurations ArrowGcs::arrow_gcs_static
IMPORTED_CONFIGURATIONS)
foreach(CONFIGURATION ${arrow_gcs_static_configurations})
string(TOUPPER "${CONFIGURATION}" CONFIGURATION)
get_target_property(arrow_gcs_static_location ArrowGcs::arrow_gcs_static
IMPORTED_LOCATION_${CONFIGURATION})
get_filename_component(arrow_gcs_lib_dir "${arrow_gcs_static_location}" DIRECTORY)
set_property(TARGET ArrowGcs::arrow_gcs_bundled_dependencies
APPEND
PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION})
set_target_properties(ArrowGcs::arrow_gcs_bundled_dependencies
PROPERTIES IMPORTED_LOCATION_${CONFIGURATION}
"${arrow_gcs_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_gcs_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
endforeach()
endif()

arrow_keep_backward_compatibility(ArrowGcs arrow_gcs)

check_required_components(ArrowGcs)
Expand Down
27 changes: 27 additions & 0 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -970,10 +970,37 @@ if(ARROW_FILESYSTEM)
endif()
if(ARROW_GCS)
set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
if(ARROW_BUILD_STATIC AND ARROW_GCS_BUNDLED_STATIC_LIBS)
arrow_car(_FIRST_GCS_LIB ${ARROW_GCS_BUNDLED_STATIC_LIBS})
arrow_cdr(_OTHER_GCS_LIBS ${ARROW_GCS_BUNDLED_STATIC_LIBS})
arrow_create_merged_static_lib(arrow_gcs_bundled_dependencies
NAME
arrow_gcs_bundled_dependencies
ROOT
${_FIRST_GCS_LIB}
TO_MERGE
${_OTHER_GCS_LIBS})
get_target_property(arrow_gcs_bundled_dependencies_path
arrow_gcs_bundled_dependencies IMPORTED_LOCATION)
install(FILES ${arrow_gcs_bundled_dependencies_path} ${INSTALL_IS_OPTIONAL}
DESTINATION ${CMAKE_INSTALL_LIBDIR})
string(PREPEND ARROW_GCS_PC_LIBS_PRIVATE " -larrow_gcs_bundled_dependencies")
# Resolve bundled Abseil after SDK
list(PREPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS
ArrowGcs::arrow_gcs_bundled_dependencies)
endif()
if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
if(WIN32)
list(APPEND
ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS
ws2_32
bcrypt
crypt32)
string(APPEND ARROW_GCS_PC_LIBS_PRIVATE " -lws2_32 -lbcrypt -lcrypt32")
endif()
if(absl_SOURCE STREQUAL "SYSTEM")
# Bundled google-cloud-cpp with system Abseil needs these for static consumers.
list(APPEND
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
usr/lib/*/cmake/ArrowGcs/
usr/lib/*/libarrow_gcs.a
usr/lib/*/libarrow_gcs*.a
usr/lib/*/libarrow_gcs.so
usr/lib/*/pkgconfig/arrow-gcs.pc
2 changes: 1 addition & 1 deletion dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ Libraries and header files for Apache Arrow GCS filesystem.
%doc README.md
%license LICENSE.txt NOTICE.txt
%{_libdir}/cmake/ArrowGcs/
%{_libdir}/libarrow_gcs.a
%{_libdir}/libarrow_gcs*.a
%{_libdir}/libarrow_gcs.so
%{_libdir}/pkgconfig/arrow-gcs.pc
%endif
Expand Down
6 changes: 5 additions & 1 deletion docs/source/developers/cpp/building.rst
Original file line number Diff line number Diff line change
Expand Up @@ -600,7 +600,7 @@ downloading them (one for each build dir!).
Statically Linking
------------------

When ``-DARROW_BUILD_STATIC=ON``, all build dependencies built as static
When ``-DARROW_BUILD_STATIC=ON``, core build dependencies built as static
libraries by the Arrow build system will be merged together to create a static
library ``arrow_bundled_dependencies``. In UNIX-like environments (Linux, macOS,
MinGW), this is called ``libarrow_bundled_dependencies.a`` and on Windows with
Expand All @@ -614,6 +614,10 @@ an `example CMake-based build configuration
<https://github.com/apache/arrow/tree/main/cpp/examples/minimal_build>`_ to
show you a working example.

With ``-DARROW_GCS=ON``, a bundled google-cloud-cpp goes into a separate
``arrow_gcs_bundled_dependencies`` library. The ``ArrowGcs::arrow_gcs_static``
CMake target includes it automatically, otherwise link it right after ``arrow_gcs``.

On Linux and macOS, if your application does not link to the ``pthread``
library already, you must include ``-pthread`` in your linker setup. In CMake
this can be accomplished with the ``Threads`` built-in package:
Expand Down
6 changes: 5 additions & 1 deletion r/configure
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,11 @@ add_feature_flags () {
if arrow_built_with ARROW_GCS; then
PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS"
PKG_CONFIG_NAMES_FEATURES="$PKG_CONFIG_NAMES_FEATURES arrow-gcs"
PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_gcs $PKG_LIBS_FEATURES_WITHOUT_PC"
if [ -n "$(find "$LIB_DIR" -name 'libarrow_gcs_bundled_dependencies.*')" ]; then
PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_gcs -larrow_gcs_bundled_dependencies $PKG_LIBS_FEATURES_WITHOUT_PC"
else
PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_gcs $PKG_LIBS_FEATURES_WITHOUT_PC"
fi
# NOTE: arrow_gcs is assumed to have the same -L flag as arrow
# so there is no need to add its location to PKG_DIRS
fi
Expand Down
5 changes: 4 additions & 1 deletion r/configure.win
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,10 @@ function configure_binaries() {
PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '
PKG_LIBS="$PKG_LIBS -L${RWINLIB}/lib"'$(R_ARCH)$(CRT) '
PKG_LIBS="$PKG_LIBS -larrow_dataset -larrow_acero -lparquet -larrow_compute -larrow_s3 \
-larrow_gcs -larrow -larrow_bundled_dependencies \
-larrow_gcs \$(if \$(wildcard \
${RWINLIB}/lib\$(subst gcc,,\$(COMPILED_BY))\$(R_ARCH)/libarrow_gcs_bundled_dependencies.a \
${RWINLIB}/lib\$(R_ARCH)\$(CRT)/libarrow_gcs_bundled_dependencies.a),-larrow_gcs_bundled_dependencies) -larrow \
-larrow_bundled_dependencies \
-lutf8proc -lsnappy -lz -lzstd -llz4 -lbz2 -lole32 \
${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS}

Expand Down
Loading