diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 4ae8e3b19bb0..c13bc925adbf 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -205,7 +205,7 @@ jobs: fail-fast: false matrix: config: - - { rtools: 40, arch: 'ucrt64' } + - { rtools: 45, arch: 'ucrt64' } steps: - run: git config --global core.autocrlf false - name: Checkout Arrow @@ -213,6 +213,12 @@ jobs: with: persist-credentials: false fetch-depth: 0 + - name: Install ccache + shell: bash + run: | + mkdir -p "$HOME/.local/bin" + ci/scripts/install_ccache.sh 4.13.6 "$HOME/.local" + echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Setup ccache shell: bash run: | @@ -228,15 +234,13 @@ jobs: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - uses: r-lib/actions/setup-r@v2 with: - # Note: we need Rtools40 here because Rtools40 + ucrt is how we build the Arrow C++ - # static library. R itself is not used, but setup-r picks the Rtools version from - # r-version, and R 4.1 was the last R to use Rtools40, so this must stay at 4.1. - r-version: "4.1" + # R is not used, but setup-r picks Rtools from r-version: 4.5 gets + # Rtools45, which builds the Arrow C++ static library. + r-version: "4.5" Ncpus: 2 - name: Build Arrow C++ shell: bash env: - MINGW_ARCH: ${{ matrix.config.arch }} RTOOLS_VERSION: ${{ matrix.config.rtools }} run: ci/scripts/r_windows_build.sh - name: Rename libarrow.zip @@ -283,14 +287,14 @@ jobs: - name: Download Timezone Database shell: bash run: | - # RTools 40 uses GCC 8.x which does not support C++20 chrono timezones, - # so Arrow uses the vendored date library which requires tzdata + # libarrow is built with ARROW_USE_STD_CHRONO=0, so Arrow uses the + # vendored date library, which requires tzdata ci/scripts/download_tz_database.sh - run: mkdir r/windows - name: Download artifacts uses: actions/download-artifact@v8 with: - name: libarrow-rtools40-ucrt64.zip + name: libarrow-rtools45-ucrt64.zip path: r/windows - name: Unzip and rezip libarrows shell: bash diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index eec1fb5bd6ed..4c53e774d22e 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -17,6 +17,14 @@ # specific language governing permissions and limitations # under the License. +# Build the static libarrow that r/configure.win downloads on Windows. +# +# * Rtools40 (MSYS2): makepkg-mingw with ci/scripts/PKGBUILD. +# * Rtools42+ (MXE, no pacman/makepkg): call CMake directly against the +# toolchain's static libraries. +# +# The two zips differ in contents; r/configure.win handles both. + set -ex : "${ARROW_HOME:=$(pwd)}" @@ -24,69 +32,236 @@ set -ex ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" export ARROW_HOME -pacman --noconfirm -Syy +: "${RTOOLS_VERSION:=45}" -: "${MINGW_ARCH:=mingw32 mingw64 ucrt64}" -export MINGW_ARCH +if [ "${RTOOLS_VERSION}" = "40" ]; then + pacman --noconfirm -Syy -cp "${ARROW_HOME}/ci/scripts/PKGBUILD" . -printenv -makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild + : "${MINGW_ARCH:=mingw32 mingw64 ucrt64}" + export MINGW_ARCH -VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" -DST_DIR="r-libarrow-windows-x86_64-$VERSION" - -# Collect the build artifacts and make the shape of zip file that rwinlib expects -ls -mkdir -p build -mv mingw* build -cd build - -# This may vary by system/CI provider -MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}" - -# Untar the builds we made -find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf -mkdir -p "${DST_DIR}" -# Grab the headers from one, either one is fine -# (if we're building twice to combine old and new toolchains, this may already exist) -if [ ! -d "${DST_DIR}/include" ]; then - mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}" -fi + cp "${ARROW_HOME}/ci/scripts/PKGBUILD" . + printenv + makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild -# mingw64 -> x64 -# mingw32 -> i386 -# ucrt64 -> x64-ucrt - -if [ -d mingw64/lib/ ]; then - ls "${MSYS_LIB_DIR}/mingw64/lib/" - # Make the rest of the directory structure - mkdir -p "${DST_DIR}/lib/x64" - # Move the 64-bit versions of libarrow into the expected location - mv mingw64/lib/*.a "${DST_DIR}/lib/x64" - # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" -fi + VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" + DST_DIR="r-libarrow-windows-x86_64-$VERSION" + + # Collect the build artifacts and make the shape of zip file that rwinlib expects + ls + mkdir -p build + mv mingw* build + cd build + + # This may vary by system/CI provider + MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}" + + # Untar the builds we made + find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf + mkdir -p "${DST_DIR}" + # Grab the headers from one, either one is fine + # (if we're building twice to combine old and new toolchains, this may already exist) + if [ ! -d "${DST_DIR}/include" ]; then + mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}" + fi + + # mingw64 -> x64 + # mingw32 -> i386 + # ucrt64 -> x64-ucrt + + if [ -d mingw64/lib/ ]; then + ls "${MSYS_LIB_DIR}/mingw64/lib/" + # Make the rest of the directory structure + mkdir -p "${DST_DIR}/lib/x64" + # Move the 64-bit versions of libarrow into the expected location + mv mingw64/lib/*.a "${DST_DIR}/lib/x64" + # These are from https://dl.bintray.com/rtools/mingw{32,64}/ + cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" + fi + + # Same for the 32-bit versions + if [ -d mingw32/lib/ ]; then + ls "${MSYS_LIB_DIR}/mingw32/lib/" + mkdir -p "${DST_DIR}/lib/i386" + mv mingw32/lib/*.a "${DST_DIR}/lib/i386" + cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" + fi + + # Do the same also for ucrt64 + if [ -d ucrt64/lib/ ]; then + ls "${MSYS_LIB_DIR}/ucrt64/lib/" + mkdir -p "${DST_DIR}/lib/x64-ucrt" + mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" + cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt + fi + + # Create build artifact + zip -r "${DST_DIR}.zip" "$DST_DIR" -# Same for the 32-bit versions -if [ -d mingw32/lib/ ]; then - ls "${MSYS_LIB_DIR}/mingw32/lib/" - mkdir -p "${DST_DIR}/lib/i386" - mv mingw32/lib/*.a "${DST_DIR}/lib/i386" - cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" + # Copy that to a file name/path that does not vary by version number so we + # can easily find it in the R package tests on CI + cp "${DST_DIR}.zip" ../libarrow.zip + + exit 0 fi -# Do the same also for ucrt64 -if [ -d ucrt64/lib/ ]; then - ls "${MSYS_LIB_DIR}/ucrt64/lib/" - mkdir -p "${DST_DIR}/lib/x64-ucrt" - mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" - cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt +: "${RTOOLS_HOME:=/c/rtools${RTOOLS_VERSION}}" +TOOLCHAIN="${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix" +export PATH="${TOOLCHAIN}/bin:${RTOOLS_HOME}/usr/bin:${PATH}" + +# CMake is a native Windows program, so give it Windows-style paths +TOOLCHAIN_WIN="$(cygpath -am "${TOOLCHAIN}")" +BUILD_DIR="$(pwd)/build" +CPP_BUILD_DIR="${BUILD_DIR}/cpp" +INSTALL_DIR="${BUILD_DIR}/install" +INSTALL_DIR_WIN="$(cygpath -am "${INSTALL_DIR}")" + +# Prefer the runner's CMake: the one in older Rtools can be too old for Arrow +CMAKE="cmake" +if [ -x "${PROGRAMFILES}/CMake/bin/cmake.exe" ]; then + CMAKE="${PROGRAMFILES}/CMake/bin/cmake.exe" fi +printenv +gcc --version +g++ --version +"${CMAKE}" --version +make --version | head -n 1 +ls "${TOOLCHAIN}/lib" +cat "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" + +# google-cloud-cpp links static cURL. CXXFLAGS, not ARROW_CXXFLAGS, because +# ARROW_CXXFLAGS aren't passed to ExternalProjects. +export CXXFLAGS="${CXXFLAGS:-} -DCURL_STATICLIB" + +rm -rf "${CPP_BUILD_DIR}" +mkdir -p "${CPP_BUILD_DIR}" "${INSTALL_DIR}" +pushd "${CPP_BUILD_DIR}" + +# * CMAKE_UNITY_BUILD=OFF: otherwise some compute tests segfault. +# * Bundled because Rtools45 doesn't ship them: AWS SDK, Thrift, +# google-cloud-cpp, snappy, utf8proc. +# * re2 bundled: the Rtools45 re2 would pull abseil onto the R link line. +# * abseil, protobuf bundled: the Rtools45 versions are too old for the +# bundled google-cloud-cpp. +# * Brotli bundled: Rtools45 has 1.2.0, without the 1.2.1 security fix +# (GH-55165). cURL's Brotli symbols resolve from the bundled copy too. +# * ARROW_USE_STD_CHRONO=0: the R package supplies the time zone database +# through tzdb, which only the vendored date library uses (GH-48593). +"${CMAKE}" \ + "$(cygpath -am "${ARROW_HOME}/cpp")" \ + -G "MSYS Makefiles" \ + -DARROW_ACERO=ON \ + -DARROW_AZURE=OFF \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_STATIC=ON \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_COMPUTE=ON \ + -DARROW_CXXFLAGS="-DARROW_USE_STD_CHRONO=0" \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_GCS=ON \ + -DARROW_HDFS=OFF \ + -DARROW_JEMALLOC=OFF \ + -DARROW_JSON=ON \ + -DARROW_LZ4_USE_SHARED=OFF \ + -DARROW_MIMALLOC=ON \ + -DARROW_PACKAGE_PREFIX="${TOOLCHAIN_WIN}" \ + -DARROW_PARQUET=ON \ + -DARROW_S3=ON \ + -DARROW_SNAPPY_USE_SHARED=OFF \ + -DARROW_USE_GLOG=OFF \ + -DARROW_UTF8PROC_USE_SHARED=OFF \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_RE2=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_ZSTD_USE_SHARED=OFF \ + -DAWSSDK_SOURCE=BUNDLED \ + -DBrotli_SOURCE=BUNDLED \ + -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR_WIN}" \ + -DCMAKE_PREFIX_PATH="${TOOLCHAIN_WIN}" \ + -DCMAKE_UNITY_BUILD=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DProtobuf_SOURCE=BUNDLED \ + -DSnappy_SOURCE=BUNDLED \ + -DThrift_SOURCE=BUNDLED \ + -Dabsl_SOURCE=BUNDLED \ + -Dgoogle_cloud_cpp_storage_SOURCE=BUNDLED \ + -Dre2_SOURCE=BUNDLED \ + -Dutf8proc_SOURCE=BUNDLED + +make -j"${NPROC:-$(nproc)}" +make install +popd + +ccache --show-stats || echo "ccache not found" + +VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" +DST_DIR="r-libarrow-windows-x86_64-${VERSION}" + +# Make the zip layout r/configure.win expects: include/ and lib/x64-ucrt/ +pushd "${BUILD_DIR}" +rm -rf "${DST_DIR}" +mkdir -p "${DST_DIR}/lib/x64-ucrt" +cp -r install/include "${DST_DIR}/include" +cp install/lib/*.a "${DST_DIR}/lib/x64-ucrt/" + +# Rtools libraries in PKG_LIBS in r/configure.win; keep in sync +for lib in z zstd lz4 bz2 crypto ssl curl; do + cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" +done + +# cURL's link dependencies vary by Rtools version, so read them from +# libcurl.pc, ship the toolchain ones, and record the flags in libcurl.libs +# for r/configure.win. Static link order matters, so: +# * system libraries (ws2_32, ...) aren't shipped and go last; +# * duplicate flags keep their last occurrence, not their first. +# Brotli is skipped: the bundled copy is used instead (see above). +CURL_LIBS="" +CURL_SYS_LIBS="" +while read -r flag; do + lib="${flag#-l}" + case "${lib}" in + brotli*) continue ;; + ws2_32 | bcrypt | advapi32 | crypt32 | gdi32 | wldap32 | secur32 | normaliz | pthread | m) + CURL_SYS_LIBS="${CURL_SYS_LIBS} ${flag}" + continue + ;; + curl | z | zstd | ssl | crypto) ;; # already copied above + *) + if [ -f "${TOOLCHAIN}/lib/lib${lib}.a" ]; then + cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" + fi + ;; + esac + CURL_LIBS="${CURL_LIBS} ${flag}" +done < <(grep -E '^Libs(\.private)?:' "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" | + grep -o -E '\-l[A-Za-z0-9_+.-]+' | + awk '{ flag[NR] = $0; last[$0] = NR } + END { for (i = 1; i <= NR; i++) if (last[flag[i]] == i) print flag[i] }') +case " ${CURL_LIBS} " in + *" -lcurl "*) ;; + *) + echo "Failed to parse link flags from ${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" >&2 + exit 1 + ;; +esac +echo "${CURL_LIBS}${CURL_SYS_LIBS}" >"${DST_DIR}/lib/x64-ucrt/libcurl.libs" +cat "${DST_DIR}/lib/x64-ucrt/libcurl.libs" +ls -la "${DST_DIR}/lib/x64-ucrt" + # Create build artifact -zip -r "${DST_DIR}.zip" "$DST_DIR" +zip -r "${DST_DIR}.zip" "${DST_DIR}" # Copy that to a file name/path that does not vary by version number so we # can easily find it in the R package tests on CI cp "${DST_DIR}.zip" ../libarrow.zip +popd diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f4aa19bab59b..6d1aab5a7505 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3745,12 +3745,24 @@ function(build_google_cloud_cpp_storage) TRUE PARENT_SCOPE) - # Workaround missing BCRYPT_RSA_ALG_HANDLE macro in older MinGW-w64 headers. - # google-cloud-cpp v3+ uses it without guards in sign_using_sha256.cc. + # Workaround missing BCRYPT_RSA_ALG_HANDLE and BCRYPT_MD5_ALG_HANDLE macros + # in older MinGW-w64 headers. google-cloud-cpp v3+ uses them without guards. set(GOOGLE_CLOUD_CPP_PATCH_COMMAND) find_program(PATCH patch) - if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11") - # This is for RTools 40. + if(MINGW) + include(CheckCXXSymbolExists) + include(CMakePushCheckState) + cmake_push_check_state() + # The macros are Windows 10+ only, so match google-cloud-cpp's _WIN32_WINNT + list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_WIN32_WINNT=0x0A01) + check_cxx_symbol_exists(BCRYPT_RSA_ALG_HANDLE "windows.h;bcrypt.h" + ARROW_HAVE_BCRYPT_RSA_HANDLE) + check_cxx_symbol_exists(BCRYPT_MD5_ALG_HANDLE "windows.h;bcrypt.h" + ARROW_HAVE_BCRYPT_MD5_HANDLE) + cmake_pop_check_state() + endif() + if(MINGW AND NOT (ARROW_HAVE_BCRYPT_RSA_HANDLE AND ARROW_HAVE_BCRYPT_MD5_HANDLE)) + # This is for the MinGW-w64 headers in RTools 40 to 45. if(PATCH) set(GOOGLE_CLOUD_CPP_PATCH_COMMAND ${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/google-cloud-cpp-bcrypt-mingw.patch) diff --git a/cpp/src/arrow/util/chrono_internal.h b/cpp/src/arrow/util/chrono_internal.h index ea4051bccf59..00b873a6d7ab 100644 --- a/cpp/src/arrow/util/chrono_internal.h +++ b/cpp/src/arrow/util/chrono_internal.h @@ -55,10 +55,20 @@ // On non-Windows, we use the vendored date library due to a GCC libstdc++ bug // where DST state is incorrectly reset during timezone rule transitions. // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116110 -#if defined(_WIN32) && defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L -# define ARROW_USE_STD_CHRONO 1 -#else -# define ARROW_USE_STD_CHRONO 0 +// +// Define ARROW_USE_STD_CHRONO (e.g. via ARROW_CXXFLAGS) to override this. +// The R package's Windows build sets it to 0 because std::chrono ignores the +// time zone database that the R package supplies. +#ifndef ARROW_USE_STD_CHRONO +# if defined(_WIN32) && defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L +# define ARROW_USE_STD_CHRONO 1 +# else +# define ARROW_USE_STD_CHRONO 0 +# endif +#endif + +#if ARROW_USE_STD_CHRONO && !(defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L) +# error "ARROW_USE_STD_CHRONO=1 requires C++20 chrono time zone support" #endif #if ARROW_USE_STD_CHRONO diff --git a/r/configure.win b/r/configure.win index 82b8c127f64f..50df4eb7c072 100755 --- a/r/configure.win +++ b/r/configure.win @@ -79,6 +79,20 @@ function configure_binaries() { RWINLIB="../windows/$(ls windows/ | grep ^r-libarrow- | tail -n 1)" fi + # A libarrow zip built with Rtools42+ (see ci/scripts/r_windows_build.sh) + # contains libcurl.libs. In that zip snappy, utf8proc and re2 are in + # arrow_bundled_dependencies, and libcurl.libs has cURL's link flags. + # -ldbghelp is for google-cloud-cpp. The Rtools Brotli is never linked: + # cURL uses the bundled Brotli (GH-55165). + UTF8PROC_SNAPPY_LIBS="-lutf8proc -lsnappy" + RE2_LIBS="-lre2" + CURL_LIBS_FILE="${RWINLIB#../}/lib/x64-ucrt/libcurl.libs" + if [ -f "${CURL_LIBS_FILE}" ]; then + UTF8PROC_SNAPPY_LIBS="" + RE2_LIBS="" + GCS_LIBS="$(cat "${CURL_LIBS_FILE}") -ldbghelp" + fi + # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DARROW_COMPUTE_STATIC -DPARQUET_STATIC \ @@ -93,8 +107,8 @@ function configure_binaries() { ${RWINLIB}/lib\$(subst gcc,,\$(COMPILED_BY))\$(R_ARCH)/libarrow_gcs_bundled_dependencies.a \ ${RWINLIB}/lib\$(R_ARCH)\$(CRT)/libarrow_gcs_bundled_dependencies.a),-larrow_gcs_bundled_dependencies) -larrow \ -larrow_bundled_dependencies \ - -lutf8proc -lsnappy -lz -lzstd -llz4 -lbz2 -lole32 \ - ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS} + ${UTF8PROC_SNAPPY_LIBS} -lz -lzstd -llz4 -lbz2 -lole32 \ + ${MIMALLOC_LIBS} ${OPENSSL_LIBS} ${RE2_LIBS} ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS} }