From 2d249673dc96abcf831574735aab1a03851a3ea9 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Fri, 13 Nov 2026 08:55:35 -0500 Subject: [PATCH 1/6] Try building with RTools45 --- .github/workflows/r.yml | 33 +-- ci/scripts/r_windows_bcrypt_compat.h | 36 +++ ci/scripts/r_windows_build.sh | 317 ++++++++++++++++++++++----- cpp/src/arrow/util/chrono_internal.h | 19 +- r/configure.win | 25 ++- 5 files changed, 350 insertions(+), 80 deletions(-) create mode 100644 ci/scripts/r_windows_bcrypt_compat.h diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 4ae8e3b19bb0..931c185f90cd 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -33,6 +33,7 @@ on: - "ci/scripts/cpp_*.sh" - "ci/scripts/install_minio.sh" - "ci/scripts/r_*.sh" + - "ci/scripts/r_windows_bcrypt_compat.h" - "cpp/**" - "compose.yaml" - "r/**" @@ -46,6 +47,7 @@ on: - "ci/scripts/cpp_*.sh" - "ci/scripts/install_minio.sh" - "ci/scripts/r_*.sh" + - "ci/scripts/r_windows_bcrypt_compat.h" - "cpp/**" - "compose.yaml" - "r/**" @@ -205,7 +207,7 @@ jobs: fail-fast: false matrix: config: - - { rtools: 40, arch: 'ucrt64' } + - { rtools: 45, arch: 'ucrt64' } steps: - run: git config --global core.autocrlf false - name: Checkout Arrow @@ -213,30 +215,16 @@ jobs: with: persist-credentials: false fetch-depth: 0 - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - - name: Cache ccache - uses: actions/cache@v6 - with: - path: ccache - key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} - restore-keys: | - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- - r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - uses: r-lib/actions/setup-r@v2 with: - # Note: we need Rtools40 here because Rtools40 + ucrt is how we build the Arrow C++ - # static library. R itself is not used, but setup-r picks the Rtools version from - # r-version, and R 4.1 was the last R to use Rtools40, so this must stay at 4.1. - r-version: "4.1" + # R itself is not used here, but setup-r picks the Rtools version + # from r-version. R 4.5 gets Rtools45, which is what builds the + # Arrow C++ static library, so this must stay at 4.5. + r-version: "4.5" Ncpus: 2 - name: Build Arrow C++ shell: bash env: - MINGW_ARCH: ${{ matrix.config.arch }} RTOOLS_VERSION: ${{ matrix.config.rtools }} run: ci/scripts/r_windows_build.sh - name: Rename libarrow.zip @@ -283,14 +271,15 @@ jobs: - name: Download Timezone Database shell: bash run: | - # RTools 40 uses GCC 8.x which does not support C++20 chrono timezones, - # so Arrow uses the vendored date library which requires tzdata + # The R libarrow is built with ARROW_USE_STD_CHRONO=0 (see + # ci/scripts/r_windows_build.sh), so Arrow uses the vendored date + # library, which requires tzdata ci/scripts/download_tz_database.sh - run: mkdir r/windows - name: Download artifacts uses: actions/download-artifact@v8 with: - name: libarrow-rtools40-ucrt64.zip + name: libarrow-rtools45-ucrt64.zip path: r/windows - name: Unzip and rezip libarrows shell: bash diff --git a/ci/scripts/r_windows_bcrypt_compat.h b/ci/scripts/r_windows_bcrypt_compat.h new file mode 100644 index 000000000000..76ad2a5e3908 --- /dev/null +++ b/ci/scripts/r_windows_bcrypt_compat.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// BCrypt algorithm pseudo-handles (available since Windows 10 1703). The +// mingw-w64 headers shipped with Rtools45 (mingw-w64 11.0) don't define them, +// but the bundled google-cloud-cpp uses them in google/cloud/internal/win32/. +// This file is force-included into every translation unit via CXXFLAGS by +// ci/scripts/r_windows_build.sh, so it deliberately includes nothing itself: +// BCRYPT_ALG_HANDLE is resolved at the point of use. Values match mingw-w64 +// master include/bcrypt.h. +#ifndef BCRYPT_MD5_ALG_HANDLE +#define BCRYPT_MD5_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x00000021) +#endif +#ifndef BCRYPT_SHA256_ALG_HANDLE +#define BCRYPT_SHA256_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x00000041) +#endif +#ifndef BCRYPT_HMAC_SHA256_ALG_HANDLE +#define BCRYPT_HMAC_SHA256_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x000000b1) +#endif +#ifndef BCRYPT_RSA_ALG_HANDLE +#define BCRYPT_RSA_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x000000e1) +#endif diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index eec1fb5bd6ed..48a17bb22a5c 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -17,6 +17,24 @@ # specific language governing permissions and limitations # under the License. +# Build the static libarrow that r/configure.win downloads for Windows users, +# using the Rtools toolchain that r-lib/actions/setup-r installs. +# +# There are two flows, chosen by RTOOLS_VERSION: +# +# * Rtools40 is MSYS2-based, so libarrow is built with makepkg-mingw from +# ci/scripts/PKGBUILD, with its dependencies installed by pacman. +# * Rtools42 and later are MXE-based: there is no pacman/makepkg, and the +# compiler plus a fixed set of static libraries live in +# ${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix. So we call CMake directly +# and point it at that directory. Libraries Rtools doesn't ship (snappy, +# utf8proc) or ships in a form we don't want to depend on (re2, which needs +# abseil; abseil and protobuf, where Arrow bundles newer versions) are built +# from source via Arrow's bundled dependency mechanism. +# +# The two flows produce zips with different contents; r/configure.win handles +# both. + set -ex : "${ARROW_HOME:=$(pwd)}" @@ -24,69 +42,264 @@ set -ex ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" export ARROW_HOME -pacman --noconfirm -Syy +: "${RTOOLS_VERSION:=45}" -: "${MINGW_ARCH:=mingw32 mingw64 ucrt64}" -export MINGW_ARCH +if [ "${RTOOLS_VERSION}" = "40" ]; then + pacman --noconfirm -Syy -cp "${ARROW_HOME}/ci/scripts/PKGBUILD" . -printenv -makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild + : "${MINGW_ARCH:=mingw32 mingw64 ucrt64}" + export MINGW_ARCH -VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" -DST_DIR="r-libarrow-windows-x86_64-$VERSION" - -# Collect the build artifacts and make the shape of zip file that rwinlib expects -ls -mkdir -p build -mv mingw* build -cd build - -# This may vary by system/CI provider -MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}" - -# Untar the builds we made -find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf -mkdir -p "${DST_DIR}" -# Grab the headers from one, either one is fine -# (if we're building twice to combine old and new toolchains, this may already exist) -if [ ! -d "${DST_DIR}/include" ]; then - mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}" -fi + cp "${ARROW_HOME}/ci/scripts/PKGBUILD" . + printenv + makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild -# mingw64 -> x64 -# mingw32 -> i386 -# ucrt64 -> x64-ucrt - -if [ -d mingw64/lib/ ]; then - ls "${MSYS_LIB_DIR}/mingw64/lib/" - # Make the rest of the directory structure - mkdir -p "${DST_DIR}/lib/x64" - # Move the 64-bit versions of libarrow into the expected location - mv mingw64/lib/*.a "${DST_DIR}/lib/x64" - # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" -fi + VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" + DST_DIR="r-libarrow-windows-x86_64-$VERSION" + + # Collect the build artifacts and make the shape of zip file that rwinlib expects + ls + mkdir -p build + mv mingw* build + cd build + + # This may vary by system/CI provider + MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}" + + # Untar the builds we made + find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf + mkdir -p "${DST_DIR}" + # Grab the headers from one, either one is fine + # (if we're building twice to combine old and new toolchains, this may already exist) + if [ ! -d "${DST_DIR}/include" ]; then + mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}" + fi + + # mingw64 -> x64 + # mingw32 -> i386 + # ucrt64 -> x64-ucrt + + if [ -d mingw64/lib/ ]; then + ls "${MSYS_LIB_DIR}/mingw64/lib/" + # Make the rest of the directory structure + mkdir -p "${DST_DIR}/lib/x64" + # Move the 64-bit versions of libarrow into the expected location + mv mingw64/lib/*.a "${DST_DIR}/lib/x64" + # These are from https://dl.bintray.com/rtools/mingw{32,64}/ + cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" + fi + + # Same for the 32-bit versions + if [ -d mingw32/lib/ ]; then + ls "${MSYS_LIB_DIR}/mingw32/lib/" + mkdir -p "${DST_DIR}/lib/i386" + mv mingw32/lib/*.a "${DST_DIR}/lib/i386" + cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" + fi + + # Do the same also for ucrt64 + if [ -d ucrt64/lib/ ]; then + ls "${MSYS_LIB_DIR}/ucrt64/lib/" + mkdir -p "${DST_DIR}/lib/x64-ucrt" + mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" + cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt + fi + + # Create build artifact + zip -r "${DST_DIR}.zip" "$DST_DIR" -# Same for the 32-bit versions -if [ -d mingw32/lib/ ]; then - ls "${MSYS_LIB_DIR}/mingw32/lib/" - mkdir -p "${DST_DIR}/lib/i386" - mv mingw32/lib/*.a "${DST_DIR}/lib/i386" - cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" + # Copy that to a file name/path that does not vary by version number so we + # can easily find it in the R package tests on CI + cp "${DST_DIR}.zip" ../libarrow.zip + + exit 0 fi -# Do the same also for ucrt64 -if [ -d ucrt64/lib/ ]; then - ls "${MSYS_LIB_DIR}/ucrt64/lib/" - mkdir -p "${DST_DIR}/lib/x64-ucrt" - mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" - cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt +: "${RTOOLS_HOME:=/c/rtools${RTOOLS_VERSION}}" +TOOLCHAIN="${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix" +# setup-r adds these to PATH already, but be explicit so this also works +# from a non-Rtools bash. +export PATH="${TOOLCHAIN}/bin:${RTOOLS_HOME}/usr/bin:${PATH}" + +# CMake is a native Windows program, so give it Windows-style paths +TOOLCHAIN_WIN="$(cygpath -am "${TOOLCHAIN}")" +BUILD_DIR="$(pwd)/build" +CPP_BUILD_DIR="${BUILD_DIR}/cpp" +INSTALL_DIR="${BUILD_DIR}/install" +INSTALL_DIR_WIN="$(cygpath -am "${INSTALL_DIR}")" + +# Prefer the CMake installed on the runner (as the old PKGBUILD did): the +# CMake inside older Rtools toolchains can predate Arrow's minimum (3.25). +CMAKE="cmake" +if [ -x "${PROGRAMFILES}/CMake/bin/cmake.exe" ]; then + CMAKE="${PROGRAMFILES}/CMake/bin/cmake.exe" fi +printenv +gcc --version +g++ --version +"${CMAKE}" --version +make --version | head -n 1 +ls "${TOOLCHAIN}/lib" +# Logged for debugging; parsed below to generate libcurl.libs for r/configure.win +cat "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" + +# We use static cURL in google-cloud-cpp. If we can use cURL's CMake +# package, we don't need to specify CURL_STATICLIB explicitly. But +# we don't have cURL's CMake package. We need to use CXXFLAGS +# instead of ARROW_CXXFLAGS because ARROW_CXXFLAGS aren't passed to +# ExternalProjects. +export CXXFLAGS="${CXXFLAGS:-} -DCURL_STATICLIB" +# The mingw-w64 11.0 headers in Rtools45 lack the BCrypt algorithm +# pseudo-handles that the bundled google-cloud-cpp uses, so force-include a +# header that defines them. See the header for details. +BCRYPT_COMPAT_WIN="$(cygpath -am "${ARROW_HOME}/ci/scripts/r_windows_bcrypt_compat.h")" +export CXXFLAGS="${CXXFLAGS} -include ${BCRYPT_COMPAT_WIN}" + +rm -rf "${CPP_BUILD_DIR}" +mkdir -p "${CPP_BUILD_DIR}" "${INSTALL_DIR}" +pushd "${CPP_BUILD_DIR}" + +# CMAKE_UNITY_BUILD is set to OFF as otherwise some compute functionality +# segfaults in tests +# +# AWS SDK, Thrift and google-cloud-cpp are bundled because Rtools doesn't +# ship them. snappy and utf8proc are bundled because Rtools45 doesn't ship +# them. re2 is bundled because the Rtools45 re2 depends on abseil, which +# would need all the absl_* libraries on the link line in r/configure.win. +# abseil and protobuf are bundled explicitly because Rtools45 ships older +# versions (20250127 and 29.5) than Arrow's bundled google-cloud-cpp expects, +# and with CMAKE_PREFIX_PATH set, AUTO would otherwise pick them up. +# ARROW_USE_STD_CHRONO=0 keeps the vendored date library for time zones +# instead of C++20 std::chrono, which Arrow otherwise selects on Windows with +# gcc 14's libstdc++ (GH-48593). The R package supplies the time zone database +# through the tzdb package via the vendored library's set_install(), which +# std::chrono ignores; libstdc++ 14 also can't format zoned_time and has +# a DST bug (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116110). This is +# the same behaviour as the Rtools40 build, whose gcc 10 has no C++20 chrono +# time zone support. +# +# Brotli is bundled because the Rtools45 Brotli (1.2.0) predates the +# security fix in 1.2.1 (GH-55165). The Rtools45 cURL is built against +# Brotli too, so we deliberately don't ship or link the Rtools Brotli; cURL's +# Brotli symbols resolve from the bundled copy in arrow_bundled_dependencies. +"${CMAKE}" \ + "$(cygpath -am "${ARROW_HOME}/cpp")" \ + -G "MSYS Makefiles" \ + -DARROW_ACERO=ON \ + -DARROW_AZURE=OFF \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_STATIC=ON \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_COMPUTE=ON \ + -DARROW_CXXFLAGS="-DARROW_USE_STD_CHRONO=0" \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_GCS=ON \ + -DARROW_HDFS=OFF \ + -DARROW_JEMALLOC=OFF \ + -DARROW_JSON=ON \ + -DARROW_LZ4_USE_SHARED=OFF \ + -DARROW_MIMALLOC=ON \ + -DARROW_PACKAGE_PREFIX="${TOOLCHAIN_WIN}" \ + -DARROW_PARQUET=ON \ + -DARROW_S3=ON \ + -DARROW_SNAPPY_USE_SHARED=OFF \ + -DARROW_USE_GLOG=OFF \ + -DARROW_UTF8PROC_USE_SHARED=OFF \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_RE2=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_ZSTD_USE_SHARED=OFF \ + -DAWSSDK_SOURCE=BUNDLED \ + -DBrotli_SOURCE=BUNDLED \ + -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR_WIN}" \ + -DCMAKE_PREFIX_PATH="${TOOLCHAIN_WIN}" \ + -DCMAKE_UNITY_BUILD=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DProtobuf_SOURCE=BUNDLED \ + -DSnappy_SOURCE=BUNDLED \ + -DThrift_SOURCE=BUNDLED \ + -Dabsl_SOURCE=BUNDLED \ + -Dgoogle_cloud_cpp_storage_SOURCE=BUNDLED \ + -Dre2_SOURCE=BUNDLED \ + -Dutf8proc_SOURCE=BUNDLED + +make -j"${NPROC:-$(nproc)}" +make install +popd + +VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" +DST_DIR="r-libarrow-windows-x86_64-${VERSION}" + +# Collect the build artifacts and make the shape of zip file that rwinlib +# expects: include/ plus lib/x64-ucrt/ (R_ARCH + CRT, see r/configure.win) +pushd "${BUILD_DIR}" +rm -rf "${DST_DIR}" +mkdir -p "${DST_DIR}/lib/x64-ucrt" +cp -r install/include "${DST_DIR}/include" +cp install/lib/*.a "${DST_DIR}/lib/x64-ucrt/" + +# Rtools libraries that r/configure.win links against. Users compiling the R +# package have an Rtools with these too, but ship them so the zip is +# self-contained. Keep in sync with PKG_LIBS in r/configure.win. +for lib in z zstd lz4 bz2 crypto ssl curl; do + cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" +done + +# cURL's link dependencies differ between Rtools versions (e.g. Rtools43's +# cURL has no nghttp2 or libpsl, Rtools45's does), so derive them from this +# toolchain's libcurl.pc, ship the ones that are toolchain libraries, and +# record the flags in the zip for r/configure.win to use as GCS_LIBS. +# Brotli is skipped: see the Brotli comment above. Windows system and mingw +# runtime import libraries (ws2_32, crypt32, ...) aren't ours to ship, and +# they are written after all the toolchain libraries: with static linking +# every library that needs Winsock etc. must come before -lws2_32, and +# pkg-config's own ordering doesn't guarantee that (Rtools43's libcurl.pc +# lists -lcrypto after the last -lws2_32). +CURL_LIBS="" +CURL_SYS_LIBS="" +while read -r flag; do + lib="${flag#-l}" + case "${lib}" in + brotli*) continue ;; + ws2_32 | bcrypt | advapi32 | crypt32 | gdi32 | wldap32 | secur32 | normaliz | pthread | m) + CURL_SYS_LIBS="${CURL_SYS_LIBS} ${flag}" + continue + ;; + curl | z | zstd | ssl | crypto) ;; # already copied above + *) + if [ -f "${TOOLCHAIN}/lib/lib${lib}.a" ]; then + cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" + fi + ;; + esac + CURL_LIBS="${CURL_LIBS} ${flag}" +done < <(grep -E '^Libs(\.private)?:' "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" | + grep -o -E '\-l[A-Za-z0-9_+.-]+' | awk '!seen[$0]++') +case " ${CURL_LIBS} " in + *" -lcurl "*) ;; + *) + echo "Failed to parse link flags from ${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" >&2 + exit 1 + ;; +esac +echo "${CURL_LIBS}${CURL_SYS_LIBS}" >"${DST_DIR}/lib/x64-ucrt/libcurl.libs" +cat "${DST_DIR}/lib/x64-ucrt/libcurl.libs" +ls -la "${DST_DIR}/lib/x64-ucrt" + # Create build artifact -zip -r "${DST_DIR}.zip" "$DST_DIR" +zip -r "${DST_DIR}.zip" "${DST_DIR}" # Copy that to a file name/path that does not vary by version number so we # can easily find it in the R package tests on CI cp "${DST_DIR}.zip" ../libarrow.zip +popd diff --git a/cpp/src/arrow/util/chrono_internal.h b/cpp/src/arrow/util/chrono_internal.h index ea4051bccf59..dc553bb028f1 100644 --- a/cpp/src/arrow/util/chrono_internal.h +++ b/cpp/src/arrow/util/chrono_internal.h @@ -55,10 +55,21 @@ // On non-Windows, we use the vendored date library due to a GCC libstdc++ bug // where DST state is incorrectly reset during timezone rule transitions. // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116110 -#if defined(_WIN32) && defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L -# define ARROW_USE_STD_CHRONO 1 -#else -# define ARROW_USE_STD_CHRONO 0 +// +// The selection can be overridden by defining ARROW_USE_STD_CHRONO on the +// command line (e.g. via ARROW_CXXFLAGS). The R package's Windows build sets +// it to 0 because the R package provides the time zone database through +// arrow::Initialize(), which only the vendored date library honours. +#ifndef ARROW_USE_STD_CHRONO +# if defined(_WIN32) && defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L +# define ARROW_USE_STD_CHRONO 1 +# else +# define ARROW_USE_STD_CHRONO 0 +# endif +#endif + +#if ARROW_USE_STD_CHRONO && !(defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L) +# error "ARROW_USE_STD_CHRONO=1 requires C++20 chrono time zone support" #endif #if ARROW_USE_STD_CHRONO diff --git a/r/configure.win b/r/configure.win index 82b8c127f64f..949fd07acdb7 100755 --- a/r/configure.win +++ b/r/configure.win @@ -79,6 +79,27 @@ function configure_binaries() { RWINLIB="../windows/$(ls windows/ | grep ^r-libarrow- | tail -n 1)" fi + # The libarrow zip has one of two layouts, depending on which flow in + # ci/scripts/r_windows_build.sh built it: + # + # * Rtools40 (ci/scripts/PKGBUILD): snappy, utf8proc and re2 are separate + # libraries in the zip, and cURL's link dependencies are the fixed + # GCS_LIBS above. + # * Rtools42 or later: snappy, utf8proc and re2 are built as part of + # arrow_bundled_dependencies, so they aren't linked separately. cURL's + # link dependencies depend on which Rtools built the zip, so they are + # recorded in libcurl.libs in the zip. -ldbghelp is for google-cloud-cpp. + # The Rtools Brotli libraries are never included: cURL's Brotli symbols + # come from the bundled Brotli in arrow_bundled_dependencies (GH-55165). + UTF8PROC_SNAPPY_LIBS="-lutf8proc -lsnappy" + RE2_LIBS="-lre2" + CURL_LIBS_FILE="${RWINLIB#../}/lib/x64-ucrt/libcurl.libs" + if [ -f "${CURL_LIBS_FILE}" ]; then + UTF8PROC_SNAPPY_LIBS="" + RE2_LIBS="" + GCS_LIBS="$(cat "${CURL_LIBS_FILE}") -ldbghelp" + fi + # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DARROW_COMPUTE_STATIC -DPARQUET_STATIC \ @@ -93,8 +114,8 @@ function configure_binaries() { ${RWINLIB}/lib\$(subst gcc,,\$(COMPILED_BY))\$(R_ARCH)/libarrow_gcs_bundled_dependencies.a \ ${RWINLIB}/lib\$(R_ARCH)\$(CRT)/libarrow_gcs_bundled_dependencies.a),-larrow_gcs_bundled_dependencies) -larrow \ -larrow_bundled_dependencies \ - -lutf8proc -lsnappy -lz -lzstd -llz4 -lbz2 -lole32 \ - ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS} + ${UTF8PROC_SNAPPY_LIBS} -lz -lzstd -llz4 -lbz2 -lole32 \ + ${MIMALLOC_LIBS} ${OPENSSL_LIBS} ${RE2_LIBS} ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS} } From ddee2fa804597b5e9f0d4db80a56f666f0e4b7dc Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Fri, 13 Nov 2026 09:41:39 -0500 Subject: [PATCH 2/6] Keep last occurrence of each cURL link flag --- ci/scripts/r_windows_build.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index 48a17bb22a5c..6fdb3e077588 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -265,6 +265,12 @@ done # every library that needs Winsock etc. must come before -lws2_32, and # pkg-config's own ordering doesn't guarantee that (Rtools43's libcurl.pc # lists -lcrypto after the last -lws2_32). +# +# libcurl.pc repeats libraries, and with static linking a library must come +# after everything that needs it, so when removing the duplicates keep the +# last occurrence of each flag, not the first (Rtools45's libcurl.pc has -lz +# and -lunistring both before and after the -lcrypto and -lpsl that need +# them). CURL_LIBS="" CURL_SYS_LIBS="" while read -r flag; do @@ -284,7 +290,9 @@ while read -r flag; do esac CURL_LIBS="${CURL_LIBS} ${flag}" done < <(grep -E '^Libs(\.private)?:' "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" | - grep -o -E '\-l[A-Za-z0-9_+.-]+' | awk '!seen[$0]++') + grep -o -E '\-l[A-Za-z0-9_+.-]+' | + awk '{ flag[NR] = $0; last[$0] = NR } + END { for (i = 1; i <= NR; i++) if (last[flag[i]] == i) print flag[i] }') case " ${CURL_LIBS} " in *" -lcurl "*) ;; *) From 02d19cbd10a821df531f0cc6acab6167effdad16 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Fri, 13 Nov 2026 10:38:18 -0500 Subject: [PATCH 3/6] Simplify bcrypt fix --- .github/workflows/r.yml | 2 -- ci/scripts/r_windows_bcrypt_compat.h | 36 --------------------- ci/scripts/r_windows_build.sh | 5 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 15 ++++++--- 4 files changed, 11 insertions(+), 47 deletions(-) delete mode 100644 ci/scripts/r_windows_bcrypt_compat.h diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 931c185f90cd..47ac5fac394c 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -33,7 +33,6 @@ on: - "ci/scripts/cpp_*.sh" - "ci/scripts/install_minio.sh" - "ci/scripts/r_*.sh" - - "ci/scripts/r_windows_bcrypt_compat.h" - "cpp/**" - "compose.yaml" - "r/**" @@ -47,7 +46,6 @@ on: - "ci/scripts/cpp_*.sh" - "ci/scripts/install_minio.sh" - "ci/scripts/r_*.sh" - - "ci/scripts/r_windows_bcrypt_compat.h" - "cpp/**" - "compose.yaml" - "r/**" diff --git a/ci/scripts/r_windows_bcrypt_compat.h b/ci/scripts/r_windows_bcrypt_compat.h deleted file mode 100644 index 76ad2a5e3908..000000000000 --- a/ci/scripts/r_windows_bcrypt_compat.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// BCrypt algorithm pseudo-handles (available since Windows 10 1703). The -// mingw-w64 headers shipped with Rtools45 (mingw-w64 11.0) don't define them, -// but the bundled google-cloud-cpp uses them in google/cloud/internal/win32/. -// This file is force-included into every translation unit via CXXFLAGS by -// ci/scripts/r_windows_build.sh, so it deliberately includes nothing itself: -// BCRYPT_ALG_HANDLE is resolved at the point of use. Values match mingw-w64 -// master include/bcrypt.h. -#ifndef BCRYPT_MD5_ALG_HANDLE -#define BCRYPT_MD5_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x00000021) -#endif -#ifndef BCRYPT_SHA256_ALG_HANDLE -#define BCRYPT_SHA256_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x00000041) -#endif -#ifndef BCRYPT_HMAC_SHA256_ALG_HANDLE -#define BCRYPT_HMAC_SHA256_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x000000b1) -#endif -#ifndef BCRYPT_RSA_ALG_HANDLE -#define BCRYPT_RSA_ALG_HANDLE ((BCRYPT_ALG_HANDLE)0x000000e1) -#endif diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index 6fdb3e077588..a81d43d3de59 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -150,11 +150,6 @@ cat "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" # instead of ARROW_CXXFLAGS because ARROW_CXXFLAGS aren't passed to # ExternalProjects. export CXXFLAGS="${CXXFLAGS:-} -DCURL_STATICLIB" -# The mingw-w64 11.0 headers in Rtools45 lack the BCrypt algorithm -# pseudo-handles that the bundled google-cloud-cpp uses, so force-include a -# header that defines them. See the header for details. -BCRYPT_COMPAT_WIN="$(cygpath -am "${ARROW_HOME}/ci/scripts/r_windows_bcrypt_compat.h")" -export CXXFLAGS="${CXXFLAGS} -include ${BCRYPT_COMPAT_WIN}" rm -rf "${CPP_BUILD_DIR}" mkdir -p "${CPP_BUILD_DIR}" "${INSTALL_DIR}" diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f4aa19bab59b..51b4d68a0d19 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3745,12 +3745,19 @@ function(build_google_cloud_cpp_storage) TRUE PARENT_SCOPE) - # Workaround missing BCRYPT_RSA_ALG_HANDLE macro in older MinGW-w64 headers. - # google-cloud-cpp v3+ uses it without guards in sign_using_sha256.cc. + # Workaround missing BCRYPT_RSA_ALG_HANDLE and BCRYPT_MD5_ALG_HANDLE macros + # in older MinGW-w64 headers. google-cloud-cpp v3+ uses them without guards. set(GOOGLE_CLOUD_CPP_PATCH_COMMAND) find_program(PATCH patch) - if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11") - # This is for RTools 40. + if(MINGW) + include(CheckCXXSymbolExists) + check_cxx_symbol_exists(BCRYPT_RSA_ALG_HANDLE "windows.h;bcrypt.h" + ARROW_HAVE_BCRYPT_RSA_HANDLE) + check_cxx_symbol_exists(BCRYPT_MD5_ALG_HANDLE "windows.h;bcrypt.h" + ARROW_HAVE_BCRYPT_MD5_HANDLE) + endif() + if(MINGW AND NOT (ARROW_HAVE_BCRYPT_RSA_HANDLE AND ARROW_HAVE_BCRYPT_MD5_HANDLE)) + # This is for the MinGW-w64 headers in RTools 40 to 45. if(PATCH) set(GOOGLE_CLOUD_CPP_PATCH_COMMAND ${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/google-cloud-cpp-bcrypt-mingw.patch) From 820c0b59826beffd17f1a0f65762a62631592c93 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Fri, 13 Nov 2026 11:08:28 -0500 Subject: [PATCH 4/6] Don't leak into other jobs --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 51b4d68a0d19..557704ca67c5 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3751,10 +3751,17 @@ function(build_google_cloud_cpp_storage) find_program(PATCH patch) if(MINGW) include(CheckCXXSymbolExists) + include(CMakePushCheckState) + cmake_push_check_state() + # MinGW-w64 headers that have these macros only define them for + # Windows 10+, so check with the same _WIN32_WINNT that google-cloud-cpp + # is built with below. + list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_WIN32_WINNT=0x0A01) check_cxx_symbol_exists(BCRYPT_RSA_ALG_HANDLE "windows.h;bcrypt.h" ARROW_HAVE_BCRYPT_RSA_HANDLE) check_cxx_symbol_exists(BCRYPT_MD5_ALG_HANDLE "windows.h;bcrypt.h" ARROW_HAVE_BCRYPT_MD5_HANDLE) + cmake_pop_check_state() endif() if(MINGW AND NOT (ARROW_HAVE_BCRYPT_RSA_HANDLE AND ARROW_HAVE_BCRYPT_MD5_HANDLE)) # This is for the MinGW-w64 headers in RTools 40 to 45. From 5d7eb4c185a9b8dc6f56c6f417b44e094c702dd4 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Sun, 15 Nov 2026 12:40:40 -0500 Subject: [PATCH 5/6] Trim comments --- .github/workflows/r.yml | 10 +-- ci/scripts/r_windows_build.sh | 95 ++++++--------------- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 +- cpp/src/arrow/util/chrono_internal.h | 7 +- r/configure.win | 17 ++-- 5 files changed, 39 insertions(+), 94 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 47ac5fac394c..038fd4c2d6d9 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -215,9 +215,8 @@ jobs: fetch-depth: 0 - uses: r-lib/actions/setup-r@v2 with: - # R itself is not used here, but setup-r picks the Rtools version - # from r-version. R 4.5 gets Rtools45, which is what builds the - # Arrow C++ static library, so this must stay at 4.5. + # R is not used, but setup-r picks Rtools from r-version: 4.5 gets + # Rtools45, which builds the Arrow C++ static library. r-version: "4.5" Ncpus: 2 - name: Build Arrow C++ @@ -269,9 +268,8 @@ jobs: - name: Download Timezone Database shell: bash run: | - # The R libarrow is built with ARROW_USE_STD_CHRONO=0 (see - # ci/scripts/r_windows_build.sh), so Arrow uses the vendored date - # library, which requires tzdata + # libarrow is built with ARROW_USE_STD_CHRONO=0, so Arrow uses the + # vendored date library, which requires tzdata ci/scripts/download_tz_database.sh - run: mkdir r/windows - name: Download artifacts diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index a81d43d3de59..fc9157558678 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -17,23 +17,13 @@ # specific language governing permissions and limitations # under the License. -# Build the static libarrow that r/configure.win downloads for Windows users, -# using the Rtools toolchain that r-lib/actions/setup-r installs. +# Build the static libarrow that r/configure.win downloads on Windows. # -# There are two flows, chosen by RTOOLS_VERSION: +# * Rtools40 (MSYS2): makepkg-mingw with ci/scripts/PKGBUILD. +# * Rtools42+ (MXE, no pacman/makepkg): call CMake directly against the +# toolchain's static libraries. # -# * Rtools40 is MSYS2-based, so libarrow is built with makepkg-mingw from -# ci/scripts/PKGBUILD, with its dependencies installed by pacman. -# * Rtools42 and later are MXE-based: there is no pacman/makepkg, and the -# compiler plus a fixed set of static libraries live in -# ${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix. So we call CMake directly -# and point it at that directory. Libraries Rtools doesn't ship (snappy, -# utf8proc) or ships in a form we don't want to depend on (re2, which needs -# abseil; abseil and protobuf, where Arrow bundles newer versions) are built -# from source via Arrow's bundled dependency mechanism. -# -# The two flows produce zips with different contents; r/configure.win handles -# both. +# The two zips differ in contents; r/configure.win handles both. set -ex @@ -117,8 +107,6 @@ fi : "${RTOOLS_HOME:=/c/rtools${RTOOLS_VERSION}}" TOOLCHAIN="${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix" -# setup-r adds these to PATH already, but be explicit so this also works -# from a non-Rtools bash. export PATH="${TOOLCHAIN}/bin:${RTOOLS_HOME}/usr/bin:${PATH}" # CMake is a native Windows program, so give it Windows-style paths @@ -128,8 +116,7 @@ CPP_BUILD_DIR="${BUILD_DIR}/cpp" INSTALL_DIR="${BUILD_DIR}/install" INSTALL_DIR_WIN="$(cygpath -am "${INSTALL_DIR}")" -# Prefer the CMake installed on the runner (as the old PKGBUILD did): the -# CMake inside older Rtools toolchains can predate Arrow's minimum (3.25). +# Prefer the runner's CMake: the one in older Rtools can be too old for Arrow CMAKE="cmake" if [ -x "${PROGRAMFILES}/CMake/bin/cmake.exe" ]; then CMAKE="${PROGRAMFILES}/CMake/bin/cmake.exe" @@ -141,43 +128,26 @@ g++ --version "${CMAKE}" --version make --version | head -n 1 ls "${TOOLCHAIN}/lib" -# Logged for debugging; parsed below to generate libcurl.libs for r/configure.win cat "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" -# We use static cURL in google-cloud-cpp. If we can use cURL's CMake -# package, we don't need to specify CURL_STATICLIB explicitly. But -# we don't have cURL's CMake package. We need to use CXXFLAGS -# instead of ARROW_CXXFLAGS because ARROW_CXXFLAGS aren't passed to -# ExternalProjects. +# google-cloud-cpp links static cURL. CXXFLAGS, not ARROW_CXXFLAGS, because +# ARROW_CXXFLAGS aren't passed to ExternalProjects. export CXXFLAGS="${CXXFLAGS:-} -DCURL_STATICLIB" rm -rf "${CPP_BUILD_DIR}" mkdir -p "${CPP_BUILD_DIR}" "${INSTALL_DIR}" pushd "${CPP_BUILD_DIR}" -# CMAKE_UNITY_BUILD is set to OFF as otherwise some compute functionality -# segfaults in tests -# -# AWS SDK, Thrift and google-cloud-cpp are bundled because Rtools doesn't -# ship them. snappy and utf8proc are bundled because Rtools45 doesn't ship -# them. re2 is bundled because the Rtools45 re2 depends on abseil, which -# would need all the absl_* libraries on the link line in r/configure.win. -# abseil and protobuf are bundled explicitly because Rtools45 ships older -# versions (20250127 and 29.5) than Arrow's bundled google-cloud-cpp expects, -# and with CMAKE_PREFIX_PATH set, AUTO would otherwise pick them up. -# ARROW_USE_STD_CHRONO=0 keeps the vendored date library for time zones -# instead of C++20 std::chrono, which Arrow otherwise selects on Windows with -# gcc 14's libstdc++ (GH-48593). The R package supplies the time zone database -# through the tzdb package via the vendored library's set_install(), which -# std::chrono ignores; libstdc++ 14 also can't format zoned_time and has -# a DST bug (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116110). This is -# the same behaviour as the Rtools40 build, whose gcc 10 has no C++20 chrono -# time zone support. -# -# Brotli is bundled because the Rtools45 Brotli (1.2.0) predates the -# security fix in 1.2.1 (GH-55165). The Rtools45 cURL is built against -# Brotli too, so we deliberately don't ship or link the Rtools Brotli; cURL's -# Brotli symbols resolve from the bundled copy in arrow_bundled_dependencies. +# * CMAKE_UNITY_BUILD=OFF: otherwise some compute tests segfault. +# * Bundled because Rtools45 doesn't ship them: AWS SDK, Thrift, +# google-cloud-cpp, snappy, utf8proc. +# * re2 bundled: the Rtools45 re2 would pull abseil onto the R link line. +# * abseil, protobuf bundled: the Rtools45 versions are too old for the +# bundled google-cloud-cpp. +# * Brotli bundled: Rtools45 has 1.2.0, without the 1.2.1 security fix +# (GH-55165). cURL's Brotli symbols resolve from the bundled copy too. +# * ARROW_USE_STD_CHRONO=0: the R package supplies the time zone database +# through tzdb, which only the vendored date library uses (GH-48593). "${CMAKE}" \ "$(cygpath -am "${ARROW_HOME}/cpp")" \ -G "MSYS Makefiles" \ @@ -235,37 +205,24 @@ popd VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" DST_DIR="r-libarrow-windows-x86_64-${VERSION}" -# Collect the build artifacts and make the shape of zip file that rwinlib -# expects: include/ plus lib/x64-ucrt/ (R_ARCH + CRT, see r/configure.win) +# Make the zip layout r/configure.win expects: include/ and lib/x64-ucrt/ pushd "${BUILD_DIR}" rm -rf "${DST_DIR}" mkdir -p "${DST_DIR}/lib/x64-ucrt" cp -r install/include "${DST_DIR}/include" cp install/lib/*.a "${DST_DIR}/lib/x64-ucrt/" -# Rtools libraries that r/configure.win links against. Users compiling the R -# package have an Rtools with these too, but ship them so the zip is -# self-contained. Keep in sync with PKG_LIBS in r/configure.win. +# Rtools libraries in PKG_LIBS in r/configure.win; keep in sync for lib in z zstd lz4 bz2 crypto ssl curl; do cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" done -# cURL's link dependencies differ between Rtools versions (e.g. Rtools43's -# cURL has no nghttp2 or libpsl, Rtools45's does), so derive them from this -# toolchain's libcurl.pc, ship the ones that are toolchain libraries, and -# record the flags in the zip for r/configure.win to use as GCS_LIBS. -# Brotli is skipped: see the Brotli comment above. Windows system and mingw -# runtime import libraries (ws2_32, crypt32, ...) aren't ours to ship, and -# they are written after all the toolchain libraries: with static linking -# every library that needs Winsock etc. must come before -lws2_32, and -# pkg-config's own ordering doesn't guarantee that (Rtools43's libcurl.pc -# lists -lcrypto after the last -lws2_32). -# -# libcurl.pc repeats libraries, and with static linking a library must come -# after everything that needs it, so when removing the duplicates keep the -# last occurrence of each flag, not the first (Rtools45's libcurl.pc has -lz -# and -lunistring both before and after the -lcrypto and -lpsl that need -# them). +# cURL's link dependencies vary by Rtools version, so read them from +# libcurl.pc, ship the toolchain ones, and record the flags in libcurl.libs +# for r/configure.win. Static link order matters, so: +# * system libraries (ws2_32, ...) aren't shipped and go last; +# * duplicate flags keep their last occurrence, not their first. +# Brotli is skipped: the bundled copy is used instead (see above). CURL_LIBS="" CURL_SYS_LIBS="" while read -r flag; do diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 557704ca67c5..6d1aab5a7505 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3753,9 +3753,7 @@ function(build_google_cloud_cpp_storage) include(CheckCXXSymbolExists) include(CMakePushCheckState) cmake_push_check_state() - # MinGW-w64 headers that have these macros only define them for - # Windows 10+, so check with the same _WIN32_WINNT that google-cloud-cpp - # is built with below. + # The macros are Windows 10+ only, so match google-cloud-cpp's _WIN32_WINNT list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_WIN32_WINNT=0x0A01) check_cxx_symbol_exists(BCRYPT_RSA_ALG_HANDLE "windows.h;bcrypt.h" ARROW_HAVE_BCRYPT_RSA_HANDLE) diff --git a/cpp/src/arrow/util/chrono_internal.h b/cpp/src/arrow/util/chrono_internal.h index dc553bb028f1..00b873a6d7ab 100644 --- a/cpp/src/arrow/util/chrono_internal.h +++ b/cpp/src/arrow/util/chrono_internal.h @@ -56,10 +56,9 @@ // where DST state is incorrectly reset during timezone rule transitions. // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116110 // -// The selection can be overridden by defining ARROW_USE_STD_CHRONO on the -// command line (e.g. via ARROW_CXXFLAGS). The R package's Windows build sets -// it to 0 because the R package provides the time zone database through -// arrow::Initialize(), which only the vendored date library honours. +// Define ARROW_USE_STD_CHRONO (e.g. via ARROW_CXXFLAGS) to override this. +// The R package's Windows build sets it to 0 because std::chrono ignores the +// time zone database that the R package supplies. #ifndef ARROW_USE_STD_CHRONO # if defined(_WIN32) && defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907L # define ARROW_USE_STD_CHRONO 1 diff --git a/r/configure.win b/r/configure.win index 949fd07acdb7..50df4eb7c072 100755 --- a/r/configure.win +++ b/r/configure.win @@ -79,18 +79,11 @@ function configure_binaries() { RWINLIB="../windows/$(ls windows/ | grep ^r-libarrow- | tail -n 1)" fi - # The libarrow zip has one of two layouts, depending on which flow in - # ci/scripts/r_windows_build.sh built it: - # - # * Rtools40 (ci/scripts/PKGBUILD): snappy, utf8proc and re2 are separate - # libraries in the zip, and cURL's link dependencies are the fixed - # GCS_LIBS above. - # * Rtools42 or later: snappy, utf8proc and re2 are built as part of - # arrow_bundled_dependencies, so they aren't linked separately. cURL's - # link dependencies depend on which Rtools built the zip, so they are - # recorded in libcurl.libs in the zip. -ldbghelp is for google-cloud-cpp. - # The Rtools Brotli libraries are never included: cURL's Brotli symbols - # come from the bundled Brotli in arrow_bundled_dependencies (GH-55165). + # A libarrow zip built with Rtools42+ (see ci/scripts/r_windows_build.sh) + # contains libcurl.libs. In that zip snappy, utf8proc and re2 are in + # arrow_bundled_dependencies, and libcurl.libs has cURL's link flags. + # -ldbghelp is for google-cloud-cpp. The Rtools Brotli is never linked: + # cURL uses the bundled Brotli (GH-55165). UTF8PROC_SNAPPY_LIBS="-lutf8proc -lsnappy" RE2_LIBS="-lre2" CURL_LIBS_FILE="${RWINLIB#../}/lib/x64-ucrt/libcurl.libs" From 75cc517e690e65edfb5251b1d7a39de9bffdef33 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Sun, 15 Nov 2026 13:09:33 -0500 Subject: [PATCH 6/6] Attempt to restore ccache --- .github/workflows/r.yml | 19 +++++++++++++++++++ ci/scripts/r_windows_build.sh | 2 ++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 038fd4c2d6d9..c13bc925adbf 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -213,6 +213,25 @@ jobs: with: persist-credentials: false fetch-depth: 0 + - name: Install ccache + shell: bash + run: | + mkdir -p "$HOME/.local/bin" + ci/scripts/install_ccache.sh 4.13.6 "$HOME/.local" + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Setup ccache + shell: bash + run: | + ci/scripts/ccache_setup.sh + echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV + - name: Cache ccache + uses: actions/cache@v6 + with: + path: ccache + key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} + restore-keys: | + r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- + r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - uses: r-lib/actions/setup-r@v2 with: # R is not used, but setup-r picks Rtools from r-version: 4.5 gets diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index fc9157558678..4c53e774d22e 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -202,6 +202,8 @@ make -j"${NPROC:-$(nproc)}" make install popd +ccache --show-stats || echo "ccache not found" + VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" DST_DIR="r-libarrow-windows-x86_64-${VERSION}"