Repository navigation
GH-55398: [R][CI] Build the Windows libarrow binary with Rtools45 instead of Rtools40 #55594
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
2d24967
ddee2fa
02d19cb
820c0b5
5d7eb4c
75cc517
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -17,76 +17,251 @@ | |||||
| # specific language governing permissions and limitations | ||||||
| # under the License. | ||||||
|
|
||||||
| # Build the static libarrow that r/configure.win downloads on Windows. | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AFAICT reading the diff, there is basically nothing shared between the old version and the new one, so maybe it would be cleaner and more readable to just have two different scripts. |
||||||
| # | ||||||
| # * Rtools40 (MSYS2): makepkg-mingw with ci/scripts/PKGBUILD. | ||||||
| # * Rtools42+ (MXE, no pacman/makepkg): call CMake directly against the | ||||||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
I'll apply this change before merging, not worth ending up rerunning CI right this sec for a comment |
||||||
| # toolchain's static libraries. | ||||||
| # | ||||||
| # The two zips differ in contents; r/configure.win handles both. | ||||||
|
|
||||||
| set -ex | ||||||
|
|
||||||
| : "${ARROW_HOME:=$(pwd)}" | ||||||
| # Make sure it is absolute and exported | ||||||
| ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" | ||||||
| export ARROW_HOME | ||||||
|
|
||||||
| pacman --noconfirm -Syy | ||||||
| : "${RTOOLS_VERSION:=45}" | ||||||
|
|
||||||
| : "${MINGW_ARCH:=mingw32 mingw64 ucrt64}" | ||||||
| export MINGW_ARCH | ||||||
| if [ "${RTOOLS_VERSION}" = "40" ]; then | ||||||
| pacman --noconfirm -Syy | ||||||
|
|
||||||
| cp "${ARROW_HOME}/ci/scripts/PKGBUILD" . | ||||||
| printenv | ||||||
| makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild | ||||||
| : "${MINGW_ARCH:=mingw32 mingw64 ucrt64}" | ||||||
| export MINGW_ARCH | ||||||
|
|
||||||
| VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" | ||||||
| DST_DIR="r-libarrow-windows-x86_64-$VERSION" | ||||||
|
|
||||||
| # Collect the build artifacts and make the shape of zip file that rwinlib expects | ||||||
| ls | ||||||
| mkdir -p build | ||||||
| mv mingw* build | ||||||
| cd build | ||||||
|
|
||||||
| # This may vary by system/CI provider | ||||||
| MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}" | ||||||
|
|
||||||
| # Untar the builds we made | ||||||
| find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf | ||||||
| mkdir -p "${DST_DIR}" | ||||||
| # Grab the headers from one, either one is fine | ||||||
| # (if we're building twice to combine old and new toolchains, this may already exist) | ||||||
| if [ ! -d "${DST_DIR}/include" ]; then | ||||||
| mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}" | ||||||
| fi | ||||||
| cp "${ARROW_HOME}/ci/scripts/PKGBUILD" . | ||||||
| printenv | ||||||
| makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild | ||||||
|
|
||||||
| # mingw64 -> x64 | ||||||
| # mingw32 -> i386 | ||||||
| # ucrt64 -> x64-ucrt | ||||||
|
|
||||||
| if [ -d mingw64/lib/ ]; then | ||||||
| ls "${MSYS_LIB_DIR}/mingw64/lib/" | ||||||
| # Make the rest of the directory structure | ||||||
| mkdir -p "${DST_DIR}/lib/x64" | ||||||
| # Move the 64-bit versions of libarrow into the expected location | ||||||
| mv mingw64/lib/*.a "${DST_DIR}/lib/x64" | ||||||
| # These are from https://dl.bintray.com/rtools/mingw{32,64}/ | ||||||
| cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" | ||||||
| fi | ||||||
| VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" | ||||||
| DST_DIR="r-libarrow-windows-x86_64-$VERSION" | ||||||
|
|
||||||
| # Collect the build artifacts and make the shape of zip file that rwinlib expects | ||||||
| ls | ||||||
| mkdir -p build | ||||||
| mv mingw* build | ||||||
| cd build | ||||||
|
|
||||||
| # This may vary by system/CI provider | ||||||
| MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}" | ||||||
|
|
||||||
| # Untar the builds we made | ||||||
| find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf | ||||||
| mkdir -p "${DST_DIR}" | ||||||
| # Grab the headers from one, either one is fine | ||||||
| # (if we're building twice to combine old and new toolchains, this may already exist) | ||||||
| if [ ! -d "${DST_DIR}/include" ]; then | ||||||
| mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}" | ||||||
| fi | ||||||
|
|
||||||
| # mingw64 -> x64 | ||||||
| # mingw32 -> i386 | ||||||
| # ucrt64 -> x64-ucrt | ||||||
|
|
||||||
| if [ -d mingw64/lib/ ]; then | ||||||
| ls "${MSYS_LIB_DIR}/mingw64/lib/" | ||||||
| # Make the rest of the directory structure | ||||||
| mkdir -p "${DST_DIR}/lib/x64" | ||||||
| # Move the 64-bit versions of libarrow into the expected location | ||||||
| mv mingw64/lib/*.a "${DST_DIR}/lib/x64" | ||||||
| # These are from https://dl.bintray.com/rtools/mingw{32,64}/ | ||||||
| cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64" | ||||||
| fi | ||||||
|
|
||||||
| # Same for the 32-bit versions | ||||||
| if [ -d mingw32/lib/ ]; then | ||||||
| ls "${MSYS_LIB_DIR}/mingw32/lib/" | ||||||
| mkdir -p "${DST_DIR}/lib/i386" | ||||||
| mv mingw32/lib/*.a "${DST_DIR}/lib/i386" | ||||||
| cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" | ||||||
| fi | ||||||
|
|
||||||
| # Do the same also for ucrt64 | ||||||
| if [ -d ucrt64/lib/ ]; then | ||||||
| ls "${MSYS_LIB_DIR}/ucrt64/lib/" | ||||||
| mkdir -p "${DST_DIR}/lib/x64-ucrt" | ||||||
| mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" | ||||||
| cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt | ||||||
| fi | ||||||
|
|
||||||
| # Create build artifact | ||||||
| zip -r "${DST_DIR}.zip" "$DST_DIR" | ||||||
|
|
||||||
| # Same for the 32-bit versions | ||||||
| if [ -d mingw32/lib/ ]; then | ||||||
| ls "${MSYS_LIB_DIR}/mingw32/lib/" | ||||||
| mkdir -p "${DST_DIR}/lib/i386" | ||||||
| mv mingw32/lib/*.a "${DST_DIR}/lib/i386" | ||||||
| cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386" | ||||||
| # Copy that to a file name/path that does not vary by version number so we | ||||||
| # can easily find it in the R package tests on CI | ||||||
| cp "${DST_DIR}.zip" ../libarrow.zip | ||||||
|
|
||||||
| exit 0 | ||||||
| fi | ||||||
|
|
||||||
| # Do the same also for ucrt64 | ||||||
| if [ -d ucrt64/lib/ ]; then | ||||||
| ls "${MSYS_LIB_DIR}/ucrt64/lib/" | ||||||
| mkdir -p "${DST_DIR}/lib/x64-ucrt" | ||||||
| mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt" | ||||||
| cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt | ||||||
| : "${RTOOLS_HOME:=/c/rtools${RTOOLS_VERSION}}" | ||||||
| TOOLCHAIN="${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix" | ||||||
| export PATH="${TOOLCHAIN}/bin:${RTOOLS_HOME}/usr/bin:${PATH}" | ||||||
|
|
||||||
| # CMake is a native Windows program, so give it Windows-style paths | ||||||
| TOOLCHAIN_WIN="$(cygpath -am "${TOOLCHAIN}")" | ||||||
| BUILD_DIR="$(pwd)/build" | ||||||
| CPP_BUILD_DIR="${BUILD_DIR}/cpp" | ||||||
| INSTALL_DIR="${BUILD_DIR}/install" | ||||||
| INSTALL_DIR_WIN="$(cygpath -am "${INSTALL_DIR}")" | ||||||
|
|
||||||
| # Prefer the runner's CMake: the one in older Rtools can be too old for Arrow | ||||||
| CMAKE="cmake" | ||||||
| if [ -x "${PROGRAMFILES}/CMake/bin/cmake.exe" ]; then | ||||||
| CMAKE="${PROGRAMFILES}/CMake/bin/cmake.exe" | ||||||
| fi | ||||||
|
|
||||||
| printenv | ||||||
| gcc --version | ||||||
| g++ --version | ||||||
| "${CMAKE}" --version | ||||||
| make --version | head -n 1 | ||||||
| ls "${TOOLCHAIN}/lib" | ||||||
| cat "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" | ||||||
|
|
||||||
| # google-cloud-cpp links static cURL. CXXFLAGS, not ARROW_CXXFLAGS, because | ||||||
| # ARROW_CXXFLAGS aren't passed to ExternalProjects. | ||||||
| export CXXFLAGS="${CXXFLAGS:-} -DCURL_STATICLIB" | ||||||
|
|
||||||
| rm -rf "${CPP_BUILD_DIR}" | ||||||
| mkdir -p "${CPP_BUILD_DIR}" "${INSTALL_DIR}" | ||||||
| pushd "${CPP_BUILD_DIR}" | ||||||
|
|
||||||
| # * CMAKE_UNITY_BUILD=OFF: otherwise some compute tests segfault. | ||||||
| # * Bundled because Rtools45 doesn't ship them: AWS SDK, Thrift, | ||||||
| # google-cloud-cpp, snappy, utf8proc. | ||||||
| # * re2 bundled: the Rtools45 re2 would pull abseil onto the R link line. | ||||||
| # * abseil, protobuf bundled: the Rtools45 versions are too old for the | ||||||
| # bundled google-cloud-cpp. | ||||||
| # * Brotli bundled: Rtools45 has 1.2.0, without the 1.2.1 security fix | ||||||
| # (GH-55165). cURL's Brotli symbols resolve from the bundled copy too. | ||||||
| # * ARROW_USE_STD_CHRONO=0: the R package supplies the time zone database | ||||||
| # through tzdb, which only the vendored date library uses (GH-48593). | ||||||
| "${CMAKE}" \ | ||||||
| "$(cygpath -am "${ARROW_HOME}/cpp")" \ | ||||||
| -G "MSYS Makefiles" \ | ||||||
| -DARROW_ACERO=ON \ | ||||||
| -DARROW_AZURE=OFF \ | ||||||
| -DARROW_BUILD_SHARED=OFF \ | ||||||
| -DARROW_BUILD_STATIC=ON \ | ||||||
| -DARROW_BUILD_UTILITIES=OFF \ | ||||||
| -DARROW_COMPUTE=ON \ | ||||||
| -DARROW_CXXFLAGS="-DARROW_USE_STD_CHRONO=0" \ | ||||||
| -DARROW_CSV=ON \ | ||||||
| -DARROW_DATASET=ON \ | ||||||
| -DARROW_FILESYSTEM=ON \ | ||||||
| -DARROW_GCS=ON \ | ||||||
| -DARROW_HDFS=OFF \ | ||||||
| -DARROW_JEMALLOC=OFF \ | ||||||
| -DARROW_JSON=ON \ | ||||||
| -DARROW_LZ4_USE_SHARED=OFF \ | ||||||
| -DARROW_MIMALLOC=ON \ | ||||||
| -DARROW_PACKAGE_PREFIX="${TOOLCHAIN_WIN}" \ | ||||||
| -DARROW_PARQUET=ON \ | ||||||
| -DARROW_S3=ON \ | ||||||
| -DARROW_SNAPPY_USE_SHARED=OFF \ | ||||||
| -DARROW_USE_GLOG=OFF \ | ||||||
| -DARROW_UTF8PROC_USE_SHARED=OFF \ | ||||||
| -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ | ||||||
| -DARROW_WITH_BROTLI=ON \ | ||||||
| -DARROW_WITH_BZ2=ON \ | ||||||
| -DARROW_WITH_LZ4=ON \ | ||||||
| -DARROW_WITH_RE2=ON \ | ||||||
| -DARROW_WITH_SNAPPY=ON \ | ||||||
| -DARROW_WITH_ZLIB=ON \ | ||||||
| -DARROW_WITH_ZSTD=ON \ | ||||||
| -DARROW_ZSTD_USE_SHARED=OFF \ | ||||||
| -DAWSSDK_SOURCE=BUNDLED \ | ||||||
| -DBrotli_SOURCE=BUNDLED \ | ||||||
| -DCMAKE_BUILD_TYPE=release \ | ||||||
| -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \ | ||||||
| -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR_WIN}" \ | ||||||
| -DCMAKE_PREFIX_PATH="${TOOLCHAIN_WIN}" \ | ||||||
| -DCMAKE_UNITY_BUILD=OFF \ | ||||||
| -DCMAKE_VERBOSE_MAKEFILE=ON \ | ||||||
| -DProtobuf_SOURCE=BUNDLED \ | ||||||
| -DSnappy_SOURCE=BUNDLED \ | ||||||
| -DThrift_SOURCE=BUNDLED \ | ||||||
| -Dabsl_SOURCE=BUNDLED \ | ||||||
| -Dgoogle_cloud_cpp_storage_SOURCE=BUNDLED \ | ||||||
| -Dre2_SOURCE=BUNDLED \ | ||||||
| -Dutf8proc_SOURCE=BUNDLED | ||||||
|
|
||||||
| make -j"${NPROC:-$(nproc)}" | ||||||
| make install | ||||||
| popd | ||||||
|
|
||||||
| ccache --show-stats || echo "ccache not found" | ||||||
|
|
||||||
| VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)" | ||||||
| DST_DIR="r-libarrow-windows-x86_64-${VERSION}" | ||||||
|
|
||||||
| # Make the zip layout r/configure.win expects: include/ and lib/x64-ucrt/ | ||||||
| pushd "${BUILD_DIR}" | ||||||
| rm -rf "${DST_DIR}" | ||||||
| mkdir -p "${DST_DIR}/lib/x64-ucrt" | ||||||
| cp -r install/include "${DST_DIR}/include" | ||||||
| cp install/lib/*.a "${DST_DIR}/lib/x64-ucrt/" | ||||||
|
|
||||||
| # Rtools libraries in PKG_LIBS in r/configure.win; keep in sync | ||||||
| for lib in z zstd lz4 bz2 crypto ssl curl; do | ||||||
| cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" | ||||||
| done | ||||||
|
|
||||||
| # cURL's link dependencies vary by Rtools version, so read them from | ||||||
| # libcurl.pc, ship the toolchain ones, and record the flags in libcurl.libs | ||||||
| # for r/configure.win. Static link order matters, so: | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. IDK the answer to this but I wonder whether this logic belongs better in configure.win |
||||||
| # * system libraries (ws2_32, ...) aren't shipped and go last; | ||||||
| # * duplicate flags keep their last occurrence, not their first. | ||||||
| # Brotli is skipped: the bundled copy is used instead (see above). | ||||||
| CURL_LIBS="" | ||||||
| CURL_SYS_LIBS="" | ||||||
| while read -r flag; do | ||||||
| lib="${flag#-l}" | ||||||
| case "${lib}" in | ||||||
| brotli*) continue ;; | ||||||
| ws2_32 | bcrypt | advapi32 | crypt32 | gdi32 | wldap32 | secur32 | normaliz | pthread | m) | ||||||
| CURL_SYS_LIBS="${CURL_SYS_LIBS} ${flag}" | ||||||
| continue | ||||||
| ;; | ||||||
| curl | z | zstd | ssl | crypto) ;; # already copied above | ||||||
| *) | ||||||
| if [ -f "${TOOLCHAIN}/lib/lib${lib}.a" ]; then | ||||||
| cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/" | ||||||
| fi | ||||||
| ;; | ||||||
| esac | ||||||
| CURL_LIBS="${CURL_LIBS} ${flag}" | ||||||
| done < <(grep -E '^Libs(\.private)?:' "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" | | ||||||
| grep -o -E '\-l[A-Za-z0-9_+.-]+' | | ||||||
| awk '{ flag[NR] = $0; last[$0] = NR } | ||||||
| END { for (i = 1; i <= NR; i++) if (last[flag[i]] == i) print flag[i] }') | ||||||
| case " ${CURL_LIBS} " in | ||||||
| *" -lcurl "*) ;; | ||||||
| *) | ||||||
| echo "Failed to parse link flags from ${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" >&2 | ||||||
| exit 1 | ||||||
| ;; | ||||||
| esac | ||||||
| echo "${CURL_LIBS}${CURL_SYS_LIBS}" >"${DST_DIR}/lib/x64-ucrt/libcurl.libs" | ||||||
| cat "${DST_DIR}/lib/x64-ucrt/libcurl.libs" | ||||||
| ls -la "${DST_DIR}/lib/x64-ucrt" | ||||||
|
|
||||||
| # Create build artifact | ||||||
| zip -r "${DST_DIR}.zip" "$DST_DIR" | ||||||
| zip -r "${DST_DIR}.zip" "${DST_DIR}" | ||||||
|
|
||||||
| # Copy that to a file name/path that does not vary by version number so we | ||||||
| # can easily find it in the R package tests on CI | ||||||
| cp "${DST_DIR}.zip" ../libarrow.zip | ||||||
| popd | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3745,12 +3745,24 @@ function(build_google_cloud_cpp_storage) | |
| TRUE | ||
| PARENT_SCOPE) | ||
|
|
||
| # Workaround missing BCRYPT_RSA_ALG_HANDLE macro in older MinGW-w64 headers. | ||
| # google-cloud-cpp v3+ uses it without guards in sign_using_sha256.cc. | ||
| # Workaround missing BCRYPT_RSA_ALG_HANDLE and BCRYPT_MD5_ALG_HANDLE macros | ||
| # in older MinGW-w64 headers. google-cloud-cpp v3+ uses them without guards. | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would turning off google-cloud-cpp in the older Windows builds simplify this? |
||
| set(GOOGLE_CLOUD_CPP_PATCH_COMMAND) | ||
| find_program(PATCH patch) | ||
| if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11") | ||
| # This is for RTools 40. | ||
| if(MINGW) | ||
| include(CheckCXXSymbolExists) | ||
| include(CMakePushCheckState) | ||
| cmake_push_check_state() | ||
| # The macros are Windows 10+ only, so match google-cloud-cpp's _WIN32_WINNT | ||
| list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_WIN32_WINNT=0x0A01) | ||
| check_cxx_symbol_exists(BCRYPT_RSA_ALG_HANDLE "windows.h;bcrypt.h" | ||
| ARROW_HAVE_BCRYPT_RSA_HANDLE) | ||
| check_cxx_symbol_exists(BCRYPT_MD5_ALG_HANDLE "windows.h;bcrypt.h" | ||
| ARROW_HAVE_BCRYPT_MD5_HANDLE) | ||
| cmake_pop_check_state() | ||
| endif() | ||
| if(MINGW AND NOT (ARROW_HAVE_BCRYPT_RSA_HANDLE AND ARROW_HAVE_BCRYPT_MD5_HANDLE)) | ||
| # This is for the MinGW-w64 headers in RTools 40 to 45. | ||
| if(PATCH) | ||
| set(GOOGLE_CLOUD_CPP_PATCH_COMMAND | ||
| ${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/google-cloud-cpp-bcrypt-mingw.patch) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Forgive my ignorance, but if we only build rtools45 here, where does the rtools40 get built?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The plan is to build RTools45 on every PR and do both RTools40 and RTools45 for binaries to distribute (in a follow-up PR). So at the moment, after this PR, we'd build with RTools40 in the
r-binary-packagesjob like it is at the moment (which is why that file doesn't show up in this PR).