Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 13 additions & 9 deletions .github/workflows/r.yml
Original file line number Diff line number Diff line change
Expand Up @@ -205,14 +205,20 @@ jobs:
fail-fast: false
matrix:
config:
- { rtools: 40, arch: 'ucrt64' }
- { rtools: 45, arch: 'ucrt64' }

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Forgive my ignorance, but if we only build rtools45 here, where does the rtools40 get built?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The plan is to build RTools45 on every PR and do both RTools40 and RTools45 for binaries to distribute (in a follow-up PR). So at the moment, after this PR, we'd build with RTools40 in the r-binary-packages job like it is at the moment (which is why that file doesn't show up in this PR).

steps:
- run: git config --global core.autocrlf false
- name: Checkout Arrow
uses: actions/checkout@v7
with:
persist-credentials: false
fetch-depth: 0
- name: Install ccache
shell: bash
run: |
mkdir -p "$HOME/.local/bin"
ci/scripts/install_ccache.sh 4.13.6 "$HOME/.local"
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: Setup ccache
shell: bash
run: |
Expand All @@ -228,15 +234,13 @@ jobs:
r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-
- uses: r-lib/actions/setup-r@v2
with:
# Note: we need Rtools40 here because Rtools40 + ucrt is how we build the Arrow C++
# static library. R itself is not used, but setup-r picks the Rtools version from
# r-version, and R 4.1 was the last R to use Rtools40, so this must stay at 4.1.
r-version: "4.1"
# R is not used, but setup-r picks Rtools from r-version: 4.5 gets
# Rtools45, which builds the Arrow C++ static library.
r-version: "4.5"
Ncpus: 2
- name: Build Arrow C++
shell: bash
env:
MINGW_ARCH: ${{ matrix.config.arch }}
RTOOLS_VERSION: ${{ matrix.config.rtools }}
run: ci/scripts/r_windows_build.sh
- name: Rename libarrow.zip
Expand Down Expand Up @@ -283,14 +287,14 @@ jobs:
- name: Download Timezone Database
shell: bash
run: |
# RTools 40 uses GCC 8.x which does not support C++20 chrono timezones,
# so Arrow uses the vendored date library which requires tzdata
# libarrow is built with ARROW_USE_STD_CHRONO=0, so Arrow uses the
# vendored date library, which requires tzdata
ci/scripts/download_tz_database.sh
- run: mkdir r/windows
- name: Download artifacts
uses: actions/download-artifact@v8
with:
name: libarrow-rtools40-ucrt64.zip
name: libarrow-rtools45-ucrt64.zip
path: r/windows
- name: Unzip and rezip libarrows
shell: bash
Expand Down
279 changes: 227 additions & 52 deletions ci/scripts/r_windows_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,76 +17,251 @@
# specific language governing permissions and limitations
# under the License.

# Build the static libarrow that r/configure.win downloads on Windows.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAICT reading the diff, there is basically nothing shared between the old version and the new one, so maybe it would be cleaner and more readable to just have two different scripts.

#
# * Rtools40 (MSYS2): makepkg-mingw with ci/scripts/PKGBUILD.
# * Rtools42+ (MXE, no pacman/makepkg): call CMake directly against the

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
# * Rtools42+ (MXE, no pacman/makepkg): call CMake directly against the
# * Rtools42+ (libraries are a fixed set built with MXE): call CMake directly against the

I'll apply this change before merging, not worth ending up rerunning CI right this sec for a comment

# toolchain's static libraries.
#
# The two zips differ in contents; r/configure.win handles both.

set -ex

: "${ARROW_HOME:=$(pwd)}"
# Make sure it is absolute and exported
ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)"
export ARROW_HOME

pacman --noconfirm -Syy
: "${RTOOLS_VERSION:=45}"

: "${MINGW_ARCH:=mingw32 mingw64 ucrt64}"
export MINGW_ARCH
if [ "${RTOOLS_VERSION}" = "40" ]; then
pacman --noconfirm -Syy

cp "${ARROW_HOME}/ci/scripts/PKGBUILD" .
printenv
makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild
: "${MINGW_ARCH:=mingw32 mingw64 ucrt64}"
export MINGW_ARCH

VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)"
DST_DIR="r-libarrow-windows-x86_64-$VERSION"

# Collect the build artifacts and make the shape of zip file that rwinlib expects
ls
mkdir -p build
mv mingw* build
cd build

# This may vary by system/CI provider
MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}"

# Untar the builds we made
find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf
mkdir -p "${DST_DIR}"
# Grab the headers from one, either one is fine
# (if we're building twice to combine old and new toolchains, this may already exist)
if [ ! -d "${DST_DIR}/include" ]; then
mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}"
fi
cp "${ARROW_HOME}/ci/scripts/PKGBUILD" .
printenv
makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild

# mingw64 -> x64
# mingw32 -> i386
# ucrt64 -> x64-ucrt

if [ -d mingw64/lib/ ]; then
ls "${MSYS_LIB_DIR}/mingw64/lib/"
# Make the rest of the directory structure
mkdir -p "${DST_DIR}/lib/x64"
# Move the 64-bit versions of libarrow into the expected location
mv mingw64/lib/*.a "${DST_DIR}/lib/x64"
# These are from https://dl.bintray.com/rtools/mingw{32,64}/
cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64"
fi
VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)"
DST_DIR="r-libarrow-windows-x86_64-$VERSION"

# Collect the build artifacts and make the shape of zip file that rwinlib expects
ls
mkdir -p build
mv mingw* build
cd build

# This may vary by system/CI provider
MSYS_LIB_DIR="/c/rtools${RTOOLS_VERSION}"

# Untar the builds we made
find . -maxdepth 1 -type f -name '*.xz' -print0 | xargs -0 -n1 tar -xJf
mkdir -p "${DST_DIR}"
# Grab the headers from one, either one is fine
# (if we're building twice to combine old and new toolchains, this may already exist)
if [ ! -d "${DST_DIR}/include" ]; then
mv "$(echo "${MINGW_ARCH}" | cut -d ' ' -f 1)/include" "${DST_DIR}"
fi

# mingw64 -> x64
# mingw32 -> i386
# ucrt64 -> x64-ucrt

if [ -d mingw64/lib/ ]; then
ls "${MSYS_LIB_DIR}/mingw64/lib/"
# Make the rest of the directory structure
mkdir -p "${DST_DIR}/lib/x64"
# Move the 64-bit versions of libarrow into the expected location
mv mingw64/lib/*.a "${DST_DIR}/lib/x64"
# These are from https://dl.bintray.com/rtools/mingw{32,64}/
cp "${MSYS_LIB_DIR}"/mingw64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/x64"
fi

# Same for the 32-bit versions
if [ -d mingw32/lib/ ]; then
ls "${MSYS_LIB_DIR}/mingw32/lib/"
mkdir -p "${DST_DIR}/lib/i386"
mv mingw32/lib/*.a "${DST_DIR}/lib/i386"
cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386"
fi

# Do the same also for ucrt64
if [ -d ucrt64/lib/ ]; then
ls "${MSYS_LIB_DIR}/ucrt64/lib/"
mkdir -p "${DST_DIR}/lib/x64-ucrt"
mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt"
cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt
fi

# Create build artifact
zip -r "${DST_DIR}.zip" "$DST_DIR"

# Same for the 32-bit versions
if [ -d mingw32/lib/ ]; then
ls "${MSYS_LIB_DIR}/mingw32/lib/"
mkdir -p "${DST_DIR}/lib/i386"
mv mingw32/lib/*.a "${DST_DIR}/lib/i386"
cp "${MSYS_LIB_DIR}"/mingw32/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "${DST_DIR}/lib/i386"
# Copy that to a file name/path that does not vary by version number so we
# can easily find it in the R package tests on CI
cp "${DST_DIR}.zip" ../libarrow.zip

exit 0
fi

# Do the same also for ucrt64
if [ -d ucrt64/lib/ ]; then
ls "${MSYS_LIB_DIR}/ucrt64/lib/"
mkdir -p "${DST_DIR}/lib/x64-ucrt"
mv ucrt64/lib/*.a "${DST_DIR}/lib/x64-ucrt"
cp "${MSYS_LIB_DIR}"/ucrt64/lib/lib{snappy,zstd,lz4,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a "$DST_DIR"/lib/x64-ucrt
: "${RTOOLS_HOME:=/c/rtools${RTOOLS_VERSION}}"
TOOLCHAIN="${RTOOLS_HOME}/x86_64-w64-mingw32.static.posix"
export PATH="${TOOLCHAIN}/bin:${RTOOLS_HOME}/usr/bin:${PATH}"

# CMake is a native Windows program, so give it Windows-style paths
TOOLCHAIN_WIN="$(cygpath -am "${TOOLCHAIN}")"
BUILD_DIR="$(pwd)/build"
CPP_BUILD_DIR="${BUILD_DIR}/cpp"
INSTALL_DIR="${BUILD_DIR}/install"
INSTALL_DIR_WIN="$(cygpath -am "${INSTALL_DIR}")"

# Prefer the runner's CMake: the one in older Rtools can be too old for Arrow
CMAKE="cmake"
if [ -x "${PROGRAMFILES}/CMake/bin/cmake.exe" ]; then
CMAKE="${PROGRAMFILES}/CMake/bin/cmake.exe"
fi

printenv
gcc --version
g++ --version
"${CMAKE}" --version
make --version | head -n 1
ls "${TOOLCHAIN}/lib"
cat "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc"

# google-cloud-cpp links static cURL. CXXFLAGS, not ARROW_CXXFLAGS, because
# ARROW_CXXFLAGS aren't passed to ExternalProjects.
export CXXFLAGS="${CXXFLAGS:-} -DCURL_STATICLIB"

rm -rf "${CPP_BUILD_DIR}"
mkdir -p "${CPP_BUILD_DIR}" "${INSTALL_DIR}"
pushd "${CPP_BUILD_DIR}"

# * CMAKE_UNITY_BUILD=OFF: otherwise some compute tests segfault.
# * Bundled because Rtools45 doesn't ship them: AWS SDK, Thrift,
# google-cloud-cpp, snappy, utf8proc.
# * re2 bundled: the Rtools45 re2 would pull abseil onto the R link line.
# * abseil, protobuf bundled: the Rtools45 versions are too old for the
# bundled google-cloud-cpp.
# * Brotli bundled: Rtools45 has 1.2.0, without the 1.2.1 security fix
# (GH-55165). cURL's Brotli symbols resolve from the bundled copy too.
# * ARROW_USE_STD_CHRONO=0: the R package supplies the time zone database
# through tzdb, which only the vendored date library uses (GH-48593).
"${CMAKE}" \
"$(cygpath -am "${ARROW_HOME}/cpp")" \
-G "MSYS Makefiles" \
-DARROW_ACERO=ON \
-DARROW_AZURE=OFF \
-DARROW_BUILD_SHARED=OFF \
-DARROW_BUILD_STATIC=ON \
-DARROW_BUILD_UTILITIES=OFF \
-DARROW_COMPUTE=ON \
-DARROW_CXXFLAGS="-DARROW_USE_STD_CHRONO=0" \
-DARROW_CSV=ON \
-DARROW_DATASET=ON \
-DARROW_FILESYSTEM=ON \
-DARROW_GCS=ON \
-DARROW_HDFS=OFF \
-DARROW_JEMALLOC=OFF \
-DARROW_JSON=ON \
-DARROW_LZ4_USE_SHARED=OFF \
-DARROW_MIMALLOC=ON \
-DARROW_PACKAGE_PREFIX="${TOOLCHAIN_WIN}" \
-DARROW_PARQUET=ON \
-DARROW_S3=ON \
-DARROW_SNAPPY_USE_SHARED=OFF \
-DARROW_USE_GLOG=OFF \
-DARROW_UTF8PROC_USE_SHARED=OFF \
-DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
-DARROW_WITH_BROTLI=ON \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_RE2=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_ZSTD=ON \
-DARROW_ZSTD_USE_SHARED=OFF \
-DAWSSDK_SOURCE=BUNDLED \
-DBrotli_SOURCE=BUNDLED \
-DCMAKE_BUILD_TYPE=release \
-DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \
-DCMAKE_INSTALL_PREFIX="${INSTALL_DIR_WIN}" \
-DCMAKE_PREFIX_PATH="${TOOLCHAIN_WIN}" \
-DCMAKE_UNITY_BUILD=OFF \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DProtobuf_SOURCE=BUNDLED \
-DSnappy_SOURCE=BUNDLED \
-DThrift_SOURCE=BUNDLED \
-Dabsl_SOURCE=BUNDLED \
-Dgoogle_cloud_cpp_storage_SOURCE=BUNDLED \
-Dre2_SOURCE=BUNDLED \
-Dutf8proc_SOURCE=BUNDLED

make -j"${NPROC:-$(nproc)}"
make install
popd

ccache --show-stats || echo "ccache not found"

VERSION="$(grep Version "${ARROW_HOME}/r/DESCRIPTION" | cut -d " " -f 2)"
DST_DIR="r-libarrow-windows-x86_64-${VERSION}"

# Make the zip layout r/configure.win expects: include/ and lib/x64-ucrt/
pushd "${BUILD_DIR}"
rm -rf "${DST_DIR}"
mkdir -p "${DST_DIR}/lib/x64-ucrt"
cp -r install/include "${DST_DIR}/include"
cp install/lib/*.a "${DST_DIR}/lib/x64-ucrt/"

# Rtools libraries in PKG_LIBS in r/configure.win; keep in sync
for lib in z zstd lz4 bz2 crypto ssl curl; do
cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/"
done

# cURL's link dependencies vary by Rtools version, so read them from
# libcurl.pc, ship the toolchain ones, and record the flags in libcurl.libs
# for r/configure.win. Static link order matters, so:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IDK the answer to this but I wonder whether this logic belongs better in configure.win

# * system libraries (ws2_32, ...) aren't shipped and go last;
# * duplicate flags keep their last occurrence, not their first.
# Brotli is skipped: the bundled copy is used instead (see above).
CURL_LIBS=""
CURL_SYS_LIBS=""
while read -r flag; do
lib="${flag#-l}"
case "${lib}" in
brotli*) continue ;;
ws2_32 | bcrypt | advapi32 | crypt32 | gdi32 | wldap32 | secur32 | normaliz | pthread | m)
CURL_SYS_LIBS="${CURL_SYS_LIBS} ${flag}"
continue
;;
curl | z | zstd | ssl | crypto) ;; # already copied above
*)
if [ -f "${TOOLCHAIN}/lib/lib${lib}.a" ]; then
cp "${TOOLCHAIN}/lib/lib${lib}.a" "${DST_DIR}/lib/x64-ucrt/"
fi
;;
esac
CURL_LIBS="${CURL_LIBS} ${flag}"
done < <(grep -E '^Libs(\.private)?:' "${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" |
grep -o -E '\-l[A-Za-z0-9_+.-]+' |
awk '{ flag[NR] = $0; last[$0] = NR }
END { for (i = 1; i <= NR; i++) if (last[flag[i]] == i) print flag[i] }')
case " ${CURL_LIBS} " in
*" -lcurl "*) ;;
*)
echo "Failed to parse link flags from ${TOOLCHAIN}/lib/pkgconfig/libcurl.pc" >&2
exit 1
;;
esac
echo "${CURL_LIBS}${CURL_SYS_LIBS}" >"${DST_DIR}/lib/x64-ucrt/libcurl.libs"
cat "${DST_DIR}/lib/x64-ucrt/libcurl.libs"
ls -la "${DST_DIR}/lib/x64-ucrt"

# Create build artifact
zip -r "${DST_DIR}.zip" "$DST_DIR"
zip -r "${DST_DIR}.zip" "${DST_DIR}"

# Copy that to a file name/path that does not vary by version number so we
# can easily find it in the R package tests on CI
cp "${DST_DIR}.zip" ../libarrow.zip
popd
20 changes: 16 additions & 4 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -3745,12 +3745,24 @@ function(build_google_cloud_cpp_storage)
TRUE
PARENT_SCOPE)

# Workaround missing BCRYPT_RSA_ALG_HANDLE macro in older MinGW-w64 headers.
# google-cloud-cpp v3+ uses it without guards in sign_using_sha256.cc.
# Workaround missing BCRYPT_RSA_ALG_HANDLE and BCRYPT_MD5_ALG_HANDLE macros
# in older MinGW-w64 headers. google-cloud-cpp v3+ uses them without guards.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would turning off google-cloud-cpp in the older Windows builds simplify this?

set(GOOGLE_CLOUD_CPP_PATCH_COMMAND)
find_program(PATCH patch)
if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11")
# This is for RTools 40.
if(MINGW)
include(CheckCXXSymbolExists)
include(CMakePushCheckState)
cmake_push_check_state()
# The macros are Windows 10+ only, so match google-cloud-cpp's _WIN32_WINNT
list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_WIN32_WINNT=0x0A01)
check_cxx_symbol_exists(BCRYPT_RSA_ALG_HANDLE "windows.h;bcrypt.h"
ARROW_HAVE_BCRYPT_RSA_HANDLE)
check_cxx_symbol_exists(BCRYPT_MD5_ALG_HANDLE "windows.h;bcrypt.h"
ARROW_HAVE_BCRYPT_MD5_HANDLE)
cmake_pop_check_state()
endif()
if(MINGW AND NOT (ARROW_HAVE_BCRYPT_RSA_HANDLE AND ARROW_HAVE_BCRYPT_MD5_HANDLE))
# This is for the MinGW-w64 headers in RTools 40 to 45.
if(PATCH)
set(GOOGLE_CLOUD_CPP_PATCH_COMMAND
${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/google-cloud-cpp-bcrypt-mingw.patch)
Expand Down
Loading
Loading