From 0060131c2b141a7b06a34d896e581225f090ab14 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 18:41:39 -0600 Subject: [PATCH 1/4] GH-55478: [R] Vendor thirdparty sources needed for Parquet, codecs, compute and JSON Ship trimmed source tarballs of the thirdparty dependencies needed for Parquet, the compression codecs, RE2/utf8proc, JSON and mimalloc in the R package so that offline source builds (e.g. on CRAN) can include those features instead of falling back to a minimal build. Brotli and Zstandard aren't vendored (yet) to keep the package small, so offline builds turn them off. * tools/vendor_dependencies.sh (run by `make sync-cpp`) downloads, trims and vendors the tarballs into tools/vendored, and records each one's upstream and trimmed checksums in tools/vendored/checksums.txt. A dependency can provide its own trim script at tools/trim/.sh. * When building the package's copy of the C++ source (tools/cpp), nixlibs.R uses the vendored tarballs that were trimmed from the versions that versions.txt pins, replacing their checksums in versions.txt with the trimmed ones. When offline, it only turns off the features whose dependencies aren't vendored. * The offline CI job (now test-r-offline-vendored) checks that the expected features are available after the build. * Add inst/COPYRIGHTS listing the vendored dependencies' licenses. Co-Authored-By: Claude Opus 5.5 --- ci/scripts/r_test.sh | 20 +++ dev/release/rat_exclude_files.txt | 1 + dev/tasks/tasks.yml | 28 ++- r/.Rbuildignore | 1 + r/.gitignore | 2 + r/DESCRIPTION | 1 + r/inst/COPYRIGHTS | 69 ++++++++ r/tools/nixlibs.R | 278 +++++++++++++++++++++++++---- r/tools/test-nixlibs.R | 284 ++++++++++++++++++++++++++---- r/tools/vendor_dependencies.sh | 235 +++++++++++++++++++++--- r/vignettes/install.Rmd | 16 ++ 11 files changed, 834 insertions(+), 101 deletions(-) create mode 100644 r/inst/COPYRIGHTS diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 20e6062e847f..ac9fbf2c479d 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then cp /etc/resolv.conf{.bak,} fi +if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then + # The tests skip what the build doesn't support, so make sure the features + # we expect (e.g. from the vendored thirdparty sources) were actually built. + # ARROW_R_EXPECTED_FEATURES is a comma-separated list of + # names(arrow::arrow_info()$capabilities) and memory pool backends + # (e.g. mimalloc). + ${R_BIN} -s -e " + library(arrow, lib.loc = 'check/arrow.Rcheck') + info <- arrow_info() + print(info\$capabilities) + print(info\$memory_pool\$available_backends) + available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends) + expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]] + missing <- setdiff(expected, available) + if (length(missing) > 0) { + stop('Expected features are not available: ', paste(missing, collapse = ', ')) + } + " +fi + AFTER=$(ls -alh ~/) if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then # Ignore ~/.TinyTex/ and ~/R/ because it has many files. diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3c5518a9596e..ef6e1a4f88dd 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md r/inst/include/cpp11.hpp r/inst/include/cpp11/*.hpp r/tools/nixlibs-allowlist.txt +r/inst/COPYRIGHTS .gitattributes ruby/red-arrow/.yardopts ruby/red-arrow-format/lib/arrow-format/org/* diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d9351d3afe0a..73cbc67564fb 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -633,14 +633,28 @@ tasks: template: r/github.linux.offline.build.yml # Simulate a CRAN builder without network access: the build should detect - # that it is offline and fall back to a minimal build without downloading - # anything. - test-r-offline-minimal: - ci: github - template: docker-tests/github.linux.yml - params: - flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true' + # that it is offline and build libarrow without downloading anything, using + # the thirdparty sources vendored in the package (tools/vendored) to still + # include Parquet, Dataset, JSON, the compression codecs, etc. + # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp + # (like the CRAN package does), whose versions.txt has the checksums of the + # trimmed tarballs. bz2 is off by default on Linux, so it is turned on to make + # sure every vendored tarball is built. Brotli and Zstandard aren't vendored. + test-r-offline-vendored: + ci: github + template: docker-tests/github.linux.yml + params: + flags: >- + -e ARROW_R_TEST_OFFLINE=true + -e NOT_CRAN=false + -e SKIP_VIGNETTES=true + -e ARROW_SOURCE_HOME='' + -e ARROW_WITH_BZ2=ON + -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,lz4,bz2 image: r + # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache + # (it's offline) + timeout: 180 # This one has -flto=auto test-r-ubuntu-22.04: diff --git a/r/.Rbuildignore b/r/.Rbuildignore index deaf5dd3948c..698ccf3f7afc 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -32,3 +32,4 @@ STYLE.md ^inst/__pycache__$ ^bootstrap.R$ air.toml +^tools/vendored-downloads$ diff --git a/r/.gitignore b/r/.gitignore index a1295ea4e0b5..e16bc75f459b 100644 --- a/r/.gitignore +++ b/r/.gitignore @@ -35,3 +35,5 @@ extra-tests/files # Source tarballs of required dependencies that we ship for offline builds. # They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh). /tools/vendored/ +# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh +/tools/vendored-downloads/ diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 1e5733c37cd5..19918beab685 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' is a cross-language package provides an interface to the 'Arrow C++' library. Depends: R (>= 4.2) License: Apache License (>= 2.0) +Copyright: See file inst/COPYRIGHTS URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://github.com/apache/arrow/issues Encoding: UTF-8 diff --git a/r/inst/COPYRIGHTS b/r/inst/COPYRIGHTS new file mode 100644 index 000000000000..8a5925d2c753 --- /dev/null +++ b/r/inst/COPYRIGHTS @@ -0,0 +1,69 @@ +The arrow R package is Copyright The Apache Software Foundation and is licensed +under the Apache License, Version 2.0. The Arrow C++ library source copied into +tools/cpp includes code from other projects; see tools/LICENSE.txt and +tools/NOTICE.txt in the source package. + +To build the Arrow C++ library without network access, the source package also +includes (trimmed) source tarballs of these thirdparty dependencies in +tools/vendored. They are only used when the dependency isn't found on the +system, and are statically linked into the package when used. Each tarball +contains the dependency's full license text. + +Boost (headers used by Apache Thrift) + Copyright the Boost authors + Boost Software License 1.0 + +bzip2 + Copyright (C) 1996-2019 Julian R Seward + BSD-style license (see LICENSE in the tarball) + +LZ4 (lib/ only) + Copyright (c) 2011-2020, Yann Collet + BSD 2-Clause License + +mimalloc + Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen + MIT License + +RapidJSON + Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip + MIT License (with code under the BSD and JSON licenses, see license.txt in + the tarball) + +RE2 + Copyright (c) 2009 The RE2 Authors + BSD 3-Clause License + +simdjson + Copyright 2018-2025 The simdjson authors + Apache License 2.0 or MIT License + +Snappy + Copyright 2011, Google Inc. + BSD 3-Clause License + +Apache Thrift + Copyright (C) 2006-2019, The Apache Software Foundation + Apache License 2.0 + +uriparser (library sources only) + Copyright (C) 2007, Weijia Song and Sebastian Pipping + BSD 3-Clause License + +utf8proc + Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman, + Jonas Fonseca, and other contributors + Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany + MIT License, with Unicode data under the Unicode Data Files and Software + License + +xsimd + Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and + Martin Renou + Copyright (c) 2016, QuantStack + Copyright (c) 2018, Serge Guelton + BSD 3-Clause License + +zlib + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + zlib License diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 226894372ec5..3fc43fba234e 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -486,7 +486,7 @@ find_local_source <- function() { # The second probably occurs if we're installing the arrow R package cpp_dir_options <- c( file.path(Sys.getenv("ARROW_SOURCE_HOME", ".."), "cpp"), - "tools/cpp" + package_cpp_dir ) for (cpp_dir in cpp_dir_options) { if (file.exists(file.path(cpp_dir, "src/arrow/api.h"))) { @@ -545,6 +545,13 @@ build_libarrow <- function(src_dir, dst_dir) { if (!quietly) { lg("Building with MAKEFLAGS=%s", makeflags) } + + thirdparty_deps_unavailable <- !download_ok && + !dir.exists(thirdparty_dependency_dir) && + !env_is("ARROW_DEPENDENCY_SOURCE", "system") + # Fail early (before e.g. downloading cmake) if the build can't work + check_vendored_deps_setting(thirdparty_deps_unavailable) + # Check for libarrow build dependencies: # * cmake cmake <- ensure_cmake() @@ -625,30 +632,38 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- with_cloud_support(env_var_list) env_var_list <- with_wasm_support(env_var_list) - # turn_off_all_optional_features() needs to happen after - # with_cloud_support(), since it might turn features ON. - thirdparty_deps_unavailable <- !download_ok && - !dir.exists(thirdparty_dependency_dir) && - !env_is("ARROW_DEPENDENCY_SOURCE", "system") + # turn_off_all_optional_features() and turn_off_unvendored_features() need to + # happen after with_cloud_support(), since it might turn features ON. do_minimal_build <- env_is("LIBARROW_MINIMAL", "true") + # Thirdparty source tarballs shipped in the package + vendored <- usable_vendored_dependencies(src_dir) if (do_minimal_build) { env_var_list <- turn_off_all_optional_features(env_var_list) } else if (thirdparty_deps_unavailable) { cat(paste0( "*** Building C++ library from source, but downloading thirdparty dependencies\n", - " is not possible, so this build will turn off all thirdparty features.\n", + " is not possible, so this build will turn off the thirdparty features\n", + " whose sources are not included in the package.\n", " See installation guide for details:\n", " https://arrow.apache.org/docs/r/articles/install.html\n" )) - env_var_list <- turn_off_all_optional_features(env_var_list) + env_var_list <- turn_off_unvendored_features(env_var_list, names(vendored)) } else if (dir.exists(thirdparty_dependency_dir)) { # Add the *_SOURCE_URL env vars env_var_list <- set_thirdparty_urls(env_var_list) } - # Some thirdparty dependencies are required even for a minimal build, so they - # can't be turned off like the ones above. - env_var_list <- with_vendored_dependencies(env_var_list) + # Point the build at the vendored sources of the thirdparty dependencies. + # Some of them are required even for a minimal build, so they can't be turned + # off like the ones above. + env_var_list <- with_vendored_dependencies(env_var_list, vendored) + original_versions <- use_vendored_checksums(src_dir, env_var_list, vendored) + if (!is.null(original_versions)) { + # Put the upstream checksums back afterwards (even if the build fails), so + # that a later build from this source (or a package built from it) accepts + # upstream tarballs, e.g. from ARROW_*_URL or thirdparty_dependency_dir + on.exit(restore_versions_txt(src_dir, original_versions), add = TRUE) + } env_vars <- env_vars_as_string(env_var_list) lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****") @@ -795,7 +810,9 @@ cmake_version <- function(cmd = "cmake") { ) } -turn_off_all_optional_features <- function(env_var_list) { +# `keep` is a vector of features (names in `turn_off` below, or +# "ARROW_SIMD_LEVEL") to leave as they are. +turn_off_all_optional_features <- function(env_var_list, keep = character()) { # Because these are done as environment variables (as opposed to build flags), # setting these to "OFF" overrides any previous setting. We don't need to # check the existing value. @@ -822,19 +839,53 @@ turn_off_all_optional_features <- function(env_var_list) { "ARROW_WITH_ZLIB" = "OFF", "ARROW_WITH_ZSTD" = "OFF", "ARROW_WITH_RE2" = "OFF", - "ARROW_WITH_UTF8PROC" = "OFF", + "ARROW_WITH_UTF8PROC" = "OFF" + ) + turn_off <- turn_off[setdiff(names(turn_off), keep)] + if (!("ARROW_SIMD_LEVEL" %in% keep)) { # The syntax to turn off XSIMD is different. # Pull existing value of EXTRA_CMAKE_FLAGS first (must be defined) - "EXTRA_CMAKE_FLAGS" = paste( - env_var_list[["EXTRA_CMAKE_FLAGS"]], - "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + turn_off <- c( + turn_off, + "EXTRA_CMAKE_FLAGS" = paste( + env_var_list[["EXTRA_CMAKE_FLAGS"]], + "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + ) ) - ) + } # Create a new env_var_list, with the values of turn_off set. # replace() also adds new values if they didn't exist before replace(env_var_list, names(turn_off), turn_off) } +# Without network access, libarrow can only build the thirdparty dependencies +# whose source tarballs are shipped in the package (see +# `with_vendored_dependencies()`), so turn off every optional feature except +# those whose dependencies are all vendored. Features that are kept are left as +# they are (e.g. as the user or the platform defaults set them), not turned on. +# `available` is the ARROW_*_URL names of the usable vendored tarballs (see +# `usable_vendored_dependencies()`). +turn_off_unvendored_features <- function(env_var_list, available) { + has_all <- vapply(vendored_features, function(deps) all(deps %in% available), logical(1)) + keep <- names(vendored_features)[has_all] + if ("ARROW_PARQUET" %in% keep) { + # Dataset needs no thirdparty dependencies itself, but it's only useful + # with Parquet (and we've always turned them on and off together) + keep <- c(keep, "ARROW_DATASET") + } + if ("ARROW_XSIMD_URL" %in% available) { + keep <- c(keep, "ARROW_SIMD_LEVEL") + } + if (!quietly) { + lg( + "Features with vendored dependencies: %s", + if (length(keep)) paste(keep, collapse = ", ") else "none", + .indent = "****" + ) + } + turn_off_all_optional_features(env_var_list, keep = keep) +} + get_component_names <- function() { if (!isTRUE(Sys.which("bash") != "")) { stop("nixlibs.R requires bash to be installed and available in your PATH") @@ -874,26 +925,86 @@ set_thirdparty_urls <- function(env_var_list) { env_var_list } -# libarrow requires some thirdparty dependencies, even for a minimal build. -# The libarrow build uses them from the system if they are available and -# otherwise builds them from source, downloading from ARROW_*_URL. To make sure -# that works without network access, `make sync-cpp` saves their source -# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. +# The libarrow build uses thirdparty dependencies from the system if they are +# available and otherwise builds them from source, downloading from ARROW_*_URL. +# Some are required even for a minimal build. To make sure that works without +# network access, `make sync-cpp` saves (trimmed) source tarballs into +# vendored_dependency_dir and we point ARROW_*_URL at them here. # This does not change whether the system versions are used. -with_vendored_dependencies <- function(env_var_list) { - if (!dir.exists(vendored_dependency_dir)) { +# `vendored` is the result of `usable_vendored_dependencies()`. +# With LIBARROW_VENDORED_DEPS=false, the build uses the full upstream sources of +# the thirdparty dependencies instead of the vendored ones, so it needs to be +# able to download them (or find them in thirdparty_dependency_dir). Some of +# them are needed even for a minimal build, so error with a clear message +# rather than failing later in the libarrow build. +check_vendored_deps_setting <- function(thirdparty_deps_unavailable) { + if (use_vendored_deps || !thirdparty_deps_unavailable) { + return(invisible(TRUE)) + } + offline_build <- env_is("ARROW_OFFLINE_BUILD", "true") + stop( + "LIBARROW_VENDORED_DEPS=false, so the thirdparty sources included in the\n", + "package won't be used, but they can't be downloaded either (", + if (offline_build) "ARROW_OFFLINE_BUILD=true" else "no network connection was found", + ").\n", + "libarrow needs some of them even for a minimal build. Either:\n", + " * unset LIBARROW_VENDORED_DEPS to use the sources included in the package,\n", + " * build with network access", + if (offline_build) " (and unset ARROW_OFFLINE_BUILD)", + ", or\n", + " * set ARROW_THIRDPARTY_DEPENDENCY_DIR to a directory with the downloaded\n", + " sources (currently '", + thirdparty_dependency_dir, + "', which doesn't exist).\n", + "See https://arrow.apache.org/docs/r/articles/install.html", + call. = FALSE + ) +} + +with_vendored_dependencies <- function(env_var_list, vendored) { + for (env_varname in names(vendored)) { + if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { + # Respect the user's (or set_thirdparty_urls()'s) choice + next + } + env_var_list <- replace(env_var_list, env_varname, vendored[[env_varname]]) + } + env_var_list +} + +# Returns the paths of the vendored tarballs that the libarrow source in +# `src_dir` can use, named by their ARROW_*_URL. +# The tarballs are trimmed, so the libarrow build only accepts them once +# `use_vendored_checksums()` has put their checksums in versions.txt. We only +# do that to the package's own copy of the source (tools/cpp), not e.g. ../cpp +# of a git checkout, and only for tarballs that were trimmed from the version +# that versions.txt pins (according to the checksums.txt file that +# tools/vendor_dependencies.sh writes). +usable_vendored_dependencies <- function(src_dir) { + if (!use_vendored_deps) { + lg("LIBARROW_VENDORED_DEPS=false, so not using vendored dependencies", .indent = "****") + return(character()) + } + manifest <- read_vendored_checksums() + if (is.null(manifest)) { # e.g. building from a git checkout without `make sync-cpp` if (!quietly) { lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****") } - return(env_var_list) + return(character()) + } + is_package_cpp_dir <- !is.null(src_dir) && + identical(normalizePath(src_dir, mustWork = FALSE), normalizePath(package_cpp_dir, mustWork = FALSE)) + if (!is_package_cpp_dir) { + if (!quietly) { + lg("Not using vendored dependencies, they are only used to build %s", package_cpp_dir, .indent = "****") + } + return(character()) } + expected <- read_thirdparty_checksums(file.path(src_dir, "thirdparty", "versions.txt")) deps_df <- get_component_names() + usable <- character() for (env_varname in vendored_dependencies) { - if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { - # Respect the user's (or set_thirdparty_urls()'s) choice - next - } filename <- deps_df$filename[deps_df$env_varname == env_varname] tarball <- file.path(vendored_dependency_dir, filename) if (length(tarball) != 1 || !file.exists(tarball)) { @@ -902,9 +1013,76 @@ with_vendored_dependencies <- function(env_var_list) { } next } - env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball)) + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + entry <- manifest[manifest$filename == filename, , drop = FALSE] + # versions.txt can still have the vendored checksum if a build was killed + # before build_libarrow() could put the upstream checksums back + matches <- nrow(entry) == 1 && + unname(expected[checksum_varname]) %in% c(entry$upstream_sha256, entry$vendored_sha256) + if (!matches) { + if (!quietly) { + lg( + "Vendored source for %s doesn't match %s, it will be downloaded if needed", + env_varname, + file.path(src_dir, "thirdparty", "versions.txt"), + .indent = "****" + ) + } + next + } + usable[[env_varname]] <- normalizePath(tarball) } - env_var_list + usable +} + +# Read the checksums.txt that tools/vendor_dependencies.sh writes, with the +# checksums of each vendored tarball before (upstream) and after trimming +read_vendored_checksums <- function() { + manifest_file <- file.path(vendored_dependency_dir, "checksums.txt") + if (!file.exists(manifest_file)) { + return(NULL) + } + read.table(manifest_file, header = TRUE, stringsAsFactors = FALSE) +} + +# The libarrow build checks the checksums of the thirdparty tarballs against +# src_dir's versions.txt, so replace the upstream checksums of the vendored +# tarballs that `env_var_list` points at with the trimmed tarballs' checksums. +# Returns the original lines of versions.txt (or NULL if it wasn't changed), to +# be put back with `restore_versions_txt()` once the build is done. +use_vendored_checksums <- function(src_dir, env_var_list, vendored) { + used <- vendored[vapply( + names(vendored), + function(env_varname) identical(env_var_list[[env_varname]], vendored[[env_varname]]), + logical(1) + )] + if (length(used) == 0) { + return(invisible(NULL)) + } + manifest <- read_vendored_checksums() + versions_txt <- file.path(src_dir, "thirdparty", "versions.txt") + original <- readLines(versions_txt) + lines <- original + for (env_varname in names(used)) { + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + checksum <- manifest$vendored_sha256[manifest$filename == basename(used[[env_varname]])] + lines <- sub(paste0("^", checksum_varname, "=.*$"), paste0(checksum_varname, "=", checksum), lines) + } + writeLines(lines, versions_txt) + invisible(original) +} + +restore_versions_txt <- function(src_dir, original) { + writeLines(original, file.path(src_dir, "thirdparty", "versions.txt")) +} + +# Parse the *_BUILD_SHA256_CHECKSUM=... lines of a versions.txt into a named vector +read_thirdparty_checksums <- function(versions_txt) { + if (!file.exists(versions_txt)) { + return(character()) + } + lines <- grep("^[A-Z0-9_]+_BUILD_SHA256_CHECKSUM=", readLines(versions_txt), value = TRUE) + setNames(sub("^[^=]*=", "", lines), sub("=.*$", "", lines)) } # this is generally about features that people asked for via environment variables, but @@ -1140,11 +1318,41 @@ if (download_libarrow_ok && !is_release && !test_mode) { # `create_package_with_all_dependencies()` in install-arrow.R thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies") -# Source tarballs of dependencies that libarrow requires even for a minimal build. -# These are always shipped in the package (see `with_vendored_dependencies()`). -# If you change this path or list, you also need to edit tools/vendor_dependencies.sh +# Trimmed source tarballs of thirdparty dependencies, shipped in the package so +# that building without network access still works and can include the features +# below (see `with_vendored_dependencies()` and `turn_off_unvendored_features()`). +# If you change this path or these lists, you also need to edit +# tools/vendor_dependencies.sh vendored_dependency_dir <- "tools/vendored" -vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL") +# Set LIBARROW_VENDORED_DEPS=false to not use them, and use the full upstream +# sources instead (downloaded, or from thirdparty_dependency_dir), e.g. if a +# build configuration needs something that was trimmed from them +use_vendored_deps <- !env_is("LIBARROW_VENDORED_DEPS", "false") +# The copy of the libarrow source in the package (see `make sync-cpp`) +package_cpp_dir <- "tools/cpp" +# Optional features and the ARROW_*_URL of each dependency they need (when the +# dependency is built from source). See "Some EP's require other EP's" in +# cpp/cmake_modules/ThirdpartyToolchain.cmake +# Brotli and Zstandard aren't vendored (to keep the package small), so they are +# turned off when building offline. +vendored_features <- list( + # Parquet turns on Thrift and simdjson; Thrift needs zlib and Boost + ARROW_PARQUET = c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL"), + ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"), + ARROW_MIMALLOC = "ARROW_MIMALLOC_URL", + ARROW_WITH_BZ2 = "ARROW_BZIP2_URL", + ARROW_WITH_LZ4 = "ARROW_LZ4_URL", + ARROW_WITH_RE2 = "ARROW_RE2_URL", + ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL", + ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL", + ARROW_WITH_ZLIB = "ARROW_ZLIB_URL" +) +# uriparser and xsimd are required even for a minimal build +vendored_dependencies <- unique(c( + "ARROW_URIPARSER_URL", + "ARROW_XSIMD_URL", + unlist(vendored_features, use.names = FALSE) +)) arrow_versioned <- paste0("arrow-", VERSION) # configure.win uses a different libarrow dir and the zip is already nested diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index 9d8abccfbb5e..c863de9ced34 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -233,58 +233,270 @@ test_that("find_latest_nightly()", { ) }) -test_that("with_vendored_dependencies()", { - vendored_dir <- tempfile() - dir.create(vendored_dir) - on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE) - original_dir <- nixlibs_env$vendored_dependency_dir - original_get_component_names <- nixlibs_env$get_component_names - original_quietly <- nixlibs_env$quietly - on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE) - on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE) - on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE) - nixlibs_env$vendored_dependency_dir <- vendored_dir - nixlibs_env$get_component_names <- function() { - data.frame( - env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"), - filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz") +# Set up a fake vendored_dependency_dir and package_cpp_dir for the vendored +# dependency tests. `vendored` is a named vector of filename = upstream +# checksum in checksums.txt (the trimmed checksum is "-trimmed"), +# `expected` is the checksums in the source dir's versions.txt, named by +# ARROW_*_URL. +local_vendored_dependencies <- function(vendored, expected, env = parent.frame()) { + vendored_dir <- withr::local_tempdir(.local_envir = env) + src_dir <- withr::local_tempdir(.local_envir = env) + for (filename in names(vendored)) { + file.create(file.path(vendored_dir, filename)) + } + if (length(vendored)) { + writeLines( + c( + "filename upstream_sha256 vendored_sha256", + paste(names(vendored), vendored, paste0(vendored, "-trimmed")) + ), + file.path(vendored_dir, "checksums.txt") ) } - withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + dir.create(file.path(src_dir, "thirdparty")) + writeLines( + c( + "# a comment", + paste0(sub("_URL$", "_BUILD_SHA256_CHECKSUM", names(expected)), "=", expected) + ), + file.path(src_dir, "thirdparty", "versions.txt") + ) + local_binding("vendored_dependency_dir", vendored_dir, env) + local_binding("package_cpp_dir", src_dir, env) + local_binding( + "get_component_names", + function() { + data.frame( + env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_LZ4_URL", "ARROW_JEMALLOC_URL"), + filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "lz4-v1.10.0.tar.gz", "jemalloc-5.3.0.tar.bz2") + ) + }, + env + ) + list(vendored_dir = vendored_dir, src_dir = src_dir) +} + +# Temporarily replace a variable in nixlibs_env +local_binding <- function(name, value, env = parent.frame()) { + original <- nixlibs_env[[name]] + assign(name, value, envir = nixlibs_env) + withr::defer(assign(name, original, envir = nixlibs_env), envir = env) +} + +test_that("usable_vendored_dependencies()", { + local_binding("quietly", TRUE) + # Nothing vendored + dirs <- local_vendored_dependencies(character(), c(ARROW_URIPARSER_URL = "aaa")) + expect_identical(usable_vendored_dependencies(dirs$src_dir), character()) - # Without the vendored tarballs, nothing is set - nixlibs_env$quietly <- TRUE - expect_silent( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")) + dirs <- local_vendored_dependencies( + c( + "uriparser-1.0.2.tar.bz2" = "aaa", + "xsimd-14.3.0.tar.gz" = "bbb", + "lz4-v1.10.0.tar.gz" = "ccc", + # Not in vendored_dependencies, so it isn't used + "jemalloc-5.3.0.tar.bz2" = "ddd" + ), + c( + ARROW_URIPARSER_URL = "aaa", + # A different version than the vendored one + ARROW_XSIMD_URL = "other", + # Still has the trimmed checksum (a build was killed before restoring it) + ARROW_LZ4_URL = "ccc-trimmed", + ARROW_JEMALLOC_URL = "ddd" + ) ) - # The missing tarballs are only reported with verbose logging - nixlibs_env$quietly <- FALSE + expect_identical( + usable_vendored_dependencies(dirs$src_dir), + c( + ARROW_URIPARSER_URL = normalizePath(file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2")), + ARROW_LZ4_URL = normalizePath(file.path(dirs$vendored_dir, "lz4-v1.10.0.tar.gz")) + ) + ) + # Mismatches are only reported with verbose logging + local_binding("quietly", FALSE) + expect_output( + usable_vendored_dependencies(dirs$src_dir), + "Vendored source for ARROW_XSIMD_URL doesn't match" + ) + + # Only used to build package_cpp_dir (not e.g. ../cpp of a git checkout) + other_src_dir <- withr::local_tempdir() + file.copy(file.path(dirs$src_dir, "thirdparty"), other_src_dir, recursive = TRUE) expect_output( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")), - "Vendored source for ARROW_URIPARSER_URL not found" + expect_identical(usable_vendored_dependencies(other_src_dir), character()), + "Not using vendored dependencies" + ) + expect_output( + expect_identical(usable_vendored_dependencies(NULL), character()), + "Not using vendored dependencies" + ) + + # A missing tarball isn't used, even if it's in checksums.txt + unlink(file.path(dirs$vendored_dir, "lz4-v1.10.0.tar.gz")) + expect_output( + expect_named(usable_vendored_dependencies(dirs$src_dir), "ARROW_URIPARSER_URL"), + "Vendored source for ARROW_LZ4_URL not found" + ) + + # None are used with LIBARROW_VENDORED_DEPS=false (logged even when quiet) + local_binding("quietly", TRUE) + local_binding("use_vendored_deps", FALSE) + expect_output( + expect_identical(usable_vendored_dependencies(dirs$src_dir), character()), + "LIBARROW_VENDORED_DEPS=false" + ) +}) + +test_that("LIBARROW_VENDORED_DEPS sets use_vendored_deps", { + use_vendored_deps_with <- function(value) { + withr::with_envvar(c(LIBARROW_VENDORED_DEPS = value), { + env <- new.env() + env$TESTING <- TRUE + capture.output(source("nixlibs.R", local = env)) + env$use_vendored_deps + }) + } + expect_true(use_vendored_deps_with(NA)) + expect_true(use_vendored_deps_with("true")) + expect_false(use_vendored_deps_with("false")) + expect_false(use_vendored_deps_with("FALSE")) +}) + +test_that("check_vendored_deps_setting()", { + local_binding("thirdparty_dependency_dir", "does/not/exist") + + # Using the vendored dependencies: always fine + local_binding("use_vendored_deps", TRUE) + expect_true(check_vendored_deps_setting(thirdparty_deps_unavailable = TRUE)) + expect_true(check_vendored_deps_setting(thirdparty_deps_unavailable = FALSE)) + + # Not using them: fine if the upstream sources can be downloaded (or are in + # thirdparty_dependency_dir) + local_binding("use_vendored_deps", FALSE) + expect_true(check_vendored_deps_setting(thirdparty_deps_unavailable = FALSE)) + + # ...but not if they can't + withr::local_envvar(c(ARROW_OFFLINE_BUILD = "true")) + expect_error( + check_vendored_deps_setting(thirdparty_deps_unavailable = TRUE), + "LIBARROW_VENDORED_DEPS=false.*ARROW_OFFLINE_BUILD=true.*does/not/exist" + ) + withr::local_envvar(c(ARROW_OFFLINE_BUILD = NA)) + expect_error( + check_vendored_deps_setting(thirdparty_deps_unavailable = TRUE), + "no network connection was found" + ) +}) + +test_that("use_vendored_checksums()", { + local_binding("quietly", TRUE) + dirs <- local_vendored_dependencies( + c("uriparser-1.0.2.tar.bz2" = "aaa", "xsimd-14.3.0.tar.gz" = "bbb", "lz4-v1.10.0.tar.gz" = "ccc"), + c(ARROW_URIPARSER_URL = "aaa", ARROW_XSIMD_URL = "bbb", ARROW_LZ4_URL = "ccc", ARROW_JEMALLOC_URL = "ddd") + ) + versions_txt <- file.path(dirs$src_dir, "thirdparty", "versions.txt") + original <- readLines(versions_txt) + vendored <- c( + ARROW_URIPARSER_URL = file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2"), + ARROW_XSIMD_URL = file.path(dirs$vendored_dir, "xsimd-14.3.0.tar.gz"), + ARROW_LZ4_URL = file.path(dirs$vendored_dir, "lz4-v1.10.0.tar.gz") ) - nixlibs_env$quietly <- TRUE - uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2") - xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz") - # Only dependencies in vendored_dependencies are used - zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz") - file.create(uriparser, xsimd, zstd) + # Nothing is pointed at the vendored tarballs + expect_null(use_vendored_checksums(dirs$src_dir, list(), vendored)) + expect_identical(readLines(versions_txt), original) + + # Only the ones that are (e.g. not when the user set ARROW_XSIMD_URL) + env_var_list <- list( + ARROW_URIPARSER_URL = vendored[["ARROW_URIPARSER_URL"]], + ARROW_XSIMD_URL = "https://example.com/xsimd.tar.gz", + ARROW_LZ4_URL = vendored[["ARROW_LZ4_URL"]] + ) + returned <- use_vendored_checksums(dirs$src_dir, env_var_list, vendored) + expect_identical(returned, original) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd)) + readLines(versions_txt), + c( + "# a comment", + "ARROW_URIPARSER_BUILD_SHA256_CHECKSUM=aaa-trimmed", + "ARROW_XSIMD_BUILD_SHA256_CHECKSUM=bbb", + "ARROW_LZ4_BUILD_SHA256_CHECKSUM=ccc-trimmed", + "ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=ddd" + ) + ) + # and they are still usable afterwards + expect_named( + usable_vendored_dependencies(dirs$src_dir), + c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_LZ4_URL") + ) + + # The upstream checksums can be put back after the build + restore_versions_txt(dirs$src_dir, returned) + expect_identical(readLines(versions_txt), original) +}) + +test_that("with_vendored_dependencies()", { + vendored <- c(ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + + expect_identical(with_vendored_dependencies(list(A = "1"), character()), list(A = "1")) + expect_identical( + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # Already set (e.g. by set_thirdparty_urls()) is respected expect_identical( - with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")), - list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2"), vendored), + list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # As is a user-set env var withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2")) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + ) +}) + +test_that("turn_off_unvendored_features()", { + local_binding("quietly", TRUE) + env_var_list <- list(EXTRA_CMAKE_FLAGS = "", ARROW_WITH_LZ4 = "ON", ARROW_S3 = "ON") + + # Nothing vendored: the same as turn_off_all_optional_features() + expect_identical( + turn_off_unvendored_features(env_var_list, character()), + turn_off_all_optional_features(env_var_list) + ) + + # Everything vendored: only features that need unvendored dependencies are + # turned off, and the rest are left as they were (not turned on) + result <- turn_off_unvendored_features(env_var_list, vendored_dependencies) + expect_identical(result$ARROW_S3, "OFF") + expect_identical(result$ARROW_JEMALLOC, "OFF") + # Brotli and Zstandard aren't vendored + expect_identical(result$ARROW_WITH_BROTLI, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "OFF") + expect_identical(result$ARROW_WITH_LZ4, "ON") + for (feature in setdiff(c(names(vendored_features), "ARROW_DATASET"), "ARROW_WITH_LZ4")) { + expect_null(result[[feature]]) + } + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + + # Parquet (and Dataset) need all of their dependencies + result <- turn_off_unvendored_features( + env_var_list, + c("ARROW_XSIMD_URL", "ARROW_THRIFT_URL", "ARROW_LZ4_URL") + ) + expect_identical(result$ARROW_PARQUET, "OFF") + expect_identical(result$ARROW_DATASET, "OFF") + expect_identical(result$ARROW_WITH_LZ4, "ON") + expect_identical(result$ARROW_WITH_SNAPPY, "OFF") + # SIMD is only turned off if xsimd isn't vendored + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + expect_match( + turn_off_unvendored_features(env_var_list, "ARROW_LZ4_URL")$EXTRA_CMAKE_FLAGS, + "-DARROW_SIMD_LEVEL=NONE" ) }) diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh index caa28bff5ce2..fdf5e1dd9f20 100644 --- a/r/tools/vendor_dependencies.sh +++ b/r/tools/vendor_dependencies.sh @@ -17,30 +17,78 @@ # specific language governing permissions and limitations # under the License. -# libarrow requires some thirdparty dependencies, even for a minimal build. -# This script downloads the source tarballs that libarrow would otherwise -# download at build time, so that they can be shipped in the R package. -# nixlibs.R uses them when the dependencies are not found on the system, so -# that building without network access (e.g. on CRAN) still works. +# libarrow downloads the source of its thirdparty dependencies at build time +# when they aren't found on the system. This script downloads those source +# tarballs ahead of time so that they can be shipped in the R package, which +# lets building without network access (e.g. on CRAN) still include features +# like Parquet. nixlibs.R uses them when the dependencies are not found on the +# system, and turns off the features whose dependencies aren't vendored. +# +# To keep the package small, each tarball is trimmed down to what the libarrow +# build uses (no tests, docs, other language bindings, ...). Trimmed tarballs +# no longer match the checksums pinned in versions.txt, so this script records +# the upstream and trimmed checksums of each tarball in DESTDIR/checksums.txt. +# When building the package's copy of the libarrow source (tools/cpp), nixlibs.R +# uses the tarballs whose upstream checksum matches versions.txt and replaces +# it with the trimmed one for the duration of the build. +# +# Trimming is done by, in order of preference: +# 1. tools/trim/.sh IN_TARBALL OUT_TARBALL, if it exists, where +# is the lowercase dependency name (e.g. ARROW_BOOST_URL -> boost). Use +# this for dependencies that need more than a few `rm`s. +# 2. Otherwise, trim_generic() plus the dependency-specific removals in +# trim_dependency() below. # # Usage: tools/vendor_dependencies.sh [DESTDIR] -# Must be run after tools/cpp has been synced (see `make sync-cpp`). +# Must be run after tools/cpp has been synced (see `make sync-cpp`). Needs curl +# and cmake (to handle .tar.bz2 without a bzip2 binary). +# +# Environment variables: +# ARROW_VENDOR_DOWNLOAD_DIR: where to cache the upstream (untrimmed) +# tarballs between runs (default: tools/vendored-downloads, which is +# ignored by git and R CMD build). set -euo pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DESTDIR="${1:-${SOURCE_DIR}/vendored}" +DOWNLOAD_DIR="${ARROW_VENDOR_DOWNLOAD_DIR:-${SOURCE_DIR}/vendored-downloads}" +VERSIONS_TXT="${SOURCE_DIR}/cpp/thirdparty/versions.txt" # If you change this list, you also need to edit -# `vendored_dependencies` in nixlibs.R +# `vendored_dependencies` (and `vendored_features`) in nixlibs.R VENDORED_DEPENDENCIES=( + # Required even for a minimal build ARROW_URIPARSER_URL ARROW_XSIMD_URL + # Parquet (ARROW_PARQUET turns on Thrift and simdjson; Thrift needs zlib and + # Boost headers) + ARROW_BOOST_URL + ARROW_SIMDJSON_URL + ARROW_THRIFT_URL + ARROW_ZLIB_URL + # Compression codecs (not Brotli or Zstandard, to keep the package small) + ARROW_BZIP2_URL + ARROW_LZ4_URL + ARROW_SNAPPY_URL + # Compute (RE2, utf8proc), JSON (RapidJSON, simdjson) and the allocator + ARROW_MIMALLOC_URL + ARROW_RAPIDJSON_URL + ARROW_RE2_URL + ARROW_UTF8PROC_URL ) +if [ ! -f "${VERSIONS_TXT}" ]; then + echo "${VERSIONS_TXT} not found. Run this from \`make sync-cpp\`." >&2 + exit 1 +fi + # Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables. # shellcheck source=/dev/null -source "${SOURCE_DIR}/cpp/thirdparty/versions.txt" +source "${VERSIONS_TXT}" + +# Don't add macOS metadata (._* files, extended attributes) to the tarballs +export COPYFILE_DISABLE=1 sha256() { if command -v sha256sum > /dev/null 2>&1; then @@ -50,8 +98,139 @@ sha256() { fi } -mkdir -p "${DESTDIR}" -keep=() +# rm -rf the given paths, relative to the current directory +rmd() { + rm -rf "$@" +} + +# Remove everything in the current directory except the given names +keep_only() { + local entry name keep + for entry in * .[!.]*; do + [ -e "${entry}" ] || continue + keep=false + for name in "$@"; do + if [ "${entry}" = "${name}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${entry}" + fi + done +} + +# Remove directories that no bundled build of our dependencies compiles. +# Arguments are paths (relative to the source root) to keep anyway. +trim_generic() { + local dir keep k + find . -depth -type d \( -name test -o -name tests -o -name testdata \ + -o -name doc -o -name docs -o -name example -o -name examples \ + -o -name benchmark -o -name benchmarks -o -name bench -o -name fuzz \) \ + -print | while read -r dir; do + keep=false + for k in "$@"; do + if [ "${dir}" = "./${k}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${dir}" + fi + done +} + +# Trim an extracted source tree (the current directory) for the dependency +# named $1 (e.g. "thrift") +trim_dependency() { + case "$1" in + bzip2) + rm -f manual.* sample* ./*.html + trim_generic + ;; + lz4) + # Only lib/ is built (LZ4_BUILD_CLI=OFF). Files outside of lib/ are + # GPL-2.0-or-later, so don't ship the ones we don't need. + rmd programs contrib ossfuzz + trim_generic + (cd build && keep_only cmake) + ;; + mimalloc) + rmd ide + trim_generic + ;; + rapidjson) + rmd bin thirdparty + trim_generic + ;; + re2) + rmd benchlog python app + trim_generic + ;; + simdjson) + # Built from src/ and include/. singleheader/ is a 20 MB amalgamation. + rmd singleheader images jsonexamples tools scripts + trim_generic + ;; + thrift) + # Only the C++ library is built (BUILD_COMPILER=OFF, WITH_*=OFF) + rmd tutorial test compiler contrib doc debian + (cd lib && keep_only cpp CMakeLists.txt) + trim_generic + ;; + zlib) + rmd contrib + # zlib's CMakeLists.txt builds test/example.c unless ZLIB_BUILD_EXAMPLES=OFF + trim_generic test + ;; + *) + trim_generic + ;; + esac +} + +# Extract $1, trim it for dependency $3 and write the result to $2 +trim_tarball() { + local in="$1" out="$2" name="$3" + local work src + if [ -f "${SOURCE_DIR}/trim/${name}.sh" ]; then + bash "${SOURCE_DIR}/trim/${name}.sh" "${in}" "${out}" + return + fi + work="$(mktemp -d)" + # Use CMake's tar (libarchive) for .tar.bz2: tar -j needs a bzip2 binary, + # which e.g. the R CI images don't have. + case "${in}" in + *.tar.bz2) (cd "${work}" && cmake -E tar xf "${in}") ;; + *) tar -xf "${in}" -C "${work}" ;; + esac + # Every tarball we vendor has a single top-level directory + src="$(cd "${work}" && ls)" + if [ "$(echo "${src}" | wc -l)" -ne 1 ] || [ ! -d "${work}/${src}" ]; then + echo "Expected a single top-level directory in ${in}" >&2 + rm -rf "${work}" + return 1 + fi + (cd "${work}/${src}" && trim_dependency "${name}") + case "${out}" in + *.tar.bz2) (cd "${work}" && cmake -E tar cjf "${out}" "${src}") ;; + *.tar.gz) tar -czf "${out}" -C "${work}" "${src}" ;; + *) + echo "Don't know how to compress ${out}" >&2 + rm -rf "${work}" + return 1 + ;; + esac + rm -rf "${work}" +} + +mkdir -p "${DESTDIR}" "${DOWNLOAD_DIR}" +# trim_tarball() changes directories, so it needs absolute paths +DOWNLOAD_DIR="$(cd "${DOWNLOAD_DIR}" && pwd)" +TRIM_DIR="$(mktemp -d)" +trap 'rm -rf "${TRIM_DIR}"' EXIT +keep=(checksums.txt) +sums="filename upstream_sha256 vendored_sha256"$'\n' for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do url="" for dep_packed in "${DEPENDENCIES[@]}"; do @@ -67,28 +246,38 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do echo "${vendored_url_var} not found in versions.txt" >&2 exit 1 fi - # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM + # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM and xsimd checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM" checksum="${!checksum_var}" + name="$(echo "${vendored_url_var}" | sed -e 's/^ARROW_//' -e 's/_URL$//' | tr '[:upper:]' '[:lower:]')" keep+=("${tar_name}") - out="${DESTDIR}/${tar_name}" - if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then - continue + upstream="${DOWNLOAD_DIR}/${tar_name}" + if [ ! -f "${upstream}" ] || [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Downloading ${url}" + curl --fail --silent --show-error --location --retry 3 --output "${upstream}" "${url}" + if [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Checksum mismatch for ${upstream}" >&2 + rm -f "${upstream}" + exit 1 + fi fi - echo "Downloading ${url}" - curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}" - - if [ "$(sha256 "${out}")" != "${checksum}" ]; then - echo "Checksum mismatch for ${out}" >&2 - rm -f "${out}" - exit 1 - fi + # Trim into a temporary directory (keeping the file name, so the extension + # says how to compress it) so that a failure doesn't leave a partial file. + echo "Trimming ${tar_name}" + trim_tarball "${upstream}" "${TRIM_DIR}/${tar_name}" "${name}" + out="${DESTDIR}/${tar_name}" + mv "${TRIM_DIR}/${tar_name}" "${out}" + sums="${sums}${tar_name} ${checksum} $(sha256 "${out}")"$'\n' done -# Remove tarballs for other versions +# See read_vendored_checksums() in nixlibs.R +printf "%s" "${sums}" > "${DESTDIR}/checksums.txt" + +# Remove tarballs for other versions (or dependencies we no longer vendor) for file in "${DESTDIR}"/*; do + [ -e "${file}" ] || continue stale=true for name in "${keep[@]}"; do if [ "$(basename "${file}")" = "${name}" ]; then diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd index 099ed6d3580c..36eacd70e9d6 100644 --- a/r/vignettes/install.Rmd +++ b/r/vignettes/install.Rmd @@ -260,6 +260,7 @@ the bundled build script. All boolean variables are case-insensitive. | `LIBARROW_BINARY` | Try to install `libarrow` binary instead of building from source | (unset) | | `LIBARROW_DOWNLOAD` | Set to `false` to explicitly forbid fetching a `libarrow` binary | (unset) | | `LIBARROW_MINIMAL` | Build with minimal features enabled | (unset) | +| `LIBARROW_VENDORED_DEPS` | Use the dependency sources included in the package | `true` | | `NOT_CRAN` | Set `LIBARROW_BINARY=true` and `LIBARROW_MINIMAL=false` | `false` | | `ARROW_R_DEV` | More verbose messaging and regenerates some code | `false` | | `ARROW_USE_PKG_CONFIG` | Use `pkg-config` to search for `libarrow` install | `true` | @@ -286,6 +287,17 @@ See below for more in-depth explanations of these environment variables. `true` turns off Parquet, Datasets, compression libraries, and other optional features. This is not commonly used but may be helpful if needing to compile on a platform that does not support these features, e.g. Solaris. +* `LIBARROW_VENDORED_DEPS` : The package includes the source code of some + libarrow dependencies (Parquet, JSON, and several compression libraries + need these), so building from source works without downloading them. + To keep the package small, these sources are trimmed to what the default + build needs. If you set `false`, the build script will not use them and will + instead download the full upstream sources (or use + `ARROW_THIRDPARTY_DEPENDENCY_DIR`, see "Offline installation (alternative)" + below). This requires network access. Use it if you have problems with the + included sources, or when building with libarrow options that need more of a + dependency than the package includes, e.g. + `EXTRA_CMAKE_FLAGS="-DARROW_USE_UBSAN=ON"`. * `NOT_CRAN` : If this variable is set to `true`, as the `devtools` package does, the build script will set `LIBARROW_BINARY=true` and `LIBARROW_MINIMAL=false` unless those environment variables are already set. This provides for a more @@ -487,6 +499,10 @@ debug messages. If building libarrow from source fails, check the error message. (If you don't see an error message, only the `----- NOTE -----`, set the environment variable `ARROW_R_DEV=TRUE` to increase verbosity and retry installation.) +If the error is about one of libarrow's thirdparty dependencies (e.g. Boost), +try again with network access and `LIBARROW_VENDORED_DEPS=false`, which uses +the full upstream sources of those dependencies instead of the ones included +in the package. The install script should work everywhere, so if libarrow fails to compile, please [report an issue](https://github.com/apache/arrow/issues/new/choose) so that we can improve the script. From d70832b3e3c51fe5449d98d15f1f160e52b176bc Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 18:42:01 -0600 Subject: [PATCH 2/4] GH-55478: [R] Trim the vendored Boost to the libraries Thrift needs The generic trim leaves a 29 MB Boost tarball. tools/trim/boost.sh keeps only the Boost libraries that build_boost() in ThirdpartyToolchain.cmake includes (minus ones the R build doesn't use, e.g. multiprecision) and their dependencies, as resolved by Boost's own CMake, and drops their tests, docs and examples. This brings the vendored Boost down to ~5 MB. Co-Authored-By: Claude Opus 5.5 --- r/.Rbuildignore | 1 + r/tools/trim/boost.sh | 207 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 r/tools/trim/boost.sh diff --git a/r/.Rbuildignore b/r/.Rbuildignore index 698ccf3f7afc..faae1aaa4480 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -33,3 +33,4 @@ STYLE.md ^bootstrap.R$ air.toml ^tools/vendored-downloads$ +^tools/trim$ diff --git a/r/tools/trim/boost.sh b/r/tools/trim/boost.sh new file mode 100644 index 000000000000..e7701f64a887 --- /dev/null +++ b/r/tools/trim/boost.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Trim the Boost CMake source tarball (boost-X.Y.Z-cmake.tar.gz, ~140 MB) +# down to the Boost libraries that libarrow's bundled Boost build +# (build_boost() in cpp/cmake_modules/ThirdpartyToolchain.cmake) uses when +# the R package builds libarrow, so it is small enough to ship in the R +# package (~5 MB). +# +# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL +# +# Called by tools/vendor_dependencies.sh (`make sync-cpp`) after tools/cpp +# has been synced. Needs cmake and a C++ compiler, because Boost's own CMake +# is used to work out which libraries are needed. +# +# How it works: +# +# 1. Read the libraries build_boost() adds to BOOST_INCLUDE_LIBRARIES from +# ThirdpartyToolchain.cmake, minus the ones that the R build never needs +# (EXCLUDED_LIBRARIES below). +# 2. Configure Boost with that BOOST_INCLUDE_LIBRARIES. Boost's CMake adds +# the transitive dependencies and logs each library it adds. +# 3. Keep only the top-level files, tools/cmake and those libraries, without +# their tests, docs, examples, etc. +# +# Set BOOST_INCLUDE_LIBRARIES (semicolon-separated) in the environment to +# skip step 1 and use that list instead. + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IN="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +OUT="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + +# Libraries that build_boost() can add to BOOST_INCLUDE_LIBRARIES but that +# the R package's libarrow build never needs. If you add a library here, +# say why. +# +# - multiprecision: only needed when ARROW_BOOST_NEED_MULTIPRECISION is +# true, i.e. with ARROW_TESTING, ARROW_GANDIVA, ARROW_USE_UBSAN or a +# compiler without a native 128-bit integer (ARROW_USE_NATIVE_INT128). +# The R package builds none of those, and gcc/clang on the 64-bit +# platforms that nixlibs.R builds for all have __int128 (32-bit targets +# don't). Including it would more than double the size of the trimmed +# tarball (~5 MB, 54 libraries -> ~11 MB, 58 libraries). If a build that +# needs it uses this tarball, configuring libarrow fails in build_boost() +# because the boost_multiprecision target doesn't exist. Such builds +# should set LIBARROW_VENDORED_DEPS=false to use the full upstream +# tarball instead (see nixlibs.R). +# - beast, variant, xpressive: only for ARROW_FLIGHT_SQL_ODBC. +EXCLUDED_LIBRARIES=(multiprecision beast variant xpressive) + +# Print the library names that build_boost() adds to BOOST_INCLUDE_LIBRARIES +# with `set(BOOST_INCLUDE_LIBRARIES ...)` or +# `list(APPEND BOOST_INCLUDE_LIBRARIES ...)`, one per line. Variable +# references such as ${ARROW_BOOST_COMPONENTS} are skipped: those are only +# non-empty when building tests, integration tests, benchmarks or ODBC. +boost_libraries_from_cmake() { + awk ' + /^function\(build_boost\)/ { inside = 1; next } + inside && /^endfunction\(\)/ { exit } + inside { + sub(/#.*/, "") + gsub(/\(/, " ( ") + gsub(/\)/, " ) ") + for (i = 1; i <= NF; i++) tok[++n] = $i + } + END { + for (i = 1; i < n; i++) { + if (tok[i + 1] != "(") continue + cmd = tolower(tok[i]) + j = i + 2 + if (cmd == "set" && tok[j] == "BOOST_INCLUDE_LIBRARIES") { + j += 1 + } else if (cmd == "list" && tok[j] == "APPEND" && + tok[j + 1] == "BOOST_INCLUDE_LIBRARIES") { + j += 2 + } else { + continue + } + for (; j <= n && tok[j] != ")"; j++) { + if (tok[j] !~ /^\$/) print tok[j] + } + } + }' "$1" +} + +if [ -z "${BOOST_INCLUDE_LIBRARIES:-}" ]; then + # tools/cpp is the synced copy of cpp/ (`make sync-cpp`); fall back to the + # repository's cpp/ so this can also be run by hand from a git checkout. + TOOLCHAIN="" + for dir in "${SCRIPT_DIR}/../cpp" "${SCRIPT_DIR}/../../../cpp"; do + if [ -f "${dir}/cmake_modules/ThirdpartyToolchain.cmake" ]; then + TOOLCHAIN="${dir}/cmake_modules/ThirdpartyToolchain.cmake" + break + fi + done + if [ -z "${TOOLCHAIN}" ]; then + echo "Can't find ThirdpartyToolchain.cmake; run \`make sync-cpp\` first" >&2 + exit 1 + fi + + ALL_LIBRARIES="$(boost_libraries_from_cmake "${TOOLCHAIN}")" + if [ -z "${ALL_LIBRARIES}" ]; then + echo "No BOOST_INCLUDE_LIBRARIES found in build_boost() in ${TOOLCHAIN}." >&2 + echo "Has build_boost() changed? Update ${BASH_SOURCE[0]}." >&2 + exit 1 + fi + for lib in "${EXCLUDED_LIBRARIES[@]}"; do + if ! printf '%s\n' "${ALL_LIBRARIES}" | grep -qxF "${lib}"; then + echo "Warning: '${lib}' is in EXCLUDED_LIBRARIES but build_boost() no" \ + "longer uses it; remove it from ${BASH_SOURCE[0]}" >&2 + fi + done + BOOST_INCLUDE_LIBRARIES="$(printf '%s\n' "${ALL_LIBRARIES}" | + grep -vxF "$(printf '%s\n' "${EXCLUDED_LIBRARIES[@]}")" | + paste -s -d ';' -)" +fi +echo "Trimming Boost to BOOST_INCLUDE_LIBRARIES=${BOOST_INCLUDE_LIBRARIES} and their dependencies" + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/arrow-trim-boost.XXXXXX")" +trap 'rm -rf "${WORK}"' EXIT + +mkdir "${WORK}/src" "${WORK}/trimmed" +tar -xzf "${IN}" -C "${WORK}/src" +# The tarball has a single top-level directory, e.g. boost-1.92.0 +SRC_NAME="$(ls "${WORK}/src")" +SRC="${WORK}/src/${SRC_NAME}" +if [ ! -f "${SRC}/tools/cmake/include/BoostRoot.cmake" ]; then + echo "${IN} doesn't look like a Boost CMake source tarball" >&2 + exit 1 +fi + +# Let Boost's CMake resolve the transitive dependencies. With +# --log-level=VERBOSE, BoostRoot.cmake logs "Adding Boost library 'x'" for +# each requested library and "Adding Boost dependency 'x'" for each +# dependency (including 'headers', which is always added). +# BUILD_TESTING=OFF matches prepare_fetchcontent(), so test-only +# dependencies are not pulled in. +cmake -S "${SRC}" -B "${WORK}/build" \ + --log-level=VERBOSE \ + -DBUILD_TESTING=OFF \ + -DBOOST_INCLUDE_LIBRARIES="${BOOST_INCLUDE_LIBRARIES}" \ + > "${WORK}/cmake.log" 2>&1 || { + cat "${WORK}/cmake.log" >&2 + echo "Configuring Boost failed" >&2 + exit 1 +} +if grep -q "has not been found" "${WORK}/cmake.log"; then + grep "has not been found" "${WORK}/cmake.log" >&2 + echo "A library in BOOST_INCLUDE_LIBRARIES doesn't exist in ${SRC_NAME}" >&2 + exit 1 +fi +# (bash 3.2 on macOS has no mapfile) +KEEP_LIBRARIES=() +while IFS= read -r lib; do + KEEP_LIBRARIES+=("${lib}") +done < <(sed -n "s/.*Adding Boost [a-z]* '\([^']*\)'.*/\1/p" "${WORK}/cmake.log" | LC_ALL=C sort -u) +if [ "${#KEEP_LIBRARIES[@]}" -eq 0 ]; then + echo "Found no \"Adding Boost ...\" messages in Boost's CMake output." >&2 + echo "Has BoostRoot.cmake changed? Update ${BASH_SOURCE[0]}." >&2 + exit 1 +fi +echo "Keeping ${#KEEP_LIBRARIES[@]} Boost libraries: ${KEEP_LIBRARIES[*]}" + +# Copy what the bundled build needs into ${WORK}/trimmed/${SRC_NAME}: +# the top-level files (CMakeLists.txt, LICENSE_1_0.txt, ...), tools/cmake +# (BoostRoot.cmake etc.) and the libraries, but not the top-level doc/, +# more/, status/ or the other tools/. +DEST="${WORK}/trimmed/${SRC_NAME}" +mkdir -p "${DEST}/tools" +find "${SRC}" -maxdepth 1 -type f -exec cp -p {} "${DEST}/" \; +cp -pR "${SRC}/tools/cmake" "${DEST}/tools/" +for lib in "${KEEP_LIBRARIES[@]}"; do + # lib can be a nested library such as numeric/conversion + mkdir -p "$(dirname "${DEST}/libs/${lib}")" + cp -pR "${SRC}/libs/${lib}" "${DEST}/libs/${lib}" + # Only remove these at the top level of each library: deeper down they + # can be header directories (e.g. include/boost/math/tools/). + for dir in test doc example examples bench benchmark perf performance meta tools; do + rm -rf "${DEST}/libs/${lib:?}/${dir}" + done +done + +# COPYFILE_DISABLE stops macOS tar from adding AppleDouble (._*) files. +(cd "${WORK}/trimmed" && COPYFILE_DISABLE=1 tar -czf "${OUT}" "${SRC_NAME}") From aebc6269c2354f65e1f70bf1ccf70a290f2a92d5 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Thu, 12 Nov 2026 08:29:10 -0600 Subject: [PATCH 3/4] Use env vars for overriding shas --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +++ docs/source/developers/cpp/building.rst | 4 ++ r/tools/nixlibs.R | 47 ++++++--------------- r/tools/test-nixlibs.R | 32 +++----------- 4 files changed, 31 insertions(+), 58 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 123def5b66c5..d92c0bc3bba2 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -469,6 +469,12 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) continue() endif() + # Allow overriding checksums by environment variables, e.g. for a source + # archive given by ARROW_$LIBRARY_URL that differs from the upstream one + if(_VARIABLE_NAME MATCHES "_CHECKSUM$" AND DEFINED ENV{${_VARIABLE_NAME}}) + set(_VARIABLE_VALUE "$ENV{${_VARIABLE_NAME}}") + endif() + # For debugging message(STATUS "${_VARIABLE_NAME}: ${_VARIABLE_VALUE}") diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index b284e2d7c9b2..6fcfe334db6a 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -581,6 +581,10 @@ internet. To enable offline builds, you can download the source artifacts yourself and use environment variables of the form ``ARROW_$LIBRARY_URL`` to direct the build system to read from a local file rather than accessing the internet. +Each archive is checked against the SHA256 checksum in +``thirdparty/versions.txt``; if you use an archive that differs from the +upstream one, you can set the matching ``ARROW_$LIBRARY_BUILD_SHA256_CHECKSUM`` +environment variable to its checksum. To make this easier for you, we have prepared a script ``thirdparty/download_dependencies.sh`` which will download the correct version diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 3fc43fba234e..2885eeb2cd91 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -657,13 +657,7 @@ build_libarrow <- function(src_dir, dst_dir) { # Some of them are required even for a minimal build, so they can't be turned # off like the ones above. env_var_list <- with_vendored_dependencies(env_var_list, vendored) - original_versions <- use_vendored_checksums(src_dir, env_var_list, vendored) - if (!is.null(original_versions)) { - # Put the upstream checksums back afterwards (even if the build fails), so - # that a later build from this source (or a package built from it) accepts - # upstream tarballs, e.g. from ARROW_*_URL or thirdparty_dependency_dir - on.exit(restore_versions_txt(src_dir, original_versions), add = TRUE) - } + env_var_list <- with_vendored_checksums(env_var_list, vendored) env_vars <- env_vars_as_string(env_var_list) lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****") @@ -974,11 +968,11 @@ with_vendored_dependencies <- function(env_var_list, vendored) { # Returns the paths of the vendored tarballs that the libarrow source in # `src_dir` can use, named by their ARROW_*_URL. -# The tarballs are trimmed, so the libarrow build only accepts them once -# `use_vendored_checksums()` has put their checksums in versions.txt. We only -# do that to the package's own copy of the source (tools/cpp), not e.g. ../cpp -# of a git checkout, and only for tarballs that were trimmed from the version -# that versions.txt pins (according to the checksums.txt file that +# The tarballs are trimmed, so the libarrow build only accepts them with the +# checksums that `with_vendored_checksums()` sets. We only use them to build the +# package's own copy of the source (tools/cpp), not e.g. ../cpp of a git +# checkout, and only for tarballs that were trimmed from the version that +# versions.txt pins (according to the checksums.txt file that # tools/vendor_dependencies.sh writes). usable_vendored_dependencies <- function(src_dir) { if (!use_vendored_deps) { @@ -1015,10 +1009,7 @@ usable_vendored_dependencies <- function(src_dir) { } checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) entry <- manifest[manifest$filename == filename, , drop = FALSE] - # versions.txt can still have the vendored checksum if a build was killed - # before build_libarrow() could put the upstream checksums back - matches <- nrow(entry) == 1 && - unname(expected[checksum_varname]) %in% c(entry$upstream_sha256, entry$vendored_sha256) + matches <- nrow(entry) == 1 && identical(unname(expected[checksum_varname]), entry$upstream_sha256) if (!matches) { if (!quietly) { lg( @@ -1045,35 +1036,25 @@ read_vendored_checksums <- function() { read.table(manifest_file, header = TRUE, stringsAsFactors = FALSE) } -# The libarrow build checks the checksums of the thirdparty tarballs against -# src_dir's versions.txt, so replace the upstream checksums of the vendored -# tarballs that `env_var_list` points at with the trimmed tarballs' checksums. -# Returns the original lines of versions.txt (or NULL if it wasn't changed), to -# be put back with `restore_versions_txt()` once the build is done. -use_vendored_checksums <- function(src_dir, env_var_list, vendored) { +# The libarrow build checks the thirdparty tarballs against the upstream +# checksums in versions.txt, so set ARROW_*_BUILD_SHA256_CHECKSUM to the trimmed +# tarballs' checksums for the vendored tarballs that `env_var_list` points at. +with_vendored_checksums <- function(env_var_list, vendored) { used <- vendored[vapply( names(vendored), function(env_varname) identical(env_var_list[[env_varname]], vendored[[env_varname]]), logical(1) )] if (length(used) == 0) { - return(invisible(NULL)) + return(env_var_list) } manifest <- read_vendored_checksums() - versions_txt <- file.path(src_dir, "thirdparty", "versions.txt") - original <- readLines(versions_txt) - lines <- original for (env_varname in names(used)) { checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) checksum <- manifest$vendored_sha256[manifest$filename == basename(used[[env_varname]])] - lines <- sub(paste0("^", checksum_varname, "=.*$"), paste0(checksum_varname, "=", checksum), lines) + env_var_list <- replace(env_var_list, checksum_varname, checksum) } - writeLines(lines, versions_txt) - invisible(original) -} - -restore_versions_txt <- function(src_dir, original) { - writeLines(original, file.path(src_dir, "thirdparty", "versions.txt")) + env_var_list } # Parse the *_BUILD_SHA256_CHECKSUM=... lines of a versions.txt into a named vector diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index c863de9ced34..6511157d513a 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -301,8 +301,7 @@ test_that("usable_vendored_dependencies()", { ARROW_URIPARSER_URL = "aaa", # A different version than the vendored one ARROW_XSIMD_URL = "other", - # Still has the trimmed checksum (a build was killed before restoring it) - ARROW_LZ4_URL = "ccc-trimmed", + ARROW_LZ4_URL = "ccc", ARROW_JEMALLOC_URL = "ddd" ) ) @@ -389,14 +388,11 @@ test_that("check_vendored_deps_setting()", { ) }) -test_that("use_vendored_checksums()", { - local_binding("quietly", TRUE) +test_that("with_vendored_checksums()", { dirs <- local_vendored_dependencies( c("uriparser-1.0.2.tar.bz2" = "aaa", "xsimd-14.3.0.tar.gz" = "bbb", "lz4-v1.10.0.tar.gz" = "ccc"), c(ARROW_URIPARSER_URL = "aaa", ARROW_XSIMD_URL = "bbb", ARROW_LZ4_URL = "ccc", ARROW_JEMALLOC_URL = "ddd") ) - versions_txt <- file.path(dirs$src_dir, "thirdparty", "versions.txt") - original <- readLines(versions_txt) vendored <- c( ARROW_URIPARSER_URL = file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2"), ARROW_XSIMD_URL = file.path(dirs$vendored_dir, "xsimd-14.3.0.tar.gz"), @@ -404,8 +400,7 @@ test_that("use_vendored_checksums()", { ) # Nothing is pointed at the vendored tarballs - expect_null(use_vendored_checksums(dirs$src_dir, list(), vendored)) - expect_identical(readLines(versions_txt), original) + expect_identical(with_vendored_checksums(list(), vendored), list()) # Only the ones that are (e.g. not when the user set ARROW_XSIMD_URL) env_var_list <- list( @@ -413,27 +408,14 @@ test_that("use_vendored_checksums()", { ARROW_XSIMD_URL = "https://example.com/xsimd.tar.gz", ARROW_LZ4_URL = vendored[["ARROW_LZ4_URL"]] ) - returned <- use_vendored_checksums(dirs$src_dir, env_var_list, vendored) - expect_identical(returned, original) expect_identical( - readLines(versions_txt), + with_vendored_checksums(env_var_list, vendored), c( - "# a comment", - "ARROW_URIPARSER_BUILD_SHA256_CHECKSUM=aaa-trimmed", - "ARROW_XSIMD_BUILD_SHA256_CHECKSUM=bbb", - "ARROW_LZ4_BUILD_SHA256_CHECKSUM=ccc-trimmed", - "ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=ddd" + env_var_list, + ARROW_URIPARSER_BUILD_SHA256_CHECKSUM = "aaa-trimmed", + ARROW_LZ4_BUILD_SHA256_CHECKSUM = "ccc-trimmed" ) ) - # and they are still usable afterwards - expect_named( - usable_vendored_dependencies(dirs$src_dir), - c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_LZ4_URL") - ) - - # The upstream checksums can be put back after the build - restore_versions_txt(dirs$src_dir, returned) - expect_identical(readLines(versions_txt), original) }) test_that("with_vendored_dependencies()", { From dcab7832fe12f840f8ba831b088158a1f2298fbe Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Fri, 13 Nov 2026 14:01:00 -0600 Subject: [PATCH 4/4] Local review iteration --- r/inst/COPYRIGHTS | 4 +- r/tools/nixlibs.R | 61 +++++++++++------------ r/tools/test-nixlibs.R | 29 ++++++----- r/tools/vendor_dependencies.sh | 88 ++++++++++++++++++++++++++-------- 4 files changed, 118 insertions(+), 64 deletions(-) diff --git a/r/inst/COPYRIGHTS b/r/inst/COPYRIGHTS index 8a5925d2c753..1c0d8bb0f929 100644 --- a/r/inst/COPYRIGHTS +++ b/r/inst/COPYRIGHTS @@ -9,7 +9,9 @@ tools/vendored. They are only used when the dependency isn't found on the system, and are statically linked into the package when used. Each tarball contains the dependency's full license text. -Boost (headers used by Apache Thrift) +Boost (the subset of libraries used by Apache Thrift; Boost.Locale and the +Boost libraries it needs are compiled and statically linked, the rest are +header-only) Copyright the Boost authors Boost Software License 1.0 diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 2885eeb2cd91..eac4031c439a 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -862,14 +862,6 @@ turn_off_all_optional_features <- function(env_var_list, keep = character()) { turn_off_unvendored_features <- function(env_var_list, available) { has_all <- vapply(vendored_features, function(deps) all(deps %in% available), logical(1)) keep <- names(vendored_features)[has_all] - if ("ARROW_PARQUET" %in% keep) { - # Dataset needs no thirdparty dependencies itself, but it's only useful - # with Parquet (and we've always turned them on and off together) - keep <- c(keep, "ARROW_DATASET") - } - if ("ARROW_XSIMD_URL" %in% available) { - keep <- c(keep, "ARROW_SIMD_LEVEL") - } if (!quietly) { lg( "Features with vendored dependencies: %s", @@ -967,13 +959,13 @@ with_vendored_dependencies <- function(env_var_list, vendored) { } # Returns the paths of the vendored tarballs that the libarrow source in -# `src_dir` can use, named by their ARROW_*_URL. +# `src_dir` (tools/cpp, or e.g. ../cpp of a git checkout) can use, named by +# their ARROW_*_URL. # The tarballs are trimmed, so the libarrow build only accepts them with the -# checksums that `with_vendored_checksums()` sets. We only use them to build the -# package's own copy of the source (tools/cpp), not e.g. ../cpp of a git -# checkout, and only for tarballs that were trimmed from the version that -# versions.txt pins (according to the checksums.txt file that -# tools/vendor_dependencies.sh writes). +# checksums that `with_vendored_checksums()` sets. We only use the tarballs +# that were trimmed from the version that `src_dir`'s versions.txt pins +# (according to the checksums.txt file that tools/vendor_dependencies.sh +# writes). usable_vendored_dependencies <- function(src_dir) { if (!use_vendored_deps) { lg("LIBARROW_VENDORED_DEPS=false, so not using vendored dependencies", .indent = "****") @@ -987,12 +979,7 @@ usable_vendored_dependencies <- function(src_dir) { } return(character()) } - is_package_cpp_dir <- !is.null(src_dir) && - identical(normalizePath(src_dir, mustWork = FALSE), normalizePath(package_cpp_dir, mustWork = FALSE)) - if (!is_package_cpp_dir) { - if (!quietly) { - lg("Not using vendored dependencies, they are only used to build %s", package_cpp_dir, .indent = "****") - } + if (is.null(src_dir)) { return(character()) } expected <- read_thirdparty_checksums(file.path(src_dir, "thirdparty", "versions.txt")) @@ -1027,7 +1014,8 @@ usable_vendored_dependencies <- function(src_dir) { } # Read the checksums.txt that tools/vendor_dependencies.sh writes, with the -# checksums of each vendored tarball before (upstream) and after trimming +# checksums of each vendored tarball before (upstream) and after trimming (and +# of how it was trimmed, which only vendor_dependencies.sh uses) read_vendored_checksums <- function() { manifest_file <- file.path(vendored_dependency_dir, "checksums.txt") if (!file.exists(manifest_file)) { @@ -1316,18 +1304,27 @@ package_cpp_dir <- "tools/cpp" # cpp/cmake_modules/ThirdpartyToolchain.cmake # Brotli and Zstandard aren't vendored (to keep the package small), so they are # turned off when building offline. -vendored_features <- list( +# The names are what `turn_off_all_optional_features()` accepts in `keep`. +vendored_features <- local({ # Parquet turns on Thrift and simdjson; Thrift needs zlib and Boost - ARROW_PARQUET = c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL"), - ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"), - ARROW_MIMALLOC = "ARROW_MIMALLOC_URL", - ARROW_WITH_BZ2 = "ARROW_BZIP2_URL", - ARROW_WITH_LZ4 = "ARROW_LZ4_URL", - ARROW_WITH_RE2 = "ARROW_RE2_URL", - ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL", - ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL", - ARROW_WITH_ZLIB = "ARROW_ZLIB_URL" -) + parquet_deps <- c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL") + list( + ARROW_PARQUET = parquet_deps, + # Dataset needs no thirdparty dependencies itself, but it's only useful + # with Parquet (and we've always turned them on and off together) + ARROW_DATASET = parquet_deps, + ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"), + ARROW_MIMALLOC = "ARROW_MIMALLOC_URL", + # Not a CMake option: SIMD is turned off with EXTRA_CMAKE_FLAGS + ARROW_SIMD_LEVEL = "ARROW_XSIMD_URL", + ARROW_WITH_BZ2 = "ARROW_BZIP2_URL", + ARROW_WITH_LZ4 = "ARROW_LZ4_URL", + ARROW_WITH_RE2 = "ARROW_RE2_URL", + ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL", + ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL", + ARROW_WITH_ZLIB = "ARROW_ZLIB_URL" + ) +}) # uriparser and xsimd are required even for a minimal build vendored_dependencies <- unique(c( "ARROW_URIPARSER_URL", diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index 6511157d513a..54a9009b0ab8 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -233,7 +233,7 @@ test_that("find_latest_nightly()", { ) }) -# Set up a fake vendored_dependency_dir and package_cpp_dir for the vendored +# Set up a fake vendored_dependency_dir and source dir for the vendored # dependency tests. `vendored` is a named vector of filename = upstream # checksum in checksums.txt (the trimmed checksum is "-trimmed"), # `expected` is the checksums in the source dir's versions.txt, named by @@ -247,8 +247,8 @@ local_vendored_dependencies <- function(vendored, expected, env = parent.frame() if (length(vendored)) { writeLines( c( - "filename upstream_sha256 vendored_sha256", - paste(names(vendored), vendored, paste0(vendored, "-trimmed")) + "filename upstream_sha256 vendored_sha256 trim_sha256", + paste(names(vendored), vendored, paste0(vendored, "-trimmed"), "recipe") ), file.path(vendored_dir, "checksums.txt") ) @@ -262,7 +262,6 @@ local_vendored_dependencies <- function(vendored, expected, env = parent.frame() file.path(src_dir, "thirdparty", "versions.txt") ) local_binding("vendored_dependency_dir", vendored_dir, env) - local_binding("package_cpp_dir", src_dir, env) local_binding( "get_component_names", function() { @@ -319,17 +318,23 @@ test_that("usable_vendored_dependencies()", { "Vendored source for ARROW_XSIMD_URL doesn't match" ) - # Only used to build package_cpp_dir (not e.g. ../cpp of a git checkout) + # Also used for other source dirs (e.g. ../cpp of a git checkout), checked + # against that dir's versions.txt + local_binding("quietly", TRUE) other_src_dir <- withr::local_tempdir() file.copy(file.path(dirs$src_dir, "thirdparty"), other_src_dir, recursive = TRUE) - expect_output( - expect_identical(usable_vendored_dependencies(other_src_dir), character()), - "Not using vendored dependencies" + expect_named( + usable_vendored_dependencies(other_src_dir), + c("ARROW_URIPARSER_URL", "ARROW_LZ4_URL") ) - expect_output( - expect_identical(usable_vendored_dependencies(NULL), character()), - "Not using vendored dependencies" + writeLines( + "ARROW_LZ4_BUILD_SHA256_CHECKSUM=ccc", + file.path(other_src_dir, "thirdparty", "versions.txt") ) + expect_named(usable_vendored_dependencies(other_src_dir), "ARROW_LZ4_URL") + # No source dir, nothing to build + expect_identical(usable_vendored_dependencies(NULL), character()) + local_binding("quietly", FALSE) # A missing tarball isn't used, even if it's in checksums.txt unlink(file.path(dirs$vendored_dir, "lz4-v1.10.0.tar.gz")) @@ -461,7 +466,7 @@ test_that("turn_off_unvendored_features()", { expect_identical(result$ARROW_WITH_BROTLI, "OFF") expect_identical(result$ARROW_WITH_ZSTD, "OFF") expect_identical(result$ARROW_WITH_LZ4, "ON") - for (feature in setdiff(c(names(vendored_features), "ARROW_DATASET"), "ARROW_WITH_LZ4")) { + for (feature in setdiff(names(vendored_features), "ARROW_WITH_LZ4")) { expect_null(result[[feature]]) } expect_identical(result$EXTRA_CMAKE_FLAGS, "") diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh index fdf5e1dd9f20..69c721144c4b 100644 --- a/r/tools/vendor_dependencies.sh +++ b/r/tools/vendor_dependencies.sh @@ -28,9 +28,13 @@ # build uses (no tests, docs, other language bindings, ...). Trimmed tarballs # no longer match the checksums pinned in versions.txt, so this script records # the upstream and trimmed checksums of each tarball in DESTDIR/checksums.txt. -# When building the package's copy of the libarrow source (tools/cpp), nixlibs.R -# uses the tarballs whose upstream checksum matches versions.txt and replaces -# it with the trimmed one for the duration of the build. +# nixlibs.R uses the tarballs whose upstream checksum matches the versions.txt +# of the libarrow source it builds (tools/cpp, or ../cpp in a git checkout) and +# replaces it with the trimmed one for the duration of the build. +# +# checksums.txt also records a checksum of how each tarball was trimmed (see +# trim_recipe_sha256()). A tarball that DESTDIR already has is only downloaded +# and trimmed again if its upstream version or the way it's trimmed changed. # # Trimming is done by, in order of preference: # 1. tools/trim/.sh IN_TARBALL OUT_TARBALL, if it exists, where @@ -90,11 +94,12 @@ source "${VERSIONS_TXT}" # Don't add macOS metadata (._* files, extended attributes) to the tarballs export COPYFILE_DISABLE=1 +# Checksum of the file $1, or of stdin without arguments sha256() { if command -v sha256sum > /dev/null 2>&1; then - sha256sum "$1" | cut -d " " -f 1 + sha256sum "${1:--}" | cut -d " " -f 1 else - shasum -a 256 "$1" | cut -d " " -f 1 + shasum -a 256 "${1:--}" | cut -d " " -f 1 fi } @@ -149,9 +154,11 @@ trim_dependency() { trim_generic ;; lz4) - # Only lib/ is built (LZ4_BUILD_CLI=OFF). Files outside of lib/ are - # GPL-2.0-or-later, so don't ship the ones we don't need. - rmd programs contrib ossfuzz + # Only lib/ is built (LZ4_BUILD_CLI=OFF), with build/cmake. LZ4's + # LICENSE says that the files outside of lib/ are GPL-2.0-or-later + # unless they say otherwise, so only keep what the build needs plus + # the license and README (which build/cmake references). + keep_only lib build LICENSE README.md trim_generic (cd build && keep_only cmake) ;; @@ -197,7 +204,8 @@ trim_tarball() { bash "${SOURCE_DIR}/trim/${name}.sh" "${in}" "${out}" return fi - work="$(mktemp -d)" + # Under TRIM_DIR, so the EXIT trap removes it if anything below fails + work="$(mktemp -d "${TRIM_DIR}/work.XXXXXX")" # Use CMake's tar (libarchive) for .tar.bz2: tar -j needs a bzip2 binary, # which e.g. the R CI images don't have. case "${in}" in @@ -208,7 +216,6 @@ trim_tarball() { src="$(cd "${work}" && ls)" if [ "$(echo "${src}" | wc -l)" -ne 1 ] || [ ! -d "${work}/${src}" ]; then echo "Expected a single top-level directory in ${in}" >&2 - rm -rf "${work}" return 1 fi (cd "${work}/${src}" && trim_dependency "${name}") @@ -217,20 +224,43 @@ trim_tarball() { *.tar.gz) tar -czf "${out}" -C "${work}" "${src}" ;; *) echo "Don't know how to compress ${out}" >&2 - rm -rf "${work}" return 1 ;; esac + # Free the space now rather than at exit: the extracted trees can be large rm -rf "${work}" } +# Print a checksum of everything that determines how dependency $1 is +# trimmed, so that a tarball is trimmed again when any of it changes: this +# script, tools/trim/.sh if there is one, and for those, the libarrow +# CMake (e.g. tools/trim/boost.sh reads the Boost libraries to keep from +# ThirdpartyToolchain.cmake) and BOOST_INCLUDE_LIBRARIES (which overrides it). +trim_recipe_sha256() { + local name="$1" + { + cat "${SOURCE_DIR}/vendor_dependencies.sh" + if [ -f "${SOURCE_DIR}/trim/${name}.sh" ]; then + cat "${SOURCE_DIR}/trim/${name}.sh" + cat "${SOURCE_DIR}/cpp/cmake_modules/ThirdpartyToolchain.cmake" + echo "BOOST_INCLUDE_LIBRARIES=${BOOST_INCLUDE_LIBRARIES:-}" + fi + } | sha256 +} + mkdir -p "${DESTDIR}" "${DOWNLOAD_DIR}" # trim_tarball() changes directories, so it needs absolute paths DOWNLOAD_DIR="$(cd "${DOWNLOAD_DIR}" && pwd)" TRIM_DIR="$(mktemp -d)" trap 'rm -rf "${TRIM_DIR}"' EXIT +# Newly trimmed tarballs are staged here and only moved into DESTDIR, together +# with the new checksums.txt, once all of them are done. So if this script +# fails, DESTDIR still has the previous tarballs with a matching checksums.txt. +STAGING_DIR="${TRIM_DIR}/staging" +mkdir "${STAGING_DIR}" +MANIFEST="${DESTDIR}/checksums.txt" keep=(checksums.txt) -sums="filename upstream_sha256 vendored_sha256"$'\n' +sums="filename upstream_sha256 vendored_sha256 trim_sha256"$'\n' for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do url="" for dep_packed in "${DEPENDENCIES[@]}"; do @@ -252,6 +282,20 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do name="$(echo "${vendored_url_var}" | sed -e 's/^ARROW_//' -e 's/_URL$//' | tr '[:upper:]' '[:lower:]')" keep+=("${tar_name}") + trim_checksum="$(trim_recipe_sha256 "${name}")" + + # Skip the download and the trimming if DESTDIR already has this tarball, + # trimmed the same way from the same upstream tarball + out="${DESTDIR}/${tar_name}" + if [ -f "${out}" ] && [ -f "${MANIFEST}" ]; then + recorded="$(awk -v f="${tar_name}" '$1 == f { print $2, $3, $4 }' "${MANIFEST}")" + if [ "${recorded}" = "${checksum} $(sha256 "${out}") ${trim_checksum}" ]; then + echo "${tar_name} is up to date" + sums="${sums}${tar_name} ${recorded}"$'\n' + continue + fi + fi + upstream="${DOWNLOAD_DIR}/${tar_name}" if [ ! -f "${upstream}" ] || [ "$(sha256 "${upstream}")" != "${checksum}" ]; then echo "Downloading ${url}" @@ -263,17 +307,23 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do fi fi - # Trim into a temporary directory (keeping the file name, so the extension - # says how to compress it) so that a failure doesn't leave a partial file. + # Keep the file name, so the extension says how to compress it echo "Trimming ${tar_name}" - trim_tarball "${upstream}" "${TRIM_DIR}/${tar_name}" "${name}" - out="${DESTDIR}/${tar_name}" - mv "${TRIM_DIR}/${tar_name}" "${out}" - sums="${sums}${tar_name} ${checksum} $(sha256 "${out}")"$'\n' + trim_tarball "${upstream}" "${STAGING_DIR}/${tar_name}" "${name}" + sums="${sums}${tar_name} ${checksum} $(sha256 "${STAGING_DIR}/${tar_name}") ${trim_checksum}"$'\n' done +# Replace the tarballs and checksums.txt together. Remove checksums.txt first: +# without it nixlibs.R uses none of the tarballs, so being interrupted part way +# through can't leave tarballs that don't match their recorded checksums. +rm -f "${MANIFEST}" +for file in "${STAGING_DIR}"/*; do + [ -e "${file}" ] || continue + mv "${file}" "${DESTDIR}/" +done # See read_vendored_checksums() in nixlibs.R -printf "%s" "${sums}" > "${DESTDIR}/checksums.txt" +printf "%s" "${sums}" > "${TRIM_DIR}/checksums.txt" +mv "${TRIM_DIR}/checksums.txt" "${MANIFEST}" # Remove tarballs for other versions (or dependencies we no longer vendor) for file in "${DESTDIR}"/*; do