Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ python/doc/

.vscode
.idea/
.icodemate/
.pytest_cache/
pkgs
docker_cache
Expand Down
7 changes: 7 additions & 0 deletions cpp/cmake_modules/DefineOptions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ takes precedence over ccache if a storage backend is configured" ON)
"SVE128" # fixed size SVE
"SVE256" # "
"SVE512" # "
"RVV" # size agnostic RVV
"RVV128" # fixed size RVV
"RVV256" # "
"RVV512" # "
"DEFAULT")

define_option_string(ARROW_RUNTIME_SIMD_LEVEL
Expand All @@ -183,6 +187,9 @@ takes precedence over ccache if a storage backend is configured" ON)
"SVE128" # fixed size SVE
"SVE256" # "
"SVE512" # "
"RVV128" # fixed size RVV
"RVV256" # "
"RVV512" # "
"MAX")

define_option(ARROW_ALTIVEC "Build with Altivec if compiler has support" ON)
Expand Down
50 changes: 50 additions & 0 deletions cpp/cmake_modules/SetupCxxFlags.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,30 @@ elseif(ARROW_CPU_FLAG STREQUAL "aarch64")
if(ARROW_SIMD_LEVEL STREQUAL "DEFAULT")
set(ARROW_SIMD_LEVEL "NEON")
endif()
elseif(ARROW_CPU_FLAG STREQUAL "riscv64")
# RISC-V compiler flags, gcc/clang only
set(ARROW_RVV_MARCH "rv64gcv")
# Fixed-size RVV is compiled with a zvl extension plus -mrvv-vector-bits=zvl,
# matching xsimd (see xsimd_get_riscv64_arch_flags in xsimd's Architecture.cmake).
set(ARROW_RVV128_FLAGS "-march=${ARROW_RVV_MARCH}_zvl128b" "-mrvv-vector-bits=zvl")
set(ARROW_RVV256_FLAGS "-march=${ARROW_RVV_MARCH}_zvl256b" "-mrvv-vector-bits=zvl")
set(ARROW_RVV512_FLAGS "-march=${ARROW_RVV_MARCH}_zvl512b" "-mrvv-vector-bits=zvl")
check_cxx_compiler_flag("-march=${ARROW_RVV_MARCH}" CXX_SUPPORTS_RVV)
if(CXX_SUPPORTS_RVV AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(RVV128|RVV256|RVV512|MAX)$")
set(ARROW_HAVE_RUNTIME_RVV128 ON)
add_definitions(-DARROW_HAVE_RUNTIME_RVV128)
endif()
if(CXX_SUPPORTS_RVV AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(RVV256|RVV512|MAX)$")
set(ARROW_HAVE_RUNTIME_RVV256 ON)
add_definitions(-DARROW_HAVE_RUNTIME_RVV256)
endif()
if(CXX_SUPPORTS_RVV AND ARROW_RUNTIME_SIMD_LEVEL MATCHES "^(RVV512|MAX)$")
set(ARROW_HAVE_RUNTIME_RVV512 ON)
add_definitions(-DARROW_HAVE_RUNTIME_RVV512)
endif()
if(ARROW_SIMD_LEVEL STREQUAL "DEFAULT")
set(ARROW_SIMD_LEVEL "NONE")
endif()
endif()

# Support C11
Expand Down Expand Up @@ -548,6 +572,32 @@ if(ARROW_CPU_FLAG STREQUAL "aarch64")
endif()
endif()

if(ARROW_CPU_FLAG STREQUAL "riscv64")
Comment thread
pitrou marked this conversation as resolved.
Resolved
if(ARROW_SIMD_LEVEL MATCHES "RVV")
if(NOT CXX_SUPPORTS_RVV)
message(FATAL_ERROR "RVV required but compiler doesn't support it.")
endif()
string(REGEX MATCH "[0-9]+" RVV_VECTOR_BITS ${ARROW_SIMD_LEVEL})
if(RVV_VECTOR_BITS)
set(ARROW_RVV_FIXED_FLAGS
"-march=${ARROW_RVV_MARCH}_zvl${RVV_VECTOR_BITS}b" "-mrvv-vector-bits=zvl")
check_cxx_compiler_flag("${ARROW_RVV_FIXED_FLAGS}" CXX_SUPPORTS_RVV_FIXED)
if(NOT CXX_SUPPORTS_RVV_FIXED)
message(FATAL_ERROR "Fixed-width RVV (zvl${RVV_VECTOR_BITS}b) required but compiler doesn't support it.")
endif()
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${ARROW_RVV_FIXED_FLAGS}")
set(ARROW_HAVE_RVV${RVV_VECTOR_BITS} ON)
add_definitions(-DARROW_HAVE_RVV${RVV_VECTOR_BITS})

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we also enable a compiler option for the given vector size, as in SVE above?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GCC documentation (https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Options.html) shows -mrvv-vector-bits only accepts scalable/zvl (and pairs with a zvlXXXb march extension), unlike SVE's direct -msve-vector-bits=N.
Fixing VLEN at compile time is also contrary to RVV's design: RVV is a variable-length architecture and the vector length is selected at runtime via vsetvl (xsimd's RVV backend is VLA as well). So the ARROW_HAVE_RVV${bits} macro is a runtime dispatch tag rather than a compile-time vector width.

@pitrou pitrou Oct 29, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, it looks like xsimd itself uses something like -march=rv64gcv_zvl${vector_bits} -mrvv-vector-bits=zvl.

I'm not sure how another approach would work. We need vectors to have a well-known length at compile-time.

@zhanchangbao-sanechips zhanchangbao-sanechips Oct 31, 2026 •

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, it looks like xsimd itself uses something like -march=rv64gcv_zvl${vector_bits} -mrvv-vector-bits=zvl.

I'm not sure how another approach would work. We need vectors to have a well-known length at compile-time.

You're right — for a static fixed-width build we should fix the vector length at compile time, matching xsimd. Updated the RVV${bits} branch to add -march=${ARROW_RVV_MARCH}_zvl${bits}b -mrvv-vector-bits=zvl (guarded by check_cxx_compiler_flag since GCC ≤ 13 doesn't support -mrvv-vector-bits), while the sizeless RVV build keeps -march=rv64gcv.
Additionally, to align with SVE's ARROW_USER_SIMD_LEVEL support, I added a RISC-V branch in cpu_info.cc so that ARROW_USER_SIMD_LEVEL=RVV/RVV128/RVV256/RVV512 takes effect.

else()
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -march=${ARROW_RVV_MARCH}")
set(ARROW_HAVE_RVV_SIZELESS ON)
add_definitions(-DARROW_HAVE_RVV_SIZELESS)
endif()
elseif(NOT ARROW_SIMD_LEVEL STREQUAL "NONE")
message(WARNING "ARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL} not supported by riscv64.")
endif()
endif()

if(NOT WIN32 AND NOT APPLE)
if(ARROW_USE_MOLD)
find_program(LD_MOLD ld.mold)
Expand Down
36 changes: 36 additions & 0 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,35 @@ macro(append_runtime_sve512_src SRCS SRC)
endif()
endmacro()

macro(append_runtime_rvv128_src SRCS SRC)
if(ARROW_HAVE_RUNTIME_RVV128)
list(APPEND ${SRCS} ${SRC})
set(_flags ${ARROW_RVV128_FLAGS})
set_property(SOURCE ${SRC}
APPEND
PROPERTY COMPILE_OPTIONS ${_flags})
endif()
endmacro()

macro(append_runtime_rvv256_src SRCS SRC)
if(ARROW_HAVE_RUNTIME_RVV256)
list(APPEND ${SRCS} ${SRC})
set(_flags ${ARROW_RVV256_FLAGS})
set_property(SOURCE ${SRC}
APPEND
PROPERTY COMPILE_OPTIONS ${_flags})
endif()
endmacro()

macro(append_runtime_rvv512_src SRCS SRC)
if(ARROW_HAVE_RUNTIME_RVV512)
list(APPEND ${SRCS} ${SRC})
set(_flags ${ARROW_RVV512_FLAGS})
set_property(SOURCE ${SRC}
APPEND
PROPERTY COMPILE_OPTIONS ${_flags})
endif()
endmacro()
# Write out compile-time configuration constants
string(REPLACE "${CMAKE_SOURCE_DIR}" "<CMAKE_SOURCE_DIR>" REDACTED_CXX_FLAGS
${CMAKE_CXX_FLAGS})
Expand Down Expand Up @@ -599,7 +628,14 @@ if(ARROW_CPU_FLAG STREQUAL "x86" AND ARROW_HAVE_RUNTIME_SSE4_2)
"${ARROW_SSE4_2_FLAG}")
endif()

# also provides the RVV128 kernels on riscv64, so it stays in ARROW_UTIL_SRCS
if(ARROW_CPU_FLAG STREQUAL "riscv64" AND ARROW_HAVE_RUNTIME_RVV128)
set_source_files_properties(util/bpacking_simd_128.cc PROPERTIES COMPILE_OPTIONS
"${ARROW_RVV128_FLAGS}")
endif()

append_runtime_sve256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)
append_runtime_rvv256_src(ARROW_UTIL_SRCS util/bpacking_simd_256.cc)

if(ARROW_WITH_BROTLI)
list(APPEND ARROW_UTIL_SRCS util/compression_brotli.cc)
Expand Down
48 changes: 46 additions & 2 deletions cpp/src/arrow/util/bpacking.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
// under the License.

#include <array>
#include <type_traits>

#include "arrow/util/bpacking_internal.h"
#include "arrow/util/bpacking_scalar_internal.h"
#include "arrow/util/bpacking_simd_internal.h"
#include "arrow/util/cpu_info.h"
#include "arrow/util/dispatch_internal.h"

namespace arrow::internal {
Expand All @@ -32,11 +34,13 @@ struct UnpackDynamicFunction {

static constexpr auto targets() {
return std::array{
ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar<Uint>) //
ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon<Uint>) //
ARROW_DISPATCH_TARGET_NONE(&bpacking::unpack_scalar<Uint>) //
ARROW_DISPATCH_TARGET_NEON(&bpacking::unpack_neon<Uint>) //
ARROW_DISPATCH_TARGET_RVV128(&bpacking::unpack_rvv128<Uint>) //
// GH-50503: No SVE128 dispatch as it increases code size without
// increasing performance vs. Neon, and can produce ODR violations.
ARROW_DISPATCH_TARGET_SVE256(&bpacking::unpack_sve256<Uint>) //
ARROW_DISPATCH_TARGET_RVV256(&bpacking::unpack_rvv256<Uint>) //
ARROW_DISPATCH_TARGET_SSE4_2(&bpacking::unpack_sse4_2<Uint>) //
ARROW_DISPATCH_TARGET_AVX2(&bpacking::unpack_avx2<Uint>) //
ARROW_DISPATCH_TARGET_AVX512(&bpacking::unpack_avx512<Uint>) //
Expand All @@ -48,6 +52,46 @@ struct UnpackDynamicFunction {

template <typename Uint>
void unpack(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
#if defined(ARROW_HAVE_RVV128) || defined(ARROW_HAVE_RVV256) || \
defined(ARROW_HAVE_RUNTIME_RVV128) || \
defined(ARROW_HAVE_RUNTIME_RVV256)
// On RISC-V RVV, the medium SIMD kernel is slower than the scalar
// implementation (which the compiler auto-vectorizes efficiently) for several
// unpack combinations. Route them directly to the scalar path to avoid both
// the slow kernel and the extra SIMD dispatch indirection.
constexpr bool kIsUint64 = sizeof(Uint) == sizeof(uint64_t);
// Whether the runtime dispatch would target the RVV256 kernel. CpuInfo::RVV256
// is set only when the hardware VLEN is 256 (see cpu_info.cc), so this is
// reliable on both VLEN=128 and VLEN=256 machines.
static const bool kRvv256 = CpuInfo::GetInstance()->IsSupported(CpuInfo::RVV256);
if (opts.bit_width == 1 && kIsUint64) {
// Uint64/1-bit regresses on all RVV VLENs.
return bpacking::unpack_scalar<Uint>(in, out, opts);
}
if (kRvv256) {
// The following combinations only regress on VLEN=256 hardware; on VLEN=128
// they are faster than scalar and must keep using the SIMD kernel.
if (kIsUint64 && opts.bit_width == 2) {
// Uint64/2-bit regresses on VLEN=256 only.
return bpacking::unpack_scalar<Uint>(in, out, opts);
}
// Large bit widths on small batches have too much fixed medium-kernel
// overhead to beat the scalar path on VLEN=256.
if constexpr (std::is_same_v<Uint, uint16_t>) {
if (opts.bit_width == 13 && opts.batch_size <= 32) {
return bpacking::unpack_scalar<Uint>(in, out, opts);
}
} else if constexpr (std::is_same_v<Uint, uint32_t>) {
if (opts.bit_width == 20 && opts.batch_size <= 32) {
return bpacking::unpack_scalar<Uint>(in, out, opts);
}
} else if constexpr (std::is_same_v<Uint, uint64_t>) {
if (opts.bit_width == 20 && opts.batch_size <= 64) {
return bpacking::unpack_scalar<Uint>(in, out, opts);
}
}
}
#endif
static const DynamicDispatch<UnpackDynamicFunction<Uint>> dispatch;
return dispatch(in, out, opts);
}
Expand Down
18 changes: 18 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_128.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
#if defined(ARROW_HAVE_NEON)
# define UNPACK_PLATFORM unpack_neon
# define KERNEL_PLATFORM KernelNeon
#elif defined(ARROW_HAVE_RVV128) || defined(ARROW_HAVE_RUNTIME_RVV128)
# define UNPACK_PLATFORM unpack_rvv128
# define KERNEL_PLATFORM KernelRvv128
# define ARROW_RVV_MEDIUM_SCALAR_FALLBACK 1
#elif defined(ARROW_HAVE_SSE4_2) || defined(ARROW_HAVE_RUNTIME_SSE4_2)
# define UNPACK_PLATFORM unpack_sse4_2
# define KERNEL_PLATFORM KernelSse42
Expand All @@ -26,6 +30,7 @@
#if defined(UNPACK_PLATFORM)

# include "arrow/util/bpacking_dispatch_internal.h"
# include "arrow/util/bpacking_scalar_internal.h"
# include "arrow/util/bpacking_simd_internal.h"
# include "arrow/util/bpacking_simd_kernel_internal.h"

Expand All @@ -36,6 +41,18 @@ using KERNEL_PLATFORM = Kernel<UnpackedUint, kPackedBitSize, xsimd::default_arch

template <typename Uint>
void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
#if defined(ARROW_RVV_MEDIUM_SCALAR_FALLBACK)
// On RISC-V RVV, unpacking single-bit packed integers into uint64_t is faster
// with the compiler auto-vectorized scalar implementation than with the generic
// medium kernel. The latter relies on byte swizzling plus per-lane variable
// shifts, which do not map efficiently to RVV instructions and cause a
// performance regression (measured on VLEN=128 and VLEN=256 hardware).
if constexpr (sizeof(Uint) == sizeof(uint64_t)) {
if (opts.bit_width == 1) {
return unpack_scalar<Uint>(in, out, opts);
}
}
#endif
return unpack_jump<KERNEL_PLATFORM>(in, out, opts);
}

Expand All @@ -47,5 +64,6 @@ template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackO

} // namespace arrow::internal::bpacking

# undef ARROW_RVV_MEDIUM_SCALAR_FALLBACK
# undef UNPACK_PLATFORM
#endif // UNPACK_PLATFORM
33 changes: 33 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_256.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
# define UNPACK_PLATFORM unpack_sve256
# define KERNEL_PLATFORM KernelSve256
#elif defined(ARROW_HAVE_RVV256) || defined(ARROW_HAVE_RUNTIME_RVV256)
# define UNPACK_PLATFORM unpack_rvv256
# define KERNEL_PLATFORM KernelRvv256
# define ARROW_RVV_MEDIUM_SCALAR_FALLBACK 1
#elif defined(ARROW_HAVE_RUNTIME_AVX2)
# define UNPACK_PLATFORM unpack_avx2
# define KERNEL_PLATFORM KernelAvx2
Expand All @@ -27,8 +31,11 @@
# error "This file must be compiled with a known SIMD micro architecture"
#endif

#include <type_traits>

#include "arrow/util/bpacking_dispatch_internal.h"
#include "arrow/util/bpacking_internal.h"
#include "arrow/util/bpacking_scalar_internal.h"
#include "arrow/util/bpacking_simd_internal.h"
#include "arrow/util/bpacking_simd_kernel_internal.h"

Expand All @@ -39,6 +46,31 @@ using KERNEL_PLATFORM = Kernel<UnpackedUint, kPackedBitSize, xsimd::default_arch

template <typename Uint>
void UNPACK_PLATFORM(const uint8_t* in, Uint* out, const UnpackOptions& opts) {
#if defined(ARROW_RVV_MEDIUM_SCALAR_FALLBACK)
// On RISC-V RVV (VLEN=256 here), several unpack combinations are faster with
// the compiler auto-vectorized scalar implementation than with the generic
// medium kernel, which relies on byte swizzling plus per-lane variable shifts
// that do not map efficiently to RVV instructions. Measured on VLEN=256
// hardware; the VLEN=128 kernel file only falls back for Uint64/1-bit.
constexpr bool kIsUint64 = sizeof(Uint) == sizeof(uint64_t);
if (kIsUint64 && (opts.bit_width == 1 || opts.bit_width == 2)) {
return unpack_scalar<Uint>(in, out, opts);
}
// Large bit widths on small batches have too much fixed medium-kernel overhead.
if constexpr (std::is_same_v<Uint, uint16_t>) {
if (opts.bit_width == 13 && opts.batch_size <= 32) {
return unpack_scalar<Uint>(in, out, opts);
}
} else if constexpr (std::is_same_v<Uint, uint32_t>) {
if (opts.bit_width == 20 && opts.batch_size <= 32) {
return unpack_scalar<Uint>(in, out, opts);
}
} else if constexpr (std::is_same_v<Uint, uint64_t>) {
if (opts.bit_width == 20 && opts.batch_size <= 64) {
return unpack_scalar<Uint>(in, out, opts);
}
}
#endif
return unpack_jump<KERNEL_PLATFORM>(in, out, opts);
}

Expand All @@ -50,4 +82,5 @@ template void UNPACK_PLATFORM<uint64_t>(const uint8_t*, uint64_t*, const UnpackO

} // namespace arrow::internal::bpacking

#undef ARROW_RVV_MEDIUM_SCALAR_FALLBACK
#undef UNPACK_PLATFORM
4 changes: 4 additions & 0 deletions cpp/src/arrow/util/bpacking_simd_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ namespace arrow::internal::bpacking {

#if defined(ARROW_HAVE_NEON)
# define UNPACK_ARCH128 unpack_neon
#elif defined(ARROW_HAVE_RVV128) || defined(ARROW_HAVE_RUNTIME_RVV128)
# define UNPACK_ARCH128 unpack_rvv128
#elif defined(ARROW_HAVE_SSE4_2) || defined(ARROW_HAVE_RUNTIME_SSE4_2)
# define UNPACK_ARCH128 unpack_sse4_2
#endif
Expand Down Expand Up @@ -55,6 +57,8 @@ extern template ARROW_TEMPLATE_EXPORT void UNPACK_ARCH128<uint64_t>(

#if defined(ARROW_HAVE_SVE256) || defined(ARROW_HAVE_RUNTIME_SVE256)
# define UNPACK_ARCH256 unpack_sve256
#elif defined(ARROW_HAVE_RVV256) || defined(ARROW_HAVE_RUNTIME_RVV256)
# define UNPACK_ARCH256 unpack_rvv256
#elif defined(UNPACK_ARCH256) || defined(ARROW_HAVE_RUNTIME_AVX2)
# define UNPACK_ARCH256 unpack_avx2
#endif
Expand Down
Loading
Loading