Skip to content

Add PCLMULQDQ-accelerated CRC32C path with large block folding #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pkdog wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add PCLMULQDQ-accelerated CRC32C path with large block folding #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,33 @@ int main() {
" HAVE_SSE42)
set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS})

# Check for PCLMULQDQ support in the compiler (requires SSE4.2).
set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS})
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /arch:AVX")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -msse4.2 -mpclmul")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
check_cxx_source_compiles("
#if defined(_MSC_VER)
#include <intrin.h>
#include <wmmintrin.h>
#else // !defined(_MSC_VER)
#include <cpuid.h>
#include <nmmintrin.h>
#include <wmmintrin.h>
#endif // defined(_MSC_VER)

int main() {
__m128i a = _mm_set_epi64x(1, 2);
__m128i b = _mm_set_epi64x(3, 4);
__m128i c = _mm_clmulepi64_si128(a, b, 0x00);
(void)c;
return 0;
}
" HAVE_PCLMUL)
set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS})

# Check for ARMv8 w/ CRC and CRYPTO extensions support in the compiler.
set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS})
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
Expand Down Expand Up @@ -262,6 +289,28 @@ if(BUILD_SHARED_LIBS)
set_property(TARGET crc32c_sse42 PROPERTY POSITION_INDEPENDENT_CODE TRUE)
endif(BUILD_SHARED_LIBS)

# PCLMULQDQ-accelerated CRC32C code is built separately with -mpclmul, so
# unsupported instructions don't leak into code that runs without PCLMUL.
add_library(crc32c_sse42_clmul OBJECT "")
target_sources(crc32c_sse42_clmul
PRIVATE
"${PROJECT_BINARY_DIR}/include/crc32c/crc32c_config.h"
"src/crc32c_sse42_clmul.cc"
"src/crc32c_sse42_clmul.h"
)
if(HAVE_PCLMUL)
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
target_compile_options(crc32c_sse42_clmul PRIVATE "/arch:AVX")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
target_compile_options(crc32c_sse42_clmul PRIVATE "-msse4.2" "-mpclmul")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
endif(HAVE_PCLMUL)

# CMake only enables PIC by default in SHARED and MODULE targets.
if(BUILD_SHARED_LIBS)
set_property(TARGET crc32c_sse42_clmul PROPERTY POSITION_INDEPENDENT_CODE TRUE)
endif(BUILD_SHARED_LIBS)

# Must be included before CMAKE_INSTALL_INCLUDEDIR is used.
include(GNUInstallDirs)

Expand All @@ -270,6 +319,7 @@ add_library(crc32c ""
# section of target_sources when cmake_minimum_required becomes 3.9 or above.
$<TARGET_OBJECTS:crc32c_arm64>
$<TARGET_OBJECTS:crc32c_sse42>
$<TARGET_OBJECTS:crc32c_sse42_clmul>
)
target_sources(crc32c
PRIVATE
Expand All @@ -283,6 +333,7 @@ target_sources(crc32c
"src/crc32c_round_up.h"
"src/crc32c_sse42.h"
"src/crc32c_sse42_check.h"
"src/crc32c_sse42_clmul.h"
"src/crc32c.cc"

# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
Expand All @@ -304,6 +355,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
set_property(TARGET crc32c APPEND PROPERTY COMPILE_OPTIONS "/WX")
set_property(TARGET crc32c_arm64 APPEND PROPERTY COMPILE_OPTIONS "/WX")
set_property(TARGET crc32c_sse42 APPEND PROPERTY COMPILE_OPTIONS "/WX")
set_property(TARGET crc32c_sse42_clmul APPEND PROPERTY COMPILE_OPTIONS "/WX")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")

if(CRC32C_BUILD_TESTS)
Expand Down
10 changes: 8 additions & 2 deletions src/crc32c.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,23 @@
#include "./crc32c_internal.h"
#include "./crc32c_sse42.h"
#include "./crc32c_sse42_check.h"
#include "./crc32c_sse42_clmul.h"

namespace crc32c {

uint32_t Extend(uint32_t crc, const uint8_t* data, size_t count) {
#if HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__))
#if HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__))
static bool can_use_clmul = CanUseClmul();
if (can_use_clmul) return ExtendSse42Clmul(crc, data, count);
static bool can_use_sse42 = CanUseSse42();
if (can_use_sse42) return ExtendSse42(crc, data, count);
#elif HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__))
static bool can_use_sse42 = CanUseSse42();
if (can_use_sse42) return ExtendSse42(crc, data, count);
#elif HAVE_ARM64_CRC32C
static bool can_use_arm64_crc32 = CanUseArm64Crc32();
if (can_use_arm64_crc32) return ExtendArm64(crc, data, count);
#endif // HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__))
#endif // HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__))

return ExtendPortable(crc, data, count);
}
Expand Down
4 changes: 4 additions & 0 deletions src/crc32c_config.h.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
// intrinsics.
#cmakedefine01 HAVE_SSE42

// Define to 1 if targeting X86 and the compiler has the _mm_clmulepi64_si128
// intrinsic (PCLMULQDQ instruction).
#cmakedefine01 HAVE_PCLMUL

// Define to 1 if targeting ARM and the compiler has the __crc32c{b,h,w,d} and
// the vmull_p64 intrinsics.
#cmakedefine01 HAVE_ARM64_CRC32C
Expand Down
11 changes: 11 additions & 0 deletions src/crc32c_sse42_check.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ inline bool CanUseSse42() {
return (cpu_info[2] & (1 << 20)) != 0;
}

inline bool CanUseClmul() {
int cpu_info[4];
__cpuid(cpu_info, 1);
return (cpu_info[2] & (1 << 1)) != 0;
}

} // namespace crc32c

#else // !defined(_MSC_VER)
Expand All @@ -39,6 +45,11 @@ inline bool CanUseSse42() {
return __get_cpuid(1, &eax, &ebx, &ecx, &edx) && ((ecx & (1 << 20)) != 0);
}

inline bool CanUseClmul() {
unsigned int eax, ebx, ecx, edx;
return __get_cpuid(1, &eax, &ebx, &ecx, &edx) && ((ecx & (1 << 1)) != 0);
}

} // namespace crc32c

#endif // defined(_MSC_VER)
Expand Down
Loading