diff --git a/CMakeLists.txt b/CMakeLists.txt index e75211f..6578a58 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,33 @@ int main() { " HAVE_SSE42) set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS}) +# Check for PCLMULQDQ support in the compiler (requires SSE4.2). +set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS}) +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /arch:AVX") +else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -msse4.2 -mpclmul") +endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") +check_cxx_source_compiles(" +#if defined(_MSC_VER) +#include +#include +#else // !defined(_MSC_VER) +#include +#include +#include +#endif // defined(_MSC_VER) + +int main() { + __m128i a = _mm_set_epi64x(1, 2); + __m128i b = _mm_set_epi64x(3, 4); + __m128i c = _mm_clmulepi64_si128(a, b, 0x00); + (void)c; + return 0; +} +" HAVE_PCLMUL) +set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS}) + # Check for ARMv8 w/ CRC and CRYPTO extensions support in the compiler. set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS}) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") @@ -262,6 +289,28 @@ if(BUILD_SHARED_LIBS) set_property(TARGET crc32c_sse42 PROPERTY POSITION_INDEPENDENT_CODE TRUE) endif(BUILD_SHARED_LIBS) +# PCLMULQDQ-accelerated CRC32C code is built separately with -mpclmul, so +# unsupported instructions don't leak into code that runs without PCLMUL. +add_library(crc32c_sse42_clmul OBJECT "") +target_sources(crc32c_sse42_clmul + PRIVATE + "${PROJECT_BINARY_DIR}/include/crc32c/crc32c_config.h" + "src/crc32c_sse42_clmul.cc" + "src/crc32c_sse42_clmul.h" +) +if(HAVE_PCLMUL) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(crc32c_sse42_clmul PRIVATE "/arch:AVX") + else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(crc32c_sse42_clmul PRIVATE "-msse4.2" "-mpclmul") + endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") +endif(HAVE_PCLMUL) + +# CMake only enables PIC by default in SHARED and MODULE targets. +if(BUILD_SHARED_LIBS) + set_property(TARGET crc32c_sse42_clmul PROPERTY POSITION_INDEPENDENT_CODE TRUE) +endif(BUILD_SHARED_LIBS) + # Must be included before CMAKE_INSTALL_INCLUDEDIR is used. include(GNUInstallDirs) @@ -270,6 +319,7 @@ add_library(crc32c "" # section of target_sources when cmake_minimum_required becomes 3.9 or above. $ $ + $ ) target_sources(crc32c PRIVATE @@ -283,6 +333,7 @@ target_sources(crc32c "src/crc32c_round_up.h" "src/crc32c_sse42.h" "src/crc32c_sse42_check.h" + "src/crc32c_sse42_clmul.h" "src/crc32c.cc" # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". @@ -304,6 +355,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") set_property(TARGET crc32c APPEND PROPERTY COMPILE_OPTIONS "/WX") set_property(TARGET crc32c_arm64 APPEND PROPERTY COMPILE_OPTIONS "/WX") set_property(TARGET crc32c_sse42 APPEND PROPERTY COMPILE_OPTIONS "/WX") + set_property(TARGET crc32c_sse42_clmul APPEND PROPERTY COMPILE_OPTIONS "/WX") endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") if(CRC32C_BUILD_TESTS) diff --git a/src/crc32c.cc b/src/crc32c.cc index 804133b..12c0a98 100644 --- a/src/crc32c.cc +++ b/src/crc32c.cc @@ -12,17 +12,23 @@ #include "./crc32c_internal.h" #include "./crc32c_sse42.h" #include "./crc32c_sse42_check.h" +#include "./crc32c_sse42_clmul.h" namespace crc32c { uint32_t Extend(uint32_t crc, const uint8_t* data, size_t count) { -#if HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__)) +#if HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__)) + static bool can_use_clmul = CanUseClmul(); + if (can_use_clmul) return ExtendSse42Clmul(crc, data, count); + static bool can_use_sse42 = CanUseSse42(); + if (can_use_sse42) return ExtendSse42(crc, data, count); +#elif HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__)) static bool can_use_sse42 = CanUseSse42(); if (can_use_sse42) return ExtendSse42(crc, data, count); #elif HAVE_ARM64_CRC32C static bool can_use_arm64_crc32 = CanUseArm64Crc32(); if (can_use_arm64_crc32) return ExtendArm64(crc, data, count); -#endif // HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__)) +#endif // HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__)) return ExtendPortable(crc, data, count); } diff --git a/src/crc32c_config.h.in b/src/crc32c_config.h.in index 4034fa5..7611070 100644 --- a/src/crc32c_config.h.in +++ b/src/crc32c_config.h.in @@ -18,6 +18,10 @@ // intrinsics. #cmakedefine01 HAVE_SSE42 +// Define to 1 if targeting X86 and the compiler has the _mm_clmulepi64_si128 +// intrinsic (PCLMULQDQ instruction). +#cmakedefine01 HAVE_PCLMUL + // Define to 1 if targeting ARM and the compiler has the __crc32c{b,h,w,d} and // the vmull_p64 intrinsics. #cmakedefine01 HAVE_ARM64_CRC32C diff --git a/src/crc32c_sse42_check.h b/src/crc32c_sse42_check.h index ad380dd..108b54f 100644 --- a/src/crc32c_sse42_check.h +++ b/src/crc32c_sse42_check.h @@ -27,6 +27,12 @@ inline bool CanUseSse42() { return (cpu_info[2] & (1 << 20)) != 0; } +inline bool CanUseClmul() { + int cpu_info[4]; + __cpuid(cpu_info, 1); + return (cpu_info[2] & (1 << 1)) != 0; +} + } // namespace crc32c #else // !defined(_MSC_VER) @@ -39,6 +45,11 @@ inline bool CanUseSse42() { return __get_cpuid(1, &eax, &ebx, &ecx, &edx) && ((ecx & (1 << 20)) != 0); } +inline bool CanUseClmul() { + unsigned int eax, ebx, ecx, edx; + return __get_cpuid(1, &eax, &ebx, &ecx, &edx) && ((ecx & (1 << 1)) != 0); +} + } // namespace crc32c #endif // defined(_MSC_VER) diff --git a/src/crc32c_sse42_clmul.cc b/src/crc32c_sse42_clmul.cc new file mode 100644 index 0000000..760fd06 --- /dev/null +++ b/src/crc32c_sse42_clmul.cc @@ -0,0 +1,318 @@ +// Copyright 2025 The CRC32C Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "./crc32c_sse42_clmul.h" + +// In a separate source file to allow this PCLMUL-accelerated CRC32C function +// to be compiled with appropriate compiler flags (-msse4.2 -mpclmul). + +// This implementation uses PCLMULQDQ (carry-less multiplication) to fold +// parallel CRC stripes, replacing the skip-table approach. Based on the +// techniques in: +// "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" +// V. Gopal, E. Ozturk, et al., 2009 + +#include +#include + +#include "./crc32c_internal.h" +#include "./crc32c_prefetch.h" +#include "./crc32c_read_le.h" +#include "./crc32c_round_up.h" +#include "crc32c/crc32c_config.h" + +#if HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__)) + +#if defined(_MSC_VER) +#include +#include +#else // !defined(_MSC_VER) +#include +#include +#endif // defined(_MSC_VER) + +namespace crc32c { + +namespace { + +// --------------------------------------------------------------------------- +// PCLMULQDQ folding helpers +// --------------------------------------------------------------------------- + +// Fold two CRC stripes (crc0, crc1) using PCLMULQDQ with constants K1, K2. +// Returns: crc32q(0, pclmul(crc0, K1)) ^ crc2 ^ crc32q(0, pclmul(crc1, K2)) +// crc2 is consumed (XORed in, not used as a CRC accumulator). +inline uint64_t FoldTwoStripes(uint64_t crc0, uint64_t crc1, uint64_t crc2, + uint64_t K1, uint64_t K2) { + __m128i xmm_crc0 = _mm_cvtsi64_si128(static_cast(crc0)); + __m128i xmm_crc1 = _mm_cvtsi64_si128(static_cast(crc1)); + __m128i xmm_K1 = _mm_cvtsi64_si128(static_cast(K1)); + __m128i xmm_K2 = _mm_cvtsi64_si128(static_cast(K2)); + + // pclmulqdq xmm, xmm with imm8=0x00: multiply low 64 bits + __m128i xmm_t0 = _mm_clmulepi64_si128(xmm_crc0, xmm_K1, 0x00); + __m128i xmm_t1 = _mm_clmulepi64_si128(xmm_crc1, xmm_K2, 0x00); + + uint64_t t0 = static_cast(_mm_cvtsi128_si64(xmm_t0)); + uint64_t t1 = static_cast(_mm_cvtsi128_si64(xmm_t1)); + + // Fold: result = crc32q(0, t0) ^ crc2 ^ crc32q(0, t1) + // All three terms are XORed together; crc2 is NOT used as an accumulator. + uint64_t folded_crc0 = _mm_crc32_u64(0, t0); + uint64_t folded_crc1 = _mm_crc32_u64(0, t1); + return folded_crc0 ^ crc2 ^ folded_crc1; +} + +// --------------------------------------------------------------------------- +// Block-level functions. +// Each processes exactly the specified number of bytes using 3 parallel +// CRC stripes folded with PCLMULQDQ. +// --------------------------------------------------------------------------- + +// Prefetch horizon in bytes: how far ahead of the current read pointer we +// request prefetch. 256 matches the SSE4.2 skip-table path. +constexpr ptrdiff_t kClmulPrefetchHorizon = 256; + +// Process exactly 256 bytes in 3 stripes of (11, 10, 10) quadwords. +// Stripe layout: [0..88) [88..168) [176..256) +inline uint32_t ExtendSse42Clmul256(const uint8_t* data, uint32_t crc) { + uint64_t crc0 = static_cast(crc); + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + // Stripe offsets: stripe0 at data+0, stripe1 at data+88, stripe2 at data+176 + const uint8_t* s0 = data; + const uint8_t* s1 = data + 88; + const uint8_t* s2 = data + 176; + + // 11 quadwords for stripe0, 10 for stripes 1 and 2 + for (int i = 0; i < 10; ++i) { + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(s0)); s0 += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + } + // Last quadword: stripe0 gets an 11th, stripe1 absorbs the "crc2" role + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(s0)); + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); + + // Magic constants for 256-byte folding + constexpr uint64_t K1 = 0x1b3d8f29ULL; + constexpr uint64_t K2 = 0x39d3b296ULL; + return static_cast(FoldTwoStripes(crc0, crc1, crc2, K1, K2)); +} + +// Process exactly 1024 bytes: 5 loops of 3×(8×8bytes) + 3×(3×8bytes) tail +// Stripe layout: [0..344) [344..680) [680..1024) +inline uint32_t ExtendSse42Clmul1024(const uint8_t* data, uint32_t crc) { + uint64_t crc0 = static_cast(crc); + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + const uint8_t* p = data; + const uint8_t* s1 = data + 344; + const uint8_t* s2 = data + 680; + + // 5 loops, each processing 64 bytes per stripe (8 quadwords) + for (int loop = 0; loop < 5; ++loop) { + RequestPrefetch(p + kClmulPrefetchHorizon); + RequestPrefetch(s1 + kClmulPrefetchHorizon); + RequestPrefetch(s2 + kClmulPrefetchHorizon); + + for (int i = 0; i < 8; ++i) { + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); p += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + } + } + + // Tail: stripe0 gets 3 quads, stripe1 gets 2 quads, stripe2 gets 3 quads + // After the loop, total per stripe: stripe0=344B, stripe1=336B, stripe2=344B + // Total = 344 + 336 + 344 = 1024 bytes + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); p += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); p += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + + // Magic constants for 1024-byte folding + constexpr uint64_t K1 = 0xe417f38aULL; + constexpr uint64_t K2 = 0x8f158014ULL; + return static_cast(FoldTwoStripes(crc0, crc1, crc2, K1, K2)); +} + +// Process exactly 3072 bytes: 16 loops of 3×(8×8bytes) +// Stripe layout: [0..1024) [1024..2048) [2048..3072) +inline uint32_t ExtendSse42Clmul3072(const uint8_t* data, uint32_t crc) { + uint64_t crc0 = static_cast(crc); + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + const uint8_t* p = data; + const uint8_t* s1 = data + 1024; + const uint8_t* s2 = data + 2048; + + // 16 loops, each processing 64 bytes per stripe (8 quadwords) + for (int loop = 0; loop < 16; ++loop) { + RequestPrefetch(p + kClmulPrefetchHorizon); + RequestPrefetch(s1 + kClmulPrefetchHorizon); + RequestPrefetch(s2 + kClmulPrefetchHorizon); + + for (int i = 0; i < 8; ++i) { + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); p += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + } + } + + // Magic constants for 3072-byte folding + constexpr uint64_t K1 = 0xa51b6135ULL; + constexpr uint64_t K2 = 0x170076faULL; + return static_cast(FoldTwoStripes(crc0, crc1, crc2, K1, K2)); +} + +// Process exactly 4032 bytes: 21 loops of 3×(8×8bytes) +// Stripe layout: [0..1344) [1344..2688) [2688..4032) +// Constant computed by tools/compute_folding_constants.cc +inline uint32_t ExtendSse42Clmul4032(const uint8_t* data, uint32_t crc) { + uint64_t crc0 = static_cast(crc); + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + const uint8_t* p = data; + const uint8_t* s1 = data + 1344; + const uint8_t* s2 = data + 2688; + + // 21 loops, each processing 64 bytes per stripe (8 quadwords) + for (int loop = 0; loop < 21; ++loop) { + RequestPrefetch(p + kClmulPrefetchHorizon); + RequestPrefetch(s1 + kClmulPrefetchHorizon); + RequestPrefetch(s2 + kClmulPrefetchHorizon); + + for (int i = 0; i < 8; ++i) { + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); p += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + } + } + + // Magic constants for 4032-byte folding + constexpr uint64_t K1 = 0x889774e1ULL; + constexpr uint64_t K2 = 0xc9c8b782ULL; + return static_cast(FoldTwoStripes(crc0, crc1, crc2, K1, K2)); +} + +// Process exactly 16128 bytes: 84 loops of 3×(8×8bytes) +// Stripe layout: [0..5376) [5376..10752) [10752..16128) +// Constant computed by tools/compute_folding_constants.cc +inline uint32_t ExtendSse42Clmul16128(const uint8_t* data, uint32_t crc) { + uint64_t crc0 = static_cast(crc); + uint64_t crc1 = 0; + uint64_t crc2 = 0; + + const uint8_t* p = data; + const uint8_t* s1 = data + 5376; + const uint8_t* s2 = data + 10752; + + // 84 loops, each processing 64 bytes per stripe (8 quadwords) + for (int loop = 0; loop < 84; ++loop) { + RequestPrefetch(p + kClmulPrefetchHorizon); + RequestPrefetch(s1 + kClmulPrefetchHorizon); + RequestPrefetch(s2 + kClmulPrefetchHorizon); + + for (int i = 0; i < 8; ++i) { + crc0 = _mm_crc32_u64(crc0, ReadUint64LE(p)); p += 8; + crc1 = _mm_crc32_u64(crc1, ReadUint64LE(s1)); s1 += 8; + crc2 = _mm_crc32_u64(crc2, ReadUint64LE(s2)); s2 += 8; + } + } + + // Magic constants for 16128-byte folding + constexpr uint64_t K1 = 0x57e82916ULL; + constexpr uint64_t K2 = 0x0e0126b2ULL; + return static_cast(FoldTwoStripes(crc0, crc1, crc2, K1, K2)); +} + +} // namespace + +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- + +uint32_t ExtendSse42Clmul(uint32_t crc, const uint8_t* data, size_t count) { + // Invert bits at entry (match the CRC32C convention used by ExtendPortable + // and ExtendSse42). + uint32_t c = crc ^ kCRC32Xor; + + const uint8_t* p = data; + intptr_t length = static_cast(count); + + // Small input: fall through to the byte/quadword loop below + if (length < 8) { + while (length-- > 0) { + c = _mm_crc32_u8(c, *p++); + } + return c ^ kCRC32Xor; + } + + // Align to 8-byte boundary + intptr_t alignment = reinterpret_cast(p) & 0x7; + intptr_t leading = (8 - alignment) & 0x7; + length -= leading; + while (leading-- > 0) { + c = _mm_crc32_u8(c, *p++); + } + + // Process large blocks with PCLMUL folding (largest first). + // Block sizes are aligned with the SSE4.2 skip-table hierarchy + // (16320/4080/1008) to keep per-byte folding overhead comparable. + while (length >= 16128) { + c = ExtendSse42Clmul16128(p, c); + p += 16128; + length -= 16128; + } + while (length >= 4032) { + c = ExtendSse42Clmul4032(p, c); + p += 4032; + length -= 4032; + } + while (length >= 3072) { + c = ExtendSse42Clmul3072(p, c); + p += 3072; + length -= 3072; + } + while (length >= 1024) { + c = ExtendSse42Clmul1024(p, c); + p += 1024; + length -= 1024; + } + while (length >= 256) { + c = ExtendSse42Clmul256(p, c); + p += 256; + length -= 256; + } + + // Process remaining aligned quadwords + while (length >= 8) { + c = static_cast( + _mm_crc32_u64(static_cast(c), ReadUint64LE(p))); + p += 8; + length -= 8; + } + + // Process trailing bytes + while (length-- > 0) { + c = _mm_crc32_u8(c, *p++); + } + + return c ^ kCRC32Xor; +} + +} // namespace crc32c + +#endif // HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__)) diff --git a/src/crc32c_sse42_clmul.h b/src/crc32c_sse42_clmul.h new file mode 100644 index 0000000..65b0062 --- /dev/null +++ b/src/crc32c_sse42_clmul.h @@ -0,0 +1,28 @@ +// Copyright 2025 The CRC32C Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef CRC32C_CRC32C_SSE42_CLMUL_H_ +#define CRC32C_CRC32C_SSE42_CLMUL_H_ + +// X86-specific code that uses PCLMULQDQ to accelerate CRC32C stripe folding. + +#include +#include + +#include "crc32c/crc32c_config.h" + +// The PCLMUL-accelerated implementation depends on SSE4.2 and is only +// enabled for 64-bit builds. +#if HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__)) + +namespace crc32c { + +// PCLMUL+SSE4.2-accelerated implementation in crc32c_sse42_clmul.cc +uint32_t ExtendSse42Clmul(uint32_t crc, const uint8_t* data, size_t count); + +} // namespace crc32c + +#endif // HAVE_SSE42 && HAVE_PCLMUL && (defined(_M_X64) || defined(__x86_64__)) + +#endif // CRC32C_CRC32C_SSE42_CLMUL_H_