crc_simd.cpp
// crc_simd.cpp - written and placed in the public domain by
// Jeffrey Walton, Uri Blumenthal and Marcel Raad.
//
// This source file uses intrinsics to gain access to SSE4.2 and
// ARMv8a CRC-32 and CRC-32C instructions. A separate source file
// is needed because additional CXXFLAGS are required to enable
// the appropriate instructions sets in some build configurations.
#include "pch.h"
#include "config.h"
#include "misc.h"
#if (CRYPTOPP_SSE42_AVAILABLE)
# include <nmmintrin.h>
#endif
#if (CRYPTOPP_ARM_ACLE_HEADER)
# include <stdint.h>
# include <arm_acle.h>
#endif
#if (CRYPTOPP_ARM_CRC32_AVAILABLE)
# include "arm_simd.h"
#endif
#ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
# include <signal.h>
# include <setjmp.h>
#endif
#if CRYPTOPP_MSC_VERSION
# pragma warning(disable: 4244)
#endif
#ifndef EXCEPTION_EXECUTE_HANDLER
# define EXCEPTION_EXECUTE_HANDLER 1
#endif
#define CONST_WORD32_CAST(x) ((const word32 *)(void*)(x))
#define CONST_WORD64_CAST(x) ((const word64 *)(void*)(x))
// Squash MS LNK4221 and libtool warnings
extern const char CRC_SIMD_FNAME[] = __FILE__;
NAMESPACE_BEGIN(CryptoPP)
#ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
extern "C" {
typedef void (*SigHandler)(int);
static jmp_buf s_jmpSIGILL;
static void SigIllHandler(int)
{
longjmp(s_jmpSIGILL, 1);
}
}
#endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
#if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
bool CPU_ProbeCRC32()
{
#if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
return false;
#elif (CRYPTOPP_ARM_CRC32_AVAILABLE)
# if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
volatile bool result = true;
__try
{
word32 w=0, x=1; byte z=3;
w = CRC32W(w,x);
w = CRC32B(w,z);
w = CRC32CW(w,x);
w = CRC32CB(w,z);
result = !!w;
}
__except (EXCEPTION_EXECUTE_HANDLER)
{
return false;
}
return result;
#else
// longjmp and clobber warnings. Volatile is required.
// http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
volatile bool result = true;
volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
if (oldHandler == SIG_ERR)
return false;
volatile sigset_t oldMask;
if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
{
signal(SIGILL, oldHandler);
return false;
}
if (setjmp(s_jmpSIGILL))
result = false;
else
{
word32 w=0, x=1; byte z=3;
w = CRC32W(w,x);
w = CRC32B(w,z);
w = CRC32CW(w,x);
w = CRC32CB(w,z);
// Hack... GCC optimizes away the code and returns true
result = !!w;
}
sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
signal(SIGILL, oldHandler);
return result;
# endif
#else
return false;
#endif // CRYPTOPP_ARM_CRC32_AVAILABLE
}
#endif // ARM32 or ARM64
#if (CRYPTOPP_ARM_CRC32_AVAILABLE)
void CRC32_Update_ARMV8(const byte *s, size_t n, word32& c)
{
for(; !IsAligned<word32>(s) && n > 0; s++, n--)
c = CRC32B(c, *s);
for(; n >= 16; s+=16, n-=16)
c = CRC32Wx4(c, CONST_WORD32_CAST(s));
for(; n >= 4; s+=4, n-=4)
c = CRC32W(c, *CONST_WORD32_CAST(s));
for(; n > 0; s++, n--)
c = CRC32B(c, *s);
}
void CRC32C_Update_ARMV8(const byte *s, size_t n, word32& c)
{
for(; !IsAligned<word32>(s) && n > 0; s++, n--)
c = CRC32CB(c, *s);
for(; n >= 16; s+=16, n-=16)
c = CRC32CWx4(c, CONST_WORD32_CAST(s));
for(; n >= 4; s+=4, n-=4)
c = CRC32CW(c, *CONST_WORD32_CAST(s));
for(; n > 0; s++, n--)
c = CRC32CB(c, *s);
}
#endif
#if (CRYPTOPP_SSE42_AVAILABLE)
void CRC32C_Update_SSE42(const byte *s, size_t n, word32& c)
{
// Temporary due to https://github.com/weidai11/cryptopp/issues/1202
word32 v = c;
// 64-bit code path due to https://github.com/weidai11/cryptopp/issues/1202
#if CRYPTOPP_BOOL_X64
for(; !IsAligned<word64>(s) && n > 0; s++, n--)
v = _mm_crc32_u8(v, *s);
#else
for(; !IsAligned<word32>(s) && n > 0; s++, n--)
v = _mm_crc32_u8(v, *s);
#endif
#if CRYPTOPP_BOOL_X64
for(; n >= 32; s+=32, n-=32)
{
v = _mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(v,
*CONST_WORD64_CAST(s+ 0)), *CONST_WORD64_CAST(s+ 8)),
*CONST_WORD64_CAST(s+16)), *CONST_WORD64_CAST(s+24));
}
#endif
for(; n >= 16; s+=16, n-=16)
{
v = _mm_crc32_u32(_mm_crc32_u32(_mm_crc32_u32(_mm_crc32_u32(v,
*CONST_WORD32_CAST(s+ 0)), *CONST_WORD32_CAST(s+ 4)),
*CONST_WORD32_CAST(s+ 8)), *CONST_WORD32_CAST(s+12));
}
for(; n >= 4; s+=4, n-=4)
v = _mm_crc32_u32(v, *CONST_WORD32_CAST(s));
for(; n > 0; s++, n--)
v = _mm_crc32_u8(v, *s);
c = static_cast<word32>(v);
}
#endif
NAMESPACE_END
arm_simd.h
Support functions for ARM and vector operations.
CRC32CWx4
uint32_t CRC32CWx4(uint32_t crc, const uint32_t vals[4])
CRC32-C checksum.
Definition arm_simd.h:118
CRC32CB
uint32_t CRC32CB(uint32_t crc, uint8_t val)
CRC32-C checksum.
Definition arm_simd.h:86
CRC32W
uint32_t CRC32W(uint32_t crc, uint32_t val)
CRC32 checksum.
Definition arm_simd.h:46
CRC32B
uint32_t CRC32B(uint32_t crc, uint8_t val)
CRC32 checksum.
Definition arm_simd.h:30
CRC32CW
uint32_t CRC32CW(uint32_t crc, uint32_t val)
CRC32-C checksum.
Definition arm_simd.h:102
CRC32Wx4
uint32_t CRC32Wx4(uint32_t crc, const uint32_t vals[4])
CRC32 checksum.
Definition arm_simd.h:62
config.h
Library configuration file.
word32
unsigned int word32
32-bit unsigned datatype
Definition config_int.h:72
misc.h
Utility functions for the Crypto++ library.
IsAligned
bool IsAligned(const void *ptr)
Determines whether ptr is minimally aligned.
Definition misc.h:1450
CryptoPP
Crypto++ library namespace.
pch.h
Precompiled header file.