Index: sys/conf/files.amd64 =================================================================== --- sys/conf/files.amd64 +++ sys/conf/files.amd64 @@ -593,6 +593,11 @@ compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx64_wrap.S optional ndisapi pci # +crc32_sse42.o standard \ + dependency "$S/libkern/x86/crc32_sse42.c" \ + compile-with "${CC} -c ${CFLAGS:N-nostdinc} ${WERROR} ${PROF} -msse4 ${.IMPSRC}" \ + no-implicit-rule \ + clean "crc32_sse42.o" libkern/memmove.c standard libkern/memset.c standard # Index: sys/conf/files.i386 =================================================================== --- sys/conf/files.i386 +++ sys/conf/files.i386 @@ -554,6 +554,11 @@ kern/imgact_aout.c optional compat_aout kern/imgact_gzip.c optional gzip kern/subr_sfbuf.c standard +crc32_sse42.o standard \ + dependency "$S/libkern/x86/crc32_sse42.c" \ + compile-with "${CC} -c ${CFLAGS:N-nostdinc} ${WERROR} ${PROF} -msse4 ${.IMPSRC}" \ + no-implicit-rule \ + clean "crc32_sse42.o" libkern/divdi3.c standard libkern/ffsll.c standard libkern/flsll.c standard Index: sys/libkern/crc32.c =================================================================== --- sys/libkern/crc32.c +++ sys/libkern/crc32.c @@ -46,8 +46,14 @@ __FBSDID("$FreeBSD$"); #include +#include #include +#if defined(__amd64__) || defined(__i386__) +#include +#include +#endif + const uint32_t crc32_tab[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, @@ -751,6 +757,10 @@ { if (length < 4) { return (singletable_crc32c(crc32c, buffer, length)); +#if defined(__amd64__) || (defined(__i386__) && !defined(PC98)) + } else if ((cpu_feature2 & CPUID2_SSE42) != 0) { + return (sse42_crc32c(crc32c, buffer, length)); +#endif } else { return (multitable_crc32c(crc32c, buffer, length)); } Index: sys/libkern/x86/crc32_sse42.c =================================================================== --- /dev/null +++ sys/libkern/x86/crc32_sse42.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2013-2015 Anand Suresh + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include + +#define CALC_CRC(op, crc, type, buf, len) do { \ + for (; (len) >= sizeof(type); \ + (len) -= sizeof(type), buf += sizeof(type)) { \ + (crc) = op((crc), *(const type *) (buf)); \ + } \ +} while(0) + +uint32_t +sse42_crc32c(uint32_t crc32c, const unsigned char *buffer, unsigned length) +{ + +#ifdef __amd64__ + const size_t align = 8; +#else + const size_t align = 4; +#endif + + KASSERT(length > 0, ("length is zero")); + + crc32c ^= UINT32_MAX; + + /* Process unaligned prefix */ + while (length > 0 && ((uintptr_t)buffer & (align - 1)) != 0) + crc32c = _mm_crc32_u8(crc32c, *buffer); + +#ifdef __amd64__ + CALC_CRC(_mm_crc32_u64, crc32c, uint64_t, buffer, length); +#endif + CALC_CRC(_mm_crc32_u32, crc32c, uint32_t, buffer, length); + CALC_CRC(_mm_crc32_u16, crc32c, uint16_t, buffer, length); + CALC_CRC(_mm_crc32_u8, crc32c, uint8_t, buffer, length); + + return (crc32c ^ UINT32_MAX); +} Index: sys/sys/libkern.h =================================================================== --- sys/sys/libkern.h +++ sys/sys/libkern.h @@ -178,6 +178,9 @@ uint32_t calculate_crc32c(uint32_t crc32c, const unsigned char *buffer, unsigned int length); +#if defined(_KERNEL) && (defined(__amd64__) || defined(__i386__)) +uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned); +#endif LIBKERN_INLINE void *memset(void *, int, size_t);