Index: head/sys/conf/files.amd64 =================================================================== --- head/sys/conf/files.amd64 +++ head/sys/conf/files.amd64 @@ -593,6 +593,11 @@ compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx64_wrap.S optional ndisapi pci # +crc32_sse42.o standard \ + dependency "$S/libkern/x86/crc32_sse42.c" \ + compile-with "${CC} -c ${CFLAGS:N-nostdinc} ${WERROR} ${PROF} -msse4 ${.IMPSRC}" \ + no-implicit-rule \ + clean "crc32_sse42.o" libkern/memmove.c standard libkern/memset.c standard # Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 +++ head/sys/conf/files.i386 @@ -554,6 +554,11 @@ kern/imgact_aout.c optional compat_aout kern/imgact_gzip.c optional gzip kern/subr_sfbuf.c standard +crc32_sse42.o standard \ + dependency "$S/libkern/x86/crc32_sse42.c" \ + compile-with "${CC} -c ${CFLAGS:N-nostdinc} ${WERROR} ${PROF} -msse4 ${.IMPSRC}" \ + no-implicit-rule \ + clean "crc32_sse42.o" libkern/divdi3.c standard libkern/ffsll.c standard libkern/flsll.c standard Index: head/sys/libkern/crc32.c =================================================================== --- head/sys/libkern/crc32.c +++ head/sys/libkern/crc32.c @@ -46,8 +46,14 @@ __FBSDID("$FreeBSD$"); #include +#include #include +#if defined(__amd64__) || defined(__i386__) +#include +#include +#endif + const uint32_t crc32_tab[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, @@ -749,6 +755,11 @@ const unsigned char *buffer, unsigned int length) { +#if defined(__amd64__) || defined(__i386__) + if ((cpu_feature2 & CPUID2_SSE42) != 0) { + return (sse42_crc32c(crc32c, buffer, length)); + } else +#endif if (length < 4) { return (singletable_crc32c(crc32c, buffer, length)); } else { Index: head/sys/libkern/x86/crc32_sse42.c =================================================================== --- head/sys/libkern/x86/crc32_sse42.c +++ head/sys/libkern/x86/crc32_sse42.c @@ -0,0 +1,288 @@ +/* + * Derived from crc32c.c version 1.1 by Mark Adler. + * + * Copyright (C) 2013 Mark Adler + * + * This software is provided 'as-is', without any express or implied warranty. + * In no event will the author be held liable for any damages arising from the + * use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + * + * Mark Adler + * madler@alumni.caltech.edu + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * This file is compiled in userspace in order to run ATF unit tests. + */ +#ifdef USERSPACE_TESTING +#include +#else +#include +#include +#include +#include +#endif + +#include + +/* CRC-32C (iSCSI) polynomial in reversed bit order. */ +#define POLY 0x82f63b78 + +/* + * Block sizes for three-way parallel crc computation. LONG and SHORT must + * both be powers of two. + */ +#define LONG 8192 +#define SHORT 256 + +/* Tables for hardware crc that shift a crc by LONG and SHORT zeros. */ +static uint32_t crc32c_long[4][256]; +static uint32_t crc32c_short[4][256]; + +/* + * Multiply a matrix times a vector over the Galois field of two elements, + * GF(2). Each element is a bit in an unsigned integer. mat must have at + * least as many entries as the power of two for most significant one bit in + * vec. + */ +static inline uint32_t +gf2_matrix_times(uint32_t *mat, uint32_t vec) +{ + uint32_t sum; + + sum = 0; + while (vec) { + if (vec & 1) + sum ^= *mat; + vec >>= 1; + mat++; + } + return (sum); +} + +/* + * Multiply a matrix by itself over GF(2). Both mat and square must have 32 + * rows. + */ +static inline void +gf2_matrix_square(uint32_t *square, uint32_t *mat) +{ + int n; + + for (n = 0; n < 32; n++) + square[n] = gf2_matrix_times(mat, mat[n]); +} + +/* + * Construct an operator to apply len zeros to a crc. len must be a power of + * two. If len is not a power of two, then the result is the same as for the + * largest power of two less than len. The result for len == 0 is the same as + * for len == 1. A version of this routine could be easily written for any + * len, but that is not needed for this application. + */ +static void +crc32c_zeros_op(uint32_t *even, size_t len) +{ + uint32_t odd[32]; /* odd-power-of-two zeros operator */ + uint32_t row; + int n; + + /* put operator for one zero bit in odd */ + odd[0] = POLY; /* CRC-32C polynomial */ + row = 1; + for (n = 1; n < 32; n++) { + odd[n] = row; + row <<= 1; + } + + /* put operator for two zero bits in even */ + gf2_matrix_square(even, odd); + + /* put operator for four zero bits in odd */ + gf2_matrix_square(odd, even); + + /* + * first square will put the operator for one zero byte (eight zero + * bits), in even -- next square puts operator for two zero bytes in + * odd, and so on, until len has been rotated down to zero + */ + do { + gf2_matrix_square(even, odd); + len >>= 1; + if (len == 0) + return; + gf2_matrix_square(odd, even); + len >>= 1; + } while (len); + + /* answer ended up in odd -- copy to even */ + for (n = 0; n < 32; n++) + even[n] = odd[n]; +} + +/* + * Take a length and build four lookup tables for applying the zeros operator + * for that length, byte-by-byte on the operand. + */ +static void +crc32c_zeros(uint32_t zeros[][256], size_t len) +{ + uint32_t op[32]; + uint32_t n; + + crc32c_zeros_op(op, len); + for (n = 0; n < 256; n++) { + zeros[0][n] = gf2_matrix_times(op, n); + zeros[1][n] = gf2_matrix_times(op, n << 8); + zeros[2][n] = gf2_matrix_times(op, n << 16); + zeros[3][n] = gf2_matrix_times(op, n << 24); + } +} + +/* Apply the zeros operator table to crc. */ +static inline uint32_t +crc32c_shift(uint32_t zeros[][256], uint32_t crc) +{ + + return (zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^ + zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24]); +} + +/* Initialize tables for shifting crcs. */ +static void +#ifdef USERSPACE_TESTING +__attribute__((__constructor__)) +#endif +crc32c_init_hw(void) +{ + crc32c_zeros(crc32c_long, LONG); + crc32c_zeros(crc32c_short, SHORT); +} +#ifdef _KERNEL +SYSINIT(crc32c_sse42, SI_SUB_LOCK, SI_ORDER_ANY, crc32c_init_hw, NULL); +#endif + +/* Compute CRC-32C using the Intel hardware instruction. */ +#ifdef USERSPACE_TESTING +uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned); +#endif +uint32_t +sse42_crc32c(uint32_t crc, const unsigned char *buf, unsigned len) +{ +#ifdef __amd64__ + const size_t align = 8; +#else + const size_t align = 4; +#endif + const unsigned char *next, *end; + uint64_t crc0, crc1, crc2; /* need to be 64 bits for crc32q */ + + next = buf; + crc0 = crc; + + /* Compute the crc to bring the data pointer to an aligned boundary. */ + while (len && ((uintptr_t)next & (align - 1)) != 0) { + crc0 = _mm_crc32_u8(crc0, *next); + next++; + len--; + } + + /* + * Compute the crc on sets of LONG*3 bytes, executing three independent + * crc instructions, each on LONG bytes -- this is optimized for the + * Nehalem, Westmere, Sandy Bridge, and Ivy Bridge architectures, which + * have a throughput of one crc per cycle, but a latency of three + * cycles. + */ + while (len >= LONG * 3) { + crc1 = 0; + crc2 = 0; + end = next + LONG; + do { +#ifdef __amd64__ + crc0 = _mm_crc32_u64(crc0, *(const uint64_t *)next); + crc1 = _mm_crc32_u64(crc1, + *(const uint64_t *)(next + LONG)); + crc2 = _mm_crc32_u64(crc2, + *(const uint64_t *)(next + (LONG * 2))); +#else + crc0 = _mm_crc32_u32(crc0, *(const uint32_t *)next); + crc1 = _mm_crc32_u32(crc1, + *(const uint32_t *)(next + LONG)); + crc2 = _mm_crc32_u32(crc2, + *(const uint32_t *)(next + (LONG * 2))); +#endif + next += align; + } while (next < end); + crc0 = crc32c_shift(crc32c_long, crc0) ^ crc1; + crc0 = crc32c_shift(crc32c_long, crc0) ^ crc2; + next += LONG * 2; + len -= LONG * 3; + } + + /* + * Do the same thing, but now on SHORT*3 blocks for the remaining data + * less than a LONG*3 block + */ + while (len >= SHORT * 3) { + crc1 = 0; + crc2 = 0; + end = next + SHORT; + do { +#ifdef __amd64__ + crc0 = _mm_crc32_u64(crc0, *(const uint64_t *)next); + crc1 = _mm_crc32_u64(crc1, + *(const uint64_t *)(next + SHORT)); + crc2 = _mm_crc32_u64(crc2, + *(const uint64_t *)(next + (SHORT * 2))); +#else + crc0 = _mm_crc32_u32(crc0, *(const uint32_t *)next); + crc1 = _mm_crc32_u32(crc1, + *(const uint32_t *)(next + SHORT)); + crc2 = _mm_crc32_u32(crc2, + *(const uint32_t *)(next + (SHORT * 2))); +#endif + next += align; + } while (next < end); + crc0 = crc32c_shift(crc32c_short, crc0) ^ crc1; + crc0 = crc32c_shift(crc32c_short, crc0) ^ crc2; + next += SHORT * 2; + len -= SHORT * 3; + } + + /* Compute the crc on the remaining bytes at native word size. */ + end = next + (len - (len & (align - 1))); + while (next < end) { +#ifdef __amd64__ + crc0 = _mm_crc32_u64(crc0, *(const uint64_t *)next); +#else + crc0 = _mm_crc32_u32(crc0, *(const uint32_t *)next); +#endif + next += align; + } + len &= (align - 1); + + /* Compute the crc for any trailing bytes. */ + while (len) { + crc0 = _mm_crc32_u8(crc0, *next); + next++; + len--; + } + + return ((uint32_t)crc0); +} Index: head/sys/sys/libkern.h =================================================================== --- head/sys/sys/libkern.h +++ head/sys/sys/libkern.h @@ -205,6 +205,11 @@ uint32_t calculate_crc32c(uint32_t crc32c, const unsigned char *buffer, unsigned int length); +#ifdef _KERNEL +#if defined(__amd64__) || defined(__i386__) +uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned); +#endif +#endif LIBKERN_INLINE void *memset(void *, int, size_t); Index: head/tests/sys/kern/Makefile =================================================================== --- head/tests/sys/kern/Makefile +++ head/tests/sys/kern/Makefile @@ -29,13 +29,19 @@ CFLAGS.mqueue_test+= -I${SRCTOP}/tests LIBADD.mqueue_test+= rt +.if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386" +ATF_TESTS_C+= libkern_crc32 +CFLAGS.libkern_crc32+= -msse4 -DUSERSPACE_TESTING +LDADD.libkern_crc32+= ${SRCTOP}/sys/libkern/x86/crc32_sse42.c +.endif + # subr_unit.c contains functions whose prototypes lie in headers that cannot be # included in userland. But as far as subr_unit_test goes, they're effectively # static. So it's ok to disable -Wmissing-prototypes for this program. CFLAGS.subr_unit.c+= -Wno-missing-prototypes SRCS.subr_unit_test+= subr_unit.c -WARNS?= 5 +WARNS?= 3 TESTS_SUBDIRS+= acct TESTS_SUBDIRS+= execve Index: head/tests/sys/kern/libkern_crc32.c =================================================================== --- head/tests/sys/kern/libkern_crc32.c +++ head/tests/sys/kern/libkern_crc32.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2017 Conrad Meyer + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +#include + +#include + +extern uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned); + +ATF_TC_WITHOUT_HEAD(crc32c_basic_correctness); +ATF_TC_BODY(crc32c_basic_correctness, tc) +{ + const uint64_t inputs[] = { + 0xf408c634b3a9142, + 0x80539e8c7c352e2b, + 0x62e9121db6e4d649, + 0x899345850ed0a286, + 0x2302df11b4a43b15, + 0xe943de7b3d35d70, + 0xdf1ff2bf41abf56b, + 0x9bc138abae315de2, + 0x31cc82e56234f0ff, + 0xce63c0cd6988e847, + 0x3e42f6b78ee352fa, + 0xfa4085436078cfa6, + 0x53349558bf670a4b, + 0x2714e10e7d722c61, + 0xc0d3261addfc6908, + 0xd1567c3181d3a1bf, + }; + const uint32_t results[] = { + 0x2ce33ede, + 0xc49cc573, + 0xb8683c96, + 0x6918660d, + 0xa904e522, + 0x52dbc42c, + 0x98863c22, + 0x894d5d2c, + 0xb003745d, + 0xfc496dbd, + 0x97d2fbb5, + 0x3c062ef1, + 0xcc2eff18, + 0x6a9b09f6, + 0x420242c1, + 0xfd562dc3, + }; + size_t i; + uint32_t act; + + ATF_REQUIRE(nitems(inputs) == nitems(results)); + + for (i = 0; i < nitems(inputs); i++) { + act = sse42_crc32c(~0, (const void *)&inputs[i], + sizeof(inputs[0])); + ATF_REQUIRE_MSG(act == results[i], + "crc32c(0x%jx) = 0x%08x, got 0x%08x", (uintmax_t)inputs[i], + results[i], act); + } +} + +ATF_TC_WITHOUT_HEAD(crc32c_alignment); +ATF_TC_BODY(crc32c_alignment, tc) +{ + const uint64_t input = 0xf408c634b3a9142; + const uint32_t result = 0x2ce33ede; + unsigned char buf[15]; + size_t i; + uint32_t act; + + + for (i = 1; i < 8; i++) { + memcpy(&buf[i], &input, sizeof(input)); + + act = sse42_crc32c(~0, (const void *)&buf[i], sizeof(input)); + ATF_REQUIRE_MSG(act == result, + "crc32c(0x%jx) = 0x%08x, got 0x%08x", (uintmax_t)input, + result, act); + } +} + +ATF_TC_WITHOUT_HEAD(crc32c_trailing_bytes); +ATF_TC_BODY(crc32c_trailing_bytes, tc) +{ + const unsigned char input[] = { + 0x87, 0x54, 0x74, 0xd2, 0xb, 0x9b, 0xdd, 0xf6, 0x68, 0x37, + 0xd4, 0x4, 0x5e, 0xa9, 0xb3 + }; + const uint32_t result = 0xec638d62; + uint32_t act; + + act = sse42_crc32c(~0, input, sizeof(input)); + ATF_REQUIRE_MSG(act == result, "expected 0x%08x, got 0x%08x", result, + act); +} + +ATF_TP_ADD_TCS(tp) +{ + + ATF_TP_ADD_TC(tp, crc32c_basic_correctness); + ATF_TP_ADD_TC(tp, crc32c_alignment); + ATF_TP_ADD_TC(tp, crc32c_trailing_bytes); + return (atf_no_error()); +}