Page MenuHomeFreeBSD
Files F144747577

D9342.id.diff
No OneTemporary
Actions

D9342.id.diff
View Options

Index: head/sys/conf/files.amd64
===================================================================
--- head/sys/conf/files.amd64
+++ head/sys/conf/files.amd64
@@ -593,6 +593,11 @@
compat/ndis/subr_usbd.c optional ndisapi pci
compat/ndis/winx64_wrap.S optional ndisapi pci
#
+crc32_sse42.o standard \
+ dependency "$S/libkern/x86/crc32_sse42.c" \
+ compile-with "${CC} -c ${CFLAGS:N-nostdinc} ${WERROR} ${PROF} -msse4 ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "crc32_sse42.o"
libkern/memmove.c standard
libkern/memset.c standard
#
Index: head/sys/conf/files.i386
===================================================================
--- head/sys/conf/files.i386
+++ head/sys/conf/files.i386
@@ -554,6 +554,11 @@
kern/imgact_aout.c optional compat_aout
kern/imgact_gzip.c optional gzip
kern/subr_sfbuf.c standard
+crc32_sse42.o standard \
+ dependency "$S/libkern/x86/crc32_sse42.c" \
+ compile-with "${CC} -c ${CFLAGS:N-nostdinc} ${WERROR} ${PROF} -msse4 ${.IMPSRC}" \
+ no-implicit-rule \
+ clean "crc32_sse42.o"
libkern/divdi3.c standard
libkern/ffsll.c standard
libkern/flsll.c standard
Index: head/sys/libkern/crc32.c
===================================================================
--- head/sys/libkern/crc32.c
+++ head/sys/libkern/crc32.c
@@ -46,8 +46,14 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
+#include <sys/libkern.h>
#include <sys/systm.h>
+#if defined(__amd64__) || defined(__i386__)
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+#endif
+
const uint32_t crc32_tab[] = {
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
@@ -749,6 +755,11 @@
const unsigned char *buffer,
unsigned int length)
{
+#if defined(__amd64__) || defined(__i386__)
+ if ((cpu_feature2 & CPUID2_SSE42) != 0) {
+ return (sse42_crc32c(crc32c, buffer, length));
+ } else
+#endif
if (length < 4) {
return (singletable_crc32c(crc32c, buffer, length));
} else {
Index: head/sys/libkern/x86/crc32_sse42.c
===================================================================
--- head/sys/libkern/x86/crc32_sse42.c
+++ head/sys/libkern/x86/crc32_sse42.c
@@ -0,0 +1,288 @@
+/*
+ * Derived from crc32c.c version 1.1 by Mark Adler.
+ *
+ * Copyright (C) 2013 Mark Adler
+ *
+ * This software is provided 'as-is', without any express or implied warranty.
+ * In no event will the author be held liable for any damages arising from the
+ * use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ * claim that you wrote the original software. If you use this software
+ * in a product, an acknowledgment in the product documentation would be
+ * appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ * misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ *
+ * Mark Adler
+ * madler@alumni.caltech.edu
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * This file is compiled in userspace in order to run ATF unit tests.
+ */
+#ifdef USERSPACE_TESTING
+#include <stdint.h>
+#else
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/libkern.h>
+#include <sys/systm.h>
+#endif
+
+#include <nmmintrin.h>
+
+/* CRC-32C (iSCSI) polynomial in reversed bit order. */
+#define POLY 0x82f63b78
+
+/*
+ * Block sizes for three-way parallel crc computation. LONG and SHORT must
+ * both be powers of two.
+ */
+#define LONG 8192
+#define SHORT 256
+
+/* Tables for hardware crc that shift a crc by LONG and SHORT zeros. */
+static uint32_t crc32c_long[4][256];
+static uint32_t crc32c_short[4][256];
+
+/*
+ * Multiply a matrix times a vector over the Galois field of two elements,
+ * GF(2). Each element is a bit in an unsigned integer. mat must have at
+ * least as many entries as the power of two for most significant one bit in
+ * vec.
+ */
+static inline uint32_t
+gf2_matrix_times(uint32_t *mat, uint32_t vec)
+{
+ uint32_t sum;
+
+ sum = 0;
+ while (vec) {
+ if (vec & 1)
+ sum ^= *mat;
+ vec >>= 1;
+ mat++;
+ }
+ return (sum);
+}
+
+/*
+ * Multiply a matrix by itself over GF(2). Both mat and square must have 32
+ * rows.
+ */
+static inline void
+gf2_matrix_square(uint32_t *square, uint32_t *mat)
+{
+ int n;
+
+ for (n = 0; n < 32; n++)
+ square[n] = gf2_matrix_times(mat, mat[n]);
+}
+
+/*
+ * Construct an operator to apply len zeros to a crc. len must be a power of
+ * two. If len is not a power of two, then the result is the same as for the
+ * largest power of two less than len. The result for len == 0 is the same as
+ * for len == 1. A version of this routine could be easily written for any
+ * len, but that is not needed for this application.
+ */
+static void
+crc32c_zeros_op(uint32_t *even, size_t len)
+{
+ uint32_t odd[32]; /* odd-power-of-two zeros operator */
+ uint32_t row;
+ int n;
+
+ /* put operator for one zero bit in odd */
+ odd[0] = POLY; /* CRC-32C polynomial */
+ row = 1;
+ for (n = 1; n < 32; n++) {
+ odd[n] = row;
+ row <<= 1;
+ }
+
+ /* put operator for two zero bits in even */
+ gf2_matrix_square(even, odd);
+
+ /* put operator for four zero bits in odd */
+ gf2_matrix_square(odd, even);
+
+ /*
+ * first square will put the operator for one zero byte (eight zero
+ * bits), in even -- next square puts operator for two zero bytes in
+ * odd, and so on, until len has been rotated down to zero
+ */
+ do {
+ gf2_matrix_square(even, odd);
+ len >>= 1;
+ if (len == 0)
+ return;
+ gf2_matrix_square(odd, even);
+ len >>= 1;
+ } while (len);
+
+ /* answer ended up in odd -- copy to even */
+ for (n = 0; n < 32; n++)
+ even[n] = odd[n];
+}
+
+/*
+ * Take a length and build four lookup tables for applying the zeros operator
+ * for that length, byte-by-byte on the operand.
+ */
+static void
+crc32c_zeros(uint32_t zeros[][256], size_t len)
+{
+ uint32_t op[32];
+ uint32_t n;
+
+ crc32c_zeros_op(op, len);
+ for (n = 0; n < 256; n++) {
+ zeros[0][n] = gf2_matrix_times(op, n);
+ zeros[1][n] = gf2_matrix_times(op, n << 8);
+ zeros[2][n] = gf2_matrix_times(op, n << 16);
+ zeros[3][n] = gf2_matrix_times(op, n << 24);
+ }
+}
+
+/* Apply the zeros operator table to crc. */
+static inline uint32_t
+crc32c_shift(uint32_t zeros[][256], uint32_t crc)
+{
+
+ return (zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^
+ zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24]);
+}
+
+/* Initialize tables for shifting crcs. */
+static void
+#ifdef USERSPACE_TESTING
+__attribute__((__constructor__))
+#endif
+crc32c_init_hw(void)
+{
+ crc32c_zeros(crc32c_long, LONG);
+ crc32c_zeros(crc32c_short, SHORT);
+}
+#ifdef _KERNEL
+SYSINIT(crc32c_sse42, SI_SUB_LOCK, SI_ORDER_ANY, crc32c_init_hw, NULL);
+#endif
+
+/* Compute CRC-32C using the Intel hardware instruction. */
+#ifdef USERSPACE_TESTING
+uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned);
+#endif
+uint32_t
+sse42_crc32c(uint32_t crc, const unsigned char *buf, unsigned len)
+{
+#ifdef __amd64__
+ const size_t align = 8;
+#else
+ const size_t align = 4;
+#endif
+ const unsigned char *next, *end;
+ uint64_t crc0, crc1, crc2; /* need to be 64 bits for crc32q */
+
+ next = buf;
+ crc0 = crc;
+
+ /* Compute the crc to bring the data pointer to an aligned boundary. */
+ while (len && ((uintptr_t)next & (align - 1)) != 0) {
+ crc0 = _mm_crc32_u8(crc0, *next);
+ next++;
+ len--;
+ }
+
+ /*
+ * Compute the crc on sets of LONG*3 bytes, executing three independent
+ * crc instructions, each on LONG bytes -- this is optimized for the
+ * Nehalem, Westmere, Sandy Bridge, and Ivy Bridge architectures, which
+ * have a throughput of one crc per cycle, but a latency of three
+ * cycles.
+ */
+ while (len >= LONG * 3) {
+ crc1 = 0;
+ crc2 = 0;
+ end = next + LONG;
+ do {
+#ifdef __amd64__
+ crc0 = _mm_crc32_u64(crc0, *(const uint64_t *)next);
+ crc1 = _mm_crc32_u64(crc1,
+ *(const uint64_t *)(next + LONG));
+ crc2 = _mm_crc32_u64(crc2,
+ *(const uint64_t *)(next + (LONG * 2)));
+#else
+ crc0 = _mm_crc32_u32(crc0, *(const uint32_t *)next);
+ crc1 = _mm_crc32_u32(crc1,
+ *(const uint32_t *)(next + LONG));
+ crc2 = _mm_crc32_u32(crc2,
+ *(const uint32_t *)(next + (LONG * 2)));
+#endif
+ next += align;
+ } while (next < end);
+ crc0 = crc32c_shift(crc32c_long, crc0) ^ crc1;
+ crc0 = crc32c_shift(crc32c_long, crc0) ^ crc2;
+ next += LONG * 2;
+ len -= LONG * 3;
+ }
+
+ /*
+ * Do the same thing, but now on SHORT*3 blocks for the remaining data
+ * less than a LONG*3 block
+ */
+ while (len >= SHORT * 3) {
+ crc1 = 0;
+ crc2 = 0;
+ end = next + SHORT;
+ do {
+#ifdef __amd64__
+ crc0 = _mm_crc32_u64(crc0, *(const uint64_t *)next);
+ crc1 = _mm_crc32_u64(crc1,
+ *(const uint64_t *)(next + SHORT));
+ crc2 = _mm_crc32_u64(crc2,
+ *(const uint64_t *)(next + (SHORT * 2)));
+#else
+ crc0 = _mm_crc32_u32(crc0, *(const uint32_t *)next);
+ crc1 = _mm_crc32_u32(crc1,
+ *(const uint32_t *)(next + SHORT));
+ crc2 = _mm_crc32_u32(crc2,
+ *(const uint32_t *)(next + (SHORT * 2)));
+#endif
+ next += align;
+ } while (next < end);
+ crc0 = crc32c_shift(crc32c_short, crc0) ^ crc1;
+ crc0 = crc32c_shift(crc32c_short, crc0) ^ crc2;
+ next += SHORT * 2;
+ len -= SHORT * 3;
+ }
+
+ /* Compute the crc on the remaining bytes at native word size. */
+ end = next + (len - (len & (align - 1)));
+ while (next < end) {
+#ifdef __amd64__
+ crc0 = _mm_crc32_u64(crc0, *(const uint64_t *)next);
+#else
+ crc0 = _mm_crc32_u32(crc0, *(const uint32_t *)next);
+#endif
+ next += align;
+ }
+ len &= (align - 1);
+
+ /* Compute the crc for any trailing bytes. */
+ while (len) {
+ crc0 = _mm_crc32_u8(crc0, *next);
+ next++;
+ len--;
+ }
+
+ return ((uint32_t)crc0);
+}
Index: head/sys/sys/libkern.h
===================================================================
--- head/sys/sys/libkern.h
+++ head/sys/sys/libkern.h
@@ -205,6 +205,11 @@
uint32_t
calculate_crc32c(uint32_t crc32c, const unsigned char *buffer,
unsigned int length);
+#ifdef _KERNEL
+#if defined(__amd64__) || defined(__i386__)
+uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned);
+#endif
+#endif
LIBKERN_INLINE void *memset(void *, int, size_t);
Index: head/tests/sys/kern/Makefile
===================================================================
--- head/tests/sys/kern/Makefile
+++ head/tests/sys/kern/Makefile
@@ -29,13 +29,19 @@
CFLAGS.mqueue_test+= -I${SRCTOP}/tests
LIBADD.mqueue_test+= rt
+.if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386"
+ATF_TESTS_C+= libkern_crc32
+CFLAGS.libkern_crc32+= -msse4 -DUSERSPACE_TESTING
+LDADD.libkern_crc32+= ${SRCTOP}/sys/libkern/x86/crc32_sse42.c
+.endif
+
# subr_unit.c contains functions whose prototypes lie in headers that cannot be
# included in userland. But as far as subr_unit_test goes, they're effectively
# static. So it's ok to disable -Wmissing-prototypes for this program.
CFLAGS.subr_unit.c+= -Wno-missing-prototypes
SRCS.subr_unit_test+= subr_unit.c
-WARNS?= 5
+WARNS?= 3
TESTS_SUBDIRS+= acct
TESTS_SUBDIRS+= execve
Index: head/tests/sys/kern/libkern_crc32.c
===================================================================
--- head/tests/sys/kern/libkern_crc32.c
+++ head/tests/sys/kern/libkern_crc32.c
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+
+#include <stdint.h>
+
+#include <atf-c.h>
+
+extern uint32_t sse42_crc32c(uint32_t, const unsigned char *, unsigned);
+
+ATF_TC_WITHOUT_HEAD(crc32c_basic_correctness);
+ATF_TC_BODY(crc32c_basic_correctness, tc)
+{
+ const uint64_t inputs[] = {
+ 0xf408c634b3a9142,
+ 0x80539e8c7c352e2b,
+ 0x62e9121db6e4d649,
+ 0x899345850ed0a286,
+ 0x2302df11b4a43b15,
+ 0xe943de7b3d35d70,
+ 0xdf1ff2bf41abf56b,
+ 0x9bc138abae315de2,
+ 0x31cc82e56234f0ff,
+ 0xce63c0cd6988e847,
+ 0x3e42f6b78ee352fa,
+ 0xfa4085436078cfa6,
+ 0x53349558bf670a4b,
+ 0x2714e10e7d722c61,
+ 0xc0d3261addfc6908,
+ 0xd1567c3181d3a1bf,
+ };
+ const uint32_t results[] = {
+ 0x2ce33ede,
+ 0xc49cc573,
+ 0xb8683c96,
+ 0x6918660d,
+ 0xa904e522,
+ 0x52dbc42c,
+ 0x98863c22,
+ 0x894d5d2c,
+ 0xb003745d,
+ 0xfc496dbd,
+ 0x97d2fbb5,
+ 0x3c062ef1,
+ 0xcc2eff18,
+ 0x6a9b09f6,
+ 0x420242c1,
+ 0xfd562dc3,
+ };
+ size_t i;
+ uint32_t act;
+
+ ATF_REQUIRE(nitems(inputs) == nitems(results));
+
+ for (i = 0; i < nitems(inputs); i++) {
+ act = sse42_crc32c(~0, (const void *)&inputs[i],
+ sizeof(inputs[0]));
+ ATF_REQUIRE_MSG(act == results[i],
+ "crc32c(0x%jx) = 0x%08x, got 0x%08x", (uintmax_t)inputs[i],
+ results[i], act);
+ }
+}
+
+ATF_TC_WITHOUT_HEAD(crc32c_alignment);
+ATF_TC_BODY(crc32c_alignment, tc)
+{
+ const uint64_t input = 0xf408c634b3a9142;
+ const uint32_t result = 0x2ce33ede;
+ unsigned char buf[15];
+ size_t i;
+ uint32_t act;
+
+
+ for (i = 1; i < 8; i++) {
+ memcpy(&buf[i], &input, sizeof(input));
+
+ act = sse42_crc32c(~0, (const void *)&buf[i], sizeof(input));
+ ATF_REQUIRE_MSG(act == result,
+ "crc32c(0x%jx) = 0x%08x, got 0x%08x", (uintmax_t)input,
+ result, act);
+ }
+}
+
+ATF_TC_WITHOUT_HEAD(crc32c_trailing_bytes);
+ATF_TC_BODY(crc32c_trailing_bytes, tc)
+{
+ const unsigned char input[] = {
+ 0x87, 0x54, 0x74, 0xd2, 0xb, 0x9b, 0xdd, 0xf6, 0x68, 0x37,
+ 0xd4, 0x4, 0x5e, 0xa9, 0xb3
+ };
+ const uint32_t result = 0xec638d62;
+ uint32_t act;
+
+ act = sse42_crc32c(~0, input, sizeof(input));
+ ATF_REQUIRE_MSG(act == result, "expected 0x%08x, got 0x%08x", result,
+ act);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+ ATF_TP_ADD_TC(tp, crc32c_basic_correctness);
+ ATF_TP_ADD_TC(tp, crc32c_alignment);
+ ATF_TP_ADD_TC(tp, crc32c_trailing_bytes);
+ return (atf_no_error());
+}

File Metadata

Mime Type
text/plain
Expires
Fri, Feb 13, 2:02 AM (9 h, 11 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28672550
Default Alt Text
D9342.id.diff (15 KB)

Event Timeline