diff --git a/sys/crypto/aesni/aesni_ghash.c b/sys/crypto/aesni/aesni_ghash.c index b0d1b6137ec6..a1295b6ccbda 100644 --- a/sys/crypto/aesni/aesni_ghash.c +++ b/sys/crypto/aesni/aesni_ghash.c @@ -1,808 +1,809 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by John-Mark Gurney under * the sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * * $FreeBSD$ * */ /* * Figure 5, 8 and 12 are copied from the Intel white paper: * Intel® Carry-Less Multiplication Instruction and its Usage for * Computing the GCM Mode * * and as such are: * Copyright © 2010 Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of Intel Corporation nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifdef _KERNEL #include #include #else #include #endif #include #include #include static inline int m128icmp(__m128i a, __m128i b) { __m128i cmp; cmp = _mm_cmpeq_epi32(a, b); return _mm_movemask_epi8(cmp) == 0xffff; } #ifdef __i386__ static inline __m128i _mm_insert_epi64(__m128i a, int64_t b, const int ndx) { if (!ndx) { a = _mm_insert_epi32(a, b, 0); a = _mm_insert_epi32(a, b >> 32, 1); } else { a = _mm_insert_epi32(a, b, 2); a = _mm_insert_epi32(a, b >> 32, 3); } return a; } #endif /* some code from carry-less-multiplication-instruction-in-gcm-mode-paper.pdf */ /* Figure 5. Code Sample - Performing Ghash Using Algorithms 1 and 5 (C) */ static void gfmul(__m128i a, __m128i b, __m128i *res) { __m128i tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9; tmp3 = _mm_clmulepi64_si128(a, b, 0x00); tmp4 = _mm_clmulepi64_si128(a, b, 0x10); tmp5 = _mm_clmulepi64_si128(a, b, 0x01); tmp6 = _mm_clmulepi64_si128(a, b, 0x11); tmp4 = _mm_xor_si128(tmp4, tmp5); tmp5 = _mm_slli_si128(tmp4, 8); tmp4 = _mm_srli_si128(tmp4, 8); tmp3 = _mm_xor_si128(tmp3, tmp5); tmp6 = _mm_xor_si128(tmp6, tmp4); tmp7 = _mm_srli_epi32(tmp3, 31); tmp8 = _mm_srli_epi32(tmp6, 31); tmp3 = _mm_slli_epi32(tmp3, 1); tmp6 = _mm_slli_epi32(tmp6, 1); tmp9 = _mm_srli_si128(tmp7, 12); tmp8 = _mm_slli_si128(tmp8, 4); tmp7 = _mm_slli_si128(tmp7, 4); tmp3 = _mm_or_si128(tmp3, tmp7); tmp6 = _mm_or_si128(tmp6, tmp8); tmp6 = _mm_or_si128(tmp6, tmp9); tmp7 = _mm_slli_epi32(tmp3, 31); tmp8 = _mm_slli_epi32(tmp3, 30); tmp9 = _mm_slli_epi32(tmp3, 25); tmp7 = _mm_xor_si128(tmp7, tmp8); tmp7 = _mm_xor_si128(tmp7, tmp9); tmp8 = _mm_srli_si128(tmp7, 4); tmp7 = _mm_slli_si128(tmp7, 12); tmp3 = _mm_xor_si128(tmp3, tmp7); tmp2 = _mm_srli_epi32(tmp3, 1); tmp4 = _mm_srli_epi32(tmp3, 2); tmp5 = _mm_srli_epi32(tmp3, 7); tmp2 = _mm_xor_si128(tmp2, tmp4); tmp2 = _mm_xor_si128(tmp2, tmp5); tmp2 = _mm_xor_si128(tmp2, tmp8); tmp3 = _mm_xor_si128(tmp3, tmp2); tmp6 = _mm_xor_si128(tmp6, tmp3); *res = tmp6; } /* * Figure 8. Code Sample - Performing Ghash Using an Aggregated Reduction * Method */ static void reduce4(__m128i H1, __m128i H2, __m128i H3, __m128i H4, __m128i X1, __m128i X2, __m128i X3, __m128i X4, __m128i *res) { /*algorithm by Krzysztof Jankowski, Pierre Laurent - Intel*/ __m128i H1_X1_lo, H1_X1_hi, H2_X2_lo, H2_X2_hi, H3_X3_lo, H3_X3_hi, H4_X4_lo, H4_X4_hi, lo, hi; __m128i tmp0, tmp1, tmp2, tmp3; __m128i tmp4, tmp5, tmp6, tmp7; __m128i tmp8, tmp9; H1_X1_lo = _mm_clmulepi64_si128(H1, X1, 0x00); H2_X2_lo = _mm_clmulepi64_si128(H2, X2, 0x00); H3_X3_lo = _mm_clmulepi64_si128(H3, X3, 0x00); H4_X4_lo = _mm_clmulepi64_si128(H4, X4, 0x00); lo = _mm_xor_si128(H1_X1_lo, H2_X2_lo); lo = _mm_xor_si128(lo, H3_X3_lo); lo = _mm_xor_si128(lo, H4_X4_lo); H1_X1_hi = _mm_clmulepi64_si128(H1, X1, 0x11); H2_X2_hi = _mm_clmulepi64_si128(H2, X2, 0x11); H3_X3_hi = _mm_clmulepi64_si128(H3, X3, 0x11); H4_X4_hi = _mm_clmulepi64_si128(H4, X4, 0x11); hi = _mm_xor_si128(H1_X1_hi, H2_X2_hi); hi = _mm_xor_si128(hi, H3_X3_hi); hi = _mm_xor_si128(hi, H4_X4_hi); tmp0 = _mm_shuffle_epi32(H1, 78); tmp4 = _mm_shuffle_epi32(X1, 78); tmp0 = _mm_xor_si128(tmp0, H1); tmp4 = _mm_xor_si128(tmp4, X1); tmp1 = _mm_shuffle_epi32(H2, 78); tmp5 = _mm_shuffle_epi32(X2, 78); tmp1 = _mm_xor_si128(tmp1, H2); tmp5 = _mm_xor_si128(tmp5, X2); tmp2 = _mm_shuffle_epi32(H3, 78); tmp6 = _mm_shuffle_epi32(X3, 78); tmp2 = _mm_xor_si128(tmp2, H3); tmp6 = _mm_xor_si128(tmp6, X3); tmp3 = _mm_shuffle_epi32(H4, 78); tmp7 = _mm_shuffle_epi32(X4, 78); tmp3 = _mm_xor_si128(tmp3, H4); tmp7 = _mm_xor_si128(tmp7, X4); tmp0 = _mm_clmulepi64_si128(tmp0, tmp4, 0x00); tmp1 = _mm_clmulepi64_si128(tmp1, tmp5, 0x00); tmp2 = _mm_clmulepi64_si128(tmp2, tmp6, 0x00); tmp3 = _mm_clmulepi64_si128(tmp3, tmp7, 0x00); tmp0 = _mm_xor_si128(tmp0, lo); tmp0 = _mm_xor_si128(tmp0, hi); tmp0 = _mm_xor_si128(tmp1, tmp0); tmp0 = _mm_xor_si128(tmp2, tmp0); tmp0 = _mm_xor_si128(tmp3, tmp0); tmp4 = _mm_slli_si128(tmp0, 8); tmp0 = _mm_srli_si128(tmp0, 8); lo = _mm_xor_si128(tmp4, lo); hi = _mm_xor_si128(tmp0, hi); tmp3 = lo; tmp6 = hi; tmp7 = _mm_srli_epi32(tmp3, 31); tmp8 = _mm_srli_epi32(tmp6, 31); tmp3 = _mm_slli_epi32(tmp3, 1); tmp6 = _mm_slli_epi32(tmp6, 1); tmp9 = _mm_srli_si128(tmp7, 12); tmp8 = _mm_slli_si128(tmp8, 4); tmp7 = _mm_slli_si128(tmp7, 4); tmp3 = _mm_or_si128(tmp3, tmp7); tmp6 = _mm_or_si128(tmp6, tmp8); tmp6 = _mm_or_si128(tmp6, tmp9); tmp7 = _mm_slli_epi32(tmp3, 31); tmp8 = _mm_slli_epi32(tmp3, 30); tmp9 = _mm_slli_epi32(tmp3, 25); tmp7 = _mm_xor_si128(tmp7, tmp8); tmp7 = _mm_xor_si128(tmp7, tmp9); tmp8 = _mm_srli_si128(tmp7, 4); tmp7 = _mm_slli_si128(tmp7, 12); tmp3 = _mm_xor_si128(tmp3, tmp7); tmp2 = _mm_srli_epi32(tmp3, 1); tmp4 = _mm_srli_epi32(tmp3, 2); tmp5 = _mm_srli_epi32(tmp3, 7); tmp2 = _mm_xor_si128(tmp2, tmp4); tmp2 = _mm_xor_si128(tmp2, tmp5); tmp2 = _mm_xor_si128(tmp2, tmp8); tmp3 = _mm_xor_si128(tmp3, tmp2); tmp6 = _mm_xor_si128(tmp6, tmp3); *res = tmp6; } /* * Figure 12. AES-GCM: Processing Four Blocks in Parallel with Aggregated * Every Four Blocks */ /* * per NIST SP-800-38D, 5.2.1.1, len(p) <= 2^39-256 (in bits), or * 2^32-256*8*16 bytes. */ void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, const unsigned char *addt, const unsigned char *ivec, unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, const unsigned char *key, int nr) { int i, j ,k; __m128i tmp1, tmp2, tmp3, tmp4; __m128i tmp5, tmp6, tmp7, tmp8; __m128i H, H2, H3, H4, Y, T; const __m128i *KEY = (const __m128i *)key; __m128i ctr1, ctr2, ctr3, ctr4; __m128i ctr5, ctr6, ctr7, ctr8; __m128i last_block = _mm_setzero_si128(); __m128i ONE = _mm_set_epi32(0, 1, 0, 0); __m128i EIGHT = _mm_set_epi32(0, 8, 0, 0); __m128i BSWAP_EPI64 = _mm_set_epi8(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6, 7); __m128i BSWAP_MASK = _mm_set_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14, 15); __m128i X = _mm_setzero_si128(); if (ibytes == 96/8) { Y = _mm_loadu_si128((const __m128i *)ivec); Y = _mm_insert_epi32(Y, 0x1000000, 3); /*(Compute E[ZERO, KS] and E[Y0, KS] together*/ tmp1 = _mm_xor_si128(X, KEY[0]); tmp2 = _mm_xor_si128(Y, KEY[0]); for (j=1; j < nr-1; j+=2) { tmp1 = _mm_aesenc_si128(tmp1, KEY[j]); tmp2 = _mm_aesenc_si128(tmp2, KEY[j]); tmp1 = _mm_aesenc_si128(tmp1, KEY[j+1]); tmp2 = _mm_aesenc_si128(tmp2, KEY[j+1]); } tmp1 = _mm_aesenc_si128(tmp1, KEY[nr-1]); tmp2 = _mm_aesenc_si128(tmp2, KEY[nr-1]); H = _mm_aesenclast_si128(tmp1, KEY[nr]); T = _mm_aesenclast_si128(tmp2, KEY[nr]); H = _mm_shuffle_epi8(H, BSWAP_MASK); } else { tmp1 = _mm_xor_si128(X, KEY[0]); for (j=1; j