diff --git a/lib/libmd/Makefile b/lib/libmd/Makefile --- a/lib/libmd/Makefile +++ b/lib/libmd/Makefile @@ -129,6 +129,11 @@ .if exists(${MACHINE_ARCH}/sha.S) || exists(${MACHINE_ARCH}/rmd160.S) || exists(${MACHINE_ARCH}/skein_block_asm.S) ACFLAGS+= -DELF -Wa,--noexecstack .endif +.if ${MACHINE_CPUARCH} == "aarch64" +SRCS+= sha256c_arm64.c +CFLAGS+= -DARM64_SHA2 +CFLAGS.sha256c_arm64.c+= -march=armv8-a+crypto +.endif .endif # ${USE_ASM_SOURCES} != 0 md4hl.c: mdXhl.c diff --git a/sys/crypto/sha2/sha256c.c b/sys/crypto/sha2/sha256c.c --- a/sys/crypto/sha2/sha256c.c +++ b/sys/crypto/sha2/sha256c.c @@ -38,6 +38,12 @@ #include "sha224.h" #include "sha256.h" +#include "sha256c_impl.h" + +#if defined(ARM64_SHA2) +#include +#include +#endif #if BYTE_ORDER == BIG_ENDIAN @@ -132,7 +138,7 @@ * the 512-bit input block to produce a new state. */ static void -SHA256_Transform(uint32_t * state, const unsigned char block[64]) +SHA256_Transform_c(uint32_t * state, const unsigned char block[64]) { uint32_t W[64]; uint32_t S[8]; @@ -188,6 +194,33 @@ state[i] += S[i]; } +#if defined(ARM64_SHA2) +static void +SHA256_Transform_arm64(uint32_t * state, const unsigned char block[64]) +{ + SHA256_Transform_arm64_impl(state, block, K); +} + +DEFINE_UIFUNC(static, void, SHA256_Transform, + (uint32_t * state, const unsigned char block[64])) +{ + u_long hwcap; + + if (elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap)) == 0) { + if ((hwcap & HWCAP_SHA2) != 0) + return (SHA256_Transform_arm64); + } + + return (SHA256_Transform_c); +} +#else +static void +SHA256_Transform(uint32_t * state, const unsigned char block[64]) +{ + SHA256_Transform_c(state, block); +} +#endif + static unsigned char PAD[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/sys/crypto/sha2/sha256c_arm64.c b/sys/crypto/sha2/sha256c_arm64.c new file mode 100644 --- /dev/null +++ b/sys/crypto/sha2/sha256c_arm64.c @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software was developed by Andrew Turner under sponsorship from + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +#include + +#include "sha256c_impl.h" + +void __hidden +SHA256_Transform_arm64_impl(uint32_t * state, const unsigned char block[64], + const uint32_t K[64]) +{ + uint32x4_t W[4]; + uint32x4_t S[2]; + uint32x4_t S_start[2]; + uint32x4_t K_tmp, S_tmp; + int i; + +#define A64_LOAD_W(x) \ + W[x] = vld1q_u32((const uint32_t *)(&block[(x) * 16])); \ + W[x] = vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(W[x]))) + + /* 1. Prepare the first part of the message schedule W. */ + A64_LOAD_W(0); + A64_LOAD_W(1); + A64_LOAD_W(2); + A64_LOAD_W(3); + + /* 2. Initialize working variables. */ + S[0] = vld1q_u32(&state[0]); + S[1] = vld1q_u32(&state[4]); + + S_start[0] = S[0]; + S_start[1] = S[1]; + + /* 3. Mix. */ + for (i = 0; i < 64; i += 16) { +#define A64_RNDr(i, ii) \ + K_tmp = vaddq_u32(W[i], vld1q_u32(&K[ii + i * 4])); \ + S_tmp = vsha256hq_u32(S[0], S[1], K_tmp); \ + S[1] = vsha256h2q_u32(S[1], S[0], K_tmp); \ + S[0] = S_tmp + + A64_RNDr(0, i); + A64_RNDr(1, i); + A64_RNDr(2, i); + A64_RNDr(3, i); + + if (i == 48) + break; + +#define A64_MSCH(x) \ + W[x] = vsha256su0q_u32(W[x], W[(x + 1) % 4]); \ + W[x] = vsha256su1q_u32(W[x], W[(x + 2) % 4], W[(x + 3) % 4]) + + A64_MSCH(0); + A64_MSCH(1); + A64_MSCH(2); + A64_MSCH(3); + } + + /* 4. Mix local working variables into global state */ + S[0] = vaddq_u32(S[0], S_start[0]); + S[1] = vaddq_u32(S[1], S_start[1]); + + vst1q_u32(&state[0], S[0]); + vst1q_u32(&state[4], S[1]); +} diff --git a/sys/crypto/sha2/sha256c_impl.h b/sys/crypto/sha2/sha256c_impl.h new file mode 100644 --- /dev/null +++ b/sys/crypto/sha2/sha256c_impl.h @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software was developed by Andrew Turner under sponsorship from + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _SHA256_IMPL_H_ +#define _SHA256_IMPL_H_ + +#if defined(ARM64_SHA2) +void SHA256_Transform_arm64_impl(uint32_t * state, + const unsigned char block[64], const uint32_t K[64]); +#endif + +#endif /* !_SHA256_IMPL_H_ */