Index: sys/crypto/armv8/ktls_armv8.c =================================================================== --- /dev/null +++ sys/crypto/armv8/ktls_armv8.c @@ -0,0 +1,558 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2020 Ampere Computing + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ +#include +__FBSDID("$FreeBSD"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KTLS_AEAD_TAGLEN 16 + +typedef struct { + uint32_t aes_ek[4*(RIJNDAEL_MAXNR + 1)]; + int aes_nr; +} AES_key_t; + +typedef union { + uint64_t u[2]; + uint32_t d[4]; + uint8_t c[16]; + size_t t[16 / sizeof(size_t)]; +} __uint128_val_t; + +typedef struct { + __uint128_val_t EK0, Xi, H, len; + __uint128_val_t Htable[16]; +} GMAC_ctx_t; + +/* Prototypes for aesv8-armx.S */ +void aes_v8_encrypt(uint8_t *in, uint8_t *out, AES_key_t *key); +int aes_v8_set_encrypt_key(const unsigned char *userKey, const int bits, AES_key_t *key); + +/* Prototypes for ghashv8-armx.S */ +void gcm_init_v8(__uint128_val_t Htable[16], const uint64_t Xi[2]); +void gcm_gmult_v8(uint64_t Xi[2], const __uint128_val_t Htable[16]); +void gcm_ghash_v8(uint64_t Xi[2], const __uint128_val_t Htable[16], const uint8_t *inp, size_t len); + +struct armv8_gcm_struct { + AES_key_t aes_key; + __uint128_val_t EKi; + GMAC_ctx_t gmac_ctx; + uint8_t aes_counter[AES_BLOCK_LEN]; +}; + +static MALLOC_DEFINE(M_ARMV8_GCM, "armv8_tls", "ARMv8 TLS"); + +static struct mtx *ctx_mtx; +static struct fpu_kern_ctx **ctx_vfp; + +#define AQUIRE_CTX(i, ctx) \ + do { \ + (i) = PCPU_GET(cpuid); \ + /*mtx_lock(&ctx_mtx[(i)]);*/ \ + (ctx) = ctx_vfp[(i)]; \ + } while (0) +#define RELEASE_CTX(i, ctx) \ + do { \ + /*mtx_unlock(&ctx_mtx[(i)]);*/ \ + (i) = -1; \ + (ctx) = NULL; \ + } while (0) + +#define AES_INC_COUNTER(sc) \ + do { \ + for (int i = AES_BLOCK_LEN - 1; \ + i >= 0; i--) \ + if (++(sc)->aes_counter[i]) \ + break; \ + } while (0) + +SYSCTL_DECL(_kern_ipc_tls); + +static int ktls_use_armv8_gcm = 1; +SYSCTL_INT(_kern_ipc_tls, OID_AUTO, armv8_gcm, CTLFLAG_RW, + &ktls_use_armv8_gcm, 1, + "Should we use the ARMv8 GCM if available"); + +SYSCTL_DECL(_kern_ipc_tls_stats); + +static counter_u64_t ktls_offload_armv8_aead; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_aead_crypts, + CTLFLAG_RD, &ktls_offload_armv8_aead, + "Total number of ARMv8 TLS AEAD encrypts called"); + +static counter_u64_t ktls_offload_armv8_tls12; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_tls12_crypts, + CTLFLAG_RD, &ktls_offload_armv8_tls12, + "Total number of ARMv8 TLSv1.2 encrypts called"); + +static counter_u64_t ktls_offload_armv8_tls13; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_tls13_crypts, + CTLFLAG_RD, &ktls_offload_armv8_tls13, + "Total number of ARMv8 TLSv1.3 encrypts called"); + +static counter_u64_t ktls_offload_armv8_unaligned_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_unaligned_bytes, + CTLFLAG_RD, &ktls_offload_armv8_unaligned_mem_b, + "Byte cnt of ARMv8 TLS unaligned"); + +static counter_u64_t ktls_offload_armv8_aligned_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_aligned_bytes, + CTLFLAG_RD, &ktls_offload_armv8_aligned_mem_b, + "Byte cnt of ARMv8 TLS aligned"); + +static counter_u64_t ktls_offload_armv8_glue_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_glue_bytes, + CTLFLAG_RD, &ktls_offload_armv8_glue_mem_b, + "Byte cnt of ARMv8 TLS \"glue\" between iovecs"); + +static counter_u64_t ktls_offload_armv8_unaligned_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_unaligned, + CTLFLAG_RD, &ktls_offload_armv8_unaligned_mem, + "Call cnt of ARMv8 TLS unaligned"); + +static counter_u64_t ktls_offload_armv8_aligned_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_aligned, + CTLFLAG_RD, &ktls_offload_armv8_aligned_mem, + "Call cnt of ARMv8 TLS aligned"); + +static counter_u64_t ktls_offload_armv8_glue_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, armv8_glue, + CTLFLAG_RD, &ktls_offload_armv8_glue_mem, + "Call cnt of ARMv8 TLS \"glue\" between iovecs"); + +static int +ktls_armv8ktls_aead_encrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, uint8_t *tag, struct iovec *iniov, + struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t tls_rtype) +{ + struct armv8_gcm_struct *sc; + struct fpu_kern_ctx *ctx; + int kt, i, iov; + uint8_t *in, *out; + uint8_t *previn, *prevout; + __uint128_val_t *in128, *out128; + uint64_t len, trailer_len, header_len; + struct tls_aead_data ad; + struct tls_aead_data_13 ad13; + struct tls_nonce_data nd; + char nonce[12]; + size_t adlen, payload_length; + uint8_t *adptr; + uint16_t tls_comp_len; + uint8_t block[AES_BLOCK_LEN]; + bool is_tls13; + + sc = (struct armv8_gcm_struct *)tls->cipher; + + kt = is_fpu_kern_thread(0); + if (!kt) { + AQUIRE_CTX(i, ctx); + fpu_kern_enter(curthread, ctx, + FPU_KERN_NORMAL | FPU_KERN_KTHR); + } + + KASSERT(sc != NULL, ("Null cipher")); + counter_u64_add(ktls_offload_armv8_aead, 1); + + is_tls13 = tls->params.tls_vminor == TLS_MINOR_VER_THREE; + if (is_tls13) + counter_u64_add(ktls_offload_armv8_tls13, 1); + else + counter_u64_add(ktls_offload_armv8_tls12, 1); + + /* Setup the associated data */ + if (is_tls13) { + ad13.type = hdr->tls_type; + ad13.tls_vmajor = hdr->tls_vmajor; + ad13.tls_vminor = hdr->tls_vminor; + ad13.tls_length = hdr->tls_length; + adlen = sizeof(ad13); + adptr = (uint8_t *)&ad13; + } else { + ad.seq = htobe64(seqno); + ad.type = hdr->tls_type; + ad.tls_vmajor = hdr->tls_vmajor; + ad.tls_vminor = hdr->tls_vminor; + tls_comp_len = ntohs(hdr->tls_length) - + (KTLS_AEAD_TAGLEN + sizeof(nd.seq)); + ad.tls_length = htons(tls_comp_len); + + adlen = sizeof(ad); + adptr = (uint8_t *)&ad; + } + + bzero(&sc->aes_counter, AES_BLOCK_LEN); + /* Setup the nonce */ + if (is_tls13) { + memcpy(nonce, tls->params.iv, tls->params.iv_len); + *(uint64_t *)(nonce + 4) ^= htobe64(seqno); + memcpy(sc->aes_counter, nonce, sizeof(nonce)); + } else { + memcpy(&nd, tls->params.iv, tls->params.iv_len); + memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); + memcpy(sc->aes_counter, &nd, sizeof(nd)); + } + /* Setup the counter */ + sc->aes_counter[AES_BLOCK_LEN - 1] = 1; + + /* EK0 for a final GMAC round */ + aes_v8_encrypt(sc->aes_counter, sc->gmac_ctx.EK0.c, &sc->aes_key); + + /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ + sc->aes_counter[AES_BLOCK_LEN - 1] = 2; + + memset(sc->gmac_ctx.Xi.c, 0, sizeof(sc->gmac_ctx.Xi.c)); + + /* + * Feed AEAD data to the GMAC + */ + KASSERT(adlen <= AES_BLOCK_LEN, ("AEAD data is larger than block")); + memset(block, 0, sizeof(block)); + memcpy(block, adptr, adlen); + gcm_ghash_v8(sc->gmac_ctx.Xi.u, sc->gmac_ctx.Htable, block, AES_BLOCK_LEN); + + payload_length = 0; + trailer_len = 0; + + for (iov = 0; iov < iovcnt; iov++) { + in = iniov[iov].iov_base; + out = outiov[iov].iov_base; + len = iniov[iov].iov_len; + + /* Previous iovec had a trailer */ + if (trailer_len) { + aes_v8_encrypt(sc->aes_counter, sc->EKi.c, &sc->aes_key); + AES_INC_COUNTER(sc); + + /* Last iov may not have enough data to fill the block */ + header_len = min(AES_BLOCK_LEN - trailer_len, len); + + /* Encode and prepare block for GMAC */ + i = 0; + while (i < trailer_len) { + block[i] = prevout[i] = previn[i] ^ sc->EKi.c[i]; + i++; + } + + while (i < header_len + trailer_len) { + block[i] = out[i - trailer_len] = in[i - trailer_len] ^ sc->EKi.c[i]; + i++; + } + + while (i < AES_BLOCK_LEN) { + block[i] = 0; + i++; + } + + gcm_ghash_v8(sc->gmac_ctx.Xi.u, sc->gmac_ctx.Htable, block, sizeof(block)); + + payload_length += trailer_len + header_len; + counter_u64_add(ktls_offload_armv8_glue_mem, 1); + counter_u64_add(ktls_offload_armv8_glue_mem_b, trailer_len + header_len); + + in += header_len; + out += header_len; + len -= header_len; + trailer_len = 0; + } + + if (len == 0) + continue; + + trailer_len = len & (AES_BLOCK_LEN - 1); + len -= trailer_len; + payload_length += len; + + in128 = (__uint128_val_t *)in; + out128 = (__uint128_val_t *)out; + for (i = 0; i < len; i += AES_BLOCK_LEN, in128++, out128++) { + aes_v8_encrypt(sc->aes_counter, sc->EKi.c, &sc->aes_key); + AES_INC_COUNTER(sc); + out128->u[0] = in128->u[0] ^ sc->EKi.u[0]; + out128->u[1] = in128->u[1] ^ sc->EKi.u[1]; + gcm_ghash_v8(sc->gmac_ctx.Xi.u, sc->gmac_ctx.Htable, out128->c, AES_BLOCK_LEN); + } + + if ((uintptr_t)in & 0xf) { + counter_u64_add(ktls_offload_armv8_unaligned_mem, 1); + counter_u64_add(ktls_offload_armv8_unaligned_mem_b, len); + } else { + counter_u64_add(ktls_offload_armv8_aligned_mem, 1); + counter_u64_add(ktls_offload_armv8_aligned_mem_b, len); + } + + in += len; + out += len; + + previn = in; + prevout = out; + } + + /* Encode the trailer of the whole data */ + if (trailer_len || is_tls13) { + aes_v8_encrypt(sc->aes_counter, sc->EKi.c, &sc->aes_key); + AES_INC_COUNTER(sc); + + i = 0; + while (i < trailer_len) { + block[i] = out[i] = in[i] ^ sc->EKi.c[i]; + i++; + } + + /* + * Although record type byte is part of the tag physically + * logically it's counted as a part of payload data + */ + if (is_tls13) { + block[i] = tag[0] = tls_rtype ^ sc->EKi.c[i]; + payload_length += 1; + i++; + } + + while (i < AES_BLOCK_LEN) { + block[i] = 0; + i++; + } + + gcm_ghash_v8(sc->gmac_ctx.Xi.u, sc->gmac_ctx.Htable, block, sizeof(block)); + payload_length += trailer_len; + } + + /* Lengths block */ + sc->gmac_ctx.len.d[1] = htobe32(adlen * 8); + sc->gmac_ctx.len.d[3] = htobe32(payload_length * 8); + gcm_ghash_v8(sc->gmac_ctx.Xi.u, sc->gmac_ctx.Htable, sc->gmac_ctx.len.c, AES_BLOCK_LEN); + sc->gmac_ctx.Xi.u[0] ^= sc->gmac_ctx.EK0.u[0]; + sc->gmac_ctx.Xi.u[1] ^= sc->gmac_ctx.EK0.u[1]; + if (is_tls13) + memcpy(tag + 1, sc->gmac_ctx.Xi.c, AES_BLOCK_LEN); + else + memcpy(tag, sc->gmac_ctx.Xi.c, AES_BLOCK_LEN); + + if (!kt) { + fpu_kern_leave(curthread, ctx); + RELEASE_CTX(i, ctx); + } + + return(0); +} + +static int +ktls_armv8ktls_setup_cipher(struct armv8_gcm_struct *sc, uint8_t *key, int keylen) +{ + struct fpu_kern_ctx *fpu_ctx; + + if (key == NULL) + return (EINVAL); + + fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NOWAIT); + if (fpu_ctx == NULL) + return (ENOMEM); + fpu_kern_enter(curthread, fpu_ctx, FPU_KERN_NORMAL); + + aes_v8_set_encrypt_key(key, keylen * 8, &sc->aes_key); + + memset(sc->gmac_ctx.H.c, 0, sizeof(sc->gmac_ctx.H.c)); + aes_v8_encrypt(sc->gmac_ctx.H.c, sc->gmac_ctx.H.c, &sc->aes_key); + sc->gmac_ctx.H.u[0] = bswap64(sc->gmac_ctx.H.u[0]); + sc->gmac_ctx.H.u[1] = bswap64(sc->gmac_ctx.H.u[1]); + gcm_init_v8(sc->gmac_ctx.Htable, sc->gmac_ctx.H.u); + + fpu_kern_leave(curthread, fpu_ctx); + fpu_kern_free_ctx(fpu_ctx); + + return (0); +} + +static void +ktls_armv8ktls_free(struct ktls_session *tls) +{ + struct armv8_gcm_struct *sc; + + sc = tls->cipher; + explicit_bzero(sc, sizeof(*sc)); + free(sc, M_ARMV8_GCM); +} + +static int +ktls_armv8ktls_try(struct socket *so, struct ktls_session *tls, int direction) +{ + struct armv8_gcm_struct *sc; + int error; + + if (direction != KTLS_TX) + return (EOPNOTSUPP); + + if (!ktls_use_armv8_gcm) + return (EOPNOTSUPP); + + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { + switch (tls->params.cipher_key_len) { + case 128 / 8: + case 256 / 8: + break; + default: + return (EOPNOTSUPP); + } + + /* Only TLS 1.2 and 1.3 are supported. */ + if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || + tls->params.tls_vminor < TLS_MINOR_VER_TWO || + tls->params.tls_vminor > TLS_MINOR_VER_THREE) + return (EPROTONOSUPPORT); + + /* TLS 1.3 is not yet supported for receive. */ + if (direction == KTLS_RX && + tls->params.tls_vminor == TLS_MINOR_VER_THREE) + return (EPROTONOSUPPORT); + + sc = malloc(sizeof (*sc), M_ARMV8_GCM, M_NOWAIT | M_ZERO); + if (sc == NULL) + return (ENOMEM); + + error = ktls_armv8ktls_setup_cipher(sc, + tls->params.cipher_key, + tls->params.cipher_key_len); + if (error) { + free(sc, M_ARMV8_GCM); + return (error); + } + + tls->cipher = sc; + tls->sw_encrypt = ktls_armv8ktls_aead_encrypt; + tls->free = ktls_armv8ktls_free; + return (0); + } + + return (EOPNOTSUPP); +} + +struct ktls_crypto_backend armv8ktls_backend = { + .name = "ARMv8 AES GCM", + .prio = 20, + .api_version = KTLS_API_VERSION, + .try = ktls_armv8ktls_try, +}; + +static int +armv8ktls_init(void) +{ + int i; + + ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_GCM, + M_WAITOK|M_ZERO); + ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_GCM, + M_WAITOK|M_ZERO); + + CPU_FOREACH(i) { + ctx_vfp[i] = fpu_kern_alloc_ctx(0); + mtx_init(&ctx_mtx[i], "armv8ktlsctx", NULL, MTX_DEF|MTX_NEW); + } + + ktls_offload_armv8_aead = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_aligned_mem = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_aligned_mem_b = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_unaligned_mem = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_unaligned_mem_b = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_glue_mem = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_glue_mem_b = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_tls12 = counter_u64_alloc(M_WAITOK); + ktls_offload_armv8_tls13 = counter_u64_alloc(M_WAITOK); + + return (ktls_crypto_backend_register(&armv8ktls_backend)); +} + +static int +armv8ktls_unload(void) +{ + int i, error; + + error = ktls_crypto_backend_deregister(&armv8ktls_backend); + if (error != 0) + return (error); + + CPU_FOREACH(i) { + if (ctx_vfp[i] != NULL) { + mtx_destroy(&ctx_mtx[i]); + fpu_kern_free_ctx(ctx_vfp[i]); + } + ctx_vfp[i] = NULL; + } + free(ctx_mtx, M_ARMV8_GCM); + ctx_mtx = NULL; + free(ctx_vfp, M_ARMV8_GCM); + ctx_vfp = NULL; + + counter_u64_free(ktls_offload_armv8_aead); + counter_u64_free(ktls_offload_armv8_aligned_mem); + counter_u64_free(ktls_offload_armv8_aligned_mem_b); + counter_u64_free(ktls_offload_armv8_unaligned_mem); + counter_u64_free(ktls_offload_armv8_unaligned_mem_b); + counter_u64_free(ktls_offload_armv8_glue_mem); + counter_u64_free(ktls_offload_armv8_glue_mem_b); + counter_u64_free(ktls_offload_armv8_tls12); + counter_u64_free(ktls_offload_armv8_tls13); + + return (0); +} + +static int +armv8ktls_module_event_handler(module_t mod, int evt, void *arg) +{ + switch (evt) { + case MOD_LOAD: + return (armv8ktls_init()); + case MOD_UNLOAD: + return (armv8ktls_unload()); + default: + return (EOPNOTSUPP); + } +} + +static moduledata_t armv8ktls_moduledata = { + "armv8ktls", + armv8ktls_module_event_handler, + NULL +}; + +DECLARE_MODULE(armv8ktls, armv8ktls_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); Index: sys/modules/Makefile =================================================================== --- sys/modules/Makefile +++ sys/modules/Makefile @@ -205,6 +205,7 @@ khelp \ krpc \ ksyms \ + ${_ktls_armv8} \ ${_ktls_ocf} \ le \ lge \ @@ -601,6 +602,7 @@ .if ${MACHINE_CPUARCH} == "aarch64" _allwinner= allwinner _armv8crypto= armv8crypto +_ktls_armv8= ktls_armv8 _em= em _rockchip= rockchip .endif Index: sys/modules/ktls_armv8/Makefile =================================================================== --- /dev/null +++ sys/modules/ktls_armv8/Makefile @@ -0,0 +1,33 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/crypto/armv8 +.PATH: ${SRCTOP}/sys/crypto/openssl/aarch64 + +KMOD= ktls_armv8 +SRCS= ktls_armv8.c +SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h + +OBJS+= aesv8-armx.o ghashv8-armx.o + + +aesv8-armx.o: aesv8-armx.S + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \ + -I${SRCTOP}/sys/crypto/armv8 \ + -I${SRCTOP}/sys/crypto/openssl/crypto \ + -I${.CURDIR} \ + -Igonzo \ + ${WERROR} ${PROF} \ + -march=armv8-a+crypto ${.IMPSRC} + ${CTFCONVERT_CMD} + +ghashv8-armx.o: ghashv8-armx.S + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \ + -I${SRCTOP}/sys/crypto/armv8 \ + -I${SRCTOP}/sys/crypto/openssl/crypto \ + -I${.CURDIR} \ + -Igonzo \ + ${WERROR} ${PROF} \ + -march=armv8-a+crypto ${.IMPSRC} + ${CTFCONVERT_CMD} + +.include Index: sys/modules/ktls_armv8/arm_arch.h =================================================================== --- /dev/null +++ sys/modules/ktls_armv8/arm_arch.h @@ -0,0 +1,84 @@ +/* + * Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef OSSL_CRYPTO_ARM_ARCH_H +# define OSSL_CRYPTO_ARM_ARCH_H + +# if !defined(__ARM_ARCH__) +# if defined(__CC_ARM) +# define __ARM_ARCH__ __TARGET_ARCH_ARM +# if defined(__BIG_ENDIAN) +# define __ARMEB__ +# else +# define __ARMEL__ +# endif +# elif defined(__GNUC__) +# if defined(__aarch64__) +# define __ARM_ARCH__ 8 +# if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define __ARMEB__ +# else +# define __ARMEL__ +# endif + /* + * Why doesn't gcc define __ARM_ARCH__? Instead it defines + * bunch of below macros. See all_architectures[] table in + * gcc/config/arm/arm.c. On a side note it defines + * __ARMEL__/__ARMEB__ for little-/big-endian. + */ +# elif defined(__ARM_ARCH) +# define __ARM_ARCH__ __ARM_ARCH +# elif defined(__ARM_ARCH_8A__) +# define __ARM_ARCH__ 8 +# elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ + defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__) || \ + defined(__ARM_ARCH_7EM__) +# define __ARM_ARCH__ 7 +# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ + defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__) || \ + defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__) || \ + defined(__ARM_ARCH_6T2__) +# define __ARM_ARCH__ 6 +# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || \ + defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__) || \ + defined(__ARM_ARCH_5TEJ__) +# define __ARM_ARCH__ 5 +# elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__) +# define __ARM_ARCH__ 4 +# else +# error "unsupported ARM architecture" +# endif +# endif +# endif + +# if !defined(__ARM_MAX_ARCH__) +# define __ARM_MAX_ARCH__ __ARM_ARCH__ +# endif + +# if __ARM_MAX_ARCH__<__ARM_ARCH__ +# error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__" +# elif __ARM_MAX_ARCH__!=__ARM_ARCH__ +# if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__) +# error "can't build universal big-endian binary" +# endif +# endif + +# ifndef __ASSEMBLER__ +extern unsigned int OPENSSL_armcap_P; +# endif + +# define ARMV7_NEON (1<<0) +# define ARMV7_TICK (1<<1) +# define ARMV8_AES (1<<2) +# define ARMV8_SHA1 (1<<3) +# define ARMV8_SHA256 (1<<4) +# define ARMV8_PMULL (1<<5) +# define ARMV8_SHA512 (1<<6) + +#endif