Page MenuHomeFreeBSD
Differential D28500 Diff 83542 sys/crypto/armv8/armv8_crypto_wrap.c

Changeset View

Standalone View

View Options

sys/crypto/armv8/armv8_crypto_wrap.c

Show First 20 Lines • Show All 228 Lines • ▼ Show 20 Lines
#define AES_INC_COUNTER(counter) \ #define AES_INC_COUNTER(counter) \
do { \ do { \
for (int pos = AES_BLOCK_LEN - 1; \ for (int pos = AES_BLOCK_LEN - 1; \
pos >= 0; pos--) \ pos >= 0; pos--) \
if (++(counter)[pos]) \ if (++(counter)[pos]) \
break; \ break; \
} while (0) } while (0)
struct armv8_gcm_state {
__uint128_val_t EK0;
__uint128_val_t EKi;
__uint128_val_t Xi;
__uint128_val_t lenblock;
uint8_t aes_counter[AES_BLOCK_LEN];
};
void void
armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len,
const uint8_t *from, uint8_t *to, const uint8_t *from, uint8_t *to,
size_t authdatalen, const uint8_t *authdata, size_t authdatalen, const uint8_t *authdata,
uint8_t tag[static GMAC_DIGEST_LEN], uint8_t tag[static GMAC_DIGEST_LEN],
const uint8_t iv[static AES_GCM_IV_LEN], const uint8_t iv[static AES_GCM_IV_LEN],
const __uint128_val_t *Htable) const __uint128_val_t *Htable)
{ {
size_t i; struct armv8_gcm_state s;
const uint64_t *from64; const uint64_t *from64;
uint64_t *to64; uint64_t *to64;
uint8_t aes_counter[AES_BLOCK_LEN];
uint8_t block[AES_BLOCK_LEN]; uint8_t block[AES_BLOCK_LEN];
size_t trailer; size_t i, trailer;
__uint128_val_t EK0, EKi, Xi, lenblock;
bzero(&aes_counter, AES_BLOCK_LEN); bzero(&s.aes_counter, AES_BLOCK_LEN);
memcpy(aes_counter, iv, AES_GCM_IV_LEN); memcpy(s.aes_counter, iv, AES_GCM_IV_LEN);
/* Setup the counter */ /* Setup the counter */
aes_counter[AES_BLOCK_LEN - 1] = 1; s.aes_counter[AES_BLOCK_LEN - 1] = 1;
/* EK0 for a final GMAC round */ /* EK0 for a final GMAC round */
aes_v8_encrypt(aes_counter, EK0.c, aes_key); aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key);
/* GCM starts with 2 as counter, 1 is used for final xor of tag. */ /* GCM starts with 2 as counter, 1 is used for final xor of tag. */
aes_counter[AES_BLOCK_LEN - 1] = 2; s.aes_counter[AES_BLOCK_LEN - 1] = 2;
memset(Xi.c, 0, sizeof(Xi.c)); memset(s.Xi.c, 0, sizeof(s.Xi.c));
trailer = authdatalen % AES_BLOCK_LEN; trailer = authdatalen % AES_BLOCK_LEN;
if (authdatalen - trailer > 0) { if (authdatalen - trailer > 0) {
gcm_ghash_v8(Xi.u, Htable, authdata, authdatalen - trailer); gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer);
authdata += authdatalen - trailer; authdata += authdatalen - trailer;
} }
if (trailer > 0 || authdatalen == 0) { if (trailer > 0 || authdatalen == 0) {
memset(block, 0, sizeof(block)); memset(block, 0, sizeof(block));
memcpy(block, authdata, trailer); memcpy(block, authdata, trailer);
gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
} }
from64 = (const uint64_t*)from; from64 = (const uint64_t*)from;
to64 = (uint64_t*)to; to64 = (uint64_t*)to;
trailer = len % AES_BLOCK_LEN; trailer = len % AES_BLOCK_LEN;
for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
aes_v8_encrypt(aes_counter, EKi.c, aes_key); aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
AES_INC_COUNTER(aes_counter); AES_INC_COUNTER(s.aes_counter);
to64[0] = from64[0] ^ EKi.u[0]; to64[0] = from64[0] ^ s.EKi.u[0];
to64[1] = from64[1] ^ EKi.u[1]; to64[1] = from64[1] ^ s.EKi.u[1];
gcm_ghash_v8(Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN);
to64 += 2; to64 += 2;
from64 += 2; from64 += 2;
} }
to += (len - trailer); to += (len - trailer);
from += (len - trailer); from += (len - trailer);
if (trailer) { if (trailer) {
aes_v8_encrypt(aes_counter, EKi.c, aes_key); aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
AES_INC_COUNTER(aes_counter); AES_INC_COUNTER(s.aes_counter);
memset(block, 0, sizeof(block)); memset(block, 0, sizeof(block));
for (i = 0; i < trailer; i++) { for (i = 0; i < trailer; i++) {
block[i] = to[i] = from[i] ^ EKi.c[i]; block[i] = to[i] = from[i] ^ s.EKi.c[i];
} }
gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
} }
/* Lengths block */ /* Lengths block */
lenblock.u[0] = lenblock.u[1] = 0; s.lenblock.u[0] = s.lenblock.u[1] = 0;
lenblock.d[1] = htobe32(authdatalen * 8); s.lenblock.d[1] = htobe32(authdatalen * 8);
lenblock.d[3] = htobe32(len * 8); s.lenblock.d[3] = htobe32(len * 8);
gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN);
Xi.u[0] ^= EK0.u[0]; s.Xi.u[0] ^= s.EK0.u[0];
Xi.u[1] ^= EK0.u[1]; s.Xi.u[1] ^= s.EK0.u[1];
memcpy(tag, Xi.c, GMAC_DIGEST_LEN); memcpy(tag, s.Xi.c, GMAC_DIGEST_LEN);
explicit_bzero(aes_counter, sizeof(aes_counter)); explicit_bzero(&s, sizeof(s));
explicit_bzero(Xi.c, sizeof(Xi.c));
explicit_bzero(EK0.c, sizeof(EK0.c));
explicit_bzero(EKi.c, sizeof(EKi.c));
explicit_bzero(lenblock.c, sizeof(lenblock.c));
} }
int int
armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len,
const uint8_t *from, uint8_t *to, const uint8_t *from, uint8_t *to,
size_t authdatalen, const uint8_t *authdata, size_t authdatalen, const uint8_t *authdata,
const uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t tag[static GMAC_DIGEST_LEN],
const uint8_t iv[static AES_GCM_IV_LEN], const uint8_t iv[static AES_GCM_IV_LEN],
const __uint128_val_t *Htable) const __uint128_val_t *Htable)
{ {
size_t i; struct armv8_gcm_state s;
const uint64_t *from64; const uint64_t *from64;
uint64_t *to64; uint64_t *to64;
uint8_t aes_counter[AES_BLOCK_LEN];
uint8_t block[AES_BLOCK_LEN]; uint8_t block[AES_BLOCK_LEN];
size_t trailer; size_t i, trailer;
__uint128_val_t EK0, EKi, Xi, lenblock;
int error; int error;
error = 0; error = 0;
bzero(&aes_counter, AES_BLOCK_LEN); bzero(&s.aes_counter, AES_BLOCK_LEN);
memcpy(aes_counter, iv, AES_GCM_IV_LEN); memcpy(s.aes_counter, iv, AES_GCM_IV_LEN);
/* Setup the counter */ /* Setup the counter */
aes_counter[AES_BLOCK_LEN - 1] = 1; s.aes_counter[AES_BLOCK_LEN - 1] = 1;
/* EK0 for a final GMAC round */ /* EK0 for a final GMAC round */
aes_v8_encrypt(aes_counter, EK0.c, aes_key); aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key);
memset(Xi.c, 0, sizeof(Xi.c)); memset(s.Xi.c, 0, sizeof(s.Xi.c));
trailer = authdatalen % AES_BLOCK_LEN; trailer = authdatalen % AES_BLOCK_LEN;
if (authdatalen - trailer > 0) { if (authdatalen - trailer > 0) {
gcm_ghash_v8(Xi.u, Htable, authdata, authdatalen - trailer); gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer);
authdata += authdatalen - trailer; authdata += authdatalen - trailer;
} }
if (trailer > 0 || authdatalen == 0) { if (trailer > 0 || authdatalen == 0) {
memset(block, 0, sizeof(block)); memset(block, 0, sizeof(block));
memcpy(block, authdata, trailer); memcpy(block, authdata, trailer);
gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
} }
trailer = len % AES_BLOCK_LEN; trailer = len % AES_BLOCK_LEN;
if (len - trailer > 0) if (len - trailer > 0)
gcm_ghash_v8(Xi.u, Htable, from, len - trailer); gcm_ghash_v8(s.Xi.u, Htable, from, len - trailer);
if (trailer > 0) { if (trailer > 0) {
memset(block, 0, sizeof(block)); memset(block, 0, sizeof(block));
memcpy(block, from + len - trailer, trailer); memcpy(block, from + len - trailer, trailer);
gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN);
} }
/* Lengths block */ /* Lengths block */
lenblock.u[0] = lenblock.u[1] = 0; s.lenblock.u[0] = s.lenblock.u[1] = 0;
lenblock.d[1] = htobe32(authdatalen * 8); s.lenblock.d[1] = htobe32(authdatalen * 8);
lenblock.d[3] = htobe32(len * 8); s.lenblock.d[3] = htobe32(len * 8);
gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN); gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN);
Xi.u[0] ^= EK0.u[0]; s.Xi.u[0] ^= s.EK0.u[0];
Xi.u[1] ^= EK0.u[1]; s.Xi.u[1] ^= s.EK0.u[1];
if (timingsafe_bcmp(tag, Xi.c, GMAC_DIGEST_LEN) != 0) { if (timingsafe_bcmp(tag, s.Xi.c, GMAC_DIGEST_LEN) != 0) {
error = EBADMSG; error = EBADMSG;
goto out; goto out;
} }
/* GCM starts with 2 as counter, 1 is used for final xor of tag. */ /* GCM starts with 2 as counter, 1 is used for final xor of tag. */
aes_counter[AES_BLOCK_LEN - 1] = 2; s.aes_counter[AES_BLOCK_LEN - 1] = 2;
from64 = (const uint64_t*)from; from64 = (const uint64_t*)from;
to64 = (uint64_t*)to; to64 = (uint64_t*)to;
for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) {
aes_v8_encrypt(aes_counter, EKi.c, aes_key); aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
AES_INC_COUNTER(aes_counter); AES_INC_COUNTER(s.aes_counter);
to64[0] = from64[0] ^ EKi.u[0]; to64[0] = from64[0] ^ s.EKi.u[0];
to64[1] = from64[1] ^ EKi.u[1]; to64[1] = from64[1] ^ s.EKi.u[1];
to64 += 2; to64 += 2;
from64 += 2; from64 += 2;
} }
to += (len - trailer); to += (len - trailer);
from += (len - trailer); from += (len - trailer);
if (trailer) { if (trailer) {
aes_v8_encrypt(aes_counter, EKi.c, aes_key); aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);
AES_INC_COUNTER(aes_counter); AES_INC_COUNTER(s.aes_counter);
for (i = 0; i < trailer; i++) for (i = 0; i < trailer; i++)
to[i] = from[i] ^ EKi.c[i]; to[i] = from[i] ^ s.EKi.c[i];
} }
out: out:
explicit_bzero(aes_counter, sizeof(aes_counter)); explicit_bzero(&s, sizeof(s));
explicit_bzero(Xi.c, sizeof(Xi.c));
explicit_bzero(EK0.c, sizeof(EK0.c));
explicit_bzero(EKi.c, sizeof(EKi.c));
explicit_bzero(lenblock.c, sizeof(lenblock.c));
return (error); return (error);
} }