diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h --- a/sys/crypto/armv8/armv8_crypto.h +++ b/sys/crypto/armv8/armv8_crypto.h @@ -63,24 +63,29 @@ void gcm_gmult_v8(uint64_t Xi[2], const __uint128_val_t Htable[16]); void gcm_ghash_v8(uint64_t Xi[2], const __uint128_val_t Htable[16], const uint8_t *inp, size_t len); -void armv8_aes_encrypt_cbc(const AES_key_t *, size_t, const uint8_t *, - uint8_t *, const uint8_t[static AES_BLOCK_LEN]); -void armv8_aes_decrypt_cbc(const AES_key_t *, size_t, uint8_t *, +void armv8_aes_encrypt_cbc(const AES_key_t *key, size_t len, + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, + const uint8_t iv[static AES_BLOCK_LEN]); +void armv8_aes_decrypt_cbc(const AES_key_t *, size_t, + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, const uint8_t[static AES_BLOCK_LEN]); -void armv8_aes_encrypt_gcm(AES_key_t *, size_t, const uint8_t *, - uint8_t *, size_t, const uint8_t*, +void armv8_aes_encrypt_gcm(AES_key_t *, size_t, + struct crypto_buffer_cursor *, struct crypto_buffer_cursor *, + size_t, const uint8_t *, uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t[static AES_BLOCK_LEN], const __uint128_val_t *); -int armv8_aes_decrypt_gcm(AES_key_t *, size_t, const uint8_t *, - uint8_t *, size_t, const uint8_t*, - const uint8_t tag[static GMAC_DIGEST_LEN], +int armv8_aes_decrypt_gcm(AES_key_t *, size_t, + struct crypto_buffer_cursor *, struct crypto_buffer_cursor *, + size_t, const uint8_t *, const uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t[static AES_BLOCK_LEN], const __uint128_val_t *); void armv8_aes_encrypt_xts(AES_key_t *, const void *, size_t, - const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); + struct crypto_buffer_cursor *, struct crypto_buffer_cursor *, + const uint8_t[AES_BLOCK_LEN]); void armv8_aes_decrypt_xts(AES_key_t *, const void *, size_t, - const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); + struct crypto_buffer_cursor *, struct crypto_buffer_cursor *, + const uint8_t[AES_BLOCK_LEN]); #endif /* _ARMV8_CRYPTO_H_ */ diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c --- a/sys/crypto/armv8/armv8_crypto.c +++ b/sys/crypto/armv8/armv8_crypto.c @@ -378,30 +378,23 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses, struct cryptop *crp) { + struct crypto_buffer_cursor fromc, toc; const struct crypto_session_params *csp; struct fpu_kern_ctx *ctx; - uint8_t *buf, *authbuf, *outbuf; + uint8_t *authbuf; uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; - int allocated, authallocated, outallocated, i; + int authallocated, i; int encflag; int kt; int error; - bool outcopy; csp = crypto_get_params(crp->crp_session); encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); - allocated = 0; - outallocated = 0; authallocated = 0; authbuf = NULL; kt = 1; - buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start, - crp->crp_payload_length, &allocated); - if (buf == NULL) - return (ENOMEM); - if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { if (crp->crp_aad != NULL) authbuf = crp->crp_aad; @@ -413,28 +406,13 @@ goto out; } } - + crypto_cursor_init(&fromc, &crp->crp_buf); + crypto_cursor_advance(&fromc, crp->crp_payload_start); if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { - outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf, - crp->crp_payload_output_start, crp->crp_payload_length); - if (outbuf == NULL) { - outcopy = true; - if (allocated) - outbuf = buf; - else { - outbuf = malloc(crp->crp_payload_length, - M_ARMV8_CRYPTO, M_NOWAIT); - if (outbuf == NULL) { - error = ENOMEM; - goto out; - } - outallocated = true; - } - } else - outcopy = false; + crypto_cursor_init(&toc, &crp->crp_obuf); + crypto_cursor_advance(&toc, crp->crp_payload_output_start); } else { - outbuf = buf; - outcopy = allocated; + crypto_cursor_copy(&fromc, &toc); } kt = is_fpu_kern_thread(0); @@ -451,7 +429,6 @@ crypto_read_iv(crp, iv); - /* Do work */ switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { @@ -460,51 +437,41 @@ } if (encflag) armv8_aes_encrypt_cbc(&ses->enc_schedule, - crp->crp_payload_length, buf, buf, iv); + crp->crp_payload_length, &fromc, &toc, iv); else armv8_aes_decrypt_cbc(&ses->dec_schedule, - crp->crp_payload_length, buf, iv); + crp->crp_payload_length, &fromc, &toc, iv); break; case CRYPTO_AES_XTS: if (encflag) armv8_aes_encrypt_xts(&ses->enc_schedule, - &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, - buf, iv); + &ses->xts_schedule.aes_key, crp->crp_payload_length, + &fromc, &toc, iv); else armv8_aes_decrypt_xts(&ses->dec_schedule, - &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, - buf, iv); + &ses->xts_schedule.aes_key, crp->crp_payload_length, + &fromc, &toc, iv); break; case CRYPTO_AES_NIST_GCM_16: if (encflag) { memset(tag, 0, sizeof(tag)); armv8_aes_encrypt_gcm(&ses->enc_schedule, - crp->crp_payload_length, - buf, outbuf, - crp->crp_aad_length, authbuf, - tag, iv, ses->Htable); + crp->crp_payload_length, &fromc, &toc, + crp->crp_aad_length, authbuf, tag, iv, ses->Htable); crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), tag); } else { crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), tag); - if (armv8_aes_decrypt_gcm(&ses->enc_schedule, - crp->crp_payload_length, - buf, outbuf, - crp->crp_aad_length, authbuf, - tag, iv, ses->Htable) != 0) { - error = EBADMSG; + error = armv8_aes_decrypt_gcm(&ses->enc_schedule, + crp->crp_payload_length, &fromc, &toc, + crp->crp_aad_length, authbuf, tag, iv, ses->Htable); + if (error != 0) goto out; - } } break; } - if (outcopy) - crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ? - crp->crp_payload_output_start : crp->crp_payload_start, - crp->crp_payload_length, outbuf); - error = 0; out: if (!kt) { @@ -512,12 +479,8 @@ RELEASE_CTX(i, ctx); } - if (allocated) - zfree(buf, M_ARMV8_CRYPTO); if (authallocated) zfree(authbuf, M_ARMV8_CRYPTO); - if (outallocated) - zfree(outbuf, M_ARMV8_CRYPTO); explicit_bzero(iv, sizeof(iv)); explicit_bzero(tag, sizeof(tag)); diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c --- a/sys/crypto/armv8/armv8_crypto_wrap.c +++ b/sys/crypto/armv8/armv8_crypto_wrap.c @@ -101,41 +101,97 @@ void armv8_aes_encrypt_cbc(const AES_key_t *key, size_t len, - const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN]) + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, + const uint8_t iv[static AES_BLOCK_LEN]) { uint8x16_t tot, ivreg, tmp; - size_t i; + uint8_t block[AES_BLOCK_LEN], *from, *to; + size_t fromseglen, oseglen, seglen, toseglen; + + KASSERT(len % AES_BLOCK_LEN == 0, + ("%s: length %zu not a multiple of the block size", __func__, len)); - len /= AES_BLOCK_LEN; ivreg = vld1q_u8(iv); - for (i = 0; i < len; i++) { - tmp = vld1q_u8(from); - tot = armv8_aes_enc(key->aes_rounds - 1, - (const void*)key->aes_key, veorq_u8(tmp, ivreg)); - ivreg = tot; - vst1q_u8(to, tot); - from += AES_BLOCK_LEN; - to += AES_BLOCK_LEN; + for (; len > 0; len -= seglen) { + from = crypto_cursor_segment(fromc, &fromseglen); + to = crypto_cursor_segment(toc, &toseglen); + + seglen = ulmin(len, ulmin(fromseglen, toseglen)); + if (seglen < AES_BLOCK_LEN) { + crypto_cursor_copydata(fromc, AES_BLOCK_LEN, block); + tmp = vld1q_u8(block); + tot = armv8_aes_enc(key->aes_rounds - 1, + (const void *)key->aes_key, veorq_u8(tmp, ivreg)); + ivreg = tot; + vst1q_u8(block, tot); + crypto_cursor_copyback(toc, AES_BLOCK_LEN, block); + seglen = AES_BLOCK_LEN; + } else { + for (oseglen = seglen; seglen >= AES_BLOCK_LEN; + seglen -= AES_BLOCK_LEN) { + tmp = vld1q_u8(from); + tot = armv8_aes_enc(key->aes_rounds - 1, + (const void *)key->aes_key, + veorq_u8(tmp, ivreg)); + ivreg = tot; + vst1q_u8(to, tot); + from += AES_BLOCK_LEN; + to += AES_BLOCK_LEN; + } + seglen = oseglen - seglen; + crypto_cursor_advance(fromc, seglen); + crypto_cursor_advance(toc, seglen); + } } + + explicit_bzero(block, sizeof(block)); } void armv8_aes_decrypt_cbc(const AES_key_t *key, size_t len, - uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN]) + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, + const uint8_t iv[static AES_BLOCK_LEN]) { uint8x16_t ivreg, nextiv, tmp; - size_t i; + uint8_t block[AES_BLOCK_LEN], *from, *to; + size_t fromseglen, oseglen, seglen, toseglen; + + KASSERT(len % AES_BLOCK_LEN == 0, + ("%s: length %zu not a multiple of the block size", __func__, len)); - len /= AES_BLOCK_LEN; ivreg = vld1q_u8(iv); - for (i = 0; i < len; i++) { - nextiv = vld1q_u8(buf); - tmp = armv8_aes_dec(key->aes_rounds - 1, - (const void*)key->aes_key, nextiv); - vst1q_u8(buf, veorq_u8(tmp, ivreg)); - ivreg = nextiv; - buf += AES_BLOCK_LEN; + for (; len > 0; len -= seglen) { + from = crypto_cursor_segment(fromc, &fromseglen); + to = crypto_cursor_segment(toc, &toseglen); + + seglen = ulmin(len, ulmin(fromseglen, toseglen)); + if (seglen < AES_BLOCK_LEN) { + crypto_cursor_copydata(fromc, AES_BLOCK_LEN, block); + nextiv = vld1q_u8(block); + tmp = armv8_aes_dec(key->aes_rounds - 1, + (const void *)key->aes_key, nextiv); + vst1q_u8(block, veorq_u8(tmp, ivreg)); + ivreg = nextiv; + crypto_cursor_copyback(toc, AES_BLOCK_LEN, block); + seglen = AES_BLOCK_LEN; + } else { + for (oseglen = seglen; seglen >= AES_BLOCK_LEN; + seglen -= AES_BLOCK_LEN) { + nextiv = vld1q_u8(from); + tmp = armv8_aes_dec(key->aes_rounds - 1, + (const void *)key->aes_key, nextiv); + vst1q_u8(to, veorq_u8(tmp, ivreg)); + ivreg = nextiv; + from += AES_BLOCK_LEN; + to += AES_BLOCK_LEN; + } + crypto_cursor_advance(fromc, oseglen - seglen); + crypto_cursor_advance(toc, oseglen - seglen); + seglen = oseglen - seglen; + } } + + explicit_bzero(block, sizeof(block)); } #define AES_XTS_BLOCKSIZE 16 @@ -180,12 +236,18 @@ static void armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule, - const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from, - uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt) + const uint8x16_t *tweak_schedule, size_t len, + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, + const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt) { uint8x16_t tweakreg; + uint8_t block[AES_XTS_BLOCKSIZE] __aligned(16); uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16); - size_t i, cnt; + uint8_t *from, *to; + size_t fromseglen, oseglen, seglen, toseglen; + + KASSERT(len % AES_XTS_BLOCKSIZE == 0, + ("%s: length %zu not a multiple of the block size", __func__, len)); /* * Prepare tweak as E_k2(IV). IV is specified as LE representation @@ -201,38 +263,57 @@ tweakreg = vld1q_u8(tweak); tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg); - cnt = len / AES_XTS_BLOCKSIZE; - for (i = 0; i < cnt; i++) { - armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg, - from, to, do_encrypt); - from += AES_XTS_BLOCKSIZE; - to += AES_XTS_BLOCKSIZE; + for (; len > 0; len -= seglen) { + from = crypto_cursor_segment(fromc, &fromseglen); + to = crypto_cursor_segment(toc, &toseglen); + + seglen = ulmin(len, ulmin(fromseglen, toseglen)); + if (seglen < AES_XTS_BLOCKSIZE) { + printf("%d seglen %zu\n", __LINE__, seglen); + crypto_cursor_copydata(fromc, AES_XTS_BLOCKSIZE, block); + armv8_aes_crypt_xts_block(rounds, data_schedule, + &tweakreg, block, block, do_encrypt); + crypto_cursor_copyback(toc, AES_XTS_BLOCKSIZE, block); + seglen = AES_XTS_BLOCKSIZE; + } else { + printf("%d seglen %zu\n", __LINE__, seglen); + for (oseglen = seglen; seglen >= AES_XTS_BLOCKSIZE; + seglen -= AES_XTS_BLOCKSIZE) { + armv8_aes_crypt_xts_block(rounds, data_schedule, + &tweakreg, from, to, do_encrypt); + from += AES_XTS_BLOCKSIZE; + to += AES_XTS_BLOCKSIZE; + } + seglen = oseglen - seglen; + crypto_cursor_advance(fromc, seglen); + crypto_cursor_advance(toc, seglen); + } } + + explicit_bzero(block, sizeof(block)); } void armv8_aes_encrypt_xts(AES_key_t *data_schedule, - const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, - const uint8_t iv[static AES_BLOCK_LEN]) + const void *tweak_schedule, size_t len, struct crypto_buffer_cursor *fromc, + struct crypto_buffer_cursor *toc, const uint8_t iv[static AES_BLOCK_LEN]) { - armv8_aes_crypt_xts(data_schedule->aes_rounds, - (const void *)&data_schedule->aes_key, tweak_schedule, len, from, - to, iv, 1); + (const void *)&data_schedule->aes_key, tweak_schedule, len, fromc, + toc, iv, 1); } void armv8_aes_decrypt_xts(AES_key_t *data_schedule, - const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const void *tweak_schedule, size_t len, + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, const uint8_t iv[static AES_BLOCK_LEN]) { - armv8_aes_crypt_xts(data_schedule->aes_rounds, - (const void *)&data_schedule->aes_key, tweak_schedule, len, from, - to,iv, 0); + (const void *)&data_schedule->aes_key, tweak_schedule, len, fromc, + toc, iv, 0); } - #define AES_INC_COUNTER(counter) \ do { \ for (int pos = AES_BLOCK_LEN - 1; \ @@ -296,115 +377,161 @@ s->Xi.u[1] ^= s->EK0.u[1]; } +static void +armv8_aes_encrypt_gcm_block(struct armv8_gcm_state *s, AES_key_t *aes_key, + const uint64_t *from, uint64_t *to) +{ + aes_v8_encrypt(s->aes_counter, s->EKi.c, aes_key); + AES_INC_COUNTER(s->aes_counter); + to[0] = from[0] ^ s->EKi.u[0]; + to[1] = from[1] ^ s->EKi.u[1]; +} + +static void +armv8_aes_decrypt_gcm_block(struct armv8_gcm_state *s, AES_key_t *aes_key, + const uint64_t *from, uint64_t *to) +{ + armv8_aes_encrypt_gcm_block(s, aes_key, from, to); +} + void armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, - const uint8_t *from, uint8_t *to, + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, size_t authdatalen, const uint8_t *authdata, uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t iv[static AES_GCM_IV_LEN], const __uint128_val_t *Htable) { struct armv8_gcm_state s; - const uint64_t *from64; - uint64_t *to64; - uint8_t block[AES_BLOCK_LEN]; - size_t i, trailer; + uint8_t block[AES_BLOCK_LEN] __aligned(AES_BLOCK_LEN); + uint64_t *from64, *to64; + size_t fromseglen, i, olen, oseglen, seglen, toseglen; armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable); - from64 = (const uint64_t *)from; - to64 = (uint64_t *)to; - trailer = len % AES_BLOCK_LEN; - - for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { - aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); - AES_INC_COUNTER(s.aes_counter); - to64[0] = from64[0] ^ s.EKi.u[0]; - to64[1] = from64[1] ^ s.EKi.u[1]; - gcm_ghash_v8(s.Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN); - - to64 += 2; - from64 += 2; - } - - to += (len - trailer); - from += (len - trailer); - - if (trailer) { - aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); - AES_INC_COUNTER(s.aes_counter); - memset(block, 0, sizeof(block)); - for (i = 0; i < trailer; i++) { - block[i] = to[i] = from[i] ^ s.EKi.c[i]; + for (olen = len; len > 0; len -= seglen) { + from64 = crypto_cursor_segment(fromc, &fromseglen); + to64 = crypto_cursor_segment(toc, &toseglen); + + seglen = ulmin(len, ulmin(fromseglen, toseglen)); + if (seglen < AES_BLOCK_LEN) { + seglen = ulmin(len, AES_BLOCK_LEN); + + memset(block, 0, sizeof(block)); + crypto_cursor_copydata(fromc, (int)seglen, block); + + if (seglen == AES_BLOCK_LEN) { + armv8_aes_encrypt_gcm_block(&s, aes_key, + (uint64_t *)block, (uint64_t *)block); + } else { + aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); + AES_INC_COUNTER(s.aes_counter); + for (i = 0; i < seglen; i++) + block[i] ^= s.EKi.c[i]; + } + gcm_ghash_v8(s.Xi.u, Htable, block, seglen); + + crypto_cursor_copyback(toc, (int)seglen, block); + } else { + for (oseglen = seglen; seglen >= AES_BLOCK_LEN; + seglen -= AES_BLOCK_LEN) { + armv8_aes_encrypt_gcm_block(&s, aes_key, from64, + to64); + gcm_ghash_v8(s.Xi.u, Htable, (uint8_t *)to64, + AES_BLOCK_LEN); + + from64 += 2; + to64 += 2; + } + + seglen = oseglen - seglen; + crypto_cursor_advance(fromc, seglen); + crypto_cursor_advance(toc, seglen); } - - gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } - armv8_aes_gmac_finish(&s, len, authdatalen, Htable); + armv8_aes_gmac_finish(&s, olen, authdatalen, Htable); memcpy(tag, s.Xi.c, GMAC_DIGEST_LEN); + explicit_bzero(block, sizeof(block)); explicit_bzero(&s, sizeof(s)); } int armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, - const uint8_t *from, uint8_t *to, + struct crypto_buffer_cursor *fromc, struct crypto_buffer_cursor *toc, size_t authdatalen, const uint8_t *authdata, const uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t iv[static AES_GCM_IV_LEN], const __uint128_val_t *Htable) { struct armv8_gcm_state s; - const uint64_t *from64; - uint64_t *to64; - uint8_t block[AES_BLOCK_LEN]; - size_t i, trailer; + struct crypto_buffer_cursor fromcc; + uint8_t block[AES_BLOCK_LEN] __aligned(AES_BLOCK_LEN), *from; + uint64_t *block64, *from64, *to64; + size_t fromseglen, olen, oseglen, seglen, toseglen; int error; - error = 0; - armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable); - trailer = len % AES_BLOCK_LEN; - if (len - trailer > 0) - gcm_ghash_v8(s.Xi.u, Htable, from, len - trailer); - if (trailer > 0) { - memset(block, 0, sizeof(block)); - memcpy(block, from + len - trailer, trailer); - gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); + crypto_cursor_copy(fromc, &fromcc); + for (olen = len; len > 0; len -= seglen) { + from = crypto_cursor_segment(&fromcc, &fromseglen); + seglen = ulmin(len, fromseglen); + seglen -= seglen % AES_BLOCK_LEN; + if (seglen > 0) { + gcm_ghash_v8(s.Xi.u, Htable, from, seglen); + crypto_cursor_advance(&fromcc, seglen); + } else { + memset(block, 0, sizeof(block)); + seglen = ulmin(len, AES_BLOCK_LEN); + crypto_cursor_copydata(&fromcc, seglen, block); + gcm_ghash_v8(s.Xi.u, Htable, block, seglen); + } } - armv8_aes_gmac_finish(&s, len, authdatalen, Htable); + armv8_aes_gmac_finish(&s, olen, authdatalen, Htable); if (timingsafe_bcmp(tag, s.Xi.c, GMAC_DIGEST_LEN) != 0) { error = EBADMSG; goto out; } - from64 = (const uint64_t *)from; - to64 = (uint64_t *)to; + block64 = (uint64_t *)block; + for (len = olen; len > 0; len -= seglen) { + from64 = crypto_cursor_segment(fromc, &fromseglen); + to64 = crypto_cursor_segment(toc, &toseglen); - for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { - aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); - AES_INC_COUNTER(s.aes_counter); - to64[0] = from64[0] ^ s.EKi.u[0]; - to64[1] = from64[1] ^ s.EKi.u[1]; - to64 += 2; - from64 += 2; - } + seglen = ulmin(len, ulmin(fromseglen, toseglen)); + if (seglen < AES_BLOCK_LEN) { + seglen = ulmin(len, AES_BLOCK_LEN); - to += (len - trailer); - from += (len - trailer); + memset(block, 0, sizeof(block)); + crypto_cursor_copydata(fromc, seglen, block); - if (trailer) { - aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key); - AES_INC_COUNTER(s.aes_counter); - for (i = 0; i < trailer; i++) - to[i] = from[i] ^ s.EKi.c[i]; + armv8_aes_decrypt_gcm_block(&s, aes_key, block64, + block64); + + crypto_cursor_copyback(toc, (int)seglen, block); + } else { + for (oseglen = seglen; seglen >= AES_BLOCK_LEN; + seglen -= AES_BLOCK_LEN) { + armv8_aes_decrypt_gcm_block(&s, aes_key, from64, + to64); + + from64 += 2; + to64 += 2; + } + + seglen = oseglen - seglen; + crypto_cursor_advance(fromc, seglen); + crypto_cursor_advance(toc, seglen); + } } + error = 0; out: + explicit_bzero(block, sizeof(block)); explicit_bzero(&s, sizeof(s)); return (error); }