Index: head/security/ktls_isa-l_crypto-kmod/files/intelisa_kern.c =================================================================== --- head/security/ktls_isa-l_crypto-kmod/files/intelisa_kern.c (revision 542974) +++ head/security/ktls_isa-l_crypto-kmod/files/intelisa_kern.c (revision 542975) @@ -1,387 +1,395 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014-2018 Netflix Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "aes_gcm.h" #define KTLS_INTELISA_AEAD_TAGLEN 16 struct isa_gcm_struct { struct gcm_key_data key_data; struct gcm_context_data ctx_data; void (*gcm_pre) (const void *key, struct gcm_key_data *); /* Done once per key */ void (*gcm_init) (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *iv, uint8_t const *aad, uint64_t aad_len); /* Done at start of crypt */ void (*gcm_upd) (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); /* With each block of data */ void (*gcm_upd_nt) (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); /* With each block of data */ void (*gcm_final) (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *tag, uint64_t tag_len); /* Pulls out the tag */ }; SYSCTL_DECL(_kern_ipc_tls); static int ktls_use_intel_isa_gcm = 1; SYSCTL_INT(_kern_ipc_tls, OID_AUTO, isa_gcm, CTLFLAG_RW, &ktls_use_intel_isa_gcm, 1, "Should we use the Intel ISA GCM if available"); SYSCTL_DECL(_kern_ipc_tls_stats); static counter_u64_t ktls_offload_isa_aead; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aead_crypts, CTLFLAG_RD, &ktls_offload_isa_aead, "Total number of Intel ISA TLS AEAD encrypts called"); static counter_u64_t ktls_offload_isa_tls_13; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_tls_13_crypts, CTLFLAG_RD, &ktls_offload_isa_tls_13, "Total number of Intel ISA TLS 1.3 encrypts called"); static counter_u64_t ktls_offload_isa_tls_12; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_tls_12_crypts, CTLFLAG_RD, &ktls_offload_isa_tls_12, "Total number of Intel ISA TLS 1.2 encrypts called"); static counter_u64_t intelisa_unaligned_mem_b; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned_bytes, CTLFLAG_RD, &intelisa_unaligned_mem_b, "Byte cnt of intel isa unaligned"); static counter_u64_t intelisa_aligned_mem_b; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned_bytes, CTLFLAG_RD, &intelisa_aligned_mem_b, "Byte cnt of intel isa aligned"); static counter_u64_t intelisa_unaligned_mem; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned, CTLFLAG_RD, &intelisa_unaligned_mem, "Call cnt of intel isa unaligned"); static counter_u64_t intelisa_aligned_mem; SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned, CTLFLAG_RD, &intelisa_aligned_mem, "Call cnt of intel isa aligned"); static MALLOC_DEFINE(M_INTEL_ISA, "isal_tls", "Intel ISA-L TLS"); static int intel_isa_seal(struct isa_gcm_struct *isa, struct iovec *outiov, int numiovs, uint8_t * iv, struct iovec *iniov, uint8_t * ad, int adlen, uint8_t * tagout, size_t *taglen, bool tls_13, uint8_t tls_13_rtype) { int i; bool nt = true; bool misaligned_len, misaligned_start; int fixup = 0; uint8_t *in; uint8_t *out; uint64_t len; isa->gcm_init(&isa->key_data, &isa->ctx_data, iv, ad, (size_t)adlen); for (i = 0; i < numiovs; i++) { in = iniov[i].iov_base; out = outiov[i].iov_base; len = iniov[i].iov_len; misaligned_start = ((uintptr_t)in & 0xf) != 0; misaligned_len = (len & 0xf) != 0; if (misaligned_start || misaligned_len) { /* * Try to do as much of a page using * non-temporals as we possibly can, and leave * a ragged tail as a separate chunk. */ if (nt && !misaligned_start && len > 0xf) { len = len & ~0xf; fixup = iniov[i].iov_len - len; } else { nt = false; } } fixup_done: if (nt) { isa->gcm_upd_nt(&isa->key_data, &isa->ctx_data, out, in, len); counter_u64_add(intelisa_aligned_mem, 1); counter_u64_add(intelisa_aligned_mem_b, len); } else { isa->gcm_upd(&isa->key_data, &isa->ctx_data, out, in, len); counter_u64_add(intelisa_unaligned_mem, 1); counter_u64_add(intelisa_unaligned_mem_b, len); } if (fixup) { in += len; out += len; len = fixup; fixup = 0; nt = false; goto fixup_done; } } if (tls_13) { *tagout = tls_13_rtype; isa->gcm_upd(&isa->key_data, &isa->ctx_data, tagout, tagout, 1); tagout += 1; } isa->gcm_final(&isa->key_data, &isa->ctx_data, tagout, *taglen); return (0); } static int ktls_intelisa_aead_encrypt(struct ktls_session *tls, const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t tls_rtype) { struct isa_gcm_struct *isa; struct tls_aead_data ad; struct tls_nonce_data nd; size_t adlen, taglen; uint8_t *adptr; int ret; uint16_t tls_comp_len; bool tls_13; isa = (struct isa_gcm_struct *)tls->cipher; KASSERT(isa != NULL, ("Null cipher")); counter_u64_add(ktls_offload_isa_aead, 1); taglen = KTLS_INTELISA_AEAD_TAGLEN; /* Setup the nonce */ memcpy(&nd, tls->params.iv, tls->params.iv_len); /* Setup the associated data */ ad.seq = htobe64(seqno); ad.type = hdr->tls_type; ad.tls_vmajor = hdr->tls_vmajor; ad.tls_vminor = hdr->tls_vminor; /* Version-specific nonce and AAD. */ if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) { tls_13 = true; counter_u64_add(ktls_offload_isa_tls_13, 1); adlen = sizeof(ad) - sizeof(ad.seq); adptr = &ad.type; ad.tls_length = hdr->tls_length; /* * RFC 8446 5.3: left pad the 64b seqno * with 0s, and xor with the IV. */ nd.seq ^= htobe64(seqno); } else { tls_13 = false; counter_u64_add(ktls_offload_isa_tls_12, 1); tls_comp_len = ntohs(hdr->tls_length) - (KTLS_INTELISA_AEAD_TAGLEN + sizeof(nd.seq)); adlen = sizeof(ad); adptr = (uint8_t *)&ad; ad.tls_length = htons(tls_comp_len); memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); } ret = intel_isa_seal(isa, outiov, iovcnt, (uint8_t *)&nd, iniov, adptr, adlen, trailer, &taglen, tls_13, tls_rtype); return(ret); } static int ktls_intelisa_setup_cipher(struct isa_gcm_struct *isa, uint8_t *key) { struct fpu_kern_ctx *fpu_ctx; if (key == NULL) { return (EINVAL); } fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NOWAIT); if (fpu_ctx == NULL) { return (ENOMEM); } fpu_kern_enter(curthread, fpu_ctx, FPU_KERN_NORMAL); isa->gcm_pre(key, &isa->key_data); fpu_kern_leave(curthread, fpu_ctx); fpu_kern_free_ctx(fpu_ctx); return (0); } static void ktls_intelisa_free(struct ktls_session *tls) { struct isa_gcm_struct *isa; isa = tls->cipher; explicit_bzero(isa, sizeof(*isa)); free(isa, M_INTEL_ISA); } static int +#if KTLS_API_VERSION >= 7 +ktls_intelisa_try(struct socket *so, struct ktls_session *tls, int direction) +#else ktls_intelisa_try(struct socket *so, struct ktls_session *tls) +#endif { struct isa_gcm_struct *isa; int error; +#if KTLS_API_VERSION >= 7 + if (direction != KTLS_TX) + return (EOPNOTSUPP); +#endif if (ktls_use_intel_isa_gcm && tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { isa = malloc(sizeof (*isa), M_INTEL_ISA, M_NOWAIT | M_ZERO); if (isa == NULL) { return (ENOMEM); } switch (tls->params.cipher_key_len) { case 16: isa->gcm_pre = aes_gcm_pre_128; isa->gcm_init = aes_gcm_init_128; isa->gcm_upd = aes_gcm_enc_128_update; isa->gcm_upd_nt = aes_gcm_enc_128_update_nt; isa->gcm_final = aes_gcm_enc_128_finalize; break; case 32: isa->gcm_pre = aes_gcm_pre_256; isa->gcm_init = aes_gcm_init_256; isa->gcm_upd = aes_gcm_enc_256_update; isa->gcm_upd_nt = aes_gcm_enc_256_update_nt; isa->gcm_final = aes_gcm_enc_256_finalize; break; default: free(isa, M_INTEL_ISA); return (EOPNOTSUPP); } error = ktls_intelisa_setup_cipher(isa, tls->params.cipher_key); if (error) { free(isa, M_INTEL_ISA); return (error); } tls->cipher = isa; tls->sw_encrypt = ktls_intelisa_aead_encrypt; tls->free = ktls_intelisa_free; return (0); } return (EOPNOTSUPP); } struct ktls_crypto_backend intelisa_backend = { .name = "Intel ISA-L", .prio = 20, .api_version = KTLS_API_VERSION, .try = ktls_intelisa_try, }; static int intelisa_init(void) { ktls_offload_isa_aead = counter_u64_alloc(M_WAITOK); ktls_offload_isa_tls_12 = counter_u64_alloc(M_WAITOK); ktls_offload_isa_tls_13 = counter_u64_alloc(M_WAITOK); intelisa_aligned_mem = counter_u64_alloc(M_WAITOK); intelisa_aligned_mem_b = counter_u64_alloc(M_WAITOK); intelisa_unaligned_mem = counter_u64_alloc(M_WAITOK); intelisa_unaligned_mem_b = counter_u64_alloc(M_WAITOK); return (ktls_crypto_backend_register(&intelisa_backend)); } static int intelisa_unload(void) { int error; error = ktls_crypto_backend_deregister(&intelisa_backend); if (error) return (error); counter_u64_free(ktls_offload_isa_aead); counter_u64_free(intelisa_aligned_mem); counter_u64_free(intelisa_aligned_mem_b); counter_u64_free(intelisa_unaligned_mem); counter_u64_free(intelisa_unaligned_mem_b); return (0); } static int intelisa_module_event_handler(module_t mod, int evt, void *arg) { switch (evt) { case MOD_LOAD: return (intelisa_init()); case MOD_UNLOAD: return (intelisa_unload()); default: return (EOPNOTSUPP); } } static moduledata_t intelisa_moduledata = { "intelisa", intelisa_module_event_handler, NULL }; DECLARE_MODULE(intelisa, intelisa_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);