Index: head/security/Makefile =================================================================== --- head/security/Makefile +++ head/security/Makefile @@ -273,6 +273,7 @@ SUBDIR += krb5-devel SUBDIR += kripp SUBDIR += kstart + SUBDIR += ktls_isa-l_crypto-kmod SUBDIR += kwalletmanager SUBDIR += l0pht-watch SUBDIR += l0phtcrack Index: head/security/ktls_isa-l_crypto-kmod/Makefile =================================================================== --- head/security/ktls_isa-l_crypto-kmod/Makefile +++ head/security/ktls_isa-l_crypto-kmod/Makefile @@ -0,0 +1,40 @@ +# Created by: Michal Bielicki +# $FreeBSD$ + +PORTNAME= isa-l_crypto +PORTVERSION= 2.21.0 +DISTVERSIONPREFIX= v +CATEGORIES= security +PKGNAMEPREFIX= ktls_ +PKGNAMESUFFIX= -kmod + +MAINTAINER= gallatin@FreeBSD.org +COMMENT= KTLS module using Intel(R) ISA-L crypto + +LICENSE= BSD3CLAUSE +LICENSE_FILE= ${WRKSRC}/LICENSE + +ONLY_FOR_ARCHS= amd64 + +BUILD_DEPENDS= yasm:devel/yasm + +USES= kmod uidfix + +USE_GITHUB= yes +GH_ACCOUNT= 01org + +MAKE_ENV+= ISASRC=${WRKSRC} + +PLIST_FILES= ${KMODDIR}/ktls_intel-isa-l.ko + +.include + +.if !exists(${SRC_BASE}/sys/sys/ktls.h) +IGNORE= requires KTLS support +.endif + +post-extract: + ${CP} ${FILESDIR}/* ${WRKSRC}/ + ${ECHO} "#include " > ${WRKSRC}/include/stdint.h + +.include Index: head/security/ktls_isa-l_crypto-kmod/distinfo =================================================================== --- head/security/ktls_isa-l_crypto-kmod/distinfo +++ head/security/ktls_isa-l_crypto-kmod/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1566856170 +SHA256 (01org-isa-l_crypto-v2.21.0_GH0.tar.gz) = 58284d7e5e60c37d6d4c1bbc2a876e95d14ca153443775a491f91c21a4243171 +SIZE (01org-isa-l_crypto-v2.21.0_GH0.tar.gz) = 502413 Index: head/security/ktls_isa-l_crypto-kmod/files/Makefile =================================================================== --- head/security/ktls_isa-l_crypto-kmod/files/Makefile +++ head/security/ktls_isa-l_crypto-kmod/files/Makefile @@ -0,0 +1,64 @@ +# +# Copyright (c) 2016-2019 Netflix, Inc +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer, +# without modification. +# 2. Redistributions in binary form must reproduce at minimum a disclaimer +# similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any +# redistribution must be conditioned upon including a substantially +# similar Disclaimer requirement for further binary redistribution. +# +# NO WARRANTY +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY +# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, +# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGES. +# +# $FreeBSD$ +# + +ISAINC=${ISASRC}/include +ISAAES=${ISASRC}/aes +INCS=-I${ISAINC} -I${ISAAES} +CFLAGS+=${INCS} +YASM=${LOCALBASE}/bin/yasm + +.PATH: ${ISASRC}/aes +.SUFFIXES: .asm +.asm.o: + ${YASM} -g dwarf2 -f elf64 ${INCS} -o ${.TARGET} ${.IMPSRC} + +KMOD= ktls_intel-isa-l +SRCS= gcm128_avx_gen2.asm \ + gcm128_avx_gen4.asm \ + gcm128_sse.asm \ + gcm256_avx_gen2.asm \ + gcm256_avx_gen4.asm \ + gcm256_sse.asm \ + gcm_multibinary.asm \ + gcm128_avx_gen2_nt.asm \ + gcm128_avx_gen4_nt.asm \ + gcm128_sse_nt.asm \ + gcm256_avx_gen2_nt.asm \ + gcm256_avx_gen4_nt.asm \ + gcm256_sse_nt.asm \ + gcm_multibinary_nt.asm \ + keyexp_128.asm \ + keyexp_192.asm \ + keyexp_256.asm \ + keyexp_multibinary.asm \ + gcm_pre.c \ + intelisa_kern.c + +.include Index: head/security/ktls_isa-l_crypto-kmod/files/intelisa_kern.c =================================================================== --- head/security/ktls_isa-l_crypto-kmod/files/intelisa_kern.c +++ head/security/ktls_isa-l_crypto-kmod/files/intelisa_kern.c @@ -0,0 +1,401 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2014-2018 Netflix Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * + */ +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aes_gcm.h" + +#define KTLS_INTELISA_AEAD_TAGLEN 16 + +struct isa_gcm_struct { + struct gcm_key_data key_data; + struct gcm_context_data ctx_data; + void (*gcm_pre) (const void *key, struct gcm_key_data *); /* Done once per key */ + void (*gcm_init) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *iv, + uint8_t const *aad, + uint64_t aad_len); /* Done at start of crypt */ + void (*gcm_upd) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *out, + const uint8_t *in, + uint64_t len); /* With each block of data */ + void (*gcm_upd_nt) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *out, + const uint8_t *in, + uint64_t len); /* With each block of data */ + void (*gcm_final) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *tag, + uint64_t tag_len); /* Pulls out the tag */ +}; + +SYSCTL_DECL(_kern_ipc_tls); + +static int ktls_use_intel_isa_gcm = 1; +SYSCTL_INT(_kern_ipc_tls, OID_AUTO, isa_gcm, CTLFLAG_RW, + &ktls_use_intel_isa_gcm, 1, + "Should we use the Intel ISA GCM if available"); + +SYSCTL_DECL(_kern_ipc_tls_stats); + +static counter_u64_t ktls_offload_isa_aead; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aead_crypts, + CTLFLAG_RD, &ktls_offload_isa_aead, + "Total number of Intel ISA TLS AEAD encrypts called"); + +static counter_u64_t ktls_offload_isa_tls_13; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_tls_13_crypts, + CTLFLAG_RD, &ktls_offload_isa_tls_13, + "Total number of Intel ISA TLS 1.3 encrypts called"); + +static counter_u64_t ktls_offload_isa_tls_12; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_tls_12_crypts, + CTLFLAG_RD, &ktls_offload_isa_tls_12, + "Total number of Intel ISA TLS 1.2 encrypts called"); + +static counter_u64_t intelisa_unaligned_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned_bytes, + CTLFLAG_RD, &intelisa_unaligned_mem_b, + "Byte cnt of intel isa unaligned"); + +static counter_u64_t intelisa_aligned_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned_bytes, + CTLFLAG_RD, &intelisa_aligned_mem_b, + "Byte cnt of intel isa aligned"); + +static counter_u64_t intelisa_unaligned_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned, + CTLFLAG_RD, &intelisa_unaligned_mem, + "Call cnt of intel isa unaligned"); + +static counter_u64_t intelisa_aligned_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned, + CTLFLAG_RD, &intelisa_aligned_mem, + "Call cnt of intel isa aligned"); + +static MALLOC_DEFINE(M_INTEL_ISA, "isal_tls", "Intel ISA-L TLS"); + +static int +intel_isa_seal(struct isa_gcm_struct *isa, + struct iovec *outiov, int numiovs, + uint8_t *static_iv, int iv_len, uint64_t seq, + struct iovec *iniov, + uint8_t * ad, int adlen, + uint8_t * tagout, size_t *taglen, + bool tls_13, uint8_t tls_13_rtype) +{ + int i; + bool nt = true; + bool misaligned_len, misaligned_start; + int fixup = 0; + size_t offset; + uint8_t *in; + uint8_t *out; + uint64_t len; + uint8_t iv[32]; + uint8_t seq_num[sizeof(seq)]; + + if (iv_len > 32 - sizeof(seq)) { + return (-1); + } + + if (tls_13) { + /* + * RFC 8446 5.3: left pad the 64b seqno + * with 0s, and xor with the IV + * + * gcm_init does not provde a way to specify the + * length of the iv, so we have hard-coded it to 12 in + * openssl + */ + memcpy(seq_num, &seq, sizeof(seq)); + + offset = iv_len - sizeof(seq); + memcpy(iv, static_iv, offset); + for (i = 0; i < sizeof(seq); i++) + iv[i + offset] = static_iv[i + offset] ^ seq_num[i]; + } else { + memcpy(iv, static_iv, iv_len); + memcpy(iv + iv_len, &seq, sizeof(seq)); + } + isa->gcm_init(&isa->key_data, &isa->ctx_data, iv, ad, (size_t)adlen); + for (i = 0; i < numiovs; i++) { + in = iniov[i].iov_base; + out = outiov[i].iov_base; + len = iniov[i].iov_len; + + misaligned_start = ((uintptr_t)in & 0xf) != 0; + misaligned_len = (len & 0xf) != 0; + + if (misaligned_start || misaligned_len) { + /* + * Try to do as much of a page using + * non-temporals as we possibly can, and leave + * a ragged tail as a separate chunk. + */ + if (nt && !misaligned_start && len > 0xf) { + len = len & ~0xf; + fixup = iniov[i].iov_len - len; + } else { + nt = false; + } + } +fixup_done: + if (nt) { + isa->gcm_upd_nt(&isa->key_data, &isa->ctx_data, out, in, len); + counter_u64_add(intelisa_aligned_mem, 1); + counter_u64_add(intelisa_aligned_mem_b, len); + } else { + isa->gcm_upd(&isa->key_data, &isa->ctx_data, out, in, len); + counter_u64_add(intelisa_unaligned_mem, 1); + counter_u64_add(intelisa_unaligned_mem_b, len); + } + if (fixup) { + in += len; + out += len; + len = fixup; + fixup = 0; + nt = false; + goto fixup_done; + } + } + if (tls_13) { + *tagout = tls_13_rtype; + isa->gcm_upd(&isa->key_data, &isa->ctx_data, tagout, + tagout, 1); + tagout += 1; + } + isa->gcm_final(&isa->key_data, &isa->ctx_data, tagout, *taglen); + return (0); +} + +static int +ktls_intelisa_aead_encrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, + struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t tls_rtype) +{ + struct isa_gcm_struct *isa; + struct tls_aead_data ad; + struct tls_nonce_data nd; + size_t adlen, taglen; + uint8_t *adptr; + int ret; + uint16_t tls_comp_len; + bool tls_13; + + isa = (struct isa_gcm_struct *)tls->cipher; + + KASSERT(isa != NULL, ("Null cipher")); + counter_u64_add(ktls_offload_isa_aead, 1); + taglen = KTLS_INTELISA_AEAD_TAGLEN; + + if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) { + tls_13 = true; + counter_u64_add(ktls_offload_isa_tls_13, 1); + adlen = sizeof(ad) - sizeof(ad.seq); + adptr = &ad.type; + ad.tls_length = hdr->tls_length; + + } else { + tls_13 = false; + counter_u64_add(ktls_offload_isa_tls_12, 1); + tls_comp_len = ntohs(hdr->tls_length) - + (KTLS_INTELISA_AEAD_TAGLEN + sizeof(nd.seq)); + adlen = sizeof(ad); + adptr = (uint8_t *)&ad; + ad.tls_length = htons(tls_comp_len); + } + /* Setup the associated data */ + ad.seq = htobe64(seqno); + ad.type = hdr->tls_type; + ad.tls_vmajor = hdr->tls_vmajor; + ad.tls_vminor = hdr->tls_vminor; + + ret = intel_isa_seal(isa, outiov, iovcnt, + tls->params.iv, tls->params.iv_len, + htobe64(seqno), iniov, + adptr, adlen, trailer, &taglen, + tls_13, tls_rtype); + + return(ret); +} + + +static int +ktls_intelisa_setup_cipher(struct isa_gcm_struct *isa, uint8_t *key) +{ + struct fpu_kern_ctx *fpu_ctx; + + if (key == NULL) { + return (EINVAL); + } + fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NOWAIT); + if (fpu_ctx == NULL) { + return (ENOMEM); + } + fpu_kern_enter(curthread, fpu_ctx, FPU_KERN_NORMAL); + isa->gcm_pre(key, &isa->key_data); + fpu_kern_leave(curthread, fpu_ctx); + fpu_kern_free_ctx(fpu_ctx); + return (0); +} + +static void +ktls_intelisa_free(struct ktls_session *tls) +{ + struct isa_gcm_struct *isa; + + isa = tls->cipher; + explicit_bzero(isa, sizeof(*isa)); + free(isa, M_INTEL_ISA); +} + +static int +ktls_intelisa_try(struct socket *so, struct ktls_session *tls) +{ + struct isa_gcm_struct *isa; + int error; + + if (ktls_use_intel_isa_gcm && + tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { + isa = malloc(sizeof (*isa), M_INTEL_ISA, M_NOWAIT | M_ZERO); + if (isa == NULL) { + return (ENOMEM); + } + switch (tls->params.cipher_key_len) { + case 16: + isa->gcm_pre = aes_gcm_pre_128; + isa->gcm_init = aes_gcm_init_128; + isa->gcm_upd = aes_gcm_enc_128_update; + isa->gcm_upd_nt = aes_gcm_enc_128_update_nt; + isa->gcm_final = aes_gcm_enc_128_finalize; + break; + case 32: + isa->gcm_pre = aes_gcm_pre_256; + isa->gcm_init = aes_gcm_init_256; + isa->gcm_upd = aes_gcm_enc_256_update; + isa->gcm_upd_nt = aes_gcm_enc_256_update_nt; + isa->gcm_final = aes_gcm_enc_256_finalize; + break; + default: + free(isa, M_INTEL_ISA); + return (EOPNOTSUPP); + } + + error = ktls_intelisa_setup_cipher(isa, tls->params.cipher_key); + if (error) { + free(isa, M_INTEL_ISA); + return (error); + } + + tls->cipher = isa; + tls->sw_encrypt = ktls_intelisa_aead_encrypt; + tls->free = ktls_intelisa_free; + return (0); + } + return (EOPNOTSUPP); +} + +struct ktls_crypto_backend intelisa_backend = { + .name = "Intel ISA-L", + .prio = 20, + .api_version = KTLS_API_VERSION, + .try = ktls_intelisa_try, +}; + +static int +intelisa_init(void) +{ + ktls_offload_isa_aead = counter_u64_alloc(M_WAITOK); + ktls_offload_isa_tls_12 = counter_u64_alloc(M_WAITOK); + ktls_offload_isa_tls_13 = counter_u64_alloc(M_WAITOK); + intelisa_aligned_mem = counter_u64_alloc(M_WAITOK); + intelisa_aligned_mem_b = counter_u64_alloc(M_WAITOK); + intelisa_unaligned_mem = counter_u64_alloc(M_WAITOK); + intelisa_unaligned_mem_b = counter_u64_alloc(M_WAITOK); + return (ktls_crypto_backend_register(&intelisa_backend)); +} + +static int +intelisa_unload(void) +{ + int error; + + error = ktls_crypto_backend_deregister(&intelisa_backend); + if (error) + return (error); + counter_u64_free(ktls_offload_isa_aead); + counter_u64_free(intelisa_aligned_mem); + counter_u64_free(intelisa_aligned_mem_b); + counter_u64_free(intelisa_unaligned_mem); + counter_u64_free(intelisa_unaligned_mem_b); + return (0); +} + +static int +intelisa_module_event_handler(module_t mod, int evt, void *arg) +{ + switch (evt) { + case MOD_LOAD: + return (intelisa_init()); + case MOD_UNLOAD: + return (intelisa_unload()); + default: + return (EOPNOTSUPP); + } +} + +static moduledata_t intelisa_moduledata = { + "intelisa", + intelisa_module_event_handler, + NULL +}; + +DECLARE_MODULE(intelisa, intelisa_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); Index: head/security/ktls_isa-l_crypto-kmod/pkg-descr =================================================================== --- head/security/ktls_isa-l_crypto-kmod/pkg-descr +++ head/security/ktls_isa-l_crypto-kmod/pkg-descr @@ -0,0 +1,8 @@ +ISA-L crypto is a collection of optimized low-level functions targeting +cryptography applications. + +This port provides a software encryption module for in-kernel TLS +(KTLS) that uses routines in the ISA-L crypto library to encrypt TLS +records. + +WWW: https://github.com/01org/isa-l_crypto