Index: security/ktls_isa-l_crypto/Makefile =================================================================== --- /dev/null +++ security/ktls_isa-l_crypto/Makefile @@ -0,0 +1,42 @@ +# Created by: Michal Bielicki +# $FreeBSD$ + +PKGNAMEPREFIX= ktls_ +PORTNAME= isa-l_crypto +PORTVERSION= 2.21.0 +DISTVERSIONPREFIX= v +CATEGORIES= security + +MAINTAINER= gallatin@FreeBSD.org +COMMENT= KTLS module using Intel(R) ISA-L crypto + +LICENSE= BSD3CLAUSE +LICENSE_FILE= ${WRKSRC}/LICENSE + +ONLY_FOR_ARCHS= amd64 + +BUILD_DEPENDS= yasm:devel/yasm + +USE_GITHUB= yes +GH_ACCOUNT= 01org + +USES= kmod + +BUILD_WRKSRC= ${WRKSRC}/kmod +INSTALL_WRKSRC= ${BUILD_WRKSRC} + +MAKE_ENV+= ISASRC=${WRKSRC} + +.include + +.if !exists(${SRC_BASE}/sys/sys/ktls.h) +BROKEN= Requires KTLS support +.endif + +post-extract: + ${MKDIR} ${BUILD_WRKSRC} + ${CP} ${FILESDIR}/Makefile ${FILESDIR}/intelisa_kern.c \ + ${BUILD_WRKSRC} + ${ECHO} "#include " > ${WRKSRC}/include/stdint.h + +.include Index: security/ktls_isa-l_crypto/distinfo =================================================================== --- /dev/null +++ security/ktls_isa-l_crypto/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1566856170 +SHA256 (01org-isa-l_crypto-v2.21.0_GH0.tar.gz) = 58284d7e5e60c37d6d4c1bbc2a876e95d14ca153443775a491f91c21a4243171 +SIZE (01org-isa-l_crypto-v2.21.0_GH0.tar.gz) = 502413 Index: security/ktls_isa-l_crypto/files/Makefile =================================================================== --- /dev/null +++ security/ktls_isa-l_crypto/files/Makefile @@ -0,0 +1,64 @@ +# +# Copyright (c) 2016-2019 Netflix, Inc +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer, +# without modification. +# 2. Redistributions in binary form must reproduce at minimum a disclaimer +# similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any +# redistribution must be conditioned upon including a substantially +# similar Disclaimer requirement for further binary redistribution. +# +# NO WARRANTY +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY +# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, +# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGES. +# +# $FreeBSD$ +# + +ISAINC=${ISASRC}/include +ISAAES=${ISASRC}/aes +INCS=-I${ISAINC} -I${ISAAES} +CFLAGS+=${INCS} +YASM=/usr/local/bin/yasm + +.PATH: ${ISASRC}/aes +.SUFFIXES: .asm +.asm.o: + ${YASM} -g dwarf2 -f elf64 ${INCS} -o ${.TARGET} ${.IMPSRC} + +KMOD= ktls_intel-isa-l +SRCS= gcm128_avx_gen2.asm \ + gcm128_avx_gen4.asm \ + gcm128_sse.asm \ + gcm256_avx_gen2.asm \ + gcm256_avx_gen4.asm \ + gcm256_sse.asm \ + gcm_multibinary.asm \ + gcm128_avx_gen2_nt.asm \ + gcm128_avx_gen4_nt.asm \ + gcm128_sse_nt.asm \ + gcm256_avx_gen2_nt.asm \ + gcm256_avx_gen4_nt.asm \ + gcm256_sse_nt.asm \ + gcm_multibinary_nt.asm \ + keyexp_128.asm \ + keyexp_192.asm \ + keyexp_256.asm \ + keyexp_multibinary.asm \ + gcm_pre.c \ + intelisa_kern.c + +.include Index: security/ktls_isa-l_crypto/files/intelisa_kern.c =================================================================== --- /dev/null +++ security/ktls_isa-l_crypto/files/intelisa_kern.c @@ -0,0 +1,350 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2014-2018 Netflix Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * + */ +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aes_gcm.h" + +#define KTLS_INTELISA_AEAD_TAGLEN 16 + +struct isa_gcm_struct { + struct gcm_key_data key_data; + struct gcm_context_data ctx_data; + void (*gcm_pre) (const void *key, struct gcm_key_data *); /* Done once per key */ + void (*gcm_init) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *iv, + uint8_t const *aad, + uint64_t aad_len); /* Done at start of crypt */ + void (*gcm_upd) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *out, + const uint8_t *in, + uint64_t len); /* With each block of data */ + void (*gcm_upd_nt) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *out, + const uint8_t *in, + uint64_t len); /* With each block of data */ + void (*gcm_final) (const struct gcm_key_data *key_data, + struct gcm_context_data *context_data, + uint8_t *tag, + uint64_t tag_len); /* Pulls out the tag */ +}; + +SYSCTL_DECL(_kern_ipc_tls); + +static int ktls_use_intel_isa_gcm = 1; +SYSCTL_INT(_kern_ipc_tls, OID_AUTO, isa_gcm, CTLFLAG_RW, + &ktls_use_intel_isa_gcm, 1, + "Should we use the Intel ISA GCM if available"); + +SYSCTL_DECL(_kern_ipc_tls_stats); + +static counter_u64_t ktls_offload_isa_aead; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aead_crypts, + CTLFLAG_RD, &ktls_offload_isa_aead, + "Total number of Intel ISA TLS AEAD encrypts called"); + +static counter_u64_t intelisa_unaligned_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned_bytes, + CTLFLAG_RD, &intelisa_unaligned_mem_b, + "Byte cnt of intel isa unaligned"); + +static counter_u64_t intelisa_aligned_mem_b; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned_bytes, + CTLFLAG_RD, &intelisa_aligned_mem_b, + "Byte cnt of intel isa aligned"); + +static counter_u64_t intelisa_unaligned_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned, + CTLFLAG_RD, &intelisa_unaligned_mem, + "Call cnt of intel isa unaligned"); + +static counter_u64_t intelisa_aligned_mem; +SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned, + CTLFLAG_RD, &intelisa_aligned_mem, + "Call cnt of intel isa aligned"); + +static MALLOC_DEFINE(M_INTEL_ISA, "isal_tls", "Intel ISA-L TLS"); + +static int +intel_isa_seal(struct isa_gcm_struct *isa, + struct iovec *outiov, int numiovs, + uint8_t * nd, int noncelen, + struct iovec *iniov, + uint8_t * ad, int adlen, + uint8_t * tagout, size_t *taglen) +{ + int i; + bool nt = true; + bool misaligned_len, misaligned_start; + int fixup = 0; + uint8_t *in; + uint8_t *out; + uint64_t len; + uint8_t iv[32]; + uint8_t const IVend[] = GCM_IV_END_MARK; + + if (noncelen > 28) { + return (-1); + } + memcpy(iv, nd, noncelen); + memcpy(&iv[noncelen], IVend, sizeof(IVend)); + isa->gcm_init(&isa->key_data, &isa->ctx_data, iv, ad, (size_t)adlen); + for (i = 0; i < numiovs; i++) { + in = iniov[i].iov_base; + out = outiov[i].iov_base; + len = iniov[i].iov_len; + + misaligned_start = ((uintptr_t)in & 0xf) != 0; + misaligned_len = (len & 0xf) != 0; + + if (misaligned_start || misaligned_len) { + /* + * Try to do as much of a page using + * non-temporals as we possibly can, and leave + * a ragged tail as a separate chunk. + */ + if (nt && !misaligned_start && len > 0xf) { + len = len & ~0xf; + fixup = iniov[i].iov_len - len; + } else { + nt = false; + } + } +fixup_done: + if (nt) { + isa->gcm_upd_nt(&isa->key_data, &isa->ctx_data, out, in, len); + counter_u64_add(intelisa_aligned_mem, 1); + counter_u64_add(intelisa_aligned_mem_b, len); + } else { + isa->gcm_upd(&isa->key_data, &isa->ctx_data, out, in, len); + counter_u64_add(intelisa_unaligned_mem, 1); + counter_u64_add(intelisa_unaligned_mem_b, len); + } + if (fixup) { + in += len; + out += len; + len = fixup; + fixup = 0; + nt = false; + goto fixup_done; + } + } + isa->gcm_final(&isa->key_data, &isa->ctx_data, tagout, *taglen); + return (0); +} + +static int +ktls_intelisa_aead_encrypt(struct ktls_session *tls, + const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov, + struct iovec *outiov, int iovcnt, uint64_t seqno) +{ + struct isa_gcm_struct *isa; + struct tls_aead_data ad; + struct tls_nonce_data nd; + size_t noncelen, adlen, taglen; + int ret; + uint16_t tls_comp_len; + + isa = (struct isa_gcm_struct *)tls->cipher; + + KASSERT(isa != NULL, ("Null cipher")); + counter_u64_add(ktls_offload_isa_aead, 1); + taglen = KTLS_INTELISA_AEAD_TAGLEN; + + /* Setup the nonce */ + memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN); + memcpy(&nd.seq, hdr + 1, sizeof(nd.seq)); + noncelen = sizeof(nd); + /* Setup the associated data */ + tls_comp_len = ntohs(hdr->tls_length) - + (KTLS_INTELISA_AEAD_TAGLEN + sizeof(nd.seq)); + ad.seq = htobe64(seqno); + ad.type = hdr->tls_type; + ad.tls_vmajor = hdr->tls_vmajor; + ad.tls_vminor = hdr->tls_vminor; + ad.tls_length = htons(tls_comp_len); + adlen = sizeof(ad); + ret = intel_isa_seal(isa, outiov, iovcnt, + (uint8_t *) & nd, noncelen, iniov, + (uint8_t *) & ad, adlen, trailer, &taglen); + + return(ret); +} + + +static int +ktls_intelisa_setup_cipher(struct isa_gcm_struct *isa, uint8_t *key) +{ + struct fpu_kern_ctx *fpu_ctx; + + if (key == NULL) { + return (EINVAL); + } + fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NOWAIT); + if (fpu_ctx == NULL) { + return (ENOMEM); + } + fpu_kern_enter(curthread, fpu_ctx, FPU_KERN_NORMAL); + isa->gcm_pre(key, &isa->key_data); + fpu_kern_leave(curthread, fpu_ctx); + fpu_kern_free_ctx(fpu_ctx); + return (0); +} + +static void +ktls_intelisa_free(struct ktls_session *tls) +{ + struct isa_gcm_struct *isa; + + isa = tls->cipher; + explicit_bzero(isa, sizeof(*isa)); + free(isa, M_INTEL_ISA); +} + +static int +ktls_intelisa_try(struct socket *so, struct ktls_session *tls) +{ + struct isa_gcm_struct *isa; + int error; + + if (ktls_use_intel_isa_gcm && + tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { + isa = malloc(sizeof (*isa), M_INTEL_ISA, M_NOWAIT | M_ZERO); + if (isa == NULL) { + return (ENOMEM); + } + switch (tls->params.cipher_key_len) { + case 16: + isa->gcm_pre = aes_gcm_pre_128; + isa->gcm_init = aes_gcm_init_128; + isa->gcm_upd = aes_gcm_enc_128_update; + isa->gcm_upd_nt = aes_gcm_enc_128_update_nt; + isa->gcm_final = aes_gcm_enc_128_finalize; + break; + case 32: + isa->gcm_pre = aes_gcm_pre_256; + isa->gcm_init = aes_gcm_init_256; + isa->gcm_upd = aes_gcm_enc_256_update; + isa->gcm_upd_nt = aes_gcm_enc_256_update_nt; + isa->gcm_final = aes_gcm_enc_256_finalize; + break; + default: + free(isa, M_INTEL_ISA); + return (EOPNOTSUPP); + } + + error = ktls_intelisa_setup_cipher(isa, tls->params.cipher_key); + if (error) { + free(isa, M_INTEL_ISA); + return (error); + } + + tls->cipher = isa; + tls->sw_encrypt = ktls_intelisa_aead_encrypt; + tls->free = ktls_intelisa_free; + return (0); + } + return (EOPNOTSUPP); +} + +struct ktls_crypto_backend intelisa_backend = { + .name = "Intel ISA-L", + .prio = 20, + .api_version = KTLS_API_VERSION, + .try = ktls_intelisa_try, +}; + +static int +intelisa_init(void) +{ + ktls_offload_isa_aead = counter_u64_alloc(M_WAITOK); + intelisa_aligned_mem = counter_u64_alloc(M_WAITOK); + intelisa_aligned_mem_b = counter_u64_alloc(M_WAITOK); + intelisa_unaligned_mem = counter_u64_alloc(M_WAITOK); + intelisa_unaligned_mem_b = counter_u64_alloc(M_WAITOK); + return (ktls_crypto_backend_register(&intelisa_backend)); +} + +static int +intelisa_unload(void) +{ + int error; + + error = ktls_crypto_backend_deregister(&intelisa_backend); + if (error) + return (error); + counter_u64_free(ktls_offload_isa_aead); + counter_u64_free(intelisa_aligned_mem); + counter_u64_free(intelisa_aligned_mem_b); + counter_u64_free(intelisa_unaligned_mem); + counter_u64_free(intelisa_unaligned_mem_b); + return (0); +} + +static int +intelisa_module_event_handler(module_t mod, int evt, void *arg) +{ + switch (evt) { + case MOD_LOAD: + return (intelisa_init()); + case MOD_UNLOAD: + return (intelisa_unload()); + default: + return (EOPNOTSUPP); + } +} + +static moduledata_t intelisa_moduledata = { + "intelisa", + intelisa_module_event_handler, + NULL +}; + +DECLARE_MODULE(intelisa, intelisa_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY); Index: security/ktls_isa-l_crypto/pkg-descr =================================================================== --- /dev/null +++ security/ktls_isa-l_crypto/pkg-descr @@ -0,0 +1,8 @@ +ISA-L crypto is a collection of optimized low-level functions targeting +cryptography applications. + +This port provides a software encryption module for in-kernel TLS +(KTLS) that uses routines in the ISA-L crypto library to encrypt TLS +records. + +WWW: https://github.com/01org/isa-l_crypto Index: security/ktls_isa-l_crypto/pkg-plist =================================================================== --- /dev/null +++ security/ktls_isa-l_crypto/pkg-plist @@ -0,0 +1 @@ +/%%KMODDIR%%/ktls_intel-isa-l.ko