diff --git a/share/man/man4/ktls.4 b/share/man/man4/ktls.4 index 56b03d45faf7..5fca616dae4d 100644 --- a/share/man/man4/ktls.4 +++ b/share/man/man4/ktls.4 @@ -1,257 +1,259 @@ .\" Copyright (c) 2020, Chelsio Inc .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions are met: .\" .\" 1. Redistributions of source code must retain the above copyright notice, .\" this list of conditions and the following disclaimer. .\" .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" 3. Neither the name of the Chelsio Inc nor the names of its .\" contributors may be used to endorse or promote products derived from .\" this software without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" .\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE .\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR .\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF .\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE .\" POSSIBILITY OF SUCH DAMAGE. .\" .\" * Other names and brands may be claimed as the property of others. .\" -.Dd October 31, 2024 +.Dd June 30, 2026 .Dt KTLS 4 .Os .Sh NAME .Nm ktls .Nd kernel Transport Layer Security .Sh SYNOPSIS .Cd options KERN_TLS .Sh DESCRIPTION The .Nm facility allows the kernel to perform Transport Layer Security (TLS) framing on TCP sockets. With .Nm , the initial handshake for a socket using TLS is performed in userland. Once the session keys are negotiated, they are provided to the kernel via the .Dv TCP_TXTLS_ENABLE and .Dv TCP_RXTLS_ENABLE socket options. Both socket options accept a .Vt struct tls_enable structure as their argument. The members of this structure describe the cipher suite used for the TLS session and provide the session keys used for the respective direction. .Pp .Nm only permits the session keys to be set once in each direction. As a result, applications must disable rekeying when using .Nm . .Ss Modes .Nm can operate in different modes. A given socket may use different modes for transmit and receive, or a socket may only offload a single direction. The available modes are: .Bl -tag -width "Dv TCP_TLS_MODE_IFNET" .It Dv TCP_TLS_MODE_NONE .Nm is not enabled. .It Dv TCP_TLS_MODE_SW TLS records are encrypted or decrypted in the kernel in the socket layer via .Xr crypto 9 . Typically the encryption or decryption is performed in software, but it may also be performed by co-processors. .It Dv TCP_TLS_MODE_IFNET TLS records are encrypted or decrypted by the network interface card (NIC). In this mode, the network stack does not work with encrypted data. Instead, the NIC encrypts TLS records as they are being transmitted, or decrypts received TLS records before providing them to the host. .Pp Network interfaces which support this feature will advertise the .Dv TXTLS4 (for IPv4) and/or .Dv TXTLS6 (for IPv6) capabilities as reported by .Xr ifconfig 8 . These capabilities can also be controlled by .Xr ifconfig 8 . .Pp If a network interface supports rate limiting (also known as packet pacing) for TLS offload, the interface will advertise the .Dv TXTLS_RTLMT capability. .It Dv TCP_TLS_MODE_TOE TLS records are encrypted by the NIC using a TCP offload engine (TOE). This is similar to .Dv TCP_TLS_MODE_IFNET in that the network stack does not work with encrypted data. However, this mode works in tandem with a TOE to handle interactions between TCP and TLS. .El .Ss Transmit Once TLS transmit is enabled by a successful set of the .Dv TCP_TXTLS_ENABLE socket option, all data written on the socket is stored in TLS records and encrypted. Most data is transmitted in application layer TLS records, and the kernel chooses how to partition data among TLS records. Individual TLS records with a fixed length and record type can be sent by .Xr sendmsg 2 with the TLS record type set in a .Dv TLS_SET_RECORD_TYPE control message. The payload of this control message is a single byte holding the desired TLS record type. This can be used to send TLS records with a type other than application data (for example, handshake messages) or to send application data records with specific contents (for example, empty fragments). .Pp The current TLS transmit mode of a socket can be queried via the .Dv TCP_TXTLS_MODE socket option. A socket using TLS transmit offload can also set the .Dv TCP_TXTLS_MODE socket option to toggle between .Dv TCP_TLS_MODE_SW and .Dv TCP_TLS_MODE_IFNET . .Ss Receive Once TLS receive is enabled by a successful set of the .Dv TCP_RXTLS_ENABLE socket option, all data read from the socket is returned as decrypted TLS records. Each received TLS record must be read from the socket using .Xr recvmsg 2 . Each received TLS record will contain a .Dv TLS_GET_RECORD control message along with the decrypted payload. The control message contains a .Vt struct tls_get_record which includes fields from the TLS record header. If an invalid or corrupted TLS record is received, .Xr recvmsg 2 will fail with one of the following errors: .Bl -tag -width Er .It Bq Er EINVAL The version fields in a TLS record's header did not match the version required by the .Vt struct tls_enable structure used to enable in-kernel TLS. .It Bq Er EMSGSIZE A TLS record's length was either too small or too large. .It Bq Er EMSGSIZE The connection was closed after sending a truncated TLS record. .It Bq Er EBADMSG The TLS record failed to match the included authentication tag. .El .Pp The current TLS receive mode of a socket can be queried via the .Dv TCP_RXTLS_MODE socket option. At present, the mode cannot be changed. .Ss Sysctl Nodes .Nm uses several sysctl nodes under the .Va kern.ipc.tls node. A few of them are described below: .Bl -tag -width ".Va kern.ipc.tls.cbc_enable" .It Va kern.ipc.tls.enable Determines if new kernel TLS sessions can be created. +.It Va kern.ipc.tls.rx_enable +Determines if new kernel TLS receive sessions can be created. .It Va kern.ipc.tls.cbc_enable Determines if new kernel TLS sessions with a cipher suite using AES-CBC can be created. .It Va kern.ipc.tls.sw A tree of nodes containing statistics for TLS sessions using .Dv TCP_TLS_MODE_SW . .It Va kern.ipc.tls.ifnet A tree of nodes containing statistics for TLS sessions using .Dv TCP_TLS_MODE_IFNET . .It Va kern.ipc.tls.toe A tree of nodes containing statistics for TLS sessions using .Dv TCP_TLS_MODE_TOE . .It Va kern.ipc.tls.stats A tree of nodes containing various kernel TLS statistics. .El .Pp The .Va kern.ipc.mb_use_ext_pgs sysctl controls whether the kernel may use unmapped mbufs. They are required for TLS transmit. .Ss Supported Hardware The .Xr cxgbe 4 and .Xr mlx5en 4 drivers include support for the .Dv TCP_TLS_MODE_IFNET mode. .Pp The .Xr cxgbe 4 driver includes support for the .Dv TCP_TLS_MODE_TOE mode. .Ss Supported Libraries OpenSSL 3.0 and later include support for .Nm . The .Fa security/openssl* and .Fa security/gnutls ports may also be built with support for .Nm by enabling the .Dv KTLS option. OpenSSL in the base system includes KTLS support when built with .Dv WITH_OPENSSL_KTLS . .Pp Applications using a supported library should generally work with .Nm without any changes provided they use standard interfaces such as .Xr SSL_read 3 and .Xr SSL_write 3 . Additional performance may be gained by the use of .Xr SSL_sendfile 3 . .Sh IMPLEMENTATION NOTES .Nm assumes the presence of a direct map of physical memory when performing software encryption and decryption. As a result, it is only supported on architectures with a direct map. .Sh SEE ALSO .Xr cxgbe 4 , .Xr mlx5en 4 , .Xr tcp 4 , .Xr src.conf 5 , .Xr ifconfig 8 , .Xr sysctl 8 , .Xr crypto 9 .Sh HISTORY Kernel TLS first appeared in .Fx 13.0 . diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c index 629fe8923466..0966c930ca0f 100644 --- a/sys/kern/uipc_ktls.c +++ b/sys/kern/uipc_ktls.c @@ -1,3475 +1,3480 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014-2019 Netflix Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_inet.h" #include "opt_inet6.h" #include "opt_kern_tls.h" #include "opt_ratelimit.h" #include "opt_rss.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__) #include #endif #include #include #include #ifdef RSS #include #include #endif #include #include #include #include #include #ifdef TCP_OFFLOAD #include #endif #include #include #include #include #include #include typedef enum { KTLS_MBUF_CRYPTO_ST_MIXED = 0, KTLS_MBUF_CRYPTO_ST_ENCRYPTED = 1, KTLS_MBUF_CRYPTO_ST_DECRYPTED = -1, KTLS_MBUF_CRYPTO_ST_SHAREDMBUF = -2, } ktls_mbuf_crypto_st_t; struct ktls_wq { struct mtx mtx; STAILQ_HEAD(, mbuf) m_head; STAILQ_HEAD(, socket) so_head; bool running; int lastallocfail; } __aligned(CACHE_LINE_SIZE); struct ktls_reclaim_thread { uint64_t wakeups; uint64_t reclaims; struct thread *td; int running; }; struct ktls_domain_info { int count; int cpu[MAXCPU]; struct ktls_reclaim_thread reclaim_td; }; struct ktls_domain_info ktls_domains[MAXMEMDOM]; static struct ktls_wq *ktls_wq; static struct proc *ktls_proc; static uma_zone_t ktls_session_zone; static uma_zone_t ktls_buffer_zone; static uint16_t ktls_cpuid_lookup[MAXCPU]; static int ktls_init_state; static struct sx ktls_init_lock; SX_SYSINIT(ktls_init_lock, &ktls_init_lock, "ktls init"); SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Kernel TLS offload"); SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Kernel TLS offload stats"); #ifdef RSS static int ktls_bind_threads = 1; #else static int ktls_bind_threads; #endif SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN, &ktls_bind_threads, 0, "Bind crypto threads to cores (1) or cores and domains (2) at boot"); static u_int ktls_maxlen = 16384; SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN, &ktls_maxlen, 0, "Maximum TLS record size"); static int ktls_number_threads; SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD, &ktls_number_threads, 0, "Number of TLS threads in thread-pool"); unsigned int ktls_ifnet_max_rexmit_pct = 2; SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, ifnet_max_rexmit_pct, CTLFLAG_RWTUN, &ktls_ifnet_max_rexmit_pct, 2, "Max percent bytes retransmitted before ifnet TLS is disabled"); static bool ktls_offload_enable = true; SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, &ktls_offload_enable, 0, "Enable support for kernel TLS offload"); +static bool ktls_rx_offload_enable = true; +SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, rx_enable, CTLFLAG_RWTUN, + &ktls_rx_offload_enable, 0, + "Enable support for kernel TLS receive offload"); + static bool ktls_cbc_enable = true; SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN, &ktls_cbc_enable, 1, "Enable support of AES-CBC crypto for kernel TLS"); static bool ktls_sw_buffer_cache = true; SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, sw_buffer_cache, CTLFLAG_RDTUN, &ktls_sw_buffer_cache, 1, "Enable caching of output buffers for SW encryption"); static int ktls_max_reclaim = 1024; SYSCTL_INT(_kern_ipc_tls, OID_AUTO, max_reclaim, CTLFLAG_RWTUN, &ktls_max_reclaim, 128, "Max number of 16k buffers to reclaim in thread context"); static COUNTER_U64_DEFINE_EARLY(ktls_tasks_active); SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD, &ktls_tasks_active, "Number of active tasks"); static COUNTER_U64_DEFINE_EARLY(ktls_cnt_tx_pending); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, sw_tx_pending, CTLFLAG_RD, &ktls_cnt_tx_pending, "Number of TLS 1.0 records waiting for earlier TLS records"); static COUNTER_U64_DEFINE_EARLY(ktls_cnt_tx_queued); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, sw_tx_inqueue, CTLFLAG_RD, &ktls_cnt_tx_queued, "Number of TLS records in queue to tasks for SW encryption"); static COUNTER_U64_DEFINE_EARLY(ktls_cnt_rx_queued); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, sw_rx_inqueue, CTLFLAG_RD, &ktls_cnt_rx_queued, "Number of TLS sockets in queue to tasks for SW decryption"); static COUNTER_U64_DEFINE_EARLY(ktls_offload_total); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, offload_total, CTLFLAG_RD, &ktls_offload_total, "Total successful TLS setups (parameters set)"); static COUNTER_U64_DEFINE_EARLY(ktls_offload_enable_calls); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, enable_calls, CTLFLAG_RD, &ktls_offload_enable_calls, "Total number of TLS enable calls made"); static COUNTER_U64_DEFINE_EARLY(ktls_offload_active); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, active, CTLFLAG_RD, &ktls_offload_active, "Total Active TLS sessions"); static COUNTER_U64_DEFINE_EARLY(ktls_offload_corrupted_records); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, corrupted_records, CTLFLAG_RD, &ktls_offload_corrupted_records, "Total corrupted TLS records received"); static COUNTER_U64_DEFINE_EARLY(ktls_offload_failed_crypto); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, failed_crypto, CTLFLAG_RD, &ktls_offload_failed_crypto, "Total TLS crypto failures"); static COUNTER_U64_DEFINE_EARLY(ktls_switch_to_ifnet); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_to_ifnet, CTLFLAG_RD, &ktls_switch_to_ifnet, "TLS sessions switched from SW to ifnet"); static COUNTER_U64_DEFINE_EARLY(ktls_switch_to_sw); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_to_sw, CTLFLAG_RD, &ktls_switch_to_sw, "TLS sessions switched from ifnet to SW"); static COUNTER_U64_DEFINE_EARLY(ktls_switch_failed); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_failed, CTLFLAG_RD, &ktls_switch_failed, "TLS sessions unable to switch between SW and ifnet"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_disable_fail); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ifnet_disable_failed, CTLFLAG_RD, &ktls_ifnet_disable_fail, "TLS sessions unable to switch to SW from ifnet"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_disable_ok); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ifnet_disable_ok, CTLFLAG_RD, &ktls_ifnet_disable_ok, "TLS sessions able to switch to SW from ifnet"); static COUNTER_U64_DEFINE_EARLY(ktls_destroy_task); SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, destroy_task, CTLFLAG_RD, &ktls_destroy_task, "Number of times ktls session was destroyed via taskqueue"); SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Software TLS session stats"); SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Hardware (ifnet) TLS session stats"); #ifdef TCP_OFFLOAD SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "TOE TLS session stats"); #endif static COUNTER_U64_DEFINE_EARLY(ktls_sw_cbc); SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, "Active number of software TLS sessions using AES-CBC"); static COUNTER_U64_DEFINE_EARLY(ktls_sw_gcm); SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, gcm, CTLFLAG_RD, &ktls_sw_gcm, "Active number of software TLS sessions using AES-GCM"); static COUNTER_U64_DEFINE_EARLY(ktls_sw_chacha20); SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, chacha20, CTLFLAG_RD, &ktls_sw_chacha20, "Active number of software TLS sessions using Chacha20-Poly1305"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_cbc); SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, cbc, CTLFLAG_RD, &ktls_ifnet_cbc, "Active number of ifnet TLS sessions using AES-CBC"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_gcm); SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, gcm, CTLFLAG_RD, &ktls_ifnet_gcm, "Active number of ifnet TLS sessions using AES-GCM"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_chacha20); SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, chacha20, CTLFLAG_RD, &ktls_ifnet_chacha20, "Active number of ifnet TLS sessions using Chacha20-Poly1305"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset); SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset, CTLFLAG_RD, &ktls_ifnet_reset, "TLS sessions updated to a new ifnet send tag"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset_dropped); SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset_dropped, CTLFLAG_RD, &ktls_ifnet_reset_dropped, "TLS sessions dropped after failing to update ifnet send tag"); static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset_failed); SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset_failed, CTLFLAG_RD, &ktls_ifnet_reset_failed, "TLS sessions that failed to allocate a new ifnet send tag"); static int ktls_ifnet_permitted = 1; SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN, &ktls_ifnet_permitted, 1, "Whether to permit hardware (ifnet) TLS sessions"); #ifdef TCP_OFFLOAD static COUNTER_U64_DEFINE_EARLY(ktls_toe_cbc); SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD, &ktls_toe_cbc, "Active number of TOE TLS sessions using AES-CBC"); static COUNTER_U64_DEFINE_EARLY(ktls_toe_gcm); SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD, &ktls_toe_gcm, "Active number of TOE TLS sessions using AES-GCM"); static COUNTER_U64_DEFINE_EARLY(ktls_toe_chacha20); SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, chacha20, CTLFLAG_RD, &ktls_toe_chacha20, "Active number of TOE TLS sessions using Chacha20-Poly1305"); #endif static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); static void ktls_reclaim_thread(void *ctx); static void ktls_reset_receive_tag(void *context, int pending); static void ktls_reset_send_tag(void *context, int pending); static void ktls_work_thread(void *ctx); int ktls_copyin_tls_enable(struct sockopt *sopt, struct tls_enable *tls) { struct tls_enable_v0 tls_v0; int error; uint8_t *cipher_key = NULL, *iv = NULL, *auth_key = NULL; if (sopt->sopt_valsize == sizeof(tls_v0)) { error = sooptcopyin(sopt, &tls_v0, sizeof(tls_v0), sizeof(tls_v0)); if (error != 0) goto done; memset(tls, 0, sizeof(*tls)); tls->cipher_key = tls_v0.cipher_key; tls->iv = tls_v0.iv; tls->auth_key = tls_v0.auth_key; tls->cipher_algorithm = tls_v0.cipher_algorithm; tls->cipher_key_len = tls_v0.cipher_key_len; tls->iv_len = tls_v0.iv_len; tls->auth_algorithm = tls_v0.auth_algorithm; tls->auth_key_len = tls_v0.auth_key_len; tls->flags = tls_v0.flags; tls->tls_vmajor = tls_v0.tls_vmajor; tls->tls_vminor = tls_v0.tls_vminor; } else error = sooptcopyin(sopt, tls, sizeof(*tls), sizeof(*tls)); if (error != 0) return (error); if (tls->cipher_key_len < 0 || tls->cipher_key_len > TLS_MAX_PARAM_SIZE) return (EINVAL); if (tls->iv_len < 0 || tls->iv_len > sizeof(((struct ktls_session *)NULL)->params.iv)) return (EINVAL); if (tls->auth_key_len < 0 || tls->auth_key_len > TLS_MAX_PARAM_SIZE) return (EINVAL); /* All supported algorithms require a cipher key. */ if (tls->cipher_key_len == 0) return (EINVAL); /* * Now do a deep copy of the variable-length arrays in the struct, so that * subsequent consumers of it can reliably assume kernel memory. This * requires doing our own allocations, which we will free in the * error paths so that our caller need only worry about outstanding * allocations existing on successful return. */ if (tls->cipher_key_len != 0) { cipher_key = malloc(tls->cipher_key_len, M_KTLS, M_WAITOK); if (sopt->sopt_td != NULL) { error = copyin(tls->cipher_key, cipher_key, tls->cipher_key_len); if (error != 0) goto done; } else { bcopy(tls->cipher_key, cipher_key, tls->cipher_key_len); } } if (tls->iv_len != 0) { iv = malloc(tls->iv_len, M_KTLS, M_WAITOK); if (sopt->sopt_td != NULL) { error = copyin(tls->iv, iv, tls->iv_len); if (error != 0) goto done; } else { bcopy(tls->iv, iv, tls->iv_len); } } if (tls->auth_key_len != 0) { auth_key = malloc(tls->auth_key_len, M_KTLS, M_WAITOK); if (sopt->sopt_td != NULL) { error = copyin(tls->auth_key, auth_key, tls->auth_key_len); if (error != 0) goto done; } else { bcopy(tls->auth_key, auth_key, tls->auth_key_len); } } tls->cipher_key = cipher_key; tls->iv = iv; tls->auth_key = auth_key; done: if (error != 0) { zfree(cipher_key, M_KTLS); zfree(iv, M_KTLS); zfree(auth_key, M_KTLS); } return (error); } void ktls_cleanup_tls_enable(struct tls_enable *tls) { zfree(__DECONST(void *, tls->cipher_key), M_KTLS); zfree(__DECONST(void *, tls->iv), M_KTLS); zfree(__DECONST(void *, tls->auth_key), M_KTLS); } static u_int ktls_get_cpu(struct socket *so) { struct inpcb *inp; #ifdef NUMA struct ktls_domain_info *di; #endif u_int cpuid; inp = sotoinpcb(so); #ifdef RSS cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); if (cpuid != NETISR_CPUID_NONE) return (cpuid); #endif /* * Just use the flowid to shard connections in a repeatable * fashion. Note that TLS 1.0 sessions rely on the * serialization provided by having the same connection use * the same queue. */ #ifdef NUMA if (ktls_bind_threads > 1 && inp->inp_numa_domain != M_NODOM) { di = &ktls_domains[inp->inp_numa_domain]; cpuid = di->cpu[inp->inp_flowid % di->count]; } else #endif cpuid = ktls_cpuid_lookup[inp->inp_flowid % ktls_number_threads]; return (cpuid); } static int ktls_buffer_import(void *arg, void **store, int count, int domain, int flags) { vm_page_t m; int i, req; KASSERT((ktls_maxlen & PAGE_MASK) == 0, ("%s: ktls max length %d is not page size-aligned", __func__, ktls_maxlen)); req = VM_ALLOC_WIRED | VM_ALLOC_NODUMP | malloc2vm_flags(flags); for (i = 0; i < count; i++) { m = vm_page_alloc_noobj_contig_domain(domain, req, atop(ktls_maxlen), 0, ~0ul, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); if (m == NULL) break; store[i] = VM_PAGE_TO_DMAP(m); } return (i); } static void ktls_buffer_release(void *arg __unused, void **store, int count) { vm_page_t m; int i, j; for (i = 0; i < count; i++) { m = DMAP_TO_VM_PAGE(store[i]); for (j = 0; j < atop(ktls_maxlen); j++) { (void)vm_page_unwire_noq(m + j); vm_page_free(m + j); } } } static void ktls_free_mext_contig(struct mbuf *m) { M_ASSERTEXTPG(m); uma_zfree(ktls_buffer_zone, PHYS_TO_DMAP(m->m_epg_pa[0])); } static int ktls_init(void) { struct thread *td; struct pcpu *pc; int count, domain, error, i; ktls_wq = malloc(sizeof(*ktls_wq) * (mp_maxid + 1), M_KTLS, M_WAITOK | M_ZERO); ktls_session_zone = uma_zcreate("ktls_session", sizeof(struct ktls_session), NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); if (ktls_sw_buffer_cache) { ktls_buffer_zone = uma_zcache_create("ktls_buffers", roundup2(ktls_maxlen, PAGE_SIZE), NULL, NULL, NULL, NULL, ktls_buffer_import, ktls_buffer_release, NULL, UMA_ZONE_FIRSTTOUCH | UMA_ZONE_NOTRIM); } /* * Initialize the workqueues to run the TLS work. We create a * work queue for each CPU. */ CPU_FOREACH(i) { STAILQ_INIT(&ktls_wq[i].m_head); STAILQ_INIT(&ktls_wq[i].so_head); mtx_init(&ktls_wq[i].mtx, "ktls work queue", NULL, MTX_DEF); if (ktls_bind_threads > 1) { pc = pcpu_find(i); domain = pc->pc_domain; count = ktls_domains[domain].count; ktls_domains[domain].cpu[count] = i; ktls_domains[domain].count++; } ktls_cpuid_lookup[ktls_number_threads] = i; ktls_number_threads++; } /* * If we somehow have an empty domain, fall back to choosing * among all KTLS threads. */ if (ktls_bind_threads > 1) { for (i = 0; i < vm_ndomains; i++) { if (ktls_domains[i].count == 0) { ktls_bind_threads = 1; break; } } } /* Start kthreads for each workqueue. */ CPU_FOREACH(i) { error = kproc_kthread_add(ktls_work_thread, &ktls_wq[i], &ktls_proc, &td, 0, 0, "KTLS", "thr_%d", i); if (error) { printf("Can't add KTLS thread %d error %d\n", i, error); return (error); } } /* * Start an allocation thread per-domain to perform blocking allocations * of 16k physically contiguous TLS crypto destination buffers. */ if (ktls_sw_buffer_cache) { for (domain = 0; domain < vm_ndomains; domain++) { if (VM_DOMAIN_EMPTY(domain)) continue; if (CPU_EMPTY(&cpuset_domain[domain])) continue; error = kproc_kthread_add(ktls_reclaim_thread, &ktls_domains[domain], &ktls_proc, &ktls_domains[domain].reclaim_td.td, 0, 0, "KTLS", "reclaim_%d", domain); if (error) { printf("Can't add KTLS reclaim thread %d error %d\n", domain, error); return (error); } } } if (bootverbose) printf("KTLS: Initialized %d threads\n", ktls_number_threads); return (0); } static int ktls_start_kthreads(void) { int error, state; start: state = atomic_load_acq_int(&ktls_init_state); if (__predict_true(state > 0)) return (0); if (state < 0) return (ENXIO); sx_xlock(&ktls_init_lock); if (ktls_init_state != 0) { sx_xunlock(&ktls_init_lock); goto start; } error = ktls_init(); if (error == 0) state = 1; else state = -1; atomic_store_rel_int(&ktls_init_state, state); sx_xunlock(&ktls_init_lock); return (error); } static int ktls_create_session(struct socket *so, struct tls_enable *en, struct ktls_session **tlsp, int direction) { struct ktls_session *tls; int error; /* Only TLS 1.0 - 1.3 are supported. */ if (en->tls_vmajor != TLS_MAJOR_VER_ONE) return (EINVAL); if (en->tls_vminor < TLS_MINOR_VER_ZERO || en->tls_vminor > TLS_MINOR_VER_THREE) return (EINVAL); /* No flags are currently supported. */ if (en->flags != 0) return (EINVAL); /* Common checks for supported algorithms. */ switch (en->cipher_algorithm) { case CRYPTO_AES_NIST_GCM_16: /* * auth_algorithm isn't used, but permit GMAC values * for compatibility. */ switch (en->auth_algorithm) { case 0: #ifdef COMPAT_FREEBSD12 /* XXX: Really 13.0-current COMPAT. */ case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: #endif break; default: return (EINVAL); } if (en->auth_key_len != 0) return (EINVAL); switch (en->tls_vminor) { case TLS_MINOR_VER_TWO: if (en->iv_len != TLS_AEAD_GCM_LEN) return (EINVAL); break; case TLS_MINOR_VER_THREE: if (en->iv_len != TLS_1_3_GCM_IV_LEN) return (EINVAL); break; default: return (EINVAL); } break; case CRYPTO_AES_CBC: if (!ktls_cbc_enable) return (EOPNOTSUPP); switch (en->auth_algorithm) { case CRYPTO_SHA1_HMAC: break; case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: if (en->tls_vminor != TLS_MINOR_VER_TWO) return (EINVAL); break; default: return (EINVAL); } if (en->auth_key_len == 0) return (EINVAL); /* * TLS 1.0 requires an implicit IV. TLS 1.1 and 1.2 * use explicit IVs. */ switch (en->tls_vminor) { case TLS_MINOR_VER_ZERO: if (en->iv_len != TLS_CBC_IMPLICIT_IV_LEN) return (EINVAL); break; case TLS_MINOR_VER_ONE: case TLS_MINOR_VER_TWO: /* Ignore any supplied IV. */ en->iv_len = 0; break; default: return (EINVAL); } break; case CRYPTO_CHACHA20_POLY1305: if (en->auth_algorithm != 0 || en->auth_key_len != 0) return (EINVAL); if (en->tls_vminor != TLS_MINOR_VER_TWO && en->tls_vminor != TLS_MINOR_VER_THREE) return (EINVAL); if (en->iv_len != TLS_CHACHA20_IV_LEN) return (EINVAL); break; default: return (EINVAL); } error = ktls_start_kthreads(); if (error != 0) return (error); tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); counter_u64_add(ktls_offload_active, 1); refcount_init(&tls->refcount, 1); if (direction == KTLS_RX) { TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_receive_tag, tls); } else { TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls); tls->inp = so->so_pcb; in_pcbref(tls->inp); tls->tx = true; } tls->wq_index = ktls_get_cpu(so); tls->params.cipher_algorithm = en->cipher_algorithm; tls->params.auth_algorithm = en->auth_algorithm; tls->params.tls_vmajor = en->tls_vmajor; tls->params.tls_vminor = en->tls_vminor; tls->params.flags = en->flags; tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2, ktls_maxlen); /* Set the header and trailer lengths. */ tls->params.tls_hlen = sizeof(struct tls_record_layer); switch (en->cipher_algorithm) { case CRYPTO_AES_NIST_GCM_16: /* * TLS 1.2 uses a 4 byte implicit IV with an explicit 8 byte * nonce. TLS 1.3 uses a 12 byte implicit IV. */ if (en->tls_vminor < TLS_MINOR_VER_THREE) tls->params.tls_hlen += sizeof(uint64_t); tls->params.tls_tlen = AES_GMAC_HASH_LEN; tls->params.tls_bs = 1; break; case CRYPTO_AES_CBC: switch (en->auth_algorithm) { case CRYPTO_SHA1_HMAC: if (en->tls_vminor == TLS_MINOR_VER_ZERO) { /* Implicit IV, no nonce. */ tls->sequential_records = true; tls->next_seqno = be64dec(en->rec_seq); STAILQ_INIT(&tls->pending_records); } else { tls->params.tls_hlen += AES_BLOCK_LEN; } tls->params.tls_tlen = AES_BLOCK_LEN + SHA1_HASH_LEN; break; case CRYPTO_SHA2_256_HMAC: tls->params.tls_hlen += AES_BLOCK_LEN; tls->params.tls_tlen = AES_BLOCK_LEN + SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384_HMAC: tls->params.tls_hlen += AES_BLOCK_LEN; tls->params.tls_tlen = AES_BLOCK_LEN + SHA2_384_HASH_LEN; break; default: panic("invalid hmac"); } tls->params.tls_bs = AES_BLOCK_LEN; break; case CRYPTO_CHACHA20_POLY1305: /* * Chacha20 uses a 12 byte implicit IV. */ tls->params.tls_tlen = POLY1305_HASH_LEN; tls->params.tls_bs = 1; break; default: panic("invalid cipher"); } /* * TLS 1.3 includes optional padding which we do not support, * and also puts the "real" record type at the end of the * encrypted data. */ if (en->tls_vminor == TLS_MINOR_VER_THREE) tls->params.tls_tlen += sizeof(uint8_t); KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN, ("TLS header length too long: %d", tls->params.tls_hlen)); KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN, ("TLS trailer length too long: %d", tls->params.tls_tlen)); if (en->auth_key_len != 0) { tls->params.auth_key_len = en->auth_key_len; tls->params.auth_key = malloc(en->auth_key_len, M_KTLS, M_WAITOK); bcopy(en->auth_key, tls->params.auth_key, en->auth_key_len); } tls->params.cipher_key_len = en->cipher_key_len; tls->params.cipher_key = malloc(en->cipher_key_len, M_KTLS, M_WAITOK); bcopy(en->cipher_key, tls->params.cipher_key, en->cipher_key_len); /* * This holds the implicit portion of the nonce for AEAD * ciphers and the initial implicit IV for TLS 1.0. The * explicit portions of the IV are generated in ktls_frame(). */ if (en->iv_len != 0) { tls->params.iv_len = en->iv_len; bcopy(en->iv, tls->params.iv, en->iv_len); /* * For TLS 1.2 with GCM, generate an 8-byte nonce as a * counter to generate unique explicit IVs. * * Store this counter in the last 8 bytes of the IV * array so that it is 8-byte aligned. */ if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16 && en->tls_vminor == TLS_MINOR_VER_TWO) arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0); } tls->gen = 0; *tlsp = tls; return (0); } static struct ktls_session * ktls_clone_session(struct ktls_session *tls, int direction) { struct ktls_session *tls_new; tls_new = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); counter_u64_add(ktls_offload_active, 1); refcount_init(&tls_new->refcount, 1); if (direction == KTLS_RX) { TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_receive_tag, tls_new); } else { TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, tls_new); tls_new->inp = tls->inp; tls_new->tx = true; in_pcbref(tls_new->inp); } /* Copy fields from existing session. */ tls_new->params = tls->params; tls_new->wq_index = tls->wq_index; /* Deep copy keys. */ if (tls_new->params.auth_key != NULL) { tls_new->params.auth_key = malloc(tls->params.auth_key_len, M_KTLS, M_WAITOK); memcpy(tls_new->params.auth_key, tls->params.auth_key, tls->params.auth_key_len); } tls_new->params.cipher_key = malloc(tls->params.cipher_key_len, M_KTLS, M_WAITOK); memcpy(tls_new->params.cipher_key, tls->params.cipher_key, tls->params.cipher_key_len); tls_new->gen = 0; return (tls_new); } #ifdef TCP_OFFLOAD static int ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction) { struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); int error; INP_WLOCK(inp); if (tp->t_flags & TF_DISCONNECTED) { INP_WUNLOCK(inp); return (ECONNRESET); } if (!(tp->t_flags & TF_TOE)) { INP_WUNLOCK(inp); return (EOPNOTSUPP); } error = tcp_offload_alloc_tls_session(tp, tls, direction); INP_WUNLOCK(inp); if (error == 0) { tls->mode = TCP_TLS_MODE_TOE; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_toe_cbc, 1); break; case CRYPTO_AES_NIST_GCM_16: counter_u64_add(ktls_toe_gcm, 1); break; case CRYPTO_CHACHA20_POLY1305: counter_u64_add(ktls_toe_chacha20, 1); break; } } return (error); } #endif /* * Common code used when first enabling ifnet TLS on a connection or * when allocating a new ifnet TLS session due to a routing change. * This function allocates a new TLS send tag on whatever interface * the connection is currently routed over. */ static int ktls_alloc_snd_tag(struct inpcb *inp, struct ktls_session *tls, bool force, struct m_snd_tag **mstp) { union if_snd_tag_alloc_params params; struct ifnet *ifp; struct nhop_object *nh; struct tcpcb *tp = intotcpcb(inp); int error; INP_RLOCK(inp); if (tp->t_flags & TF_DISCONNECTED) { INP_RUNLOCK(inp); return (ECONNRESET); } /* * Check administrative controls on ifnet TLS to determine if * ifnet TLS should be denied. * * - Always permit 'force' requests. * - ktls_ifnet_permitted == 0: always deny. */ if (!force && ktls_ifnet_permitted == 0) { INP_RUNLOCK(inp); return (ENXIO); } /* * XXX: Use the cached route in the inpcb to find the * interface. This should perhaps instead use * rtalloc1_fib(dst, 0, 0, fibnum). Since KTLS is only * enabled after a connection has completed key negotiation in * userland, the cached route will be present in practice. */ nh = inp->inp_route.ro_nh; if (nh == NULL) { INP_RUNLOCK(inp); return (ENXIO); } ifp = nh->nh_ifp; if_ref(ifp); /* * Allocate a TLS + ratelimit tag if the connection has an * existing pacing rate. */ if (tp->t_pacing_rate != -1 && (if_getcapenable(ifp) & IFCAP_TXTLS_RTLMT) != 0) { params.hdr.type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT; params.tls_rate_limit.inp = inp; params.tls_rate_limit.tls = tls; params.tls_rate_limit.max_rate = tp->t_pacing_rate; } else { params.hdr.type = IF_SND_TAG_TYPE_TLS; params.tls.inp = inp; params.tls.tls = tls; } params.hdr.flowid = inp->inp_flowid; params.hdr.flowtype = inp->inp_flowtype; params.hdr.numa_domain = inp->inp_numa_domain; INP_RUNLOCK(inp); if ((if_getcapenable(ifp) & IFCAP_MEXTPG) == 0) { error = EOPNOTSUPP; goto out; } if (inp->inp_vflag & INP_IPV6) { if ((if_getcapenable(ifp) & IFCAP_TXTLS6) == 0) { error = EOPNOTSUPP; goto out; } } else { if ((if_getcapenable(ifp) & IFCAP_TXTLS4) == 0) { error = EOPNOTSUPP; goto out; } } error = m_snd_tag_alloc(ifp, ¶ms, mstp); out: if_rele(ifp); return (error); } /* * Allocate an initial TLS receive tag for doing HW decryption of TLS * data. * * This function allocates a new TLS receive tag on whatever interface * the connection is currently routed over. If the connection ends up * using a different interface for receive this will get fixed up via * ktls_input_ifp_mismatch as future packets arrive. */ static int ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls, struct m_snd_tag **mstp) { union if_snd_tag_alloc_params params; struct ifnet *ifp; struct nhop_object *nh; int error; if (!ktls_ocf_recrypt_supported(tls)) return (ENXIO); INP_RLOCK(inp); if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) { INP_RUNLOCK(inp); return (ECONNRESET); } /* * Check administrative controls on ifnet TLS to determine if * ifnet TLS should be denied. */ if (ktls_ifnet_permitted == 0) { INP_RUNLOCK(inp); return (ENXIO); } /* * XXX: As with ktls_alloc_snd_tag, use the cached route in * the inpcb to find the interface. */ nh = inp->inp_route.ro_nh; if (nh == NULL) { INP_RUNLOCK(inp); return (ENXIO); } ifp = nh->nh_ifp; if_ref(ifp); tls->rx_ifp = ifp; params.hdr.type = IF_SND_TAG_TYPE_TLS_RX; params.hdr.flowid = inp->inp_flowid; params.hdr.flowtype = inp->inp_flowtype; params.hdr.numa_domain = inp->inp_numa_domain; params.tls_rx.inp = inp; params.tls_rx.tls = tls; params.tls_rx.vlan_id = 0; INP_RUNLOCK(inp); if (inp->inp_vflag & INP_IPV6) { if ((if_getcapenable2(ifp) & IFCAP2_BIT(IFCAP2_RXTLS6)) == 0) { error = EOPNOTSUPP; goto out; } } else { if ((if_getcapenable2(ifp) & IFCAP2_BIT(IFCAP2_RXTLS4)) == 0) { error = EOPNOTSUPP; goto out; } } error = m_snd_tag_alloc(ifp, ¶ms, mstp); /* * If this connection is over a vlan, vlan_snd_tag_alloc * rewrites vlan_id with the saved interface. Save the VLAN * ID for use in ktls_reset_receive_tag which allocates new * receive tags directly from the leaf interface bypassing * if_vlan. */ if (error == 0) tls->rx_vlan_id = params.tls_rx.vlan_id; out: return (error); } static int ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction, bool force) { struct m_snd_tag *mst; int error; switch (direction) { case KTLS_TX: error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); if (__predict_false(error != 0)) goto done; break; case KTLS_RX: KASSERT(!force, ("%s: forced receive tag", __func__)); error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst); if (__predict_false(error != 0)) goto done; break; default: __assert_unreachable(); } tls->mode = TCP_TLS_MODE_IFNET; tls->snd_tag = mst; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_ifnet_cbc, 1); break; case CRYPTO_AES_NIST_GCM_16: counter_u64_add(ktls_ifnet_gcm, 1); break; case CRYPTO_CHACHA20_POLY1305: counter_u64_add(ktls_ifnet_chacha20, 1); break; default: break; } done: return (error); } static void ktls_use_sw(struct ktls_session *tls) { tls->mode = TCP_TLS_MODE_SW; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_sw_cbc, 1); break; case CRYPTO_AES_NIST_GCM_16: counter_u64_add(ktls_sw_gcm, 1); break; case CRYPTO_CHACHA20_POLY1305: counter_u64_add(ktls_sw_chacha20, 1); break; } } static int ktls_try_sw(struct ktls_session *tls, int direction) { int error; error = ktls_ocf_try(tls, direction); if (error) return (error); ktls_use_sw(tls); return (0); } /* * KTLS RX stores data in the socket buffer as a list of TLS records, * where each record is stored as a control message containg the TLS * header followed by data mbufs containing the decrypted data. This * is different from KTLS TX which always uses an mb_ext_pgs mbuf for * both encrypted and decrypted data. TLS records decrypted by a NIC * should be queued to the socket buffer as records, but encrypted * data which needs to be decrypted by software arrives as a stream of * regular mbufs which need to be converted. In addition, there may * already be pending encrypted data in the socket buffer when KTLS RX * is enabled. * * To manage not-yet-decrypted data for KTLS RX, the following scheme * is used: * * - A single chain of NOTREADY mbufs is hung off of sb_mtls. * * - ktls_check_rx checks this chain of mbufs reading the TLS header * from the first mbuf. Once all of the data for that TLS record is * queued, the socket is queued to a worker thread. * * - The worker thread calls ktls_decrypt to decrypt TLS records in * the TLS chain. Each TLS record is detached from the TLS chain, * decrypted, and inserted into the regular socket buffer chain as * record starting with a control message holding the TLS header and * a chain of mbufs holding the encrypted data. */ static void sb_mark_notready(struct sockbuf *sb) { struct mbuf *m; m = sb->sb_mb; sb->sb_mtls = m; sb->sb_mb = NULL; sb->sb_mbtail = NULL; sb->sb_lastrecord = NULL; for (; m != NULL; m = m->m_next) { KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt != NULL", __func__)); KASSERT((m->m_flags & M_NOTREADY) == 0, ("%s: mbuf not ready", __func__)); KASSERT(sb->sb_acc >= m->m_len, ("%s: sb_acc < m->m_len", __func__)); m->m_flags |= M_NOTREADY; sb->sb_acc -= m->m_len; sb->sb_tlscc += m->m_len; sb->sb_mtlstail = m; } KASSERT(sb->sb_acc == 0 && sb->sb_tlscc == sb->sb_ccc, ("%s: acc %u tlscc %u ccc %u", __func__, sb->sb_acc, sb->sb_tlscc, sb->sb_ccc)); } /* * Return information about the pending TLS data in a socket * buffer. On return, 'seqno' is set to the sequence number * of the next TLS record to be received, 'resid' is set to * the amount of bytes still needed for the last pending * record. The function returns 'false' if the last pending * record contains a partial TLS header. In that case, 'resid' * is the number of bytes needed to complete the TLS header. */ bool ktls_pending_rx_info(struct sockbuf *sb, uint64_t *seqnop, size_t *residp) { struct tls_record_layer hdr; struct mbuf *m; uint64_t seqno; size_t resid; u_int offset, record_len; SOCKBUF_LOCK_ASSERT(sb); MPASS(sb->sb_flags & SB_TLS_RX); seqno = sb->sb_tls_seqno; resid = sb->sb_tlscc; m = sb->sb_mtls; offset = 0; if (resid == 0) { *seqnop = seqno; *residp = 0; return (true); } for (;;) { seqno++; if (resid < sizeof(hdr)) { *seqnop = seqno; *residp = sizeof(hdr) - resid; return (false); } m_copydata(m, offset, sizeof(hdr), (void *)&hdr); record_len = sizeof(hdr) + ntohs(hdr.tls_length); if (resid <= record_len) { *seqnop = seqno; *residp = record_len - resid; return (true); } resid -= record_len; while (record_len != 0) { if (m->m_len - offset > record_len) { offset += record_len; break; } record_len -= (m->m_len - offset); offset = 0; m = m->m_next; } } } int ktls_enable_rx(struct socket *so, struct tls_enable *en) { struct ktls_session *tls; int error; - if (!ktls_offload_enable) + if (!ktls_offload_enable || !ktls_rx_offload_enable) return (ENOTSUP); counter_u64_add(ktls_offload_enable_calls, 1); /* * This should always be true since only the TCP socket option * invokes this function. */ if (so->so_proto->pr_protocol != IPPROTO_TCP) return (EINVAL); /* * XXX: Don't overwrite existing sessions. We should permit * this to support rekeying in the future. */ if (so->so_rcv.sb_tls_info != NULL) return (EALREADY); error = ktls_create_session(so, en, &tls, KTLS_RX); if (error) return (error); error = ktls_ocf_try(tls, KTLS_RX); if (error) { ktls_free(tls); return (error); } /* * Serialize with soreceive_generic() and make sure that we're not * operating on a listening socket. */ error = SOCK_IO_RECV_LOCK(so, SBL_WAIT); if (error) { ktls_free(tls); return (error); } /* Mark the socket as using TLS offload. */ SOCK_RECVBUF_LOCK(so); if (__predict_false(so->so_rcv.sb_tls_info != NULL)) error = EALREADY; else if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) error = EINVAL; if (error != 0) { SOCK_RECVBUF_UNLOCK(so); SOCK_IO_RECV_UNLOCK(so); ktls_free(tls); return (EALREADY); } so->so_rcv.sb_tls_seqno = be64dec(en->rec_seq); so->so_rcv.sb_tls_info = tls; so->so_rcv.sb_flags |= SB_TLS_RX; /* Mark existing data as not ready until it can be decrypted. */ sb_mark_notready(&so->so_rcv); ktls_check_rx(&so->so_rcv); SOCK_RECVBUF_UNLOCK(so); SOCK_IO_RECV_UNLOCK(so); /* Prefer TOE -> ifnet TLS -> software TLS. */ #ifdef TCP_OFFLOAD error = ktls_try_toe(so, tls, KTLS_RX); if (error) #endif error = ktls_try_ifnet(so, tls, KTLS_RX, false); if (error) ktls_use_sw(tls); counter_u64_add(ktls_offload_total, 1); return (0); } int ktls_enable_tx(struct socket *so, struct tls_enable *en) { struct ktls_session *tls; struct inpcb *inp; struct tcpcb *tp; int error; if (!ktls_offload_enable) return (ENOTSUP); counter_u64_add(ktls_offload_enable_calls, 1); /* * This should always be true since only the TCP socket option * invokes this function. */ if (so->so_proto->pr_protocol != IPPROTO_TCP) return (EINVAL); /* * XXX: Don't overwrite existing sessions. We should permit * this to support rekeying in the future. */ if (so->so_snd.sb_tls_info != NULL) return (EALREADY); /* TLS requires ext pgs */ if (mb_use_ext_pgs == 0) return (ENXIO); error = ktls_create_session(so, en, &tls, KTLS_TX); if (error) return (error); /* some ktls offload NICs require initial seqno to start offload */ tls->initial_offload_seqno = be64dec(en->rec_seq); /* Prefer TOE -> ifnet TLS -> software TLS. */ #ifdef TCP_OFFLOAD error = ktls_try_toe(so, tls, KTLS_TX); if (error) #endif error = ktls_try_ifnet(so, tls, KTLS_TX, false); if (error) error = ktls_try_sw(tls, KTLS_TX); if (error) { ktls_free(tls); return (error); } /* * Serialize with sosend_generic() and make sure that we're not * operating on a listening socket. */ error = SOCK_IO_SEND_LOCK(so, SBL_WAIT); if (error) { ktls_free(tls); return (error); } /* * Write lock the INP when setting sb_tls_info so that * routines in tcp_ratelimit.c can read sb_tls_info while * holding the INP lock. */ inp = so->so_pcb; INP_WLOCK(inp); SOCK_SENDBUF_LOCK(so); if (__predict_false(so->so_snd.sb_tls_info != NULL)) error = EALREADY; else if ((so->so_snd.sb_flags & SB_SPLICED) != 0) error = EINVAL; if (error != 0) { SOCK_SENDBUF_UNLOCK(so); INP_WUNLOCK(inp); SOCK_IO_SEND_UNLOCK(so); ktls_free(tls); return (error); } so->so_snd.sb_tls_seqno = be64dec(en->rec_seq); so->so_snd.sb_tls_info = tls; if (tls->mode != TCP_TLS_MODE_SW) { tp = intotcpcb(inp); MPASS(tp->t_nic_ktls_xmit == 0); tp->t_nic_ktls_xmit = 1; if (tp->t_fb->tfb_hwtls_change != NULL) (*tp->t_fb->tfb_hwtls_change)(tp, 1); } SOCK_SENDBUF_UNLOCK(so); INP_WUNLOCK(inp); SOCK_IO_SEND_UNLOCK(so); counter_u64_add(ktls_offload_total, 1); return (0); } int ktls_get_rx_mode(struct socket *so, int *modep) { struct ktls_session *tls; struct inpcb *inp __diagused; if (SOLISTENING(so)) return (EINVAL); inp = so->so_pcb; INP_WLOCK_ASSERT(inp); SOCK_RECVBUF_LOCK(so); tls = so->so_rcv.sb_tls_info; if (tls == NULL) *modep = TCP_TLS_MODE_NONE; else *modep = tls->mode; SOCK_RECVBUF_UNLOCK(so); return (0); } /* * ktls_get_rx_sequence - get the next TCP- and TLS- sequence number. * * This function gets information about the next TCP- and TLS- * sequence number to be processed by the TLS receive worker * thread. The information is extracted from the given "inpcb" * structure. The values are stored in host endian format at the two * given output pointer locations. The TCP sequence number points to * the beginning of the TLS header. * * This function returns zero on success, else a non-zero error code * is returned. */ int ktls_get_rx_sequence(struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq) { struct socket *so = inp->inp_socket; struct tcpcb *tp = intotcpcb(inp); INP_RLOCK(inp); if (tp->t_flags & TF_DISCONNECTED) { INP_RUNLOCK(inp); return (ECONNRESET); } SOCKBUF_LOCK(&so->so_rcv); *tcpseq = tp->rcv_nxt - so->so_rcv.sb_tlscc; *tlsseq = so->so_rcv.sb_tls_seqno; SOCKBUF_UNLOCK(&so->so_rcv); INP_RUNLOCK(inp); return (0); } int ktls_get_tx_mode(struct socket *so, int *modep) { struct ktls_session *tls; struct inpcb *inp __diagused; if (SOLISTENING(so)) return (EINVAL); inp = so->so_pcb; INP_WLOCK_ASSERT(inp); SOCK_SENDBUF_LOCK(so); tls = so->so_snd.sb_tls_info; if (tls == NULL) *modep = TCP_TLS_MODE_NONE; else *modep = tls->mode; SOCK_SENDBUF_UNLOCK(so); return (0); } /* * Switch between SW and ifnet TLS sessions as requested. */ int ktls_set_tx_mode(struct socket *so, int mode) { struct ktls_session *tls, *tls_new; struct inpcb *inp; struct tcpcb *tp; int error; if (SOLISTENING(so)) return (EINVAL); switch (mode) { case TCP_TLS_MODE_SW: case TCP_TLS_MODE_IFNET: break; default: return (EINVAL); } inp = so->so_pcb; INP_WLOCK_ASSERT(inp); tp = intotcpcb(inp); if (mode == TCP_TLS_MODE_IFNET) { /* Don't allow enabling ifnet ktls multiple times */ if (tp->t_nic_ktls_xmit) return (EALREADY); /* * Don't enable ifnet ktls if we disabled it due to an * excessive retransmission rate */ if (tp->t_nic_ktls_xmit_dis) return (ENXIO); } SOCKBUF_LOCK(&so->so_snd); tls = so->so_snd.sb_tls_info; if (tls == NULL) { SOCKBUF_UNLOCK(&so->so_snd); return (0); } if (tls->mode == mode) { SOCKBUF_UNLOCK(&so->so_snd); return (0); } tls = ktls_hold(tls); SOCKBUF_UNLOCK(&so->so_snd); INP_WUNLOCK(inp); tls_new = ktls_clone_session(tls, KTLS_TX); if (mode == TCP_TLS_MODE_IFNET) error = ktls_try_ifnet(so, tls_new, KTLS_TX, true); else error = ktls_try_sw(tls_new, KTLS_TX); if (error) { counter_u64_add(ktls_switch_failed, 1); ktls_free(tls_new); ktls_free(tls); INP_WLOCK(inp); return (error); } error = SOCK_IO_SEND_LOCK(so, SBL_WAIT); if (error) { counter_u64_add(ktls_switch_failed, 1); ktls_free(tls_new); ktls_free(tls); INP_WLOCK(inp); return (error); } /* * If we raced with another session change, keep the existing * session. */ if (tls != so->so_snd.sb_tls_info) { counter_u64_add(ktls_switch_failed, 1); SOCK_IO_SEND_UNLOCK(so); ktls_free(tls_new); ktls_free(tls); INP_WLOCK(inp); return (EBUSY); } INP_WLOCK(inp); SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls_new; if (tls_new->mode != TCP_TLS_MODE_SW) { MPASS(tp->t_nic_ktls_xmit == 0); tp->t_nic_ktls_xmit = 1; if (tp->t_fb->tfb_hwtls_change != NULL) (*tp->t_fb->tfb_hwtls_change)(tp, 1); } SOCKBUF_UNLOCK(&so->so_snd); SOCK_IO_SEND_UNLOCK(so); /* * Drop two references on 'tls'. The first is for the * ktls_hold() above. The second drops the reference from the * socket buffer. */ KASSERT(tls->refcount >= 2, ("too few references on old session")); ktls_free(tls); ktls_free(tls); if (mode == TCP_TLS_MODE_IFNET) counter_u64_add(ktls_switch_to_ifnet, 1); else counter_u64_add(ktls_switch_to_sw, 1); return (0); } /* * Try to allocate a new TLS receive tag. This task is scheduled when * sbappend_ktls_rx detects an input path change. If a new tag is * allocated, replace the tag in the TLS session. If a new tag cannot * be allocated, let the session fall back to software decryption. */ static void ktls_reset_receive_tag(void *context, int pending) { union if_snd_tag_alloc_params params; struct ktls_session *tls; struct m_snd_tag *mst; struct inpcb *inp; struct ifnet *ifp; struct socket *so; int error; MPASS(pending == 1); tls = context; so = tls->so; inp = so->so_pcb; ifp = NULL; INP_RLOCK(inp); if (intotcpcb(inp)->t_flags & TF_DISCONNECTED) { INP_RUNLOCK(inp); goto out; } SOCKBUF_LOCK(&so->so_rcv); mst = tls->snd_tag; tls->snd_tag = NULL; if (mst != NULL) m_snd_tag_rele(mst); ifp = tls->rx_ifp; if_ref(ifp); SOCKBUF_UNLOCK(&so->so_rcv); params.hdr.type = IF_SND_TAG_TYPE_TLS_RX; params.hdr.flowid = inp->inp_flowid; params.hdr.flowtype = inp->inp_flowtype; params.hdr.numa_domain = inp->inp_numa_domain; params.tls_rx.inp = inp; params.tls_rx.tls = tls; params.tls_rx.vlan_id = tls->rx_vlan_id; INP_RUNLOCK(inp); if (inp->inp_vflag & INP_IPV6) { if ((if_getcapenable2(ifp) & IFCAP2_RXTLS6) == 0) goto out; } else { if ((if_getcapenable2(ifp) & IFCAP2_RXTLS4) == 0) goto out; } error = m_snd_tag_alloc(ifp, ¶ms, &mst); if (error == 0) { SOCKBUF_LOCK(&so->so_rcv); tls->snd_tag = mst; SOCKBUF_UNLOCK(&so->so_rcv); counter_u64_add(ktls_ifnet_reset, 1); } else { /* * Just fall back to software decryption if a tag * cannot be allocated leaving the connection intact. * If a future input path change switches to another * interface this connection will resume ifnet TLS. */ counter_u64_add(ktls_ifnet_reset_failed, 1); } out: mtx_pool_lock(mtxpool_sleep, tls); tls->reset_pending = false; mtx_pool_unlock(mtxpool_sleep, tls); if (ifp != NULL) if_rele(ifp); CURVNET_SET(so->so_vnet); sorele(so); CURVNET_RESTORE(); ktls_free(tls); } /* * Try to allocate a new TLS send tag. This task is scheduled when * ip_output detects a route change while trying to transmit a packet * holding a TLS record. If a new tag is allocated, replace the tag * in the TLS session. Subsequent packets on the connection will use * the new tag. If a new tag cannot be allocated, drop the * connection. */ static void ktls_reset_send_tag(void *context, int pending) { struct epoch_tracker et; struct ktls_session *tls; struct m_snd_tag *old, *new; struct inpcb *inp; struct tcpcb *tp; int error; MPASS(pending == 1); tls = context; inp = tls->inp; /* * Free the old tag first before allocating a new one. * ip[6]_output_send() will treat a NULL send tag the same as * an ifp mismatch and drop packets until a new tag is * allocated. * * Write-lock the INP when changing tls->snd_tag since * ip[6]_output_send() holds a read-lock when reading the * pointer. */ INP_WLOCK(inp); old = tls->snd_tag; tls->snd_tag = NULL; INP_WUNLOCK(inp); if (old != NULL) m_snd_tag_rele(old); error = ktls_alloc_snd_tag(inp, tls, true, &new); if (error == 0) { INP_WLOCK(inp); tls->snd_tag = new; mtx_pool_lock(mtxpool_sleep, tls); tls->reset_pending = false; mtx_pool_unlock(mtxpool_sleep, tls); INP_WUNLOCK(inp); counter_u64_add(ktls_ifnet_reset, 1); /* * XXX: Should we kick tcp_output explicitly now that * the send tag is fixed or just rely on timers? */ } else { NET_EPOCH_ENTER(et); INP_WLOCK(inp); tp = intotcpcb(inp); if (!(tp->t_flags & TF_DISCONNECTED)) { CURVNET_SET(inp->inp_socket->so_vnet); tp = tcp_drop(tp, ECONNABORTED); CURVNET_RESTORE(); if (tp != NULL) { counter_u64_add(ktls_ifnet_reset_dropped, 1); INP_WUNLOCK(inp); } } else INP_WUNLOCK(inp); NET_EPOCH_EXIT(et); counter_u64_add(ktls_ifnet_reset_failed, 1); /* * Leave reset_pending true to avoid future tasks while * the socket goes away. */ } ktls_free(tls); } void ktls_input_ifp_mismatch(struct sockbuf *sb, struct ifnet *ifp) { struct ktls_session *tls; struct socket *so; SOCKBUF_LOCK_ASSERT(sb); KASSERT(sb->sb_flags & SB_TLS_RX, ("%s: sockbuf %p isn't TLS RX", __func__, sb)); so = __containerof(sb, struct socket, so_rcv); tls = sb->sb_tls_info; if_rele(tls->rx_ifp); if_ref(ifp); tls->rx_ifp = ifp; /* * See if we should schedule a task to update the receive tag for * this session. */ mtx_pool_lock(mtxpool_sleep, tls); if (!tls->reset_pending) { (void) ktls_hold(tls); soref(so); tls->so = so; tls->reset_pending = true; taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task); } mtx_pool_unlock(mtxpool_sleep, tls); } int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls) { if (inp == NULL) return (ENOBUFS); INP_LOCK_ASSERT(inp); /* * See if we should schedule a task to update the send tag for * this session. */ mtx_pool_lock(mtxpool_sleep, tls); if (!tls->reset_pending) { (void) ktls_hold(tls); tls->reset_pending = true; taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task); } mtx_pool_unlock(mtxpool_sleep, tls); return (ENOBUFS); } #ifdef RATELIMIT int ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate) { union if_snd_tag_modify_params params = { .rate_limit.max_rate = max_pacing_rate, .rate_limit.flags = M_NOWAIT, }; struct m_snd_tag *mst; /* Can't get to the inp, but it should be locked. */ /* INP_LOCK_ASSERT(inp); */ MPASS(tls->mode == TCP_TLS_MODE_IFNET); if (tls->snd_tag == NULL) { /* * Resetting send tag, ignore this change. The * pending reset may or may not see this updated rate * in the tcpcb. If it doesn't, we will just lose * this rate change. */ return (0); } mst = tls->snd_tag; MPASS(mst != NULL); MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT); return (mst->sw->snd_tag_modify(mst, ¶ms)); } #endif static void ktls_destroy_help(void *context, int pending __unused) { ktls_destroy(context); } void ktls_destroy(struct ktls_session *tls) { struct inpcb *inp; struct tcpcb *tp; bool wlocked; MPASS(tls->refcount == 0); inp = tls->inp; if (tls->tx) { wlocked = INP_WLOCKED(inp); if (!wlocked && !INP_TRY_WLOCK(inp)) { /* * rwlocks read locks are anonymous, and there * is no way to know if our current thread * holds an rlock on the inp. As a rough * estimate, check to see if the thread holds * *any* rlocks at all. If it does not, then we * know that we don't hold the inp rlock, and * can safely take the wlock */ if (curthread->td_rw_rlocks == 0) { INP_WLOCK(inp); } else { /* * We might hold the rlock, so let's * do the destroy in a taskqueue * context to avoid a potential * deadlock. This should be very * rare. */ counter_u64_add(ktls_destroy_task, 1); TASK_INIT(&tls->destroy_task, 0, ktls_destroy_help, tls); (void)taskqueue_enqueue(taskqueue_thread, &tls->destroy_task); return; } } } if (tls->sequential_records) { struct mbuf *m, *n; int page_count; STAILQ_FOREACH_SAFE(m, &tls->pending_records, m_epg_stailq, n) { page_count = m->m_epg_enc_cnt; while (page_count > 0) { KASSERT(page_count >= m->m_epg_nrdy, ("%s: too few pages", __func__)); page_count -= m->m_epg_nrdy; m = m_free(m); } } } counter_u64_add(ktls_offload_active, -1); switch (tls->mode) { case TCP_TLS_MODE_SW: switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_sw_cbc, -1); break; case CRYPTO_AES_NIST_GCM_16: counter_u64_add(ktls_sw_gcm, -1); break; case CRYPTO_CHACHA20_POLY1305: counter_u64_add(ktls_sw_chacha20, -1); break; } break; case TCP_TLS_MODE_IFNET: switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_ifnet_cbc, -1); break; case CRYPTO_AES_NIST_GCM_16: counter_u64_add(ktls_ifnet_gcm, -1); break; case CRYPTO_CHACHA20_POLY1305: counter_u64_add(ktls_ifnet_chacha20, -1); break; } if (tls->snd_tag != NULL) m_snd_tag_rele(tls->snd_tag); if (tls->rx_ifp != NULL) if_rele(tls->rx_ifp); if (tls->tx) { INP_WLOCK_ASSERT(inp); tp = intotcpcb(inp); MPASS(tp->t_nic_ktls_xmit == 1); tp->t_nic_ktls_xmit = 0; } break; #ifdef TCP_OFFLOAD case TCP_TLS_MODE_TOE: switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_toe_cbc, -1); break; case CRYPTO_AES_NIST_GCM_16: counter_u64_add(ktls_toe_gcm, -1); break; case CRYPTO_CHACHA20_POLY1305: counter_u64_add(ktls_toe_chacha20, -1); break; } break; #endif } if (tls->ocf_session != NULL) ktls_ocf_free(tls); if (tls->params.auth_key != NULL) { zfree(tls->params.auth_key, M_KTLS); tls->params.auth_key = NULL; tls->params.auth_key_len = 0; } if (tls->params.cipher_key != NULL) { zfree(tls->params.cipher_key, M_KTLS); tls->params.cipher_key = NULL; tls->params.cipher_key_len = 0; } if (tls->tx) { INP_WLOCK_ASSERT(inp); if (!in_pcbrele_wlocked(inp) && !wlocked) INP_WUNLOCK(inp); } explicit_bzero(tls->params.iv, sizeof(tls->params.iv)); uma_zfree(ktls_session_zone, tls); } void ktls_seq(struct sockbuf *sb, struct mbuf *m) { for (; m != NULL; m = m->m_next) { KASSERT((m->m_flags & M_EXTPG) != 0, ("ktls_seq: mapped mbuf %p", m)); m->m_epg_seqno = sb->sb_tls_seqno; sb->sb_tls_seqno++; } } /* * Add TLS framing (headers and trailers) to a chain of mbufs. Each * mbuf in the chain must be an unmapped mbuf. The payload of the * mbuf must be populated with the payload of each TLS record. * * The record_type argument specifies the TLS record type used when * populating the TLS header. * * The enq_count argument on return is set to the number of pages of * payload data for this entire chain that need to be encrypted via SW * encryption. The returned value should be passed to ktls_enqueue * when scheduling encryption of this chain of mbufs. To handle the * special case of empty fragments for TLS 1.0 sessions, an empty * fragment counts as one page. */ void ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt, uint8_t record_type) { struct tls_record_layer *tlshdr; struct mbuf *m; uint64_t *noncep; uint16_t tls_len; int maxlen __diagused; maxlen = tls->params.max_frame_len; *enq_cnt = 0; for (m = top; m != NULL; m = m->m_next) { /* * All mbufs in the chain should be TLS records whose * payload does not exceed the maximum frame length. * * Empty TLS 1.0 records are permitted when using CBC. */ KASSERT(m->m_len <= maxlen && m->m_len >= 0 && (m->m_len > 0 || ktls_permit_empty_frames(tls)), ("ktls_frame: m %p len %d", m, m->m_len)); /* * TLS frames require unmapped mbufs to store session * info. */ KASSERT((m->m_flags & M_EXTPG) != 0, ("ktls_frame: mapped mbuf %p (top = %p)", m, top)); tls_len = m->m_len; /* Save a reference to the session. */ m->m_epg_tls = ktls_hold(tls); m->m_epg_hdrlen = tls->params.tls_hlen; m->m_epg_trllen = tls->params.tls_tlen; if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) { int bs, delta; /* * AES-CBC pads messages to a multiple of the * block size. Note that the padding is * applied after the digest and the encryption * is done on the "plaintext || mac || padding". * At least one byte of padding is always * present. * * Compute the final trailer length assuming * at most one block of padding. * tls->params.tls_tlen is the maximum * possible trailer length (padding + digest). * delta holds the number of excess padding * bytes if the maximum were used. Those * extra bytes are removed. */ bs = tls->params.tls_bs; delta = (tls_len + tls->params.tls_tlen) & (bs - 1); m->m_epg_trllen -= delta; } m->m_len += m->m_epg_hdrlen + m->m_epg_trllen; /* Populate the TLS header. */ tlshdr = (void *)m->m_epg_hdr; tlshdr->tls_vmajor = tls->params.tls_vmajor; /* * TLS 1.3 masquarades as TLS 1.2 with a record type * of TLS_RLTYPE_APP. */ if (tls->params.tls_vminor == TLS_MINOR_VER_THREE && tls->params.tls_vmajor == TLS_MAJOR_VER_ONE) { tlshdr->tls_vminor = TLS_MINOR_VER_TWO; tlshdr->tls_type = TLS_RLTYPE_APP; /* save the real record type for later */ m->m_epg_record_type = record_type; m->m_epg_trail[0] = record_type; } else { tlshdr->tls_vminor = tls->params.tls_vminor; tlshdr->tls_type = record_type; } tlshdr->tls_length = htons(m->m_len - sizeof(*tlshdr)); /* * Store nonces / explicit IVs after the end of the * TLS header. * * For GCM with TLS 1.2, an 8 byte nonce is copied * from the end of the IV. The nonce is then * incremented for use by the next record. * * For CBC, a random nonce is inserted for TLS 1.1+. */ if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 && tls->params.tls_vminor == TLS_MINOR_VER_TWO) { noncep = (uint64_t *)(tls->params.iv + 8); be64enc(tlshdr + 1, *noncep); (*noncep)++; } else if (tls->params.cipher_algorithm == CRYPTO_AES_CBC && tls->params.tls_vminor >= TLS_MINOR_VER_ONE) arc4rand(tlshdr + 1, AES_BLOCK_LEN, 0); /* * When using SW encryption, mark the mbuf not ready. * It will be marked ready via sbready() after the * record has been encrypted. * * When using ifnet TLS, unencrypted TLS records are * sent down the stack to the NIC. */ if (tls->mode == TCP_TLS_MODE_SW) { m->m_flags |= M_NOTREADY; if (__predict_false(tls_len == 0)) { /* TLS 1.0 empty fragment. */ m->m_epg_nrdy = 1; } else m->m_epg_nrdy = m->m_epg_npgs; *enq_cnt += m->m_epg_nrdy; } } } bool ktls_permit_empty_frames(struct ktls_session *tls) { return (tls->params.cipher_algorithm == CRYPTO_AES_CBC && tls->params.tls_vminor == TLS_MINOR_VER_ZERO); } void ktls_check_rx(struct sockbuf *sb) { struct tls_record_layer hdr; struct ktls_wq *wq; struct socket *so; bool running; SOCKBUF_LOCK_ASSERT(sb); KASSERT(sb->sb_flags & SB_TLS_RX, ("%s: sockbuf %p isn't TLS RX", __func__, sb)); so = __containerof(sb, struct socket, so_rcv); if (sb->sb_flags & SB_TLS_RX_RUNNING) return; /* Is there enough queued for a TLS header? */ if (sb->sb_tlscc < sizeof(hdr)) { if ((sb->sb_state & SBS_CANTRCVMORE) != 0 && sb->sb_tlscc != 0) so->so_error = EMSGSIZE; return; } m_copydata(sb->sb_mtls, 0, sizeof(hdr), (void *)&hdr); /* Is the entire record queued? */ if (sb->sb_tlscc < sizeof(hdr) + ntohs(hdr.tls_length)) { if ((sb->sb_state & SBS_CANTRCVMORE) != 0) so->so_error = EMSGSIZE; return; } sb->sb_flags |= SB_TLS_RX_RUNNING; soref(so); wq = &ktls_wq[so->so_rcv.sb_tls_info->wq_index]; mtx_lock(&wq->mtx); STAILQ_INSERT_TAIL(&wq->so_head, so, so_ktls_rx_list); running = wq->running; mtx_unlock(&wq->mtx); if (!running) wakeup(wq); counter_u64_add(ktls_cnt_rx_queued, 1); } static struct mbuf * ktls_detach_record(struct sockbuf *sb, int len) { struct mbuf *m, *n, *top; int remain; SOCKBUF_LOCK_ASSERT(sb); MPASS(len <= sb->sb_tlscc); /* * If TLS chain is the exact size of the record, * just grab the whole record. */ top = sb->sb_mtls; if (sb->sb_tlscc == len) { sb->sb_mtls = NULL; sb->sb_mtlstail = NULL; goto out; } /* * While it would be nice to use m_split() here, we need * to know exactly what m_split() allocates to update the * accounting, so do it inline instead. */ remain = len; for (m = top; remain > m->m_len; m = m->m_next) remain -= m->m_len; /* Easy case: don't have to split 'm'. */ if (remain == m->m_len) { sb->sb_mtls = m->m_next; if (sb->sb_mtls == NULL) sb->sb_mtlstail = NULL; m->m_next = NULL; goto out; } /* * Need to allocate an mbuf to hold the remainder of 'm'. Try * with M_NOWAIT first. */ n = m_get(M_NOWAIT, MT_DATA); if (n == NULL) { /* * Use M_WAITOK with socket buffer unlocked. If * 'sb_mtls' changes while the lock is dropped, return * NULL to force the caller to retry. */ SOCKBUF_UNLOCK(sb); n = m_get(M_WAITOK, MT_DATA); SOCKBUF_LOCK(sb); if (sb->sb_mtls != top) { m_free(n); return (NULL); } } n->m_flags |= (m->m_flags & (M_NOTREADY | M_DECRYPTED)); /* Store remainder in 'n'. */ n->m_len = m->m_len - remain; if (m->m_flags & M_EXT) { n->m_data = m->m_data + remain; mb_dupcl(n, m); } else { bcopy(mtod(m, caddr_t) + remain, mtod(n, caddr_t), n->m_len); } /* Trim 'm' and update accounting. */ m->m_len -= n->m_len; sb->sb_tlscc -= n->m_len; sb->sb_ccc -= n->m_len; /* Account for 'n'. */ sballoc_ktls_rx(sb, n); /* Insert 'n' into the TLS chain. */ sb->sb_mtls = n; n->m_next = m->m_next; if (sb->sb_mtlstail == m) sb->sb_mtlstail = n; /* Detach the record from the TLS chain. */ m->m_next = NULL; out: MPASS(m_length(top, NULL) == len); for (m = top; m != NULL; m = m->m_next) sbfree_ktls_rx(sb, m); sb->sb_tlsdcc = len; sb->sb_ccc += len; SBCHECK(sb); return (top); } /* * Determine the length of the trailing zero padding and find the real * record type in the byte before the padding. * * Walking the mbuf chain backwards is clumsy, so another option would * be to scan forwards remembering the last non-zero byte before the * trailer. However, it would be expensive to scan the entire record. * Instead, find the last non-zero byte of each mbuf in the chain * keeping track of the relative offset of that nonzero byte. * * trail_len is the size of the MAC/tag on input and is set to the * size of the full trailer including padding and the record type on * return. */ static int tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len, int *trailer_len, uint8_t *record_typep) { char *cp; u_int digest_start, last_offset, m_len, offset; uint8_t record_type; digest_start = tls_len - *trailer_len; last_offset = 0; offset = 0; for (; m != NULL && offset < digest_start; offset += m->m_len, m = m->m_next) { /* Don't look for padding in the tag. */ m_len = min(digest_start - offset, m->m_len); cp = mtod(m, char *); /* Find last non-zero byte in this mbuf. */ while (m_len > 0 && cp[m_len - 1] == 0) m_len--; if (m_len > 0) { record_type = cp[m_len - 1]; last_offset = offset + m_len; } } if (last_offset < tls->params.tls_hlen) return (EBADMSG); *record_typep = record_type; *trailer_len = tls_len - last_offset + 1; return (0); } /* * Check if a mbuf chain is fully decrypted. Returns * KTLS_MBUF_CRYPTO_ST_DECRYPTED if all data is decrypted. * KTLS_MBUF_CRYPTO_ST_MIXED if there is a mix of encrypted and * decrypted data. KTLS_MBUF_CRYPTO_ST_ENCRYPTED if all data is * encrypted. KTLS_MBUF_CRYPTO_ST_SHAREDMBUF if any mbuf points at * shared data that must not be modified in place (non-anonymous * M_EXTPG or sendfile M_EXT buffers). */ static ktls_mbuf_crypto_st_t ktls_mbuf_crypto_state(struct mbuf *mb) { bool seen_decrypted, seen_encrypted; seen_decrypted = false; seen_encrypted = false; for (; mb != NULL; mb = mb->m_next) { if ((mb->m_flags & M_EXTPG) != 0 && (mb->m_epg_flags & EPG_FLAG_ANON) == 0) return (KTLS_MBUF_CRYPTO_ST_SHAREDMBUF); if ((mb->m_flags & M_EXT) != 0 && mb->m_ext.ext_type == EXT_SFBUF) return (KTLS_MBUF_CRYPTO_ST_SHAREDMBUF); if (mb->m_flags & M_DECRYPTED) seen_decrypted = true; else seen_encrypted = true; } if (seen_decrypted && seen_encrypted) return (KTLS_MBUF_CRYPTO_ST_MIXED); else return (seen_decrypted ? KTLS_MBUF_CRYPTO_ST_DECRYPTED : KTLS_MBUF_CRYPTO_ST_ENCRYPTED); } /* * ktls_resync_ifnet - get HW TLS RX back on track after packet loss */ static int ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num) { union if_snd_tag_modify_params params; struct m_snd_tag *mst; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); mst = so->so_rcv.sb_tls_info->snd_tag; if (__predict_false(mst == NULL)) return (EINVAL); INP_RLOCK(inp); if (tp->t_flags & TF_DISCONNECTED) { INP_RUNLOCK(inp); return (ECONNRESET); } /* Get the TCP sequence number of the next valid TLS header. */ SOCKBUF_LOCK(&so->so_rcv); params.tls_rx.tls_hdr_tcp_sn = tp->rcv_nxt - so->so_rcv.sb_tlscc - tls_len; params.tls_rx.tls_rec_length = tls_len; params.tls_rx.tls_seq_number = tls_rcd_num; SOCKBUF_UNLOCK(&so->so_rcv); INP_RUNLOCK(inp); MPASS(mst->sw->type == IF_SND_TAG_TYPE_TLS_RX); return (mst->sw->snd_tag_modify(mst, ¶ms)); } static void ktls_drop(struct socket *so, int error) { struct epoch_tracker et; struct inpcb *inp = sotoinpcb(so); struct tcpcb *tp = intotcpcb(inp); NET_EPOCH_ENTER(et); INP_WLOCK(inp); if (!(tp->t_flags & TF_DISCONNECTED)) { CURVNET_SET(inp->inp_socket->so_vnet); tp = tcp_drop(tp, error); CURVNET_RESTORE(); if (tp != NULL) INP_WUNLOCK(inp); } else { so->so_error = error; SOCK_RECVBUF_LOCK(so); sorwakeup_locked(so); INP_WUNLOCK(inp); } NET_EPOCH_EXIT(et); } static void ktls_decrypt(struct socket *so) { char tls_header[MBUF_PEXT_HDR_LEN]; struct ktls_session *tls; struct sockbuf *sb; struct tls_record_layer *hdr; struct tls_get_record tgr; struct mbuf *control, *data, *m; ktls_mbuf_crypto_st_t state; uint64_t seqno; int error, remain, tls_len, trail_len; bool tls13; uint8_t vminor, record_type; hdr = (struct tls_record_layer *)tls_header; sb = &so->so_rcv; SOCKBUF_LOCK(sb); KASSERT(sb->sb_flags & SB_TLS_RX_RUNNING, ("%s: socket %p not running", __func__, so)); tls = sb->sb_tls_info; MPASS(tls != NULL); tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE); if (tls13) vminor = TLS_MINOR_VER_TWO; else vminor = tls->params.tls_vminor; for (;;) { /* Is there enough queued for a TLS header? */ if (sb->sb_tlscc < tls->params.tls_hlen) break; m_copydata(sb->sb_mtls, 0, tls->params.tls_hlen, tls_header); tls_len = sizeof(*hdr) + ntohs(hdr->tls_length); if (hdr->tls_vmajor != tls->params.tls_vmajor || hdr->tls_vminor != vminor) error = EINVAL; else if (tls13 && hdr->tls_type != TLS_RLTYPE_APP) error = EINVAL; else if (tls_len < tls->params.tls_hlen || tls_len > tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 + tls->params.tls_tlen) error = EMSGSIZE; else error = 0; if (__predict_false(error != 0)) { /* * We have a corrupted record and are likely * out of sync. The connection isn't * recoverable at this point, so abort it. */ SOCKBUF_UNLOCK(sb); counter_u64_add(ktls_offload_corrupted_records, 1); ktls_drop(so, error); goto deref; } /* Is the entire record queued? */ if (sb->sb_tlscc < tls_len) break; /* * Split out the portion of the mbuf chain containing * this TLS record. */ data = ktls_detach_record(sb, tls_len); if (data == NULL) continue; MPASS(sb->sb_tlsdcc == tls_len); seqno = sb->sb_tls_seqno; sb->sb_tls_seqno++; SBCHECK(sb); SOCKBUF_UNLOCK(sb); /* get crypto state for this TLS record */ state = ktls_mbuf_crypto_state(data); switch (state) { case KTLS_MBUF_CRYPTO_ST_MIXED: error = ktls_ocf_recrypt(tls, hdr, data, seqno); if (error) break; /* FALLTHROUGH */ case KTLS_MBUF_CRYPTO_ST_ENCRYPTED: error = ktls_ocf_decrypt(tls, hdr, data, seqno, &trail_len); if (__predict_true(error == 0)) { if (tls13) { error = tls13_find_record_type(tls, data, tls_len, &trail_len, &record_type); } else { record_type = hdr->tls_type; } } break; case KTLS_MBUF_CRYPTO_ST_DECRYPTED: /* * NIC TLS is only supported for AEAD * ciphersuites which used a fixed sized * trailer. */ if (tls13) { trail_len = tls->params.tls_tlen - 1; error = tls13_find_record_type(tls, data, tls_len, &trail_len, &record_type); } else { trail_len = tls->params.tls_tlen; error = 0; record_type = hdr->tls_type; } break; case KTLS_MBUF_CRYPTO_ST_SHAREDMBUF: error = EINVAL; break; default: __assert_unreachable(); } if (error) { counter_u64_add(ktls_offload_failed_crypto, 1); SOCKBUF_LOCK(sb); if (sb->sb_tlsdcc == 0) { /* * sbcut/drop/flush discarded these * mbufs. */ m_freem(data); break; } /* * Drop this TLS record's data, but keep * decrypting subsequent records. */ sb->sb_ccc -= tls_len; sb->sb_tlsdcc = 0; if (error != EMSGSIZE) error = EBADMSG; CURVNET_SET(so->so_vnet); so->so_error = error; sorwakeup_locked(so); CURVNET_RESTORE(); m_freem(data); SOCKBUF_LOCK(sb); continue; } /* Allocate the control mbuf. */ memset(&tgr, 0, sizeof(tgr)); tgr.tls_type = record_type; tgr.tls_vmajor = hdr->tls_vmajor; tgr.tls_vminor = hdr->tls_vminor; tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen - trail_len); control = sbcreatecontrol(&tgr, sizeof(tgr), TLS_GET_RECORD, IPPROTO_TCP, M_WAITOK); SOCKBUF_LOCK(sb); if (sb->sb_tlsdcc == 0) { /* sbcut/drop/flush discarded these mbufs. */ MPASS(sb->sb_tlscc == 0); m_freem(data); m_freem(control); break; } /* * Clear the 'dcc' accounting in preparation for * adding the decrypted record. */ sb->sb_ccc -= tls_len; sb->sb_tlsdcc = 0; SBCHECK(sb); /* If there is no payload, drop all of the data. */ if (tgr.tls_length == htobe16(0)) { m_freem(data); data = NULL; } else { /* Trim header. */ remain = tls->params.tls_hlen; while (remain > 0) { if (data->m_len > remain) { data->m_data += remain; data->m_len -= remain; break; } remain -= data->m_len; data = m_free(data); } /* Trim trailer and clear M_NOTREADY. */ remain = be16toh(tgr.tls_length); m = data; for (m = data; remain > m->m_len; m = m->m_next) { m->m_flags &= ~(M_NOTREADY | M_DECRYPTED); remain -= m->m_len; } m->m_len = remain; m_freem(m->m_next); m->m_next = NULL; m->m_flags &= ~(M_NOTREADY | M_DECRYPTED); /* Set EOR on the final mbuf. */ m->m_flags |= M_EOR; } sbappendcontrol_locked(sb, data, control, 0); if (__predict_false(state != KTLS_MBUF_CRYPTO_ST_DECRYPTED)) { sb->sb_flags |= SB_TLS_RX_RESYNC; SOCKBUF_UNLOCK(sb); ktls_resync_ifnet(so, tls_len, seqno); SOCKBUF_LOCK(sb); } else if (__predict_false(sb->sb_flags & SB_TLS_RX_RESYNC)) { sb->sb_flags &= ~SB_TLS_RX_RESYNC; SOCKBUF_UNLOCK(sb); ktls_resync_ifnet(so, 0, seqno); SOCKBUF_LOCK(sb); } } sb->sb_flags &= ~SB_TLS_RX_RUNNING; if ((sb->sb_state & SBS_CANTRCVMORE) != 0 && sb->sb_tlscc > 0) so->so_error = EMSGSIZE; sorwakeup_locked(so); deref: SOCKBUF_UNLOCK_ASSERT(sb); CURVNET_SET(so->so_vnet); sorele(so); CURVNET_RESTORE(); } void ktls_enqueue_to_free(struct mbuf *m) { struct ktls_wq *wq; bool running; /* Mark it for freeing. */ m->m_epg_flags |= EPG_FLAG_2FREE; wq = &ktls_wq[m->m_epg_tls->wq_index]; mtx_lock(&wq->mtx); STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq); running = wq->running; mtx_unlock(&wq->mtx); if (!running) wakeup(wq); } static void * ktls_buffer_alloc(struct ktls_wq *wq, struct mbuf *m) { void *buf; int domain, running; if (m->m_epg_npgs <= 2) return (NULL); if (ktls_buffer_zone == NULL) return (NULL); if ((u_int)(ticks - wq->lastallocfail) < hz) { /* * Rate-limit allocation attempts after a failure. * ktls_buffer_import() will acquire a per-domain mutex to check * the free page queues and may fail consistently if memory is * fragmented. */ return (NULL); } buf = uma_zalloc(ktls_buffer_zone, M_NOWAIT | M_NORECLAIM); if (buf == NULL) { domain = PCPU_GET(domain); wq->lastallocfail = ticks; /* * Note that this check is "racy", but the races are * harmless, and are either a spurious wakeup if * multiple threads fail allocations before the alloc * thread wakes, or waiting an extra second in case we * see an old value of running == true. */ if (!VM_DOMAIN_EMPTY(domain)) { running = atomic_load_int(&ktls_domains[domain].reclaim_td.running); if (!running) wakeup(&ktls_domains[domain].reclaim_td); } } return (buf); } static int ktls_encrypt_record(struct ktls_wq *wq, struct mbuf *m, struct ktls_session *tls, struct ktls_ocf_encrypt_state *state) { vm_page_t pg; int error, i, len, off; KASSERT((m->m_flags & (M_EXTPG | M_NOTREADY)) == (M_EXTPG | M_NOTREADY), ("%p not unready & nomap mbuf\n", m)); KASSERT(ptoa(m->m_epg_npgs) <= ktls_maxlen, ("page count %d larger than maximum frame length %d", m->m_epg_npgs, ktls_maxlen)); /* Anonymous mbufs are encrypted in place. */ if ((m->m_epg_flags & EPG_FLAG_ANON) != 0) return (ktls_ocf_encrypt(state, tls, m, NULL, 0)); /* * For file-backed mbufs (from sendfile), anonymous wired * pages are allocated and used as the encryption destination. */ if ((state->cbuf = ktls_buffer_alloc(wq, m)) != NULL) { len = ptoa(m->m_epg_npgs - 1) + m->m_epg_last_len - m->m_epg_1st_off; state->dst_iov[0].iov_base = (char *)state->cbuf + m->m_epg_1st_off; state->dst_iov[0].iov_len = len; state->parray[0] = DMAP_TO_PHYS(state->cbuf); i = 1; } else { off = m->m_epg_1st_off; for (i = 0; i < m->m_epg_npgs; i++, off = 0) { pg = vm_page_alloc_noobj(VM_ALLOC_NODUMP | VM_ALLOC_WIRED | VM_ALLOC_WAITOK); len = m_epg_pagelen(m, i, off); state->parray[i] = VM_PAGE_TO_PHYS(pg); state->dst_iov[i].iov_base = (char *)PHYS_TO_DMAP(state->parray[i]) + off; state->dst_iov[i].iov_len = len; } } KASSERT(i + 1 <= nitems(state->dst_iov), ("dst_iov is too small")); state->dst_iov[i].iov_base = m->m_epg_trail; state->dst_iov[i].iov_len = m->m_epg_trllen; error = ktls_ocf_encrypt(state, tls, m, state->dst_iov, i + 1); if (__predict_false(error != 0)) { /* Free the anonymous pages. */ if (state->cbuf != NULL) uma_zfree(ktls_buffer_zone, state->cbuf); else { for (i = 0; i < m->m_epg_npgs; i++) { pg = PHYS_TO_VM_PAGE(state->parray[i]); (void)vm_page_unwire_noq(pg); vm_page_free(pg); } } } return (error); } /* Number of TLS records in a batch passed to ktls_enqueue(). */ static u_int ktls_batched_records(struct mbuf *m) { int page_count, records; records = 0; page_count = m->m_epg_enc_cnt; while (page_count > 0) { records++; page_count -= m->m_epg_nrdy; m = m->m_next; } KASSERT(page_count == 0, ("%s: mismatched page count", __func__)); return (records); } void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count) { struct ktls_session *tls; struct ktls_wq *wq; int queued; bool running; KASSERT(((m->m_flags & (M_EXTPG | M_NOTREADY)) == (M_EXTPG | M_NOTREADY)), ("ktls_enqueue: %p not unready & nomap mbuf\n", m)); KASSERT(page_count != 0, ("enqueueing TLS mbuf with zero page count")); KASSERT(m->m_epg_tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf")); m->m_epg_enc_cnt = page_count; /* * Save a pointer to the socket. The caller is responsible * for taking an additional reference via soref(). */ m->m_epg_so = so; queued = 1; tls = m->m_epg_tls; wq = &ktls_wq[tls->wq_index]; mtx_lock(&wq->mtx); if (__predict_false(tls->sequential_records)) { /* * For TLS 1.0, records must be encrypted * sequentially. For a given connection, all records * queued to the associated work queue are processed * sequentially. However, sendfile(2) might complete * I/O requests spanning multiple TLS records out of * order. Here we ensure TLS records are enqueued to * the work queue in FIFO order. * * tls->next_seqno holds the sequence number of the * next TLS record that should be enqueued to the work * queue. If this next record is not tls->next_seqno, * it must be a future record, so insert it, sorted by * TLS sequence number, into tls->pending_records and * return. * * If this TLS record matches tls->next_seqno, place * it in the work queue and then check * tls->pending_records to see if any * previously-queued records are now ready for * encryption. */ if (m->m_epg_seqno != tls->next_seqno) { struct mbuf *n, *p; p = NULL; STAILQ_FOREACH(n, &tls->pending_records, m_epg_stailq) { if (n->m_epg_seqno > m->m_epg_seqno) break; p = n; } if (n == NULL) STAILQ_INSERT_TAIL(&tls->pending_records, m, m_epg_stailq); else if (p == NULL) STAILQ_INSERT_HEAD(&tls->pending_records, m, m_epg_stailq); else STAILQ_INSERT_AFTER(&tls->pending_records, p, m, m_epg_stailq); mtx_unlock(&wq->mtx); counter_u64_add(ktls_cnt_tx_pending, 1); return; } tls->next_seqno += ktls_batched_records(m); STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq); while (!STAILQ_EMPTY(&tls->pending_records)) { struct mbuf *n; n = STAILQ_FIRST(&tls->pending_records); if (n->m_epg_seqno != tls->next_seqno) break; queued++; STAILQ_REMOVE_HEAD(&tls->pending_records, m_epg_stailq); tls->next_seqno += ktls_batched_records(n); STAILQ_INSERT_TAIL(&wq->m_head, n, m_epg_stailq); } counter_u64_add(ktls_cnt_tx_pending, -(queued - 1)); } else STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq); running = wq->running; mtx_unlock(&wq->mtx); if (!running) wakeup(wq); counter_u64_add(ktls_cnt_tx_queued, queued); } /* * Once a file-backed mbuf (from sendfile) has been encrypted, free * the pages from the file and replace them with the anonymous pages * allocated in ktls_encrypt_record(). */ static void ktls_finish_nonanon(struct mbuf *m, struct ktls_ocf_encrypt_state *state) { int i; MPASS((m->m_epg_flags & EPG_FLAG_ANON) == 0); /* Free the old pages. */ m->m_ext.ext_free(m); /* Replace them with the new pages. */ if (state->cbuf != NULL) { for (i = 0; i < m->m_epg_npgs; i++) m->m_epg_pa[i] = state->parray[0] + ptoa(i); /* Contig pages should go back to the cache. */ m->m_ext.ext_free = ktls_free_mext_contig; } else { for (i = 0; i < m->m_epg_npgs; i++) m->m_epg_pa[i] = state->parray[i]; /* Use the basic free routine. */ m->m_ext.ext_free = mb_free_mext_pgs; } /* Pages are now writable. */ m->m_epg_flags |= EPG_FLAG_ANON; } static __noinline void ktls_encrypt(struct ktls_wq *wq, struct mbuf *top) { struct ktls_ocf_encrypt_state state; struct ktls_session *tls; struct socket *so; struct mbuf *m; int error, npages, total_pages; so = top->m_epg_so; tls = top->m_epg_tls; KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top)); KASSERT(so != NULL, ("so = NULL, top = %p\n", top)); #ifdef INVARIANTS top->m_epg_so = NULL; #endif total_pages = top->m_epg_enc_cnt; npages = 0; /* * Encrypt the TLS records in the chain of mbufs starting with * 'top'. 'total_pages' gives us a total count of pages and is * used to know when we have finished encrypting the TLS * records originally queued with 'top'. * * NB: These mbufs are queued in the socket buffer and * 'm_next' is traversing the mbufs in the socket buffer. The * socket buffer lock is not held while traversing this chain. * Since the mbufs are all marked M_NOTREADY their 'm_next' * pointers should be stable. However, the 'm_next' of the * last mbuf encrypted is not necessarily NULL. It can point * to other mbufs appended while 'top' was on the TLS work * queue. * * Each mbuf holds an entire TLS record. */ error = 0; for (m = top; npages != total_pages; m = m->m_next) { KASSERT(m->m_epg_tls == tls, ("different TLS sessions in a single mbuf chain: %p vs %p", tls, m->m_epg_tls)); KASSERT(npages + m->m_epg_npgs <= total_pages, ("page count mismatch: top %p, total_pages %d, m %p", top, total_pages, m)); error = ktls_encrypt_record(wq, m, tls, &state); if (error) { counter_u64_add(ktls_offload_failed_crypto, 1); break; } if ((m->m_epg_flags & EPG_FLAG_ANON) == 0) ktls_finish_nonanon(m, &state); m->m_flags |= M_RDONLY; npages += m->m_epg_nrdy; /* * Drop a reference to the session now that it is no * longer needed. Existing code depends on encrypted * records having no associated session vs * yet-to-be-encrypted records having an associated * session. */ m->m_epg_tls = NULL; ktls_free(tls); } CURVNET_SET(so->so_vnet); if (error == 0) { (void)so->so_proto->pr_ready(so, top, npages); } else { ktls_drop(so, EIO); mb_free_notready(top, total_pages); } sorele(so); CURVNET_RESTORE(); } void ktls_encrypt_cb(struct ktls_ocf_encrypt_state *state, int error) { struct ktls_session *tls; struct socket *so; struct mbuf *m; int npages; m = state->m; if ((m->m_epg_flags & EPG_FLAG_ANON) == 0) ktls_finish_nonanon(m, state); m->m_flags |= M_RDONLY; so = state->so; free(state, M_KTLS); /* * Drop a reference to the session now that it is no longer * needed. Existing code depends on encrypted records having * no associated session vs yet-to-be-encrypted records having * an associated session. */ tls = m->m_epg_tls; m->m_epg_tls = NULL; ktls_free(tls); if (error != 0) counter_u64_add(ktls_offload_failed_crypto, 1); CURVNET_SET(so->so_vnet); npages = m->m_epg_nrdy; if (error == 0) { (void)so->so_proto->pr_ready(so, m, npages); } else { ktls_drop(so, EIO); mb_free_notready(m, npages); } sorele(so); CURVNET_RESTORE(); } /* * Similar to ktls_encrypt, but used with asynchronous OCF backends * (coprocessors) where encryption does not use host CPU resources and * it can be beneficial to queue more requests than CPUs. */ static __noinline void ktls_encrypt_async(struct ktls_wq *wq, struct mbuf *top) { struct ktls_ocf_encrypt_state *state; struct ktls_session *tls; struct socket *so; struct mbuf *m, *n; int error, mpages, npages, total_pages; so = top->m_epg_so; tls = top->m_epg_tls; KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top)); KASSERT(so != NULL, ("so = NULL, top = %p\n", top)); #ifdef INVARIANTS top->m_epg_so = NULL; #endif total_pages = top->m_epg_enc_cnt; npages = 0; error = 0; for (m = top; npages != total_pages; m = n) { KASSERT(m->m_epg_tls == tls, ("different TLS sessions in a single mbuf chain: %p vs %p", tls, m->m_epg_tls)); KASSERT(npages + m->m_epg_npgs <= total_pages, ("page count mismatch: top %p, total_pages %d, m %p", top, total_pages, m)); state = malloc(sizeof(*state), M_KTLS, M_WAITOK | M_ZERO); soref(so); state->so = so; state->m = m; mpages = m->m_epg_nrdy; n = m->m_next; error = ktls_encrypt_record(wq, m, tls, state); if (error) { counter_u64_add(ktls_offload_failed_crypto, 1); free(state, M_KTLS); CURVNET_SET(so->so_vnet); sorele(so); CURVNET_RESTORE(); break; } npages += mpages; } CURVNET_SET(so->so_vnet); if (error != 0) { ktls_drop(so, EIO); mb_free_notready(m, total_pages - npages); } sorele(so); CURVNET_RESTORE(); } static int ktls_bind_domain(int domain) { int error; error = cpuset_setthread(curthread->td_tid, &cpuset_domain[domain]); if (error != 0) return (error); curthread->td_domain.dr_policy = DOMAINSET_PREF(domain); return (0); } static void ktls_reclaim_thread(void *ctx) { struct ktls_domain_info *ktls_domain = ctx; struct ktls_reclaim_thread *sc = &ktls_domain->reclaim_td; struct sysctl_oid *oid; char name[80]; int error, domain; domain = ktls_domain - ktls_domains; if (bootverbose) printf("Starting KTLS reclaim thread for domain %d\n", domain); error = ktls_bind_domain(domain); if (error) printf("Unable to bind KTLS reclaim thread for domain %d: error %d\n", domain, error); snprintf(name, sizeof(name), "domain%d", domain); oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_kern_ipc_tls), OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "reclaims", CTLFLAG_RD, &sc->reclaims, 0, "buffers reclaimed"); SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "wakeups", CTLFLAG_RD, &sc->wakeups, 0, "thread wakeups"); SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "running", CTLFLAG_RD, &sc->running, 0, "thread running"); for (;;) { atomic_store_int(&sc->running, 0); tsleep(sc, PZERO | PNOLOCK, "-", 0); atomic_store_int(&sc->running, 1); sc->wakeups++; /* * Below we attempt to reclaim ktls_max_reclaim * buffers using vm_page_reclaim_contig_domain_ext(). * We do this here, as this function can take several * seconds to scan all of memory and it does not * matter if this thread pauses for a while. If we * block a ktls worker thread, we risk developing * backlogs of buffers to be encrypted, leading to * surges of traffic and potential NIC output drops. */ if (vm_page_reclaim_contig_domain_ext(domain, VM_ALLOC_NORMAL, atop(ktls_maxlen), 0, ~0ul, PAGE_SIZE, 0, ktls_max_reclaim) != 0) { vm_wait_domain(domain); } else { sc->reclaims += ktls_max_reclaim; } } } static void ktls_work_thread(void *ctx) { struct ktls_wq *wq = ctx; struct mbuf *m, *n; struct socket *so, *son; STAILQ_HEAD(, mbuf) local_m_head; STAILQ_HEAD(, socket) local_so_head; int cpu; cpu = wq - ktls_wq; if (bootverbose) printf("Starting KTLS worker thread for CPU %d\n", cpu); /* * Bind to a core. If ktls_bind_threads is > 1, then * we bind to the NUMA domain instead. */ if (ktls_bind_threads) { int error; if (ktls_bind_threads > 1) { struct pcpu *pc = pcpu_find(cpu); error = ktls_bind_domain(pc->pc_domain); } else { cpuset_t mask; CPU_SETOF(cpu, &mask); error = cpuset_setthread(curthread->td_tid, &mask); } if (error) printf("Unable to bind KTLS worker thread for CPU %d: error %d\n", cpu, error); } #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__) fpu_kern_thread(0); #endif for (;;) { mtx_lock(&wq->mtx); while (STAILQ_EMPTY(&wq->m_head) && STAILQ_EMPTY(&wq->so_head)) { wq->running = false; mtx_sleep(wq, &wq->mtx, 0, "-", 0); wq->running = true; } STAILQ_INIT(&local_m_head); STAILQ_CONCAT(&local_m_head, &wq->m_head); STAILQ_INIT(&local_so_head); STAILQ_CONCAT(&local_so_head, &wq->so_head); mtx_unlock(&wq->mtx); STAILQ_FOREACH_SAFE(m, &local_m_head, m_epg_stailq, n) { if (m->m_epg_flags & EPG_FLAG_2FREE) { ktls_free(m->m_epg_tls); m_free_raw(m); } else { if (m->m_epg_tls->sync_dispatch) ktls_encrypt(wq, m); else ktls_encrypt_async(wq, m); counter_u64_add(ktls_cnt_tx_queued, -1); } } STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) { ktls_decrypt(so); counter_u64_add(ktls_cnt_rx_queued, -1); } } } static void ktls_disable_ifnet_help(void *context, int pending __unused) { struct ktls_session *tls; struct inpcb *inp; struct tcpcb *tp; struct socket *so; int err; tls = context; inp = tls->inp; if (inp == NULL) return; INP_WLOCK(inp); so = inp->inp_socket; MPASS(so != NULL); tp = intotcpcb(inp); if (tp->t_flags & TF_DISCONNECTED) { goto out; } if (so->so_snd.sb_tls_info != NULL) err = ktls_set_tx_mode(so, TCP_TLS_MODE_SW); else err = ENXIO; if (err == 0) { counter_u64_add(ktls_ifnet_disable_ok, 1); /* ktls_set_tx_mode() drops inp wlock, so recheck flags */ if ((tp->t_flags & TF_DISCONNECTED) == 0 && tp->t_fb->tfb_hwtls_change != NULL) (*tp->t_fb->tfb_hwtls_change)(tp, 0); } else { counter_u64_add(ktls_ifnet_disable_fail, 1); } out: CURVNET_SET(so->so_vnet); sorele(so); CURVNET_RESTORE(); INP_WUNLOCK(inp); ktls_free(tls); } /* * Called when re-transmits are becoming a substantial portion of the * sends on this connection. When this happens, we transition the * connection to software TLS. This is needed because most inline TLS * NICs keep crypto state only for in-order transmits. This means * that to handle a TCP rexmit (which is out-of-order), the NIC must * re-DMA the entire TLS record up to and including the current * segment. This means that when re-transmitting the last ~1448 byte * segment of a 16KB TLS record, we could wind up re-DMA'ing an order * of magnitude more data than we are sending. This can cause the * PCIe link to saturate well before the network, which can cause * output drops, and a general loss of capacity. */ void ktls_disable_ifnet(void *arg) { struct tcpcb *tp; struct inpcb *inp; struct socket *so; struct ktls_session *tls; tp = arg; inp = tptoinpcb(tp); INP_WLOCK_ASSERT(inp); so = inp->inp_socket; SOCK_LOCK(so); tls = so->so_snd.sb_tls_info; if (tp->t_nic_ktls_xmit_dis == 1) { SOCK_UNLOCK(so); return; } /* * note that t_nic_ktls_xmit_dis is never cleared; disabling * ifnet can only be done once per connection, so we never want * to do it again */ (void)ktls_hold(tls); soref(so); tp->t_nic_ktls_xmit_dis = 1; SOCK_UNLOCK(so); TASK_INIT(&tls->disable_ifnet_task, 0, ktls_disable_ifnet_help, tls); (void)taskqueue_enqueue(taskqueue_thread, &tls->disable_ifnet_task); } void ktls_session_to_xktls_onedir(const struct ktls_session *ktls, bool export_keys, struct xktls_session_onedir *xk) { if_t ifp; struct m_snd_tag *st; xk->gen = ktls->gen; #define A(m) xk->m = ktls->params.m A(cipher_algorithm); A(auth_algorithm); A(cipher_key_len); A(auth_key_len); A(max_frame_len); A(tls_vmajor); A(tls_vminor); A(tls_hlen); A(tls_tlen); A(tls_bs); A(flags); if (export_keys) { memcpy(&xk->iv, &ktls->params.iv, XKTLS_SESSION_IV_BUF_LEN); A(iv_len); } else { memset(&xk->iv, 0, XKTLS_SESSION_IV_BUF_LEN); xk->iv_len = 0; } #undef A if ((st = ktls->snd_tag) != NULL && (ifp = ktls->snd_tag->ifp) != NULL) strncpy(xk->ifnet, if_name(ifp), sizeof(xk->ifnet)); } void ktls_session_copy_keys(const struct ktls_session *ktls, uint8_t *data, size_t *sz) { size_t t, ta, tc; if (ktls == NULL) { *sz = 0; return; } t = *sz; tc = MIN(t, ktls->params.cipher_key_len); if (data != NULL) memcpy(data, ktls->params.cipher_key, tc); ta = MIN(t - tc, ktls->params.auth_key_len); if (data != NULL) memcpy(data + tc, ktls->params.auth_key, ta); *sz = ta + tc; } diff --git a/tests/sys/kern/ktls_test.c b/tests/sys/kern/ktls_test.c index 7003b86fb4e2..e0bcf17262f2 100644 --- a/tests/sys/kern/ktls_test.c +++ b/tests/sys/kern/ktls_test.c @@ -1,3047 +1,3073 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Netflix Inc. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void require_ktls(void) { size_t len; bool enable; len = sizeof(enable); if (sysctlbyname("kern.ipc.tls.enable", &enable, &len, NULL, 0) == -1) { if (errno == ENOENT) atf_tc_skip("kernel does not support TLS offload"); atf_libc_error(errno, "Failed to read kern.ipc.tls.enable"); } if (!enable) atf_tc_skip("Kernel TLS is disabled"); } #define ATF_REQUIRE_KTLS() require_ktls() +static void +require_ktls_rx(void) +{ + size_t len; + bool enable; + + ATF_REQUIRE_KTLS(); + + len = sizeof(enable); + if (sysctlbyname("kern.ipc.tls.rx_enable", &enable, &len, NULL, 0) == + -1) { + if (errno == ENOENT) + atf_tc_skip("kernel does not support TLS offload"); + atf_libc_error(errno, "Failed to read kern.ipc.tls.rx_enable"); + } + + if (!enable) + atf_tc_skip("Kernel TLS receive is disabled"); +} + +#define ATF_REQUIRE_KTLS_RX() require_ktls_rx() + static void check_tls_mode(const atf_tc_t *tc, int s, int sockopt) { if (atf_tc_get_config_var_as_bool_wd(tc, "ktls.require_ifnet", false)) { socklen_t len; int mode; len = sizeof(mode); if (getsockopt(s, IPPROTO_TCP, sockopt, &mode, &len) == -1) atf_libc_error(errno, "Failed to fetch TLS mode"); if (mode != TCP_TLS_MODE_IFNET) atf_tc_skip("connection did not use ifnet TLS"); } if (atf_tc_get_config_var_as_bool_wd(tc, "ktls.require_toe", false)) { socklen_t len; int mode; len = sizeof(mode); if (getsockopt(s, IPPROTO_TCP, sockopt, &mode, &len) == -1) atf_libc_error(errno, "Failed to fetch TLS mode"); if (mode != TCP_TLS_MODE_TOE) atf_tc_skip("connection did not use TOE TLS"); } } static void __printflike(2, 3) debug(const atf_tc_t *tc, const char *fmt, ...) { if (!atf_tc_get_config_var_as_bool_wd(tc, "ktls.debug", false)) return; va_list ap; va_start(ap, fmt); vprintf(fmt, ap); va_end(ap); } static void debug_hexdump(const atf_tc_t *tc, const void *buf, int length, const char *label) { if (!atf_tc_get_config_var_as_bool_wd(tc, "ktls.debug", false)) return; if (label != NULL) printf("%s:\n", label); hexdump(buf, length, NULL, 0); } static char rdigit(void) { /* ASCII printable values between 0x20 and 0x7e */ return (0x20 + random() % (0x7f - 0x20)); } static char * alloc_buffer(size_t len) { char *buf; size_t i; if (len == 0) return (NULL); buf = malloc(len); for (i = 0; i < len; i++) buf[i] = rdigit(); return (buf); } static bool socketpair_tcp(int sv[2]) { struct pollfd pfd; struct sockaddr_in sin; socklen_t len; int as, cs, ls; ls = socket(PF_INET, SOCK_STREAM, 0); if (ls == -1) { warn("socket() for listen"); return (false); } memset(&sin, 0, sizeof(sin)); sin.sin_len = sizeof(sin); sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); if (bind(ls, (struct sockaddr *)&sin, sizeof(sin)) == -1) { warn("bind"); close(ls); return (false); } if (listen(ls, 1) == -1) { warn("listen"); close(ls); return (false); } len = sizeof(sin); if (getsockname(ls, (struct sockaddr *)&sin, &len) == -1) { warn("getsockname"); close(ls); return (false); } cs = socket(PF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0); if (cs == -1) { warn("socket() for connect"); close(ls); return (false); } if (connect(cs, (struct sockaddr *)&sin, sizeof(sin)) == -1) { if (errno != EINPROGRESS) { warn("connect"); close(ls); close(cs); return (false); } } as = accept4(ls, NULL, NULL, SOCK_NONBLOCK); if (as == -1) { warn("accept4"); close(ls); close(cs); return (false); } close(ls); pfd.fd = cs; pfd.events = POLLOUT; pfd.revents = 0; ATF_REQUIRE_INTEQ(1, poll(&pfd, 1, INFTIM)); ATF_REQUIRE_INTEQ(POLLOUT, pfd.revents); sv[0] = cs; sv[1] = as; return (true); } static bool echo_socket(const atf_tc_t *tc, int sv[2]) { const char *cause, *host, *port; struct addrinfo hints, *ai, *tofree; int error, flags, s; host = atf_tc_get_config_var(tc, "ktls.host"); port = atf_tc_get_config_var_wd(tc, "ktls.port", "echo"); memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; error = getaddrinfo(host, port, &hints, &tofree); if (error != 0) { warnx("getaddrinfo(%s:%s) failed: %s", host, port, gai_strerror(error)); return (false); } cause = NULL; for (ai = tofree; ai != NULL; ai = ai->ai_next) { s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); if (s == -1) { cause = "socket"; error = errno; continue; } if (connect(s, ai->ai_addr, ai->ai_addrlen) == -1) { cause = "connect"; error = errno; close(s); continue; } freeaddrinfo(tofree); ATF_REQUIRE((flags = fcntl(s, F_GETFL)) != -1); flags |= O_NONBLOCK; ATF_REQUIRE(fcntl(s, F_SETFL, flags) != -1); sv[0] = s; sv[1] = s; return (true); } warnc(error, "%s", cause); freeaddrinfo(tofree); return (false); } static bool open_sockets(const atf_tc_t *tc, int sv[2]) { if (atf_tc_has_config_var(tc, "ktls.host")) return (echo_socket(tc, sv)); else return (socketpair_tcp(sv)); } static void close_sockets(int sv[2]) { if (sv[0] != sv[1]) ATF_REQUIRE(close(sv[1]) == 0); ATF_REQUIRE(close(sv[0]) == 0); } static void close_sockets_ignore_errors(int sv[2]) { if (sv[0] != sv[1]) close(sv[1]); close(sv[0]); } static void fd_set_blocking(int fd) { int flags; ATF_REQUIRE((flags = fcntl(fd, F_GETFL)) != -1); flags &= ~O_NONBLOCK; ATF_REQUIRE(fcntl(fd, F_SETFL, flags) != -1); } static void tcp_nodelay(int fd) { int nodelay = 1; ATF_REQUIRE(setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay)) == 0); } static bool cbc_crypt(const EVP_CIPHER *cipher, const char *key, const char *iv, const char *input, char *output, size_t size, int enc) { EVP_CIPHER_CTX *ctx; int outl, total; ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { warnx("EVP_CIPHER_CTX_new failed: %s", ERR_error_string(ERR_get_error(), NULL)); return (false); } if (EVP_CipherInit_ex(ctx, cipher, NULL, (const u_char *)key, (const u_char *)iv, enc) != 1) { warnx("EVP_CipherInit_ex failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } EVP_CIPHER_CTX_set_padding(ctx, 0); if (EVP_CipherUpdate(ctx, (u_char *)output, &outl, (const u_char *)input, size) != 1) { warnx("EVP_CipherUpdate failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } total = outl; if (EVP_CipherFinal_ex(ctx, (u_char *)output + outl, &outl) != 1) { warnx("EVP_CipherFinal_ex failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } total += outl; if ((size_t)total != size) { warnx("decrypt size mismatch: %zu vs %d", size, total); EVP_CIPHER_CTX_free(ctx); return (false); } EVP_CIPHER_CTX_free(ctx); return (true); } static bool cbc_encrypt(const EVP_CIPHER *cipher, const char *key, const char *iv, const char *input, char *output, size_t size) { return (cbc_crypt(cipher, key, iv, input, output, size, 1)); } static bool cbc_decrypt(const EVP_CIPHER *cipher, const char *key, const char *iv, const char *input, char *output, size_t size) { return (cbc_crypt(cipher, key, iv, input, output, size, 0)); } static bool compute_hash(const EVP_MD *md, const void *key, size_t key_len, const void *aad, size_t aad_len, const void *buffer, size_t len, void *digest, u_int *digest_len) { HMAC_CTX *ctx; ctx = HMAC_CTX_new(); if (ctx == NULL) { warnx("HMAC_CTX_new failed: %s", ERR_error_string(ERR_get_error(), NULL)); return (false); } if (HMAC_Init_ex(ctx, key, key_len, md, NULL) != 1) { warnx("HMAC_Init_ex failed: %s", ERR_error_string(ERR_get_error(), NULL)); HMAC_CTX_free(ctx); return (false); } if (HMAC_Update(ctx, aad, aad_len) != 1) { warnx("HMAC_Update (aad) failed: %s", ERR_error_string(ERR_get_error(), NULL)); HMAC_CTX_free(ctx); return (false); } if (HMAC_Update(ctx, buffer, len) != 1) { warnx("HMAC_Update (payload) failed: %s", ERR_error_string(ERR_get_error(), NULL)); HMAC_CTX_free(ctx); return (false); } if (HMAC_Final(ctx, digest, digest_len) != 1) { warnx("HMAC_Final failed: %s", ERR_error_string(ERR_get_error(), NULL)); HMAC_CTX_free(ctx); return (false); } HMAC_CTX_free(ctx); return (true); } static bool verify_hash(const EVP_MD *md, const void *key, size_t key_len, const void *aad, size_t aad_len, const void *buffer, size_t len, const void *digest) { unsigned char digest2[EVP_MAX_MD_SIZE]; u_int digest_len; if (!compute_hash(md, key, key_len, aad, aad_len, buffer, len, digest2, &digest_len)) return (false); if (memcmp(digest, digest2, digest_len) != 0) { warnx("HMAC mismatch"); return (false); } return (true); } static bool aead_encrypt(const EVP_CIPHER *cipher, const char *key, const char *nonce, const void *aad, size_t aad_len, const char *input, char *output, size_t size, char *tag, size_t tag_len) { EVP_CIPHER_CTX *ctx; int outl, total; ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { warnx("EVP_CIPHER_CTX_new failed: %s", ERR_error_string(ERR_get_error(), NULL)); return (false); } if (EVP_EncryptInit_ex(ctx, cipher, NULL, (const u_char *)key, (const u_char *)nonce) != 1) { warnx("EVP_EncryptInit_ex failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } EVP_CIPHER_CTX_set_padding(ctx, 0); if (aad != NULL) { if (EVP_EncryptUpdate(ctx, NULL, &outl, (const u_char *)aad, aad_len) != 1) { warnx("EVP_EncryptUpdate for AAD failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } } if (EVP_EncryptUpdate(ctx, (u_char *)output, &outl, (const u_char *)input, size) != 1) { warnx("EVP_EncryptUpdate failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } total = outl; if (EVP_EncryptFinal_ex(ctx, (u_char *)output + outl, &outl) != 1) { warnx("EVP_EncryptFinal_ex failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } total += outl; if ((size_t)total != size) { warnx("encrypt size mismatch: %zu vs %d", size, total); EVP_CIPHER_CTX_free(ctx); return (false); } if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, tag_len, tag) != 1) { warnx("EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_GET_TAG) failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } EVP_CIPHER_CTX_free(ctx); return (true); } static bool aead_decrypt(const EVP_CIPHER *cipher, const char *key, const char *nonce, const void *aad, size_t aad_len, const char *input, char *output, size_t size, const char *tag, size_t tag_len) { EVP_CIPHER_CTX *ctx; int outl, total; bool valid; ctx = EVP_CIPHER_CTX_new(); if (ctx == NULL) { warnx("EVP_CIPHER_CTX_new failed: %s", ERR_error_string(ERR_get_error(), NULL)); return (false); } if (EVP_DecryptInit_ex(ctx, cipher, NULL, (const u_char *)key, (const u_char *)nonce) != 1) { warnx("EVP_DecryptInit_ex failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } EVP_CIPHER_CTX_set_padding(ctx, 0); if (aad != NULL) { if (EVP_DecryptUpdate(ctx, NULL, &outl, (const u_char *)aad, aad_len) != 1) { warnx("EVP_DecryptUpdate for AAD failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } } if (EVP_DecryptUpdate(ctx, (u_char *)output, &outl, (const u_char *)input, size) != 1) { warnx("EVP_DecryptUpdate failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } total = outl; if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, tag_len, __DECONST(char *, tag)) != 1) { warnx("EVP_CIPHER_CTX_ctrl(EVP_CTRL_AEAD_SET_TAG) failed: %s", ERR_error_string(ERR_get_error(), NULL)); EVP_CIPHER_CTX_free(ctx); return (false); } valid = (EVP_DecryptFinal_ex(ctx, (u_char *)output + outl, &outl) == 1); total += outl; if ((size_t)total != size) { warnx("decrypt size mismatch: %zu vs %d", size, total); EVP_CIPHER_CTX_free(ctx); return (false); } if (!valid) warnx("tag mismatch"); EVP_CIPHER_CTX_free(ctx); return (valid); } static void build_tls_enable(const atf_tc_t *tc, int cipher_alg, size_t cipher_key_len, int auth_alg, int minor, uint64_t seqno, struct tls_enable *en) { u_int auth_key_len, iv_len; memset(en, 0, sizeof(*en)); switch (cipher_alg) { case CRYPTO_AES_CBC: if (minor == TLS_MINOR_VER_ZERO) iv_len = AES_BLOCK_LEN; else iv_len = 0; break; case CRYPTO_AES_NIST_GCM_16: if (minor == TLS_MINOR_VER_TWO) iv_len = TLS_AEAD_GCM_LEN; else iv_len = TLS_1_3_GCM_IV_LEN; break; case CRYPTO_CHACHA20_POLY1305: iv_len = TLS_CHACHA20_IV_LEN; break; default: iv_len = 0; break; } switch (auth_alg) { case CRYPTO_SHA1_HMAC: auth_key_len = SHA1_HASH_LEN; break; case CRYPTO_SHA2_256_HMAC: auth_key_len = SHA2_256_HASH_LEN; break; case CRYPTO_SHA2_384_HMAC: auth_key_len = SHA2_384_HASH_LEN; break; default: auth_key_len = 0; break; } en->cipher_key = alloc_buffer(cipher_key_len); debug_hexdump(tc, en->cipher_key, cipher_key_len, "cipher key"); en->iv = alloc_buffer(iv_len); if (iv_len != 0) debug_hexdump(tc, en->iv, iv_len, "iv"); en->auth_key = alloc_buffer(auth_key_len); if (auth_key_len != 0) debug_hexdump(tc, en->auth_key, auth_key_len, "auth key"); en->cipher_algorithm = cipher_alg; en->cipher_key_len = cipher_key_len; en->iv_len = iv_len; en->auth_algorithm = auth_alg; en->auth_key_len = auth_key_len; en->tls_vmajor = TLS_MAJOR_VER_ONE; en->tls_vminor = minor; be64enc(en->rec_seq, seqno); debug(tc, "seqno: %ju\n", (uintmax_t)seqno); } static void free_tls_enable(struct tls_enable *en) { free(__DECONST(void *, en->cipher_key)); free(__DECONST(void *, en->iv)); free(__DECONST(void *, en->auth_key)); } static const EVP_CIPHER * tls_EVP_CIPHER(const struct tls_enable *en) { switch (en->cipher_algorithm) { case CRYPTO_AES_CBC: switch (en->cipher_key_len) { case 128 / 8: return (EVP_aes_128_cbc()); case 256 / 8: return (EVP_aes_256_cbc()); default: return (NULL); } break; case CRYPTO_AES_NIST_GCM_16: switch (en->cipher_key_len) { case 128 / 8: return (EVP_aes_128_gcm()); case 256 / 8: return (EVP_aes_256_gcm()); default: return (NULL); } break; case CRYPTO_CHACHA20_POLY1305: return (EVP_chacha20_poly1305()); default: return (NULL); } } static const EVP_MD * tls_EVP_MD(const struct tls_enable *en) { switch (en->auth_algorithm) { case CRYPTO_SHA1_HMAC: return (EVP_sha1()); case CRYPTO_SHA2_256_HMAC: return (EVP_sha256()); case CRYPTO_SHA2_384_HMAC: return (EVP_sha384()); default: return (NULL); } } static size_t tls_header_len(struct tls_enable *en) { size_t len; len = sizeof(struct tls_record_layer); switch (en->cipher_algorithm) { case CRYPTO_AES_CBC: if (en->tls_vminor != TLS_MINOR_VER_ZERO) len += AES_BLOCK_LEN; return (len); case CRYPTO_AES_NIST_GCM_16: if (en->tls_vminor == TLS_MINOR_VER_TWO) len += sizeof(uint64_t); return (len); case CRYPTO_CHACHA20_POLY1305: return (len); default: return (0); } } static size_t tls_mac_len(struct tls_enable *en) { switch (en->cipher_algorithm) { case CRYPTO_AES_CBC: switch (en->auth_algorithm) { case CRYPTO_SHA1_HMAC: return (SHA1_HASH_LEN); case CRYPTO_SHA2_256_HMAC: return (SHA2_256_HASH_LEN); case CRYPTO_SHA2_384_HMAC: return (SHA2_384_HASH_LEN); default: return (0); } case CRYPTO_AES_NIST_GCM_16: return (AES_GMAC_HASH_LEN); case CRYPTO_CHACHA20_POLY1305: return (POLY1305_HASH_LEN); default: return (0); } } /* Includes maximum padding for MTE. */ static size_t tls_trailer_len(struct tls_enable *en) { size_t len; len = tls_mac_len(en); if (en->cipher_algorithm == CRYPTO_AES_CBC) len += AES_BLOCK_LEN; if (en->tls_vminor == TLS_MINOR_VER_THREE) len++; return (len); } /* Minimum valid record payload size for a given cipher suite. */ static size_t tls_minimum_record_payload(struct tls_enable *en) { size_t len; len = tls_header_len(en); if (en->cipher_algorithm == CRYPTO_AES_CBC) len += roundup2(tls_mac_len(en) + 1, AES_BLOCK_LEN); else len += tls_mac_len(en); if (en->tls_vminor == TLS_MINOR_VER_THREE) len++; return (len - sizeof(struct tls_record_layer)); } /* 'len' is the length of the payload application data. */ static void tls_mte_aad(struct tls_enable *en, size_t len, const struct tls_record_layer *hdr, uint64_t seqno, struct tls_mac_data *ad) { ad->seq = htobe64(seqno); ad->type = hdr->tls_type; ad->tls_vmajor = hdr->tls_vmajor; ad->tls_vminor = hdr->tls_vminor; ad->tls_length = htons(len); } static void tls_12_aead_aad(struct tls_enable *en, size_t len, const struct tls_record_layer *hdr, uint64_t seqno, struct tls_aead_data *ad) { ad->seq = htobe64(seqno); ad->type = hdr->tls_type; ad->tls_vmajor = hdr->tls_vmajor; ad->tls_vminor = hdr->tls_vminor; ad->tls_length = htons(len); } static void tls_13_aad(struct tls_enable *en, const struct tls_record_layer *hdr, uint64_t seqno, struct tls_aead_data_13 *ad) { ad->type = hdr->tls_type; ad->tls_vmajor = hdr->tls_vmajor; ad->tls_vminor = hdr->tls_vminor; ad->tls_length = hdr->tls_length; } static void tls_12_gcm_nonce(struct tls_enable *en, const struct tls_record_layer *hdr, char *nonce) { memcpy(nonce, en->iv, TLS_AEAD_GCM_LEN); memcpy(nonce + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t)); } static void tls_13_nonce(struct tls_enable *en, uint64_t seqno, char *nonce) { static_assert(TLS_1_3_GCM_IV_LEN == TLS_CHACHA20_IV_LEN, "TLS 1.3 nonce length mismatch"); memcpy(nonce, en->iv, TLS_1_3_GCM_IV_LEN); *(uint64_t *)(nonce + 4) ^= htobe64(seqno); } /* * Decrypt a TLS record 'len' bytes long at 'src' and store the result at * 'dst'. If the TLS record header length doesn't match or 'dst' doesn't * have sufficient room ('avail'), fail the test. */ static size_t decrypt_tls_aes_cbc_mte(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, const void *src, size_t len, void *dst, size_t avail, uint8_t *record_type) { const struct tls_record_layer *hdr; struct tls_mac_data aad; const char *iv; char *buf; size_t hdr_len, mac_len, payload_len; int padding; hdr = src; hdr_len = tls_header_len(en); mac_len = tls_mac_len(en); ATF_REQUIRE_INTEQ(TLS_MAJOR_VER_ONE, hdr->tls_vmajor); ATF_REQUIRE_INTEQ(en->tls_vminor, hdr->tls_vminor); debug(tc, "decrypting MTE record seqno %ju:\n", (uintmax_t)seqno); debug_hexdump(tc, src, len, NULL); /* First, decrypt the outer payload into a temporary buffer. */ payload_len = len - hdr_len; buf = malloc(payload_len); if (en->tls_vminor == TLS_MINOR_VER_ZERO) iv = en->iv; else iv = (void *)(hdr + 1); debug_hexdump(tc, iv, AES_BLOCK_LEN, "iv"); ATF_REQUIRE(cbc_decrypt(tls_EVP_CIPHER(en), en->cipher_key, iv, (const u_char *)src + hdr_len, buf, payload_len)); debug_hexdump(tc, buf, payload_len, "decrypted buffer"); /* * Copy the last encrypted block to use as the IV for the next * record for TLS 1.0. */ if (en->tls_vminor == TLS_MINOR_VER_ZERO) memcpy(__DECONST(uint8_t *, en->iv), (const u_char *)src + (len - AES_BLOCK_LEN), AES_BLOCK_LEN); /* * Verify trailing padding and strip. * * The kernel always generates the smallest amount of padding. */ padding = buf[payload_len - 1] + 1; ATF_REQUIRE_MSG(padding > 0 && padding <= AES_BLOCK_LEN, "invalid padding %d", padding); ATF_REQUIRE_MSG(payload_len >= mac_len + padding, "payload_len (%zu) < mac_len (%zu) + padding (%d)", payload_len, mac_len, padding); payload_len -= padding; /* Verify HMAC. */ payload_len -= mac_len; tls_mte_aad(en, payload_len, hdr, seqno, &aad); debug_hexdump(tc, &aad, sizeof(aad), "aad"); ATF_REQUIRE(verify_hash(tls_EVP_MD(en), en->auth_key, en->auth_key_len, &aad, sizeof(aad), buf, payload_len, buf + payload_len)); ATF_REQUIRE_MSG(payload_len <= avail, "payload_len (%zu) < avail (%zu)", payload_len, avail); memcpy(dst, buf, payload_len); *record_type = hdr->tls_type; return (payload_len); } static size_t decrypt_tls_12_aead(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, const void *src, size_t len, void *dst, uint8_t *record_type) { const struct tls_record_layer *hdr; struct tls_aead_data aad; char nonce[12]; size_t hdr_len, mac_len, payload_len; hdr = src; hdr_len = tls_header_len(en); mac_len = tls_mac_len(en); payload_len = len - (hdr_len + mac_len); ATF_REQUIRE_INTEQ(TLS_MAJOR_VER_ONE, hdr->tls_vmajor); ATF_REQUIRE_INTEQ(TLS_MINOR_VER_TWO, hdr->tls_vminor); debug(tc, "decrypting TLS 1.2 record seqno %ju:\n", (uintmax_t)seqno); debug_hexdump(tc, src, len, NULL); tls_12_aead_aad(en, payload_len, hdr, seqno, &aad); debug_hexdump(tc, &aad, sizeof(aad), "aad"); if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16) tls_12_gcm_nonce(en, hdr, nonce); else tls_13_nonce(en, seqno, nonce); debug_hexdump(tc, nonce, sizeof(nonce), "nonce"); ATF_REQUIRE(aead_decrypt(tls_EVP_CIPHER(en), en->cipher_key, nonce, &aad, sizeof(aad), (const char *)src + hdr_len, dst, payload_len, (const char *)src + hdr_len + payload_len, mac_len)); *record_type = hdr->tls_type; return (payload_len); } static size_t decrypt_tls_13_aead(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, const void *src, size_t len, void *dst, uint8_t *record_type) { const struct tls_record_layer *hdr; struct tls_aead_data_13 aad; char nonce[12]; char *buf; size_t hdr_len, mac_len, payload_len; hdr = src; hdr_len = tls_header_len(en); mac_len = tls_mac_len(en); payload_len = len - (hdr_len + mac_len); ATF_REQUIRE_MSG(payload_len >= 1, "payload_len (%zu) too short: len %zu hdr_len %zu mac_len %zu", payload_len, len, hdr_len, mac_len); ATF_REQUIRE_INTEQ(TLS_RLTYPE_APP, hdr->tls_type); ATF_REQUIRE_INTEQ(TLS_MAJOR_VER_ONE, hdr->tls_vmajor); ATF_REQUIRE_INTEQ(TLS_MINOR_VER_TWO, hdr->tls_vminor); debug(tc, "decrypting TLS 1.3 record seqno %ju:\n", (uintmax_t)seqno); debug_hexdump(tc, src, len, NULL); tls_13_aad(en, hdr, seqno, &aad); debug_hexdump(tc, &aad, sizeof(aad), "aad"); tls_13_nonce(en, seqno, nonce); debug_hexdump(tc, nonce, sizeof(nonce), "nonce"); /* * Have to use a temporary buffer for the output due to the * record type as the last byte of the trailer. */ buf = malloc(payload_len); ATF_REQUIRE(aead_decrypt(tls_EVP_CIPHER(en), en->cipher_key, nonce, &aad, sizeof(aad), (const char *)src + hdr_len, buf, payload_len, (const char *)src + hdr_len + payload_len, mac_len)); debug_hexdump(tc, buf, payload_len, "decrypted buffer"); /* Trim record type. */ *record_type = buf[payload_len - 1]; payload_len--; memcpy(dst, buf, payload_len); free(buf); return (payload_len); } static size_t decrypt_tls_aead(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, const void *src, size_t len, void *dst, size_t avail, uint8_t *record_type) { const struct tls_record_layer *hdr; size_t payload_len; hdr = src; ATF_REQUIRE_INTEQ(len, ntohs(hdr->tls_length) + sizeof(*hdr)); payload_len = len - (tls_header_len(en) + tls_trailer_len(en)); ATF_REQUIRE_MSG(payload_len <= avail, "payload_len (%zu) > avail (%zu)", payload_len, avail); if (en->tls_vminor == TLS_MINOR_VER_TWO) { ATF_REQUIRE_INTEQ(payload_len, decrypt_tls_12_aead(tc, en, seqno, src, len, dst, record_type)); } else { ATF_REQUIRE_INTEQ(payload_len, decrypt_tls_13_aead(tc, en, seqno, src, len, dst, record_type)); } return (payload_len); } static size_t decrypt_tls_record(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, const void *src, size_t len, void *dst, size_t avail, uint8_t *record_type) { if (en->cipher_algorithm == CRYPTO_AES_CBC) return (decrypt_tls_aes_cbc_mte(tc, en, seqno, src, len, dst, avail, record_type)); else return (decrypt_tls_aead(tc, en, seqno, src, len, dst, avail, record_type)); } /* * Encrypt a TLS record of type 'record_type' with payload 'len' bytes * long at 'src' and store the result at 'dst'. If 'dst' doesn't have * sufficient room ('avail'), fail the test. 'padding' is the amount * of additional padding to include beyond any amount mandated by the * cipher suite. */ static size_t encrypt_tls_aes_cbc_mte(const atf_tc_t *tc, struct tls_enable *en, uint8_t record_type, uint64_t seqno, const void *src, size_t len, void *dst, size_t avail, size_t padding) { struct tls_record_layer *hdr; struct tls_mac_data aad; char *buf, *iv; size_t hdr_len, mac_len, record_len; u_int digest_len, i; ATF_REQUIRE_INTEQ(0, padding % 16); hdr = dst; buf = dst; debug(tc, "encrypting MTE record seqno %ju:\n", (uintmax_t)seqno); hdr_len = tls_header_len(en); mac_len = tls_mac_len(en); padding += (AES_BLOCK_LEN - (len + mac_len) % AES_BLOCK_LEN); ATF_REQUIRE_MSG(padding > 0 && padding <= 255, "invalid padding (%zu)", padding); record_len = hdr_len + len + mac_len + padding; ATF_REQUIRE_MSG(record_len <= avail, "record_len (%zu) > avail (%zu): " "hdr_len %zu, len %zu, mac_len %zu, padding %zu", record_len, avail, hdr_len, len, mac_len, padding); hdr->tls_type = record_type; hdr->tls_vmajor = TLS_MAJOR_VER_ONE; hdr->tls_vminor = en->tls_vminor; hdr->tls_length = htons(record_len - sizeof(*hdr)); iv = (char *)(hdr + 1); for (i = 0; i < AES_BLOCK_LEN; i++) iv[i] = rdigit(); debug_hexdump(tc, iv, AES_BLOCK_LEN, "explicit IV"); /* Copy plaintext to ciphertext region. */ memcpy(buf + hdr_len, src, len); /* Compute HMAC. */ tls_mte_aad(en, len, hdr, seqno, &aad); debug_hexdump(tc, &aad, sizeof(aad), "aad"); debug_hexdump(tc, src, len, "plaintext"); ATF_REQUIRE(compute_hash(tls_EVP_MD(en), en->auth_key, en->auth_key_len, &aad, sizeof(aad), src, len, buf + hdr_len + len, &digest_len)); ATF_REQUIRE_INTEQ(mac_len, digest_len); /* Store padding. */ for (i = 0; i < padding; i++) buf[hdr_len + len + mac_len + i] = padding - 1; debug_hexdump(tc, buf + hdr_len + len, mac_len + padding, "MAC and padding"); /* Encrypt the record. */ ATF_REQUIRE(cbc_encrypt(tls_EVP_CIPHER(en), en->cipher_key, iv, buf + hdr_len, buf + hdr_len, len + mac_len + padding)); debug_hexdump(tc, dst, record_len, "encrypted record"); return (record_len); } static size_t encrypt_tls_12_aead(const atf_tc_t *tc, struct tls_enable *en, uint8_t record_type, uint64_t seqno, const void *src, size_t len, void *dst) { struct tls_record_layer *hdr; struct tls_aead_data aad; char nonce[12]; size_t hdr_len, mac_len, record_len; hdr = dst; debug(tc, "encrypting TLS 1.2 record seqno %ju:\n", (uintmax_t)seqno); hdr_len = tls_header_len(en); mac_len = tls_mac_len(en); record_len = hdr_len + len + mac_len; hdr->tls_type = record_type; hdr->tls_vmajor = TLS_MAJOR_VER_ONE; hdr->tls_vminor = TLS_MINOR_VER_TWO; hdr->tls_length = htons(record_len - sizeof(*hdr)); if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16) memcpy(hdr + 1, &seqno, sizeof(seqno)); tls_12_aead_aad(en, len, hdr, seqno, &aad); debug_hexdump(tc, &aad, sizeof(aad), "aad"); if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16) tls_12_gcm_nonce(en, hdr, nonce); else tls_13_nonce(en, seqno, nonce); debug_hexdump(tc, nonce, sizeof(nonce), "nonce"); debug_hexdump(tc, src, len, "plaintext"); ATF_REQUIRE(aead_encrypt(tls_EVP_CIPHER(en), en->cipher_key, nonce, &aad, sizeof(aad), src, (char *)dst + hdr_len, len, (char *)dst + hdr_len + len, mac_len)); debug_hexdump(tc, dst, record_len, "encrypted record"); return (record_len); } static size_t encrypt_tls_13_aead(const atf_tc_t *tc, struct tls_enable *en, uint8_t record_type, uint64_t seqno, const void *src, size_t len, void *dst, size_t padding) { struct tls_record_layer *hdr; struct tls_aead_data_13 aad; char nonce[12]; char *buf; size_t hdr_len, mac_len, record_len; hdr = dst; debug(tc, "encrypting TLS 1.3 record seqno %ju:\n", (uintmax_t)seqno); hdr_len = tls_header_len(en); mac_len = tls_mac_len(en); record_len = hdr_len + len + 1 + padding + mac_len; hdr->tls_type = TLS_RLTYPE_APP; hdr->tls_vmajor = TLS_MAJOR_VER_ONE; hdr->tls_vminor = TLS_MINOR_VER_TWO; hdr->tls_length = htons(record_len - sizeof(*hdr)); tls_13_aad(en, hdr, seqno, &aad); debug_hexdump(tc, &aad, sizeof(aad), "aad"); tls_13_nonce(en, seqno, nonce); debug_hexdump(tc, nonce, sizeof(nonce), "nonce"); /* * Have to use a temporary buffer for the input so that the record * type can be appended. */ buf = malloc(len + 1 + padding); memcpy(buf, src, len); buf[len] = record_type; memset(buf + len + 1, 0, padding); debug_hexdump(tc, buf, len + 1 + padding, "plaintext + type + padding"); ATF_REQUIRE(aead_encrypt(tls_EVP_CIPHER(en), en->cipher_key, nonce, &aad, sizeof(aad), buf, (char *)dst + hdr_len, len + 1 + padding, (char *)dst + hdr_len + len + 1 + padding, mac_len)); debug_hexdump(tc, dst, record_len, "encrypted record"); free(buf); return (record_len); } static size_t encrypt_tls_aead(const atf_tc_t *tc, struct tls_enable *en, uint8_t record_type, uint64_t seqno, const void *src, size_t len, void *dst, size_t avail, size_t padding) { size_t record_len; record_len = tls_header_len(en) + len + padding + tls_trailer_len(en); ATF_REQUIRE_MSG(record_len <= avail, "record_len (%zu) > avail (%zu): " "header %zu len %zu padding %zu trailer %zu", record_len, avail, tls_header_len(en), len, padding, tls_trailer_len(en)); if (en->tls_vminor == TLS_MINOR_VER_TWO) { ATF_REQUIRE_INTEQ(0, padding); ATF_REQUIRE_INTEQ(record_len, encrypt_tls_12_aead(tc, en, record_type, seqno, src, len, dst)); } else ATF_REQUIRE_INTEQ(record_len, encrypt_tls_13_aead(tc, en, record_type, seqno, src, len, dst, padding)); return (record_len); } static size_t encrypt_tls_record(const atf_tc_t *tc, struct tls_enable *en, uint8_t record_type, uint64_t seqno, const void *src, size_t len, void *dst, size_t avail, size_t padding) { if (en->cipher_algorithm == CRYPTO_AES_CBC) return (encrypt_tls_aes_cbc_mte(tc, en, record_type, seqno, src, len, dst, avail, padding)); else return (encrypt_tls_aead(tc, en, record_type, seqno, src, len, dst, avail, padding)); } static void test_ktls_transmit_app_data(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { struct kevent ev; struct tls_record_layer *hdr; char *plaintext, *decrypted, *outbuf; size_t decrypted_len, outbuf_len, outbuf_cap, record_len, written; ssize_t rv; int kq, sockets[2]; uint8_t record_type; plaintext = alloc_buffer(len); debug_hexdump(tc, plaintext, len, "plaintext"); decrypted = malloc(len); outbuf_cap = tls_header_len(en) + TLS_MAX_MSG_SIZE_V10_2 + tls_trailer_len(en); outbuf = malloc(outbuf_cap); hdr = (struct tls_record_layer *)outbuf; ATF_REQUIRE((kq = kqueue()) != -1); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_TXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[1], TCP_TXTLS_MODE); EV_SET(&ev, sockets[0], EVFILT_READ, EV_ADD, 0, 0, NULL); ATF_REQUIRE(kevent(kq, &ev, 1, NULL, 0, NULL) == 0); EV_SET(&ev, sockets[1], EVFILT_WRITE, EV_ADD, 0, 0, NULL); ATF_REQUIRE(kevent(kq, &ev, 1, NULL, 0, NULL) == 0); decrypted_len = 0; outbuf_len = 0; written = 0; while (decrypted_len != len) { ATF_REQUIRE(kevent(kq, NULL, 0, &ev, 1, NULL) == 1); switch (ev.filter) { case EVFILT_WRITE: /* Try to write any remaining data. */ rv = write(ev.ident, plaintext + written, len - written); ATF_REQUIRE_MSG(rv > 0, "failed to write to socket"); written += rv; if (written == len) { ev.flags = EV_DISABLE; ATF_REQUIRE(kevent(kq, &ev, 1, NULL, 0, NULL) == 0); } break; case EVFILT_READ: ATF_REQUIRE((ev.flags & EV_EOF) == 0); /* * Try to read data for the next TLS record * into outbuf. Start by reading the header * to determine how much additional data to * read. */ if (outbuf_len < sizeof(struct tls_record_layer)) { rv = read(ev.ident, outbuf + outbuf_len, sizeof(struct tls_record_layer) - outbuf_len); ATF_REQUIRE_MSG(rv > 0, "failed to read from socket"); outbuf_len += rv; if (outbuf_len == sizeof(struct tls_record_layer)) { debug(tc, "TLS header for seqno %ju:\n", (uintmax_t)seqno); debug_hexdump(tc, outbuf, outbuf_len, NULL); } } if (outbuf_len < sizeof(struct tls_record_layer)) break; record_len = sizeof(struct tls_record_layer) + ntohs(hdr->tls_length); debug(tc, "record_len %zu outbuf_cap %zu\n", record_len, outbuf_cap); ATF_REQUIRE(record_len <= outbuf_cap); ATF_REQUIRE(record_len > outbuf_len); rv = read(ev.ident, outbuf + outbuf_len, record_len - outbuf_len); if (rv == -1 && errno == EAGAIN) break; ATF_REQUIRE_MSG(rv > 0, "failed to read from socket: %s", strerror(errno)); outbuf_len += rv; if (outbuf_len == record_len) { decrypted_len += decrypt_tls_record(tc, en, seqno, outbuf, outbuf_len, decrypted + decrypted_len, len - decrypted_len, &record_type); ATF_REQUIRE_INTEQ(TLS_RLTYPE_APP, record_type); seqno++; outbuf_len = 0; } break; } } ATF_REQUIRE_MSG(written == decrypted_len, "read %zu decrypted bytes, but wrote %zu", decrypted_len, written); ATF_REQUIRE(memcmp(plaintext, decrypted, len) == 0); free(outbuf); free(decrypted); free(plaintext); close_sockets(sockets); ATF_REQUIRE(close(kq) == 0); } static void ktls_send_control_message(int fd, uint8_t type, void *data, size_t len) { struct msghdr msg; struct cmsghdr *cmsg; char cbuf[CMSG_SPACE(sizeof(type))]; struct iovec iov; memset(&msg, 0, sizeof(msg)); msg.msg_control = cbuf; msg.msg_controllen = sizeof(cbuf); cmsg = CMSG_FIRSTHDR(&msg); cmsg->cmsg_level = IPPROTO_TCP; cmsg->cmsg_type = TLS_SET_RECORD_TYPE; cmsg->cmsg_len = CMSG_LEN(sizeof(type)); *(uint8_t *)CMSG_DATA(cmsg) = type; iov.iov_base = data; iov.iov_len = len; msg.msg_iov = &iov; msg.msg_iovlen = 1; ATF_REQUIRE_INTEQ((ssize_t)len, sendmsg(fd, &msg, 0)); } static void test_ktls_transmit_control(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, uint8_t type, size_t len) { struct tls_record_layer *hdr; char *plaintext, *decrypted, *outbuf; size_t outbuf_cap, payload_len, record_len; ssize_t rv; int sockets[2]; uint8_t record_type; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); plaintext = alloc_buffer(len); decrypted = malloc(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); hdr = (struct tls_record_layer *)outbuf; ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_TXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[1], TCP_TXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); ktls_send_control_message(sockets[1], type, plaintext, len); /* * First read the header to determine how much additional data * to read. */ rv = read(sockets[0], outbuf, sizeof(struct tls_record_layer)); ATF_REQUIRE_INTEQ(sizeof(struct tls_record_layer), rv); payload_len = ntohs(hdr->tls_length); record_len = payload_len + sizeof(struct tls_record_layer); ATF_REQUIRE_MSG(record_len <= outbuf_cap, "record_len (%zu) > outbuf_cap (%zu)", record_len, outbuf_cap); rv = read(sockets[0], outbuf + sizeof(struct tls_record_layer), payload_len); ATF_REQUIRE_INTEQ((ssize_t)payload_len, rv); rv = decrypt_tls_record(tc, en, seqno, outbuf, record_len, decrypted, len, &record_type); ATF_REQUIRE_MSG((ssize_t)len == rv, "read %zd decrypted bytes, but wrote %zu", rv, len); ATF_REQUIRE_INTEQ(type, record_type); ATF_REQUIRE(memcmp(plaintext, decrypted, len) == 0); free(outbuf); free(decrypted); free(plaintext); close_sockets(sockets); } static void test_ktls_transmit_empty_fragment(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno) { struct tls_record_layer *hdr; char *outbuf; size_t outbuf_cap, payload_len, record_len; ssize_t rv; int sockets[2]; uint8_t record_type; outbuf_cap = tls_header_len(en) + tls_trailer_len(en); outbuf = malloc(outbuf_cap); hdr = (struct tls_record_layer *)outbuf; ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[1], IPPROTO_TCP, TCP_TXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[1], TCP_TXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); /* * A write of zero bytes should send an empty fragment only for * TLS 1.0, otherwise an error should be raised. */ rv = write(sockets[1], NULL, 0); if (rv == 0) { ATF_REQUIRE_INTEQ(CRYPTO_AES_CBC, en->cipher_algorithm); ATF_REQUIRE_INTEQ(TLS_MINOR_VER_ZERO, en->tls_vminor); } else { ATF_REQUIRE_INTEQ(-1, rv); ATF_REQUIRE_ERRNO(EINVAL, true); goto out; } /* * First read the header to determine how much additional data * to read. */ rv = read(sockets[0], outbuf, sizeof(struct tls_record_layer)); ATF_REQUIRE_INTEQ(sizeof(struct tls_record_layer), rv); payload_len = ntohs(hdr->tls_length); record_len = payload_len + sizeof(struct tls_record_layer); ATF_REQUIRE_MSG(record_len <= outbuf_cap, "record_len (%zu) > outbuf_cap (%zu)", record_len, outbuf_cap); rv = read(sockets[0], outbuf + sizeof(struct tls_record_layer), payload_len); ATF_REQUIRE_INTEQ((ssize_t)payload_len, rv); rv = decrypt_tls_record(tc, en, seqno, outbuf, record_len, NULL, 0, &record_type); ATF_REQUIRE_MSG(rv == 0, "read %zd decrypted bytes for an empty fragment", rv); ATF_REQUIRE_INTEQ(TLS_RLTYPE_APP, record_type); out: free(outbuf); close_sockets(sockets); } static size_t ktls_receive_tls_record(struct tls_enable *en, int fd, uint8_t record_type, void *data, size_t len) { struct msghdr msg; struct cmsghdr *cmsg; struct tls_get_record *tgr; char cbuf[CMSG_SPACE(sizeof(*tgr))]; struct iovec iov; ssize_t rv; memset(&msg, 0, sizeof(msg)); msg.msg_control = cbuf; msg.msg_controllen = sizeof(cbuf); iov.iov_base = data; iov.iov_len = len; msg.msg_iov = &iov; msg.msg_iovlen = 1; ATF_REQUIRE((rv = recvmsg(fd, &msg, 0)) > 0); ATF_REQUIRE((msg.msg_flags & (MSG_EOR | MSG_CTRUNC)) == MSG_EOR); cmsg = CMSG_FIRSTHDR(&msg); ATF_REQUIRE(cmsg != NULL); ATF_REQUIRE_INTEQ(IPPROTO_TCP, cmsg->cmsg_level); ATF_REQUIRE_INTEQ(TLS_GET_RECORD, cmsg->cmsg_type); ATF_REQUIRE_INTEQ(CMSG_LEN(sizeof(*tgr)), cmsg->cmsg_len); tgr = (struct tls_get_record *)CMSG_DATA(cmsg); ATF_REQUIRE_INTEQ(record_type, tgr->tls_type); ATF_REQUIRE_INTEQ(en->tls_vmajor, tgr->tls_vmajor); /* XXX: Not sure if this is what OpenSSL expects? */ if (en->tls_vminor == TLS_MINOR_VER_THREE) ATF_REQUIRE_INTEQ(TLS_MINOR_VER_TWO, tgr->tls_vminor); else ATF_REQUIRE_INTEQ(en->tls_vminor, tgr->tls_vminor); ATF_REQUIRE_INTEQ(htons(rv), tgr->tls_length); return (rv); } static void test_ktls_receive_app_data(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len, size_t padding) { struct kevent ev; char *plaintext, *received, *outbuf; size_t outbuf_cap, outbuf_len, outbuf_sent, received_len, todo, written; ssize_t rv; int kq, sockets[2]; plaintext = alloc_buffer(len); received = malloc(len); outbuf_cap = tls_header_len(en) + TLS_MAX_MSG_SIZE_V10_2 + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE((kq = kqueue()) != -1); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); EV_SET(&ev, sockets[0], EVFILT_READ, EV_ADD, 0, 0, NULL); ATF_REQUIRE(kevent(kq, &ev, 1, NULL, 0, NULL) == 0); EV_SET(&ev, sockets[1], EVFILT_WRITE, EV_ADD, 0, 0, NULL); ATF_REQUIRE(kevent(kq, &ev, 1, NULL, 0, NULL) == 0); received_len = 0; outbuf_len = 0; written = 0; while (received_len != len) { ATF_REQUIRE(kevent(kq, NULL, 0, &ev, 1, NULL) == 1); switch (ev.filter) { case EVFILT_WRITE: /* * Compose the next TLS record to send. */ if (outbuf_len == 0) { ATF_REQUIRE(written < len); todo = len - written; if (todo > TLS_MAX_MSG_SIZE_V10_2 - padding) todo = TLS_MAX_MSG_SIZE_V10_2 - padding; outbuf_len = encrypt_tls_record(tc, en, TLS_RLTYPE_APP, seqno, plaintext + written, todo, outbuf, outbuf_cap, padding); outbuf_sent = 0; written += todo; seqno++; } /* * Try to write the remainder of the current * TLS record. */ rv = write(ev.ident, outbuf + outbuf_sent, outbuf_len - outbuf_sent); ATF_REQUIRE_MSG(rv > 0, "failed to write to socket: %s", strerror(errno)); outbuf_sent += rv; if (outbuf_sent == outbuf_len) { outbuf_len = 0; if (written == len) { ev.flags = EV_DISABLE; ATF_REQUIRE(kevent(kq, &ev, 1, NULL, 0, NULL) == 0); } } break; case EVFILT_READ: ATF_REQUIRE((ev.flags & EV_EOF) == 0); rv = ktls_receive_tls_record(en, ev.ident, TLS_RLTYPE_APP, received + received_len, len - received_len); received_len += rv; break; } } ATF_REQUIRE_MSG(written == received_len, "read %zu decrypted bytes, but wrote %zu", received_len, written); ATF_REQUIRE(memcmp(plaintext, received, len) == 0); free(outbuf); free(received); free(plaintext); close_sockets(sockets); ATF_REQUIRE(close(kq) == 0); } static void ktls_receive_tls_error(int fd, int expected_error) { struct msghdr msg; struct tls_get_record *tgr; char cbuf[CMSG_SPACE(sizeof(*tgr))]; char buf[64]; struct iovec iov; memset(&msg, 0, sizeof(msg)); msg.msg_control = cbuf; msg.msg_controllen = sizeof(cbuf); iov.iov_base = buf; iov.iov_len = sizeof(buf); msg.msg_iov = &iov; msg.msg_iovlen = 1; ATF_REQUIRE(recvmsg(fd, &msg, 0) == -1); if (expected_error != 0) ATF_REQUIRE_ERRNO(expected_error, true); } static void test_ktls_receive_corrupted_record(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len, ssize_t offset) { char *plaintext, *outbuf; size_t outbuf_cap, outbuf_len; ssize_t rv; int sockets[2]; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); plaintext = alloc_buffer(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); outbuf_len = encrypt_tls_record(tc, en, TLS_RLTYPE_APP, seqno, plaintext, len, outbuf, outbuf_cap, 0); /* A negative offset is an offset from the end. */ if (offset < 0) offset += outbuf_len; outbuf[offset] ^= 0x01; rv = write(sockets[1], outbuf, outbuf_len); ATF_REQUIRE_INTEQ((ssize_t)outbuf_len, rv); ktls_receive_tls_error(sockets[0], EBADMSG); free(outbuf); free(plaintext); close_sockets_ignore_errors(sockets); } static void test_ktls_receive_corrupted_iv(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { ATF_REQUIRE(tls_header_len(en) > sizeof(struct tls_record_layer)); /* Corrupt the first byte of the explicit IV after the header. */ test_ktls_receive_corrupted_record(tc, en, seqno, len, sizeof(struct tls_record_layer)); } static void test_ktls_receive_corrupted_data(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { ATF_REQUIRE(len > 0); /* Corrupt the first ciphertext byte after the header. */ test_ktls_receive_corrupted_record(tc, en, seqno, len, tls_header_len(en)); } static void test_ktls_receive_corrupted_mac(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { size_t offset; /* Corrupt the first byte of the MAC. */ if (en->cipher_algorithm == CRYPTO_AES_CBC) offset = tls_header_len(en) + len; else offset = -tls_mac_len(en); test_ktls_receive_corrupted_record(tc, en, seqno, len, offset); } static void test_ktls_receive_corrupted_padding(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { ATF_REQUIRE_INTEQ(CRYPTO_AES_CBC, en->cipher_algorithm); /* Corrupt the last byte of the padding. */ test_ktls_receive_corrupted_record(tc, en, seqno, len, -1); } static void test_ktls_receive_truncated_record(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { char *plaintext, *outbuf; size_t outbuf_cap, outbuf_len; ssize_t rv; int sockets[2]; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); plaintext = alloc_buffer(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); outbuf_len = encrypt_tls_record(tc, en, TLS_RLTYPE_APP, seqno, plaintext, len, outbuf, outbuf_cap, 0); rv = write(sockets[1], outbuf, outbuf_len / 2); ATF_REQUIRE_INTEQ((ssize_t)(outbuf_len / 2), rv); ATF_REQUIRE(shutdown(sockets[1], SHUT_WR) == 0); ktls_receive_tls_error(sockets[0], EMSGSIZE); free(outbuf); free(plaintext); close_sockets_ignore_errors(sockets); } static void test_ktls_receive_bad_major(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { struct tls_record_layer *hdr; char *plaintext, *outbuf; size_t outbuf_cap, outbuf_len; ssize_t rv; int sockets[2]; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); plaintext = alloc_buffer(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); outbuf_len = encrypt_tls_record(tc, en, TLS_RLTYPE_APP, seqno, plaintext, len, outbuf, outbuf_cap, 0); hdr = (void *)outbuf; hdr->tls_vmajor++; rv = write(sockets[1], outbuf, outbuf_len); ATF_REQUIRE_INTEQ((ssize_t)outbuf_len, rv); ktls_receive_tls_error(sockets[0], EINVAL); free(outbuf); free(plaintext); close_sockets_ignore_errors(sockets); } static void test_ktls_receive_bad_minor(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { struct tls_record_layer *hdr; char *plaintext, *outbuf; size_t outbuf_cap, outbuf_len; ssize_t rv; int sockets[2]; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); plaintext = alloc_buffer(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); outbuf_len = encrypt_tls_record(tc, en, TLS_RLTYPE_APP, seqno, plaintext, len, outbuf, outbuf_cap, 0); hdr = (void *)outbuf; hdr->tls_vminor++; rv = write(sockets[1], outbuf, outbuf_len); ATF_REQUIRE_INTEQ((ssize_t)outbuf_len, rv); ktls_receive_tls_error(sockets[0], EINVAL); free(outbuf); free(plaintext); close_sockets_ignore_errors(sockets); } static void test_ktls_receive_bad_type(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { struct tls_record_layer *hdr; char *plaintext, *outbuf; size_t outbuf_cap, outbuf_len; ssize_t rv; int sockets[2]; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); ATF_REQUIRE_INTEQ(TLS_MINOR_VER_THREE, en->tls_vminor); plaintext = alloc_buffer(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); outbuf_len = encrypt_tls_record(tc, en, 0x21 /* Alert */, seqno, plaintext, len, outbuf, outbuf_cap, 0); hdr = (void *)outbuf; hdr->tls_type = TLS_RLTYPE_APP + 1; rv = write(sockets[1], outbuf, outbuf_len); ATF_REQUIRE_INTEQ((ssize_t)outbuf_len, rv); ktls_receive_tls_error(sockets[0], EINVAL); free(outbuf); free(plaintext); close_sockets_ignore_errors(sockets); } static void test_ktls_receive_bad_size(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len) { struct tls_record_layer *hdr; char *outbuf; size_t outbuf_len; ssize_t rv; int sockets[2]; outbuf_len = sizeof(*hdr) + len; outbuf = calloc(1, outbuf_len); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); hdr = (void *)outbuf; hdr->tls_vmajor = en->tls_vmajor; if (en->tls_vminor == TLS_MINOR_VER_THREE) hdr->tls_vminor = TLS_MINOR_VER_TWO; else hdr->tls_vminor = en->tls_vminor; hdr->tls_type = TLS_RLTYPE_APP; hdr->tls_length = htons(len); rv = write(sockets[1], outbuf, outbuf_len); ATF_REQUIRE_INTEQ((ssize_t)outbuf_len, rv); /* * The other end may notice the error and drop the connection * before this executes resulting in shutdown() failing with * either ENOTCONN or ECONNRESET. Ignore this error if it * occurs. */ if (shutdown(sockets[1], SHUT_WR) != 0) { ATF_REQUIRE_MSG(errno == ENOTCONN || errno == ECONNRESET, "shutdown() failed: %s", strerror(errno)); } ktls_receive_tls_error(sockets[0], EMSGSIZE); free(outbuf); close_sockets_ignore_errors(sockets); } static void test_ktls_receive_split_record(const atf_tc_t *tc, struct tls_enable *en, uint64_t seqno, size_t len, size_t first_len) { char *plaintext, *received, *outbuf; size_t outbuf_cap, outbuf_len; ssize_t rv; int sockets[2]; ATF_REQUIRE(len <= TLS_MAX_MSG_SIZE_V10_2); plaintext = alloc_buffer(len); received = malloc(len); outbuf_cap = tls_header_len(en) + len + tls_trailer_len(en); outbuf = malloc(outbuf_cap); ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == 0); check_tls_mode(tc, sockets[0], TCP_RXTLS_MODE); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); outbuf_len = encrypt_tls_record(tc, en, TLS_RLTYPE_APP, seqno, plaintext, len, outbuf, outbuf_cap, 0); ATF_REQUIRE(first_len < outbuf_len); tcp_nodelay(sockets[1]); rv = write(sockets[1], outbuf, first_len); ATF_REQUIRE_INTEQ((ssize_t)(first_len), rv); rv = write(sockets[1], outbuf + first_len, outbuf_len - first_len); ATF_REQUIRE_INTEQ((ssize_t)(outbuf_len - first_len), rv); rv = ktls_receive_tls_record(en, sockets[0], TLS_RLTYPE_APP, received, len); ATF_REQUIRE_INTEQ((ssize_t)len, rv); ATF_REQUIRE(memcmp(plaintext, received, len) == 0); free(outbuf); free(received); free(plaintext); close_sockets(sockets); } #define TLS_10_TESTS(M) \ M(aes128_cbc_1_0_sha1, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_ZERO) \ M(aes256_cbc_1_0_sha1, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_ZERO) #define TLS_13_TESTS(M) \ M(aes128_gcm_1_3, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_THREE) \ M(aes256_gcm_1_3, CRYPTO_AES_NIST_GCM_16, 256 / 8, 0, \ TLS_MINOR_VER_THREE) \ M(chacha20_poly1305_1_3, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ TLS_MINOR_VER_THREE) #define AES_CBC_NONZERO_TESTS(M) \ M(aes128_cbc_1_1_sha1, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_ONE) \ M(aes256_cbc_1_1_sha1, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_ONE) \ M(aes128_cbc_1_2_sha1, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_TWO) \ M(aes256_cbc_1_2_sha1, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_TWO) \ M(aes128_cbc_1_2_sha256, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_256_HMAC, TLS_MINOR_VER_TWO) \ M(aes256_cbc_1_2_sha256, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA2_256_HMAC, TLS_MINOR_VER_TWO) \ M(aes128_cbc_1_2_sha384, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_384_HMAC, TLS_MINOR_VER_TWO) \ M(aes256_cbc_1_2_sha384, CRYPTO_AES_CBC, 256 / 8, \ CRYPTO_SHA2_384_HMAC, TLS_MINOR_VER_TWO) \ #define AES_CBC_TESTS(M) \ TLS_10_TESTS(M) \ AES_CBC_NONZERO_TESTS(M) #define AES_GCM_12_TESTS(M) \ M(aes128_gcm_1_2, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_TWO) \ M(aes256_gcm_1_2, CRYPTO_AES_NIST_GCM_16, 256 / 8, 0, \ TLS_MINOR_VER_TWO) #define AES_GCM_TESTS(M) \ AES_GCM_12_TESTS(M) \ M(aes128_gcm_1_3, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_THREE) \ M(aes256_gcm_1_3, CRYPTO_AES_NIST_GCM_16, 256 / 8, 0, \ TLS_MINOR_VER_THREE) #define CHACHA20_TESTS(M) \ M(chacha20_poly1305_1_2, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ TLS_MINOR_VER_TWO) \ M(chacha20_poly1305_1_3, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ TLS_MINOR_VER_THREE) #define GEN_TRANSMIT_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name, len) \ ATF_TC_WITHOUT_HEAD(ktls_transmit_##cipher_name##_##name); \ ATF_TC_BODY(ktls_transmit_##cipher_name##_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ ATF_REQUIRE_KTLS(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_transmit_app_data(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_TRANSMIT_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name) \ ATF_TP_ADD_TC(tp, ktls_transmit_##cipher_name##_##name); #define GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name, type, len) \ ATF_TC_WITHOUT_HEAD(ktls_transmit_##cipher_name##_##name); \ ATF_TC_BODY(ktls_transmit_##cipher_name##_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ ATF_REQUIRE_KTLS(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_transmit_control(tc, &en, seqno, type, len); \ free_tls_enable(&en); \ } #define ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name) \ ATF_TP_ADD_TC(tp, ktls_transmit_##cipher_name##_##name); #define GEN_TRANSMIT_EMPTY_FRAGMENT_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ ATF_TC_WITHOUT_HEAD(ktls_transmit_##cipher_name##_empty_fragment); \ ATF_TC_BODY(ktls_transmit_##cipher_name##_empty_fragment, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ ATF_REQUIRE_KTLS(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_transmit_empty_fragment(tc, &en, seqno); \ free_tls_enable(&en); \ } #define ADD_TRANSMIT_EMPTY_FRAGMENT_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_transmit_##cipher_name##_empty_fragment); #define GEN_TRANSMIT_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ GEN_TRANSMIT_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short, 64) \ GEN_TRANSMIT_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, long, 64 * 1024) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, control, 0x21 /* Alert */, 32) #define ADD_TRANSMIT_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ ADD_TRANSMIT_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short) \ ADD_TRANSMIT_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, long) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, control) /* * For each supported cipher suite, run three transmit tests: * * - a short test which sends 64 bytes of application data (likely as * a single TLS record) * * - a long test which sends 64KB of application data (split across * multiple TLS records) * * - a control test which sends a single record with a specific * content type via sendmsg() */ AES_CBC_TESTS(GEN_TRANSMIT_TESTS); AES_GCM_TESTS(GEN_TRANSMIT_TESTS); CHACHA20_TESTS(GEN_TRANSMIT_TESTS); #define GEN_TRANSMIT_PADDING_TESTS(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_1, 0x21 /* Alert */, 1) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_2, 0x21 /* Alert */, 2) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_3, 0x21 /* Alert */, 3) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_4, 0x21 /* Alert */, 4) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_5, 0x21 /* Alert */, 5) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_6, 0x21 /* Alert */, 6) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_7, 0x21 /* Alert */, 7) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_8, 0x21 /* Alert */, 8) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_9, 0x21 /* Alert */, 9) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_10, 0x21 /* Alert */, 10) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_11, 0x21 /* Alert */, 11) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_12, 0x21 /* Alert */, 12) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_13, 0x21 /* Alert */, 13) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_14, 0x21 /* Alert */, 14) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_15, 0x21 /* Alert */, 15) \ GEN_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_16, 0x21 /* Alert */, 16) #define ADD_TRANSMIT_PADDING_TESTS(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_1) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_2) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_3) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_4) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_5) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_6) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_7) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_8) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_9) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_10) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_11) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_12) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_13) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_14) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_15) \ ADD_TRANSMIT_CONTROL_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_16) /* * For AES-CBC MTE cipher suites using padding, add tests of messages * with each possible padding size. Note that the padding_ tests * do not necessarily test bytes of padding as the padding is a * function of the cipher suite's MAC length. However, cycling * through all of the payload sizes from 1 to 16 should exercise all * of the possible padding lengths for each suite. */ AES_CBC_TESTS(GEN_TRANSMIT_PADDING_TESTS); /* * Test "empty fragments" which are TLS records with no payload that * OpenSSL can send for TLS 1.0 connections. */ AES_CBC_TESTS(GEN_TRANSMIT_EMPTY_FRAGMENT_TEST); AES_GCM_TESTS(GEN_TRANSMIT_EMPTY_FRAGMENT_TEST); CHACHA20_TESTS(GEN_TRANSMIT_EMPTY_FRAGMENT_TEST); static void test_ktls_invalid_transmit_cipher_suite(const atf_tc_t *tc, struct tls_enable *en) { int sockets[2]; ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE_ERRNO(EINVAL, setsockopt(sockets[1], IPPROTO_TCP, TCP_TXTLS_ENABLE, en, sizeof(*en)) == -1); close_sockets(sockets); } #define GEN_INVALID_TRANSMIT_TEST(name, cipher_alg, key_size, auth_alg, \ minor) \ ATF_TC_WITHOUT_HEAD(ktls_transmit_invalid_##name); \ ATF_TC_BODY(ktls_transmit_invalid_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ ATF_REQUIRE_KTLS(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_invalid_transmit_cipher_suite(tc, &en); \ free_tls_enable(&en); \ } #define ADD_INVALID_TRANSMIT_TEST(name, cipher_alg, key_size, auth_alg, \ minor) \ ATF_TP_ADD_TC(tp, ktls_transmit_invalid_##name); #define INVALID_CIPHER_SUITES(M) \ M(aes128_cbc_1_0_sha256, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_256_HMAC, TLS_MINOR_VER_ZERO) \ M(aes128_cbc_1_0_sha384, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_384_HMAC, TLS_MINOR_VER_ZERO) \ M(aes128_gcm_1_0, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_ZERO) \ M(chacha20_poly1305_1_0, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ TLS_MINOR_VER_ZERO) \ M(aes128_cbc_1_1_sha256, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_256_HMAC, TLS_MINOR_VER_ONE) \ M(aes128_cbc_1_1_sha384, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_384_HMAC, TLS_MINOR_VER_ONE) \ M(aes128_gcm_1_1, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, \ TLS_MINOR_VER_ONE) \ M(chacha20_poly1305_1_1, CRYPTO_CHACHA20_POLY1305, 256 / 8, 0, \ TLS_MINOR_VER_ONE) \ M(aes128_cbc_1_3_sha1, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA1_HMAC, TLS_MINOR_VER_THREE) \ M(aes128_cbc_1_3_sha256, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_256_HMAC, TLS_MINOR_VER_THREE) \ M(aes128_cbc_1_3_sha384, CRYPTO_AES_CBC, 128 / 8, \ CRYPTO_SHA2_384_HMAC, TLS_MINOR_VER_THREE) /* * Ensure that invalid cipher suites are rejected for transmit. */ INVALID_CIPHER_SUITES(GEN_INVALID_TRANSMIT_TEST); #define GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name, len, padding) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_##name); \ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_app_data(tc, &en, seqno, len, padding); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_##name); #define GEN_RECEIVE_BAD_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_data); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_data, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_corrupted_data(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_data); #define GEN_RECEIVE_BAD_MAC_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_mac); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_mac, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_corrupted_mac(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_MAC_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_mac); #define GEN_RECEIVE_TRUNCATED_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_truncated_record); \ ATF_TC_BODY(ktls_receive_##cipher_name##_truncated_record, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_truncated_record(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_TRUNCATED_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_truncated_record); #define GEN_RECEIVE_BAD_MAJOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_major); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_major, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_bad_major(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_MAJOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_major); #define GEN_RECEIVE_BAD_MINOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_minor); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_minor, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_bad_minor(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_MINOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_minor); #define GEN_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_##name); \ ATF_TC_BODY(ktls_receive_##cipher_name##_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_bad_size(tc, &en, seqno, (len)); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, name) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_##name); #define GEN_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, name, len, first_len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_split_##name); \ ATF_TC_BODY(ktls_receive_##cipher_name##_split_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_split_record(tc, &en, seqno, len, first_len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, name) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_split_##name); #define GEN_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short, 64, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, long, 64 * 1024, 0) \ GEN_RECEIVE_BAD_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) \ GEN_RECEIVE_BAD_MAC_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) \ GEN_RECEIVE_TRUNCATED_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) \ GEN_RECEIVE_BAD_MAJOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) \ GEN_RECEIVE_BAD_MINOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) \ GEN_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, small_record, \ tls_minimum_record_payload(&en) - 1) \ GEN_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, oversized_record, \ TLS_MAX_MSG_SIZE_V10_2 * 2) \ GEN_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, header, 64, \ sizeof(struct tls_record_layer)); \ GEN_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, full_header, 64, \ tls_header_len(&en)); \ GEN_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, half, 64, \ tls_header_len(&en) + 32); \ GEN_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, trailer_start, 64, \ tls_header_len(&en) + 64); \ GEN_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, trailer_middle, 64, \ tls_header_len(&en) + 64 + tls_trailer_len(&en) / 2); #define ADD_RECEIVE_TESTS(cipher_name, cipher_alg, key_size, auth_alg, \ minor) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, long) \ ADD_RECEIVE_BAD_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_BAD_MAC_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_TRUNCATED_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_BAD_MAJOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_BAD_MINOR_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, small_record) \ ADD_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, oversized_record) \ ADD_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, header) \ ADD_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, full_header) \ ADD_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, half) \ ADD_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, trailer_start) \ ADD_RECEIVE_SPLIT_RECORD_TEST(cipher_name, cipher_alg, \ key_size, auth_alg, minor, trailer_middle) \ /* * For each supported cipher suite, run several receive tests: * * - a short test which sends 64 bytes of application data (likely as * a single TLS record) * * - a long test which sends 64KB of application data (split across * multiple TLS records) * * - a test with corrupted payload data in a single TLS record * * - a test with a corrupted MAC in a single TLS record * * - a test with a truncated TLS record * * - tests with invalid TLS major and minor versions * * - a tests with a record whose is one less than the smallest valid * size * * - a test with an oversized TLS record * * - tests of a single record whose data is split across two writes, * with each test using a different split point */ AES_CBC_NONZERO_TESTS(GEN_RECEIVE_TESTS); AES_GCM_TESTS(GEN_RECEIVE_TESTS); CHACHA20_TESTS(GEN_RECEIVE_TESTS); #define GEN_RECEIVE_MTE_PADDING_TESTS(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_1, 1, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_2, 2, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_3, 3, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_4, 4, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_5, 5, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_6, 6, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_7, 7, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_8, 8, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_9, 9, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_10, 10, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_11, 11, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_12, 12, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_13, 13, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_14, 14, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_15, 15, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_16, 16, 0) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_16_extra, 16, 16) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_32_extra, 16, 32) #define ADD_RECEIVE_MTE_PADDING_TESTS(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_1) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_2) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_3) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_4) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_5) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_6) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_7) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_8) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_9) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_10) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_11) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_12) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_13) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_14) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_15) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_16) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_16_extra) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, padding_32_extra) #define GEN_RECEIVE_BAD_PADDING_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_padding); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_padding, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_corrupted_padding(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_PADDING_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_padding); #define GEN_RECEIVE_MTE_TESTS(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ GEN_RECEIVE_MTE_PADDING_TESTS(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ GEN_RECEIVE_BAD_PADDING_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) \ GEN_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, non_block_size, \ tls_minimum_record_payload(&en) + 1) #define ADD_RECEIVE_MTE_TESTS(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_MTE_PADDING_TESTS(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ ADD_RECEIVE_BAD_PADDING_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, non_block_size) /* * For AES-CBC MTE cipher suites using padding, add tests of messages * with each possible padding size. Note that the padding_ tests * do not necessarily test bytes of padding as the padding is a * function of the cipher suite's MAC length. However, cycling * through all of the payload sizes from 1 to 16 should exercise all * of the possible padding lengths for each suite. * * Two additional tests check for additional padding with an extra * 16 or 32 bytes beyond the normal padding. * * Another test checks for corrupted padding. * * Another test checks for a record whose payload is not a multiple of * the AES block size. */ AES_CBC_NONZERO_TESTS(GEN_RECEIVE_MTE_TESTS); #define GEN_RECEIVE_BAD_IV_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_iv); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_iv, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_corrupted_iv(tc, &en, seqno, 64); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_IV_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_iv); #define GEN_RECEIVE_EXPLICIT_IV_TESTS(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ GEN_RECEIVE_BAD_IV_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ GEN_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short_header, \ sizeof(struct tls_record_layer) + 1) #define ADD_RECEIVE_EXPLICIT_IV_TESTS(cipher_name, cipher_alg, \ key_size, auth_alg, minor) \ ADD_RECEIVE_BAD_IV_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_BAD_SIZE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short_header) /* * For cipher suites with an explicit IV, run a receive test where the * explicit IV has been corrupted. Also run a receive test that sends * a short record without a complete IV. */ AES_CBC_NONZERO_TESTS(GEN_RECEIVE_EXPLICIT_IV_TESTS); AES_GCM_12_TESTS(GEN_RECEIVE_EXPLICIT_IV_TESTS); #define GEN_RECEIVE_BAD_TYPE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, len) \ ATF_TC_WITHOUT_HEAD(ktls_receive_##cipher_name##_bad_type); \ ATF_TC_BODY(ktls_receive_##cipher_name##_bad_type, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_receive_bad_type(tc, &en, seqno, len); \ free_tls_enable(&en); \ } #define ADD_RECEIVE_BAD_TYPE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_##cipher_name##_bad_type); #define GEN_RECEIVE_TLS13_TESTS(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short_padded, 64, 16) \ GEN_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, long_padded, 64 * 1024, 15) \ GEN_RECEIVE_BAD_TYPE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, 64) #define ADD_RECEIVE_TLS13_TESTS(cipher_name, cipher_alg, key_size, \ auth_alg, minor) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, short_padded) \ ADD_RECEIVE_APP_DATA_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor, long_padded) \ ADD_RECEIVE_BAD_TYPE_TEST(cipher_name, cipher_alg, key_size, \ auth_alg, minor) /* * For TLS 1.3 cipher suites, run two additional receive tests which * use add padding to each record. Also run a test that uses an * invalid "outer" record type. */ TLS_13_TESTS(GEN_RECEIVE_TLS13_TESTS); static void test_ktls_invalid_receive_cipher_suite(const atf_tc_t *tc, struct tls_enable *en) { int sockets[2]; ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE_ERRNO(EINVAL, setsockopt(sockets[1], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == -1); close_sockets(sockets); } #define GEN_INVALID_RECEIVE_TEST(name, cipher_alg, key_size, auth_alg, \ minor) \ ATF_TC_WITHOUT_HEAD(ktls_receive_invalid_##name); \ ATF_TC_BODY(ktls_receive_invalid_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_invalid_receive_cipher_suite(tc, &en); \ free_tls_enable(&en); \ } #define ADD_INVALID_RECEIVE_TEST(name, cipher_alg, key_size, auth_alg, \ minor) \ ATF_TP_ADD_TC(tp, ktls_receive_invalid_##name); /* * Ensure that invalid cipher suites are rejected for receive. */ INVALID_CIPHER_SUITES(GEN_INVALID_RECEIVE_TEST); static void test_ktls_unsupported_receive_cipher_suite(const atf_tc_t *tc, struct tls_enable *en) { int sockets[2]; ATF_REQUIRE_MSG(open_sockets(tc, sockets), "failed to create sockets"); ATF_REQUIRE_ERRNO(EPROTONOSUPPORT, setsockopt(sockets[1], IPPROTO_TCP, TCP_RXTLS_ENABLE, en, sizeof(*en)) == -1); close_sockets(sockets); } #define GEN_UNSUPPORTED_RECEIVE_TEST(name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TC_WITHOUT_HEAD(ktls_receive_unsupported_##name); \ ATF_TC_BODY(ktls_receive_unsupported_##name, tc) \ { \ struct tls_enable en; \ uint64_t seqno; \ \ - ATF_REQUIRE_KTLS(); \ + ATF_REQUIRE_KTLS_RX(); \ seqno = random(); \ build_tls_enable(tc, cipher_alg, key_size, auth_alg, minor, \ seqno, &en); \ test_ktls_unsupported_receive_cipher_suite(tc, &en); \ free_tls_enable(&en); \ } #define ADD_UNSUPPORTED_RECEIVE_TEST(name, cipher_alg, key_size, \ auth_alg, minor) \ ATF_TP_ADD_TC(tp, ktls_receive_unsupported_##name); /* * Ensure that valid cipher suites not supported for receive are * rejected. */ TLS_10_TESTS(GEN_UNSUPPORTED_RECEIVE_TEST); /* * Try to perform an invalid sendto(2) on a TXTLS-enabled socket, to exercise * KTLS error handling in the socket layer. */ ATF_TC_WITHOUT_HEAD(ktls_sendto_baddst); ATF_TC_BODY(ktls_sendto_baddst, tc) { char buf[32]; struct sockaddr_in dst; struct tls_enable en; ssize_t n; int s; ATF_REQUIRE_KTLS(); s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); ATF_REQUIRE(s >= 0); build_tls_enable(tc, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, TLS_MINOR_VER_THREE, (uint64_t)random(), &en); ATF_REQUIRE(setsockopt(s, IPPROTO_TCP, TCP_TXTLS_ENABLE, &en, sizeof(en)) == 0); memset(&dst, 0, sizeof(dst)); dst.sin_family = AF_INET; dst.sin_len = sizeof(dst); dst.sin_addr.s_addr = htonl(INADDR_BROADCAST); dst.sin_port = htons(12345); memset(buf, 0, sizeof(buf)); n = sendto(s, buf, sizeof(buf), 0, (struct sockaddr *)&dst, sizeof(dst)); /* Can't transmit to the broadcast address over TCP. */ ATF_REQUIRE_ERRNO(EACCES, n == -1); ATF_REQUIRE(close(s) == 0); } /* * Make sure that listen(2) returns an error for KTLS-enabled sockets, and * verify that an attempt to enable KTLS on a listening socket fails. */ -ATF_TC_WITHOUT_HEAD(ktls_listening_socket); -ATF_TC_BODY(ktls_listening_socket, tc) +static void +ktls_listening_socket(const atf_tc_t *tc, int optname) { struct tls_enable en; struct sockaddr_in sin; int s; - ATF_REQUIRE_KTLS(); - - s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - ATF_REQUIRE(s >= 0); - build_tls_enable(tc, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, - TLS_MINOR_VER_THREE, (uint64_t)random(), &en); - ATF_REQUIRE(setsockopt(s, IPPROTO_TCP, TCP_TXTLS_ENABLE, &en, - sizeof(en)) == 0); - ATF_REQUIRE_ERRNO(EINVAL, listen(s, 1) == -1); - ATF_REQUIRE(close(s) == 0); - s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); ATF_REQUIRE(s >= 0); build_tls_enable(tc, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, TLS_MINOR_VER_THREE, (uint64_t)random(), &en); - ATF_REQUIRE(setsockopt(s, IPPROTO_TCP, TCP_RXTLS_ENABLE, &en, + ATF_REQUIRE(setsockopt(s, IPPROTO_TCP, optname, &en, sizeof(en)) == 0); ATF_REQUIRE_ERRNO(EINVAL, listen(s, 1) == -1); ATF_REQUIRE(close(s) == 0); s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); ATF_REQUIRE(s >= 0); memset(&sin, 0, sizeof(sin)); sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); ATF_REQUIRE(bind(s, (struct sockaddr *)&sin, sizeof(sin)) == 0); ATF_REQUIRE(listen(s, 1) == 0); build_tls_enable(tc, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, TLS_MINOR_VER_THREE, (uint64_t)random(), &en); ATF_REQUIRE_ERRNO(ENOTCONN, - setsockopt(s, IPPROTO_TCP, TCP_TXTLS_ENABLE, &en, sizeof(en)) != 0); - ATF_REQUIRE_ERRNO(ENOTCONN, - setsockopt(s, IPPROTO_TCP, TCP_RXTLS_ENABLE, &en, sizeof(en)) != 0); + setsockopt(s, IPPROTO_TCP, optname, &en, sizeof(en)) != 0); ATF_REQUIRE(close(s) == 0); } +ATF_TC_WITHOUT_HEAD(ktls_listening_socket_tx); +ATF_TC_BODY(ktls_listening_socket_tx, tc) +{ + ATF_REQUIRE_KTLS(); + + ktls_listening_socket(tc, TCP_TXTLS_ENABLE); +} + +ATF_TC_WITHOUT_HEAD(ktls_listening_socket_rx); +ATF_TC_BODY(ktls_listening_socket_rx, tc) +{ + ATF_REQUIRE_KTLS_RX(); + + ktls_listening_socket(tc, TCP_RXTLS_ENABLE); +} + /* * Verify that the KTLS receive path does not overwrite data belonging * to a file whose payload is transmitted over a loopback connection * via plain sendfile. */ ATF_TC_WITHOUT_HEAD(ktls_receive_loopback_sendfile); ATF_TC_BODY(ktls_receive_loopback_sendfile, tc) { struct tls_enable en; struct msghdr msg; struct sf_hdtr hdtr; struct iovec iov[2]; uint64_t seqno; off_t sbytes; char cbuf[CMSG_SPACE(sizeof(struct tls_get_record))]; char *plaintext, *ciphertext, *outbuf; void *p; const size_t payload_len = PAGE_SIZE; ssize_t rv; size_t len; int mode, shm, sockets[2]; socklen_t slen; - ATF_REQUIRE_KTLS(); + ATF_REQUIRE_KTLS_RX(); seqno = random(); build_tls_enable(tc, CRYPTO_AES_NIST_GCM_16, 128 / 8, 0, TLS_MINOR_VER_TWO, seqno, &en); len = tls_header_len(&en) + payload_len + tls_trailer_len(&en); plaintext = alloc_buffer(payload_len); ciphertext = malloc(len); ATF_REQUIRE_INTEQ(len, encrypt_tls_record(tc, &en, TLS_RLTYPE_APP, seqno, plaintext, payload_len, ciphertext, len, 0)); ATF_REQUIRE((shm = shm_open(SHM_ANON, O_RDWR, 0600)) > 0); ATF_REQUIRE_INTEQ(0, ftruncate(shm, payload_len)); ATF_REQUIRE((p = mmap(NULL, payload_len, PROT_READ | PROT_WRITE, MAP_SHARED, shm, 0)) != MAP_FAILED); memcpy(p, ciphertext + tls_header_len(&en), payload_len); ATF_REQUIRE_MSG(socketpair_tcp(sockets), "failed to create sockets"); ATF_REQUIRE(setsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_ENABLE, &en, sizeof(en)) == 0); slen = sizeof(mode); ATF_REQUIRE_INTEQ(0, getsockopt(sockets[0], IPPROTO_TCP, TCP_RXTLS_MODE, &mode, &slen)); ATF_REQUIRE_INTEQ(TCP_TLS_MODE_SW, mode); fd_set_blocking(sockets[0]); fd_set_blocking(sockets[1]); iov[0].iov_base = ciphertext; iov[0].iov_len = tls_header_len(&en); iov[1].iov_base = ciphertext + tls_header_len(&en) + payload_len; iov[1].iov_len = tls_trailer_len(&en); hdtr.headers = iov; hdtr.hdr_cnt = 1; hdtr.trailers = iov + 1; hdtr.trl_cnt = 1; debug_hexdump(tc, p, payload_len, "shm buffer before"); ATF_REQUIRE_INTEQ(0, sendfile(shm, sockets[1], 0, payload_len, &hdtr, &sbytes, 0)); ATF_REQUIRE_INTEQ(sbytes, len); outbuf = calloc(payload_len, 1); memset(&msg, 0, sizeof(msg)); msg.msg_control = cbuf; msg.msg_controllen = sizeof(cbuf); iov[0].iov_base = outbuf; iov[0].iov_len = payload_len; msg.msg_iov = iov; msg.msg_iovlen = 1; rv = recvmsg(sockets[0], &msg, 0); if (rv >= 0) { ATF_REQUIRE_INTEQ(payload_len, rv); ATF_REQUIRE_INTEQ(0, memcmp(outbuf, plaintext, payload_len)); } else ATF_REQUIRE_ERRNO(EBADMSG, true); debug_hexdump(tc, p, payload_len, "shm buffer after"); ATF_REQUIRE_INTEQ(0, memcmp(p, ciphertext + tls_header_len(&en), payload_len)); close_sockets_ignore_errors(sockets); (void)close(shm); } ATF_TP_ADD_TCS(tp) { /* Transmit tests */ AES_CBC_TESTS(ADD_TRANSMIT_TESTS); AES_GCM_TESTS(ADD_TRANSMIT_TESTS); CHACHA20_TESTS(ADD_TRANSMIT_TESTS); AES_CBC_TESTS(ADD_TRANSMIT_PADDING_TESTS); AES_CBC_TESTS(ADD_TRANSMIT_EMPTY_FRAGMENT_TEST); AES_GCM_TESTS(ADD_TRANSMIT_EMPTY_FRAGMENT_TEST); CHACHA20_TESTS(ADD_TRANSMIT_EMPTY_FRAGMENT_TEST); INVALID_CIPHER_SUITES(ADD_INVALID_TRANSMIT_TEST); /* Receive tests */ TLS_10_TESTS(ADD_UNSUPPORTED_RECEIVE_TEST); AES_CBC_NONZERO_TESTS(ADD_RECEIVE_TESTS); AES_GCM_TESTS(ADD_RECEIVE_TESTS); CHACHA20_TESTS(ADD_RECEIVE_TESTS); AES_CBC_NONZERO_TESTS(ADD_RECEIVE_MTE_TESTS); AES_CBC_NONZERO_TESTS(ADD_RECEIVE_EXPLICIT_IV_TESTS); AES_GCM_12_TESTS(ADD_RECEIVE_EXPLICIT_IV_TESTS); TLS_13_TESTS(ADD_RECEIVE_TLS13_TESTS); INVALID_CIPHER_SUITES(ADD_INVALID_RECEIVE_TEST); /* Miscellaneous */ ATF_TP_ADD_TC(tp, ktls_sendto_baddst); - ATF_TP_ADD_TC(tp, ktls_listening_socket); + ATF_TP_ADD_TC(tp, ktls_listening_socket_tx); + ATF_TP_ADD_TC(tp, ktls_listening_socket_rx); ATF_TP_ADD_TC(tp, ktls_receive_loopback_sendfile); return (atf_no_error()); }