Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/uipc_ktls.c
Show First 20 Lines • Show All 193 Lines • ▼ Show 20 Lines | |||||
static COUNTER_U64_DEFINE_EARLY(ktls_sw_cbc); | static COUNTER_U64_DEFINE_EARLY(ktls_sw_cbc); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, | SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, | ||||
"Active number of software TLS sessions using AES-CBC"); | "Active number of software TLS sessions using AES-CBC"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_sw_gcm); | static COUNTER_U64_DEFINE_EARLY(ktls_sw_gcm); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, gcm, CTLFLAG_RD, &ktls_sw_gcm, | SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, gcm, CTLFLAG_RD, &ktls_sw_gcm, | ||||
"Active number of software TLS sessions using AES-GCM"); | "Active number of software TLS sessions using AES-GCM"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_sw_chacha20); | |||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, chacha20, CTLFLAG_RD, | |||||
&ktls_sw_chacha20, | |||||
"Active number of software TLS sessions using Chacha20-Poly1305"); | |||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_cbc); | static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_cbc); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, cbc, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, cbc, CTLFLAG_RD, | ||||
&ktls_ifnet_cbc, | &ktls_ifnet_cbc, | ||||
"Active number of ifnet TLS sessions using AES-CBC"); | "Active number of ifnet TLS sessions using AES-CBC"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_gcm); | static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_gcm); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, gcm, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, gcm, CTLFLAG_RD, | ||||
&ktls_ifnet_gcm, | &ktls_ifnet_gcm, | ||||
"Active number of ifnet TLS sessions using AES-GCM"); | "Active number of ifnet TLS sessions using AES-GCM"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_chacha20); | |||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, chacha20, CTLFLAG_RD, | |||||
&ktls_ifnet_chacha20, | |||||
"Active number of ifnet TLS sessions using Chacha20-Poly1305"); | |||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset); | static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset, CTLFLAG_RD, | ||||
&ktls_ifnet_reset, "TLS sessions updated to a new ifnet send tag"); | &ktls_ifnet_reset, "TLS sessions updated to a new ifnet send tag"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset_dropped); | static COUNTER_U64_DEFINE_EARLY(ktls_ifnet_reset_dropped); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset_dropped, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset_dropped, CTLFLAG_RD, | ||||
&ktls_ifnet_reset_dropped, | &ktls_ifnet_reset_dropped, | ||||
"TLS sessions dropped after failing to update ifnet send tag"); | "TLS sessions dropped after failing to update ifnet send tag"); | ||||
Show All 13 Lines | |||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD, | ||||
&ktls_toe_cbc, | &ktls_toe_cbc, | ||||
"Active number of TOE TLS sessions using AES-CBC"); | "Active number of TOE TLS sessions using AES-CBC"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_toe_gcm); | static COUNTER_U64_DEFINE_EARLY(ktls_toe_gcm); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD, | ||||
&ktls_toe_gcm, | &ktls_toe_gcm, | ||||
"Active number of TOE TLS sessions using AES-GCM"); | "Active number of TOE TLS sessions using AES-GCM"); | ||||
static counter_u64_t ktls_toe_chacha20; | |||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, chacha20, CTLFLAG_RD, | |||||
&ktls_toe_chacha20, | |||||
"Active number of TOE TLS sessions using Chacha20-Poly1305"); | |||||
#endif | #endif | ||||
static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); | static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); | ||||
static void ktls_cleanup(struct ktls_session *tls); | static void ktls_cleanup(struct ktls_session *tls); | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
static void ktls_reset_send_tag(void *context, int pending); | static void ktls_reset_send_tag(void *context, int pending); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 253 Lines • ▼ Show 20 Lines | case CRYPTO_SHA2_384_HMAC: | ||||
en->iv_len = 0; | en->iv_len = 0; | ||||
break; | break; | ||||
default: | default: | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
if (en->auth_key_len == 0) | if (en->auth_key_len == 0) | ||||
return (EINVAL); | return (EINVAL); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
if (en->auth_algorithm != 0 || en->auth_key_len != 0) | |||||
return (EINVAL); | |||||
if (en->tls_vminor != TLS_MINOR_VER_TWO && | |||||
en->tls_vminor != TLS_MINOR_VER_THREE) | |||||
return (EINVAL); | |||||
if (en->iv_len != TLS_CHACHA20_IV_LEN) | |||||
return (EINVAL); | |||||
break; | |||||
default: | default: | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); | tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); | ||||
counter_u64_add(ktls_offload_active, 1); | counter_u64_add(ktls_offload_active, 1); | ||||
Show All 15 Lines | #endif | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
/* | /* | ||||
* TLS 1.2 uses a 4 byte implicit IV with an explicit 8 byte | * TLS 1.2 uses a 4 byte implicit IV with an explicit 8 byte | ||||
* nonce. TLS 1.3 uses a 12 byte implicit IV. | * nonce. TLS 1.3 uses a 12 byte implicit IV. | ||||
*/ | */ | ||||
if (en->tls_vminor < TLS_MINOR_VER_THREE) | if (en->tls_vminor < TLS_MINOR_VER_THREE) | ||||
tls->params.tls_hlen += sizeof(uint64_t); | tls->params.tls_hlen += sizeof(uint64_t); | ||||
tls->params.tls_tlen = AES_GMAC_HASH_LEN; | tls->params.tls_tlen = AES_GMAC_HASH_LEN; | ||||
/* | |||||
* TLS 1.3 includes optional padding which we | |||||
* do not support, and also puts the "real" record | |||||
* type at the end of the encrypted data. | |||||
*/ | |||||
if (en->tls_vminor == TLS_MINOR_VER_THREE) | |||||
tls->params.tls_tlen += sizeof(uint8_t); | |||||
tls->params.tls_bs = 1; | tls->params.tls_bs = 1; | ||||
break; | break; | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
switch (en->auth_algorithm) { | switch (en->auth_algorithm) { | ||||
case CRYPTO_SHA1_HMAC: | case CRYPTO_SHA1_HMAC: | ||||
if (en->tls_vminor == TLS_MINOR_VER_ZERO) { | if (en->tls_vminor == TLS_MINOR_VER_ZERO) { | ||||
/* Implicit IV, no nonce. */ | /* Implicit IV, no nonce. */ | ||||
} else { | } else { | ||||
Show All 12 Lines | case CRYPTO_SHA2_384_HMAC: | ||||
tls->params.tls_tlen = AES_BLOCK_LEN + | tls->params.tls_tlen = AES_BLOCK_LEN + | ||||
SHA2_384_HASH_LEN; | SHA2_384_HASH_LEN; | ||||
break; | break; | ||||
default: | default: | ||||
panic("invalid hmac"); | panic("invalid hmac"); | ||||
} | } | ||||
tls->params.tls_bs = AES_BLOCK_LEN; | tls->params.tls_bs = AES_BLOCK_LEN; | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
/* | |||||
* Chacha20 uses a 12 byte implicit IV. | |||||
*/ | |||||
tls->params.tls_tlen = POLY1305_HASH_LEN; | |||||
tls->params.tls_bs = 1; | |||||
break; | |||||
default: | default: | ||||
panic("invalid cipher"); | panic("invalid cipher"); | ||||
} | } | ||||
/* | |||||
* TLS 1.3 includes optional padding which we do not support, | |||||
* and also puts the "real" record type at the end of the | |||||
* encrypted data. | |||||
*/ | |||||
if (en->tls_vminor == TLS_MINOR_VER_THREE) | |||||
tls->params.tls_tlen += sizeof(uint8_t); | |||||
KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN, | KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN, | ||||
("TLS header length too long: %d", tls->params.tls_hlen)); | ("TLS header length too long: %d", tls->params.tls_hlen)); | ||||
KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN, | KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN, | ||||
("TLS trailer length too long: %d", tls->params.tls_tlen)); | ("TLS trailer length too long: %d", tls->params.tls_tlen)); | ||||
if (en->auth_key_len != 0) { | if (en->auth_key_len != 0) { | ||||
tls->params.auth_key_len = en->auth_key_len; | tls->params.auth_key_len = en->auth_key_len; | ||||
tls->params.auth_key = malloc(en->auth_key_len, M_KTLS, | tls->params.auth_key = malloc(en->auth_key_len, M_KTLS, | ||||
M_WAITOK); | M_WAITOK); | ||||
error = copyin(en->auth_key, tls->params.auth_key, | error = copyin(en->auth_key, tls->params.auth_key, | ||||
en->auth_key_len); | en->auth_key_len); | ||||
if (error) | if (error) | ||||
goto out; | goto out; | ||||
} | } | ||||
tls->params.cipher_key_len = en->cipher_key_len; | tls->params.cipher_key_len = en->cipher_key_len; | ||||
tls->params.cipher_key = malloc(en->cipher_key_len, M_KTLS, M_WAITOK); | tls->params.cipher_key = malloc(en->cipher_key_len, M_KTLS, M_WAITOK); | ||||
error = copyin(en->cipher_key, tls->params.cipher_key, | error = copyin(en->cipher_key, tls->params.cipher_key, | ||||
en->cipher_key_len); | en->cipher_key_len); | ||||
if (error) | if (error) | ||||
goto out; | goto out; | ||||
/* | /* | ||||
* This holds the implicit portion of the nonce for GCM and | * This holds the implicit portion of the nonce for AEAD | ||||
* the initial implicit IV for TLS 1.0. The explicit portions | * ciphers and the initial implicit IV for TLS 1.0. The | ||||
* of the IV are generated in ktls_frame(). | * explicit portions of the IV are generated in ktls_frame(). | ||||
*/ | */ | ||||
if (en->iv_len != 0) { | if (en->iv_len != 0) { | ||||
tls->params.iv_len = en->iv_len; | tls->params.iv_len = en->iv_len; | ||||
error = copyin(en->iv, tls->params.iv, en->iv_len); | error = copyin(en->iv, tls->params.iv, en->iv_len); | ||||
if (error) | if (error) | ||||
goto out; | goto out; | ||||
/* | /* | ||||
* For TLS 1.2, generate an 8-byte nonce as a counter | * For TLS 1.2 with GCM, generate an 8-byte nonce as a | ||||
* to generate unique explicit IVs. | * counter to generate unique explicit IVs. | ||||
* | * | ||||
* Store this counter in the last 8 bytes of the IV | * Store this counter in the last 8 bytes of the IV | ||||
* array so that it is 8-byte aligned. | * array so that it is 8-byte aligned. | ||||
*/ | */ | ||||
if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16 && | if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16 && | ||||
en->tls_vminor == TLS_MINOR_VER_TWO) | en->tls_vminor == TLS_MINOR_VER_TWO) | ||||
arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0); | arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | case TCP_TLS_MODE_SW: | ||||
MPASS(tls->be != NULL); | MPASS(tls->be != NULL); | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_sw_cbc, -1); | counter_u64_add(ktls_sw_cbc, -1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_sw_gcm, -1); | counter_u64_add(ktls_sw_gcm, -1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
counter_u64_add(ktls_sw_chacha20, -1); | |||||
break; | |||||
} | } | ||||
tls->free(tls); | tls->free(tls); | ||||
break; | break; | ||||
case TCP_TLS_MODE_IFNET: | case TCP_TLS_MODE_IFNET: | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_ifnet_cbc, -1); | counter_u64_add(ktls_ifnet_cbc, -1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_ifnet_gcm, -1); | counter_u64_add(ktls_ifnet_gcm, -1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
counter_u64_add(ktls_ifnet_chacha20, -1); | |||||
break; | |||||
} | } | ||||
if (tls->snd_tag != NULL) | if (tls->snd_tag != NULL) | ||||
m_snd_tag_rele(tls->snd_tag); | m_snd_tag_rele(tls->snd_tag); | ||||
break; | break; | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
case TCP_TLS_MODE_TOE: | case TCP_TLS_MODE_TOE: | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_toe_cbc, -1); | counter_u64_add(ktls_toe_cbc, -1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_toe_gcm, -1); | counter_u64_add(ktls_toe_gcm, -1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
counter_u64_add(ktls_toe_chacha20, -1); | |||||
break; | |||||
} | } | ||||
break; | break; | ||||
#endif | #endif | ||||
} | } | ||||
if (tls->params.auth_key != NULL) { | if (tls->params.auth_key != NULL) { | ||||
zfree(tls->params.auth_key, M_KTLS); | zfree(tls->params.auth_key, M_KTLS); | ||||
tls->params.auth_key = NULL; | tls->params.auth_key = NULL; | ||||
tls->params.auth_key_len = 0; | tls->params.auth_key_len = 0; | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | if (error == 0) { | ||||
tls->mode = TCP_TLS_MODE_TOE; | tls->mode = TCP_TLS_MODE_TOE; | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_toe_cbc, 1); | counter_u64_add(ktls_toe_cbc, 1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_toe_gcm, 1); | counter_u64_add(ktls_toe_gcm, 1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
counter_u64_add(ktls_toe_chacha20, 1); | |||||
break; | |||||
} | } | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Common code used when first enabling ifnet TLS on a connection or | * Common code used when first enabling ifnet TLS on a connection or | ||||
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Lines | if (error == 0) { | ||||
tls->snd_tag = mst; | tls->snd_tag = mst; | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_ifnet_cbc, 1); | counter_u64_add(ktls_ifnet_cbc, 1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_ifnet_gcm, 1); | counter_u64_add(ktls_ifnet_gcm, 1); | ||||
break; | break; | ||||
case CRYPTO_CHACHA20_POLY1305: | |||||
counter_u64_add(ktls_ifnet_chacha20, 1); | |||||
break; | |||||
} | } | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
ktls_try_sw(struct socket *so, struct ktls_session *tls, int direction) | ktls_try_sw(struct socket *so, struct ktls_session *tls, int direction) | ||||
{ | { | ||||
Show All 26 Lines | if (be == NULL) | ||||
return (EOPNOTSUPP); | return (EOPNOTSUPP); | ||||
tls->mode = TCP_TLS_MODE_SW; | tls->mode = TCP_TLS_MODE_SW; | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
case CRYPTO_AES_CBC: | case CRYPTO_AES_CBC: | ||||
counter_u64_add(ktls_sw_cbc, 1); | counter_u64_add(ktls_sw_cbc, 1); | ||||
break; | break; | ||||
case CRYPTO_AES_NIST_GCM_16: | case CRYPTO_AES_NIST_GCM_16: | ||||
counter_u64_add(ktls_sw_gcm, 1); | counter_u64_add(ktls_sw_gcm, 1); | ||||
break; | |||||
case CRYPTO_CHACHA20_POLY1305: | |||||
counter_u64_add(ktls_sw_chacha20, 1); | |||||
break; | break; | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* KTLS RX stores data in the socket buffer as a list of TLS records, | * KTLS RX stores data in the socket buffer as a list of TLS records, | ||||
* where each record is stored as a control message containg the TLS | * where each record is stored as a control message containg the TLS | ||||
▲ Show 20 Lines • Show All 1,219 Lines • Show Last 20 Lines |