Changeset View
Standalone View
sys/opencrypto/ktls_ocf.c
Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/uio.h> | #include <sys/uio.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <opencrypto/cryptodev.h> | #include <opencrypto/cryptodev.h> | ||||
#include <opencrypto/ktls.h> | #include <opencrypto/ktls.h> | ||||
struct ocf_session { | struct ktls_ocf_session { | ||||
crypto_session_t sid; | crypto_session_t sid; | ||||
crypto_session_t mac_sid; | crypto_session_t mac_sid; | ||||
struct mtx lock; | struct mtx lock; | ||||
int mac_len; | int mac_len; | ||||
bool implicit_iv; | bool implicit_iv; | ||||
/* Only used for TLS 1.0 with the implicit IV. */ | /* Only used for TLS 1.0 with the implicit IV. */ | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bool in_progress; | bool in_progress; | ||||
uint64_t next_seqno; | uint64_t next_seqno; | ||||
#endif | #endif | ||||
char iv[AES_BLOCK_LEN]; | char iv[AES_BLOCK_LEN]; | ||||
}; | }; | ||||
struct ocf_operation { | struct ocf_operation { | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
bool done; | bool done; | ||||
}; | }; | ||||
static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); | static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS"); | ||||
SYSCTL_DECL(_kern_ipc_tls); | SYSCTL_DECL(_kern_ipc_tls); | ||||
SYSCTL_DECL(_kern_ipc_tls_stats); | SYSCTL_DECL(_kern_ipc_tls_stats); | ||||
Show All 16 Lines | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts, | ||||
CTLFLAG_RD, &ocf_tls12_gcm_crypts, | CTLFLAG_RD, &ocf_tls12_gcm_crypts, | ||||
"Total number of OCF TLS 1.2 GCM encryption operations"); | "Total number of OCF TLS 1.2 GCM encryption operations"); | ||||
static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_crypts); | static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_crypts); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_crypts, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_crypts, | ||||
CTLFLAG_RD, &ocf_tls12_chacha20_crypts, | CTLFLAG_RD, &ocf_tls12_chacha20_crypts, | ||||
"Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); | "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations"); | ||||
static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts); | static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_crypts); | ||||
jhb: Does the Mellanox NIC support hardware NIC TLS for ChaCha20-Poly1305? | |||||
Done Inline ActionsNo. hselasky: No. | |||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts, | ||||
CTLFLAG_RD, &ocf_tls13_gcm_crypts, | CTLFLAG_RD, &ocf_tls13_gcm_crypts, | ||||
"Total number of OCF TLS 1.3 GCM encryption operations"); | "Total number of OCF TLS 1.3 GCM encryption operations"); | ||||
static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_crypts); | static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_crypts); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_crypts, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_crypts, | ||||
CTLFLAG_RD, &ocf_tls13_chacha20_crypts, | CTLFLAG_RD, &ocf_tls13_chacha20_crypts, | ||||
"Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations"); | "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations"); | ||||
Show All 28 Lines | ktls_ocf_callback_async(struct cryptop *crp) | ||||
mtx_lock(&oo->os->lock); | mtx_lock(&oo->os->lock); | ||||
oo->done = true; | oo->done = true; | ||||
mtx_unlock(&oo->os->lock); | mtx_unlock(&oo->os->lock); | ||||
wakeup(oo); | wakeup(oo); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp) | ktls_ocf_dispatch(struct ktls_ocf_session *os, struct cryptop *crp) | ||||
{ | { | ||||
struct ocf_operation oo; | struct ocf_operation oo; | ||||
int error; | int error; | ||||
bool async; | bool async; | ||||
oo.os = os; | oo.os = os; | ||||
oo.done = false; | oo.done = false; | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | |||||
ktls_ocf_tls_cbc_encrypt(struct ktls_ocf_encrypt_state *state, | ktls_ocf_tls_cbc_encrypt(struct ktls_ocf_encrypt_state *state, | ||||
struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | ||||
int outiovcnt) | int outiovcnt) | ||||
{ | { | ||||
const struct tls_record_layer *hdr; | const struct tls_record_layer *hdr; | ||||
struct uio *uio; | struct uio *uio; | ||||
struct tls_mac_data *ad; | struct tls_mac_data *ad; | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
struct iovec iov[m->m_epg_npgs + 2]; | struct iovec iov[m->m_epg_npgs + 2]; | ||||
u_int pgoff; | u_int pgoff; | ||||
int i, error; | int i, error; | ||||
uint16_t tls_comp_len; | uint16_t tls_comp_len; | ||||
uint8_t pad; | uint8_t pad; | ||||
MPASS(outiovcnt + 1 <= nitems(iov)); | MPASS(outiovcnt + 1 <= nitems(iov)); | ||||
os = tls->cipher; | os = tls->ocf_session; | ||||
hdr = (const struct tls_record_layer *)m->m_epg_hdr; | hdr = (const struct tls_record_layer *)m->m_epg_hdr; | ||||
crp = &state->crp; | crp = &state->crp; | ||||
uio = &state->uio; | uio = &state->uio; | ||||
MPASS(tls->sync_dispatch); | MPASS(tls->sync_dispatch); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (os->implicit_iv) { | if (os->implicit_iv) { | ||||
mtx_lock(&os->lock); | mtx_lock(&os->lock); | ||||
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | |||||
ktls_ocf_tls12_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ktls_ocf_tls12_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ||||
struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | ||||
int outiovcnt) | int outiovcnt) | ||||
{ | { | ||||
const struct tls_record_layer *hdr; | const struct tls_record_layer *hdr; | ||||
struct uio *uio; | struct uio *uio; | ||||
struct tls_aead_data *ad; | struct tls_aead_data *ad; | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
int error; | int error; | ||||
uint16_t tls_comp_len; | uint16_t tls_comp_len; | ||||
os = tls->cipher; | os = tls->ocf_session; | ||||
hdr = (const struct tls_record_layer *)m->m_epg_hdr; | hdr = (const struct tls_record_layer *)m->m_epg_hdr; | ||||
crp = &state->crp; | crp = &state->crp; | ||||
uio = &state->uio; | uio = &state->uio; | ||||
crypto_initreq(crp, os->sid); | crypto_initreq(crp, os->sid); | ||||
/* Setup the IV. */ | /* Setup the IV. */ | ||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { | if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { | ||||
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | ktls_ocf_tls12_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ||||
} else | } else | ||||
error = ktls_ocf_dispatch_async(state, crp); | error = ktls_ocf_dispatch_async(state, crp); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls, | ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls, | ||||
const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, | const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno, | ||||
int *trailer_len) | int *trailer_len, bool verify_digest) | ||||
{ | { | ||||
struct tls_aead_data ad; | struct tls_aead_data ad; | ||||
struct cryptop crp; | struct cryptop crp; | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
struct ocf_operation oo; | struct ocf_operation oo; | ||||
int error; | int error; | ||||
uint16_t tls_comp_len; | uint16_t tls_comp_len; | ||||
os = tls->cipher; | os = tls->ocf_session; | ||||
oo.os = os; | oo.os = os; | ||||
oo.done = false; | oo.done = false; | ||||
crypto_initreq(&crp, os->sid); | crypto_initreq(&crp, os->sid); | ||||
/* Setup the IV. */ | /* Setup the IV. */ | ||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { | if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { | ||||
Show All 23 Lines | ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls, | ||||
ad.tls_length = htons(tls_comp_len); | ad.tls_length = htons(tls_comp_len); | ||||
crp.crp_aad = &ad; | crp.crp_aad = &ad; | ||||
crp.crp_aad_length = sizeof(ad); | crp.crp_aad_length = sizeof(ad); | ||||
crp.crp_payload_start = tls->params.tls_hlen; | crp.crp_payload_start = tls->params.tls_hlen; | ||||
crp.crp_payload_length = tls_comp_len; | crp.crp_payload_length = tls_comp_len; | ||||
crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; | crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length; | ||||
crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; | crp.crp_op = CRYPTO_OP_DECRYPT | | ||||
(verify_digest ? CRYPTO_OP_VERIFY_DIGEST : CRYPTO_OP_SKIP_DIGEST); | |||||
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; | crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE; | ||||
crypto_use_mbuf(&crp, m); | crypto_use_mbuf(&crp, m); | ||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) | if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) | ||||
counter_u64_add(ocf_tls12_gcm_crypts, 1); | counter_u64_add(ocf_tls12_gcm_crypts, 1); | ||||
else | else | ||||
counter_u64_add(ocf_tls12_chacha20_crypts, 1); | counter_u64_add(ocf_tls12_chacha20_crypts, 1); | ||||
error = ktls_ocf_dispatch(os, &crp); | error = ktls_ocf_dispatch(os, &crp); | ||||
crypto_destroyreq(&crp); | crypto_destroyreq(&crp); | ||||
*trailer_len = tls->params.tls_tlen; | *trailer_len = tls->params.tls_tlen; | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ||||
struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | struct ktls_session *tls, struct mbuf *m, struct iovec *outiov, | ||||
int outiovcnt) | int outiovcnt) | ||||
{ | { | ||||
const struct tls_record_layer *hdr; | const struct tls_record_layer *hdr; | ||||
struct uio *uio; | struct uio *uio; | ||||
struct tls_aead_data_13 *ad; | struct tls_aead_data_13 *ad; | ||||
struct cryptop *crp; | struct cryptop *crp; | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
char nonce[12]; | char nonce[12]; | ||||
int error; | int error; | ||||
os = tls->cipher; | os = tls->ocf_session; | ||||
hdr = (const struct tls_record_layer *)m->m_epg_hdr; | hdr = (const struct tls_record_layer *)m->m_epg_hdr; | ||||
crp = &state->crp; | crp = &state->crp; | ||||
uio = &state->uio; | uio = &state->uio; | ||||
crypto_initreq(crp, os->sid); | crypto_initreq(crp, os->sid); | ||||
/* Setup the nonce. */ | /* Setup the nonce. */ | ||||
memcpy(nonce, tls->params.iv, tls->params.iv_len); | memcpy(nonce, tls->params.iv, tls->params.iv_len); | ||||
Show All 38 Lines | ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state, | ||||
memcpy(crp->crp_iv, nonce, sizeof(nonce)); | memcpy(crp->crp_iv, nonce, sizeof(nonce)); | ||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) | if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) | ||||
counter_u64_add(ocf_tls13_gcm_crypts, 1); | counter_u64_add(ocf_tls13_gcm_crypts, 1); | ||||
else | else | ||||
counter_u64_add(ocf_tls13_chacha20_crypts, 1); | counter_u64_add(ocf_tls13_chacha20_crypts, 1); | ||||
if (outiov != NULL) | if (outiov != NULL) | ||||
counter_u64_add(ocf_separate_output, 1); | counter_u64_add(ocf_separate_output, 1); | ||||
else | else | ||||
Not Done Inline ActionsAs I said earlier, this is way more complicated than you need. I would make each kthread pre-allocate a buffer sized to one TLS record and just memset it to 0 and use AES-CTR to encrypt the zeroed buffer, then xor the relevant portion of the mbuf chain with the resulting buffer. I would probably not even bother with OCF for this but just use the 'enc_xform_aes_ctr' directly to do this (especially once the multi-block hooks review I have lands). You might even be able to re-encrypt the mbufs in place in some cases without needing the temporary buffer. As it is you are doing all the GMAC computations just to throw them away. jhb: As I said earlier, this is way more complicated than you need. I would make each kthread pre… | |||||
Done Inline ActionsOK, I'll try to re-do this part. hselasky: OK, I'll try to re-do this part. | |||||
Done Inline Actions@jhb : From what I can see the current encrypt functions are tightly bound to single / EXT PG mbufs. Can you help decouple the current decrypt functions from EXT PG mbufs only? hselasky: @jhb : From what I can see the current encrypt functions are tightly bound to single / EXT PG… | |||||
Not Done Inline ActionsIf you use enc_xform they are not. Also, I still think you should have each ktls worker thread just malloc() a single 16KB block during startup and pass a pointer down to the decrypt worker routine instead having to do malloc/frees. If you use enc_xform you can bypass OCF entirely and generally not have to touch anything at all in ktls_ocf.c. If you do use OCF you need to allocate a completely different session since it is doing AES-CTR, not AES-GCM. However, you could also just create a single session for this in uipc_ktls.c and use per-op keys instead of a session key. But the idea is that back in ktls_decrypt in the MIXED case you would simply do the encrypt zeros + XOR and then fall through to the "ENCRYPTED" case that calls the normal callback here. But in particular, you shouldn't be trying to reuse the existing routines here to encrypt, you should be writing completely new code to encrypt the block of zeroes. You want to just do the encryption (AES-CTR for AES-GCM) without the MAC. If it's a single span that is block aligned at the start (but doesn't have to be at the end) you can even construct the request (if you use OCF) to just encrypt (or decrypt, in AES-CTR it's all the same) that one span of the mbuf chain directly without even using the 16KB block. (This optimization is probably only worth doing if you know it's always a single span at the end, which I don't see how it would be otherwise unless the NIC decrypts the packet _after_ a drop that is still within the known TLS header length which would be awfully weird since you know in that case you have to undo the work anyway). In fact, I still don't understand how the case can ever be anything but that the you have decrypted the start of a TLS record and had to stop somewhere in the middle due to a drop, and the rest of the TLS record is then still encrypted (as presumably the NIC just stops decrypting everything else in the stream once a drop happens (TCP sequence mismatch) until you get back in sync again after fixing up a record). In that case you can use an AES-CTR request on the first part of the TLS record (it will always start at offset 0 in terms of the encryption) and a length just of the decrypted bits without the need for the block of zeroes. jhb: If you use enc_xform they are not. Also, I still think you should have each ktls worker thread… | |||||
counter_u64_add(ocf_inplace, 1); | counter_u64_add(ocf_inplace, 1); | ||||
if (tls->sync_dispatch) { | if (tls->sync_dispatch) { | ||||
error = ktls_ocf_dispatch(os, crp); | error = ktls_ocf_dispatch(os, crp); | ||||
crypto_destroyreq(crp); | crypto_destroyreq(crp); | ||||
} else | } else | ||||
error = ktls_ocf_dispatch_async(state, crp); | error = ktls_ocf_dispatch_async(state, crp); | ||||
return (error); | return (error); | ||||
} | } | ||||
void | void | ||||
ktls_ocf_free(struct ktls_session *tls) | ktls_ocf_free(struct ktls_session *tls) | ||||
{ | { | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
os = tls->cipher; | os = tls->ocf_session; | ||||
crypto_freesession(os->sid); | crypto_freesession(os->sid); | ||||
mtx_destroy(&os->lock); | mtx_destroy(&os->lock); | ||||
zfree(os, M_KTLS_OCF); | zfree(os, M_KTLS_OCF); | ||||
} | } | ||||
int | int | ||||
ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) | ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction) | ||||
{ | { | ||||
struct crypto_session_params csp, mac_csp; | struct crypto_session_params csp, mac_csp; | ||||
struct ocf_session *os; | struct ktls_ocf_session *os; | ||||
int error, mac_len; | int error, mac_len; | ||||
memset(&csp, 0, sizeof(csp)); | memset(&csp, 0, sizeof(csp)); | ||||
memset(&mac_csp, 0, sizeof(mac_csp)); | memset(&mac_csp, 0, sizeof(mac_csp)); | ||||
mac_csp.csp_mode = CSP_MODE_NONE; | mac_csp.csp_mode = CSP_MODE_NONE; | ||||
mac_len = 0; | mac_len = 0; | ||||
switch (tls->params.cipher_algorithm) { | switch (tls->params.cipher_algorithm) { | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | if (error) { | ||||
crypto_freesession(os->sid); | crypto_freesession(os->sid); | ||||
free(os, M_KTLS_OCF); | free(os, M_KTLS_OCF); | ||||
return (error); | return (error); | ||||
} | } | ||||
os->mac_len = mac_len; | os->mac_len = mac_len; | ||||
} | } | ||||
mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); | mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF); | ||||
tls->cipher = os; | tls->ocf_session = os; | ||||
if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 || | if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 || | ||||
tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) { | tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) { | ||||
if (direction == KTLS_TX) { | if (direction == KTLS_TX) { | ||||
if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) | if (tls->params.tls_vminor == TLS_MINOR_VER_THREE) | ||||
tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt; | tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt; | ||||
else | else | ||||
tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; | tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt; | ||||
} else { | } else { | ||||
Show All 19 Lines |
Does the Mellanox NIC support hardware NIC TLS for ChaCha20-Poly1305?